This is an automated email from the ASF dual-hosted git repository.
jimjag pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/openoffice-devtools.git
The following commit(s) were added to refs/heads/main by this push:
new 7a4cb47 translation-scripts: add update_translations.py to update
every language without Pootle
7a4cb47 is described below
commit 7a4cb47771fa34cac9be1d846a47fbac2fa4001d
Author: Jim Jagielski <[email protected]>
AuthorDate: Thu Nov 12 10:16:10 2026 -0500
translation-scripts: add update_translations.py to update every language
without Pootle
rekey now finds the template the tree is keyed from, so it can be rerun,
and the
tools split SDF lines at newlines only: some translations contain
U+2028/U+2029.
---
translation-scripts/integrate/README.md | 36 ++
translation-scripts/integrate/sdf_integrate.py | 99 ++++--
.../integrate/update_translations.py | 370 +++++++++++++++++++++
3 files changed, 478 insertions(+), 27 deletions(-)
diff --git a/translation-scripts/integrate/README.md
b/translation-scripts/integrate/README.md
index 03b38bd..c6d7af5 100644
--- a/translation-scripts/integrate/README.md
+++ b/translation-scripts/integrate/README.md
@@ -11,6 +11,42 @@ so a whole-file replacement loses them (15,441 entries for
`am` in 2026).
Needs Python 3.7+, translate-toolkit (`po2oo`) and `gsicheck` from a built
OpenOffice tree. No other dependencies.
+## All languages in one command
+
+`update_translations.py` does what the Pootle server's template update and an
+integration round do together, for every tree language or for `--langs de,fr`:
+
+ update_translations.py --aoo ~/src/openoffice # report only
+ update_translations.py --aoo ~/src/openoffice --apply # write the tree
files
+
+- English template: `../scripts/en-US.sdf`, or `--extract` to make a fresh one
+ from `--aoo` (Linux, configured and built; `localize -e` fails on macOS).
+- PO files: cloned from apache/openoffice-translation into the work directory,
+ only the languages asked for, or `--po-repo` for an existing checkout.
+- translate-toolkit: 2.2.5, the server's version, installed into a venv in the
+ work directory unless this Python already has it.
+- `gsicheck`: from `main/solver` in `--aoo`, or `--solver`.
+
+Per language it re-keys the tree file (the steps below), makes templates, runs
+`fix_msgctxt.py`, `pomerge` and `po2oo`, and gates the merge on `gsicheck`. The
+tree is matched against every committed version of `../scripts/en-US.sdf` and
+the 2022 one in openoffice-translation; if it is already keyed to the current
+template, nothing moves.
+
+`--mode add-only` (the default) fills only entries the tree has no translation
+for. `--mode update` also replaces tree text with newer PO text. Use it with
+`--history` and a few older openoffice-translation revisions, so that text from
+a Pootle rollback is not taken; this needs the repository's full history. In
+2026 the revisions `ebbd469a1c a5e2ff9195 a962caea70 4b986fdc1b 3a07d1c27e
+60e3e58b9e c1358ae8ce` held back 1,086 `uk` and 789 `pl` entries.
+
+Each run gets its own directory under `--workdir` with `report.md` (entries
+moved, updated, added and left out, and translated entries before and after),
+`langs/<lang>/` with the intermediate files and a log, `pot/`, and with
+`--export-po` the updated PO files that Pootle would offer translators. Nothing
+is committed. Languages that have PO files but no tree file are listed, not
+added. A second run on the result reports no changes.
+
## Steps, for one language
1. Convert the PO files with the English SDF they were made from:
diff --git a/translation-scripts/integrate/sdf_integrate.py
b/translation-scripts/integrate/sdf_integrate.py
index d7ce227..df525c0 100755
--- a/translation-scripts/integrate/sdf_integrate.py
+++ b/translation-scripts/integrate/sdf_integrate.py
@@ -48,13 +48,22 @@ def key(cols):
return (cols[0], cols[1], cols[3], cols[4], cols[5], cols[6], cols[7])
+def read_lines(path):
+ """Lines with their endings, split at \n only.
+
+ str.splitlines() also splits at U+2028, U+2029 and other separators, and
+ some translations contain them."""
+ with open(path, encoding="utf-8", newline="") as f:
+ parts = f.read().split("\n")
+ return [p + "\n" for p in parts[:-1]] + ([parts[-1]] if parts[-1] else [])
+
+
def load(path, lang):
entries = {}
- with open(path, encoding="utf-8") as f:
- for line in f:
- cols = line.rstrip("\r\n").split("\t")
- if len(cols) == 15 and cols[9] == lang:
- entries[key(cols)] = cols
+ for line in read_lines(path):
+ cols = line.rstrip("\r\n").split("\t")
+ if len(cols) == 15 and cols[9] == lang:
+ entries[key(cols)] = cols
return entries
@@ -96,16 +105,28 @@ def combine(lang, template, base_path, newer_path,
history_paths):
return "".join(out), stats
-def merge(lang, tree_path, new_path, template, skip=(), add_only=False):
- """Return (text, changed keys, added keys)."""
+def history_values(lang, paths):
+ values = collections.defaultdict(set)
+ for path in paths:
+ for k, cols in load(path, lang).items():
+ for i in TEXT_COLUMNS:
+ values[(k, i)].add(cols[i])
+ return values
+
+
+def merge(lang, tree_path, new_path, template, skip=(), add_only=False,
history=()):
+ """Return (text, changed keys, added keys).
+
+ A translation in the tree is not replaced by text found in a history SDF
+ for the same key, which is how a Pootle rollback shows up."""
new = load(new_path, lang)
if not new:
sys.exit("no %s lines in %s; po2oo --language must use the tree's code
(pt-BR, not pt_BR)"
% (lang, new_path))
en = load(template, "en-US")
skip = set(skip)
- with open(tree_path, encoding="utf-8", newline="") as f:
- lines = f.read().splitlines(keepends=True)
+ old = history_values(lang, history)
+ lines = read_lines(tree_path)
seen, changed, last_of_file = set(), [], {}
timestamp, eol = None, "\n"
@@ -122,7 +143,7 @@ def merge(lang, tree_path, new_path, template, skip=(),
add_only=False):
if n and k not in skip:
updated = list(cols)
for i in TEXT_COLUMNS:
- if add_only and is_translation(cols[i], e[i] if e else
None):
+ if is_translation(cols[i], e[i] if e else None) and
(add_only or n[i] in old[(k, i)]):
continue
if n[i] != cols[i] and is_translation(n[i], e[i] if e else
None):
updated[i] = n[i]
@@ -159,13 +180,27 @@ def with_key(cols, k):
return row
-def rekey(lang, tree_path, old_template, template):
+def rekey(lang, tree_path, old_templates, template):
"""Move tree entries whose key changed to the key their English has now.
- A move needs the English to be the same under both keys: first the same
- key without the help lid (oldref), else the only key in the same file with
- that English. Return (text, moves, stats)."""
- en_old, en_new = load(old_template, "en-US"), load(template, "en-US")
+ The tree is taken to be keyed from whichever of the old templates, or the
+ current one, holds most of its keys; if that is the current template,
+ nothing moves. A move needs the English to be the same under both keys:
+ first the same key without the help lid (oldref), else the only key in the
+ same file with that English. Return (text, moves, stats)."""
+ en_new = load(template, "en-US")
+ own = set(key(c) for c in (l.rstrip("\r\n").split("\t") for l in
read_lines(tree_path))
+ if len(c) == 15 and c[9] == lang)
+ best, en_old = None, en_new
+ hits = len(own & en_new.keys())
+ for path in old_templates:
+ cand = load(path, "en-US")
+ n = len(own & cand.keys())
+ if n > hits:
+ best, en_old, hits = path, cand, n
+ if best is None:
+ with open(tree_path, encoding="utf-8", newline="") as f:
+ return f.read(), {}, collections.Counter({"keyed to the current
template": len(own)})
by_file = collections.defaultdict(lambda: collections.defaultdict(list))
old_by_file = collections.defaultdict(collections.Counter)
for k, cols in en_new.items():
@@ -173,8 +208,7 @@ def rekey(lang, tree_path, old_template, template):
for k, cols in en_old.items():
old_by_file[k[:2]][english(cols)] += 1
- with open(tree_path, encoding="utf-8", newline="") as f:
- lines = f.read().splitlines(keepends=True)
+ lines = read_lines(tree_path)
# Some tree files carry an en-US line paired with the translation; gsicheck
# needs the pair to keep one key, so both lines move together.
tree_keys = []
@@ -183,7 +217,7 @@ def rekey(lang, tree_path, old_template, template):
ok = len(cols) == 15 and cols[9] in (lang, "en-US")
tree_keys.append(key(cols) if ok else None)
- stats = collections.Counter()
+ stats = collections.Counter({"keyed from " + best: hits})
plan = {}
for k in dict.fromkeys(filter(None, tree_keys)):
old, new = en_old.get(k), en_new.get(k)
@@ -278,23 +312,31 @@ def cmd_rekey(a):
def cmd_merge(a):
skip = [tuple(k) for k in json.load(open(a.skip))] if a.skip else []
- text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip,
a.add_only)
+ text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip,
a.add_only, a.history)
write(a.out, text)
print("%s: %d entries updated, %d keys added" % (a.lang, len(changed),
len(added)))
-def cmd_gate(a):
- baseline = run_gsicheck(a.gsicheck, a.libdir, a.tree, a.lang)
+def gate(lang, tree, new, template, out, gsicheck, libdir, rounds=4,
add_only=False, history=()):
+ """Merge, leaving out every entry that adds a gsicheck error.
+
+ Return (changed, added, left-out keys, new errors not tied to an entry)."""
+ baseline = run_gsicheck(gsicheck, libdir, tree, lang)
skip = []
- for _ in range(a.rounds):
- text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip,
a.add_only)
- write(a.out, text)
- new_errors = run_gsicheck(a.gsicheck, a.libdir, a.out, a.lang) -
baseline
+ for _ in range(rounds):
+ text, changed, added = merge(lang, tree, new, template, skip,
add_only, history)
+ write(out, text)
+ new_errors = run_gsicheck(gsicheck, libdir, out, lang) - baseline
keyed = {uid_to_key(uid) for _, uid in new_errors if uid} - set(skip)
if not keyed:
break
skip += sorted(keyed)
- unkeyed = [msg for msg, uid in new_errors.elements() if not uid]
+ return changed, added, skip, new_errors
+
+
+def cmd_gate(a):
+ changed, added, skip, new_errors = gate(a.lang, a.tree, a.new, a.template,
a.out, a.gsicheck, a.libdir,
+ a.rounds, a.add_only, a.history)
if a.skip_out:
with open(a.skip_out, "w", encoding="utf-8") as f:
json.dump(skip, f, ensure_ascii=False, indent=1)
@@ -323,7 +365,8 @@ def main():
r = sub.add_parser("rekey", help="move tree entries to the keys of a newer
en-US.sdf")
r.add_argument("--lang", required=True)
r.add_argument("--tree", required=True,
help="extras/l10n/source/<lang>/localize.sdf")
- r.add_argument("--old-template", required=True, help="en-US.sdf the tree
keys come from")
+ r.add_argument("--old-template", required=True, nargs="+",
+ help="en-US.sdf files the tree keys may come from; the best
match is used")
r.add_argument("--template", required=True, help="en-US.sdf extracted from
the current source")
r.add_argument("--out", required=True)
r.set_defaults(func=cmd_rekey)
@@ -338,6 +381,8 @@ def main():
m.add_argument("--out", required=True)
m.add_argument("--add-only", action="store_true",
help="fill only entries the tree has no translation
for")
+ m.add_argument("--history", nargs="*", default=[],
+ help="po2oo output from older snapshots; their text
never replaces a tree translation")
if name == "merge":
m.add_argument("--skip", help="JSON list of keys to leave as they
are")
else:
diff --git a/translation-scripts/integrate/update_translations.py
b/translation-scripts/integrate/update_translations.py
new file mode 100755
index 0000000..7078238
--- /dev/null
+++ b/translation-scripts/integrate/update_translations.py
@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+################################################################
+"""Update extras/l10n/source/*/localize.sdf from the Pootle PO files, without
Pootle.
+
+For each language this does what the Pootle server and the integration
+round do together: re-key the tree file to the current en-US.sdf, make
+templates, re-key and pomerge the PO files from apache/openoffice-translation,
+convert them with po2oo, and merge the result into the tree file under a
+gsicheck gate. Nothing is committed; --apply copies the results into the tree.
+
+See README.md for the steps it runs and how to check the result.
+"""
+
+import argparse
+import collections
+import concurrent.futures
+import datetime
+import filecmp
+import glob
+import json
+import os
+import platform
+import shutil
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import sdf_integrate as si # noqa: E402
+
+DEFAULT_TEMPLATE = os.path.join(HERE, os.pardir, "scripts", "en-US.sdf")
+PO_REPO_URL = "https://github.com/apache/openoffice-translation.git";
+TOOLKIT = "translate-toolkit==2.2.5" # what the Pootle server runs
+
+
+def log_run(cmd, log, **kw):
+ log.write("$ " + " ".join(cmd) + "\n")
+ log.flush()
+ return subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT,
**kw).returncode
+
+
+def toolkit_python(args, workdir):
+ if args.python:
+ return args.python
+ if subprocess.run([sys.executable, "-c", "import translate"],
capture_output=True).returncode == 0:
+ return sys.executable
+ env = os.path.join(workdir, "venv")
+ py = os.path.join(env, "bin", "python")
+ if not os.path.exists(py):
+ print("installing %s into %s" % (TOOLKIT, env))
+ subprocess.run([sys.executable, "-m", "venv", env], check=True)
+ subprocess.run([py, "-m", "pip", "install", "-q",
"--disable-pip-version-check", TOOLKIT], check=True)
+ return py
+
+
+def extract_template(aoo, out):
+ if platform.system() == "Darwin":
+ sys.exit("localize -e does not work on macOS; extract on a Linux build
or pass --template")
+ envs = glob.glob(os.path.join(aoo, "main", "*Env.Set.sh"))
+ if len(envs) != 1:
+ sys.exit("need exactly one main/*Env.Set.sh in a configured and built
tree, found %d" % len(envs))
+ cmd = 'cd "%s" && . "./%s" >/dev/null && localize -e -l en-US -f "%s"' % (
+ os.path.join(aoo, "main"), os.path.basename(envs[0]), out)
+ with open(out + ".log", "w") as log:
+ rc = log_run(["bash", "-c", cmd], log)
+ if rc or not os.path.getsize(out):
+ sys.exit("localize -e failed, see %s.log" % out)
+
+
+def old_templates(template, po_repo, extra, outdir):
+ """en-US.sdf files the tree may still be keyed from: every committed
version
+ of scripts/en-US.sdf, the legacy copy in openoffice-translation, and
extra."""
+ found = list(extra)
+ committed = os.path.realpath(DEFAULT_TEMPLATE)
+ top = subprocess.run(["git", "-C", os.path.dirname(committed),
"rev-parse", "--show-toplevel"],
+ capture_output=True, text=True)
+ if top.returncode == 0:
+ rel = os.path.relpath(committed, top.stdout.strip())
+ revs = subprocess.run(["git", "-C", top.stdout.strip(), "log",
"--format=%h", "--", rel],
+ capture_output=True, text=True).stdout.split()
+ for rev in revs:
+ path = os.path.join(outdir, "en-US.%s.sdf" % rev)
+ with open(path, "wb") as f:
+ subprocess.run(["git", "-C", top.stdout.strip(), "show",
"%s:%s" % (rev, rel)], stdout=f)
+ if os.path.getsize(path) and not filecmp.cmp(path, template,
shallow=False):
+ found.append(path)
+ legacy = os.path.join(po_repo, "en-US.sdf") if po_repo else None
+ if legacy and os.path.exists(legacy) and not filecmp.cmp(legacy, template,
shallow=False):
+ found.append(legacy)
+ return found
+
+
+def get_po_repo(args, workdir, langs):
+ if args.no_po:
+ return None
+ repo = args.po_repo or os.path.join(workdir, "openoffice-translation")
+ dirs = ["PO-Files/" + x.replace("-", "_") for x in langs] if args.langs
else ["PO-Files"]
+ if args.po_repo:
+ if not os.path.isdir(os.path.join(repo, "PO-Files")):
+ sys.exit("%s has no PO-Files directory" % repo)
+ elif not os.path.isdir(os.path.join(repo, ".git")):
+ print("cloning %s (%s) into %s" % (PO_REPO_URL, " ".join(dirs) if
args.langs else "PO-Files", repo))
+ depth = [] if args.history else ["--depth", "1"]
+ subprocess.run(["git", "clone", "--filter=blob:none", "--sparse"] +
depth + [PO_REPO_URL, repo], check=True)
+ subprocess.run(["git", "-C", repo, "sparse-checkout", "set"] + dirs,
check=True)
+ elif subprocess.run(["git", "-C", repo, "sparse-checkout", "list"],
capture_output=True).returncode == 0:
+ subprocess.run(["git", "-C", repo, "sparse-checkout", "add"] + dirs,
check=True)
+ if args.history and subprocess.run(["git", "-C", repo, "rev-parse",
"--is-shallow-repository"],
+ capture_output=True,
text=True).stdout.strip() == "true":
+ sys.exit("--history needs the full history of %s (git fetch
--unshallow)" % repo)
+ return repo
+
+
+def po_to_sdf(py, lang, podir, pot, template, out, log):
+ """Pootle's template update and mksdf.sh for one language: re-key,
pomerge, po2oo."""
+ fixed, merged = out + ".fixed", out + ".merged"
+ steps = [
+ [py, os.path.join(HERE, "fix_msgctxt.py"), "--template", pot,
"--input", podir, "--output", fixed],
+ [py, "-m", "translate.tools.pomerge", "--template=" + pot + "/",
"--input=" + fixed + "/",
+ "--output=" + merged + "/"],
+ [py, "-m", "translate.convert.po2oo", "--template=" + template,
"--skipsource", "--keeptimestamp",
+ "--language=" + lang, merged + "/", out],
+ ]
+ for cmd in steps:
+ if log_run(cmd, log):
+ raise RuntimeError("%s failed" % os.path.basename(cmd[1] if cmd[1]
!= "-m" else cmd[2]))
+ return merged
+
+
+def check_rekey(tree, rekeyed, old, template):
+ """Every changed line keeps its text, timestamp and line ending, and its
English."""
+ en_new, en_old = si.load(template, "en-US"), si.load(old, "en-US")
+ a, b = si.read_lines(tree), si.read_lines(rekeyed)
+ if len(a) != len(b):
+ return "line count changed"
+ for x, y in zip(a, b):
+ if x == y:
+ continue
+ p, q = x.rstrip("\r\n").split("\t"), y.rstrip("\r\n").split("\t")
+ same = [p[i] for i in (2, 8, 9, 10, 11, 12, 13, 14)] == [q[i] for i in
(2, 8, 9, 10, 11, 12, 13, 14)]
+ if not same or x[len(x.rstrip("\r\n")):] != y[len(y.rstrip("\r\n")):]:
+ return "a moved line changed outside its key: %s" % p[4]
+ if si.english(en_old.get(si.key(p), [""] * 15)) !=
si.english(en_new.get(si.key(q), ["-"] * 15)):
+ return "a line moved to a key with different English: %s -> %s" %
(p[4], q[4])
+ return None
+
+
+def usable(path, lang, en):
+ return sum(1 for k, c in si.load(path, lang).items()
+ if k in en and any(si.is_translation(c[i], en[k][i]) for i in
si.TEXT_COLUMNS))
+
+
+def do_language(lang, ctx):
+ a = ctx
+ d = os.path.join(a.run, "langs", lang)
+ os.makedirs(d)
+ tree = os.path.join(a.l10n, lang, "localize.sdf")
+ result = {"lang": lang, "status": "ok"}
+ with open(os.path.join(d, "log.txt"), "w") as log:
+ try:
+ rekeyed = os.path.join(d, "rekeyed.sdf")
+ text, plan, stats = si.rekey(lang, tree, a.olds, a.template)
+ si.write(rekeyed, text)
+ log.write("rekey: %s\n" % dict(stats))
+ chosen = next((k[len("keyed from "):] for k in stats if
k.startswith("keyed from ")), None)
+ result["keyed from"] = os.path.basename(chosen) if chosen else
"current"
+ result["moved"] = len(plan)
+ if plan:
+ problem = check_rekey(tree, rekeyed, chosen, a.template)
+ if problem:
+ raise RuntimeError("re-key check: " + problem)
+ final = rekeyed
+ podir = os.path.join(a.po_repo, "PO-Files", lang.replace("-",
"_")) if a.po_repo else None
+ if podir and os.path.isdir(podir):
+ po2oo = os.path.join(d, "po2oo.sdf")
+ merged = po_to_sdf(a.py, lang, podir, a.pot, a.template,
po2oo, log)
+ history = []
+ for rev in a.history:
+ hdir = os.path.join(d, "history", rev)
+ os.makedirs(hdir)
+ arch = subprocess.run(["git", "-C", a.po_repo, "archive",
rev,
+ "PO-Files/" + lang.replace("-",
"_")], capture_output=True)
+ if arch.returncode:
+ log.write("history %s: no PO files for %s\n" % (rev,
lang))
+ continue
+ subprocess.run(["tar", "-x", "-C", hdir],
input=arch.stdout, check=True)
+ hsdf = os.path.join(d, "history-%s.sdf" % rev)
+ po_to_sdf(a.py, lang, os.path.join(hdir, "PO-Files",
lang.replace("-", "_")),
+ a.pot, a.template, hsdf, log)
+ history.append(hsdf)
+ if si.load(po2oo, lang):
+ final = os.path.join(d, "final.sdf")
+ if a.gsicheck:
+ changed, added, skip, new_errors = si.gate(
+ lang, rekeyed, po2oo, a.template, final,
a.gsicheck, a.libdir,
+ add_only=a.mode == "add-only", history=history)
+ result["left out"] = len(skip)
+ for k in skip:
+ log.write("left out (adds a gsicheck error): %s %s
%s\n" % (k[1], k[3], k[4]))
+ if new_errors:
+ result["status"] = "new gsicheck errors not tied
to an entry"
+ for msg, uid in sorted(new_errors):
+ log.write("new gsicheck error: %s %s\n" %
(msg, uid or ""))
+ si.write(os.path.join(d, "left-out.json"),
json.dumps(skip, ensure_ascii=False, indent=1))
+ else:
+ text, changed, added = si.merge(lang, rekeyed, po2oo,
a.template, (),
+ a.mode == "add-only",
history)
+ si.write(final, text)
+ result["updated"], result["added"] = len(changed),
len(added)
+ if a.export_po:
+ shutil.copytree(merged, os.path.join(a.run, "po",
lang.replace("-", "_")))
+ else:
+ result["note"] = "no PO files"
+ result["before"] = usable(tree, lang, a.en)
+ result["after"] = usable(final, lang, a.en)
+ result["final"] = final
+ result["changed"] = not filecmp.cmp(final, tree, shallow=False)
+ except Exception as e: # report and go on with the other languages
+ result["status"] = "failed: %s" % e
+ log.write("FAILED: %s\n" % e)
+ return result
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
+ p.add_argument("--aoo", required=True, help="AOO checkout (the directory
with main/ and extras/)")
+ p.add_argument("--template", default=DEFAULT_TEMPLATE,
+ help="en-US.sdf of the current source (default:
%(default)s)")
+ p.add_argument("--extract", action="store_true",
+ help="extract en-US.sdf from --aoo with localize -e first
(Linux, built tree)")
+ p.add_argument("--old-template", action="append", default=[],
+ help="more en-US.sdf files the tree may be keyed from
(earlier versions of --template "
+ "and openoffice-translation/en-US.sdf are added
automatically)")
+ p.add_argument("--po-repo", help="checkout of
apache/openoffice-translation (default: clone one into the work dir)")
+ p.add_argument("--no-po", action="store_true", help="only re-key the tree
files")
+ p.add_argument("--langs", help="comma-separated tree language codes
(default: every tree language)")
+ p.add_argument("--mode", choices=("add-only", "update"),
default="add-only",
+ help="add-only: fill entries the tree has no translation
for (default); "
+ "update: also replace tree text with newer PO text")
+ p.add_argument("--history", nargs="*", default=[],
+ help="openoffice-translation revisions whose text must not
replace a tree translation "
+ "(update mode; guards against Pootle rollbacks)")
+ p.add_argument("--solver", help="solver platform directory with
bin/gsicheck and lib/ (default: from --aoo)")
+ p.add_argument("--no-gsicheck", action="store_true", help="merge without
the gsicheck gate")
+ p.add_argument("--workdir", default="aoo-l10n-work", help="where runs are
kept (default: %(default)s)")
+ p.add_argument("--jobs", type=int, default=os.cpu_count() or 4)
+ p.add_argument("--python", help="Python with translate-toolkit (default:
this one, else a venv in the work dir)")
+ p.add_argument("--export-po", action="store_true", help="also write the
updated PO files per language")
+ p.add_argument("--apply", action="store_true", help="copy the results into
the tree")
+ a = p.parse_args()
+ sys.stdout.reconfigure(line_buffering=True)
+
+ a.aoo = os.path.abspath(a.aoo)
+ a.l10n = os.path.join(a.aoo, "extras", "l10n", "source")
+ if not os.path.isdir(a.l10n):
+ sys.exit("no extras/l10n/source in %s" % a.aoo)
+ if a.history and a.mode != "update":
+ sys.exit("--history only applies to --mode update")
+ a.history = list(dict.fromkeys(a.history))
+ workdir = os.path.abspath(a.workdir)
+ a.run = os.path.join(workdir,
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
+ os.makedirs(a.run)
+ print("work directory:", a.run)
+
+ if a.extract:
+ a.template = os.path.join(a.run, "en-US.sdf")
+ extract_template(a.aoo, a.template)
+ a.template = os.path.abspath(a.template)
+ a.en = si.load(a.template, "en-US")
+ if not a.en:
+ sys.exit("no en-US lines in %s" % a.template)
+ print("template: %s (%d entries)" % (a.template, len(a.en)))
+
+ langs = sorted(d for d in os.listdir(a.l10n) if
os.path.isfile(os.path.join(a.l10n, d, "localize.sdf")))
+ if a.langs:
+ wanted = a.langs.split(",")
+ missing = [x for x in wanted if x not in langs]
+ if missing:
+ sys.exit("no tree file for: %s" % " ".join(missing))
+ langs = wanted
+
+ a.po_repo = get_po_repo(a, workdir, langs)
+ if a.po_repo:
+ rev = subprocess.run(["git", "-C", a.po_repo, "log", "-1",
"--format=%h %cs"],
+ capture_output=True, text=True).stdout.strip()
+ print("PO files: %s at %s" % (a.po_repo, rev))
+ os.makedirs(os.path.join(a.run, "old"))
+ a.olds = old_templates(a.template, a.po_repo, [os.path.abspath(x) for x in
a.old_template],
+ os.path.join(a.run, "old"))
+
+ a.gsicheck = a.libdir = None
+ if not a.no_gsicheck:
+ solver = a.solver or next(iter(sorted(glob.glob(os.path.join(a.aoo,
"main", "solver", "*", "*", "bin",
+
"gsicheck")))), None)
+ if solver and solver.endswith("gsicheck"):
+ solver = os.path.dirname(os.path.dirname(solver))
+ if not solver or not os.access(os.path.join(solver, "bin",
"gsicheck"), os.X_OK):
+ sys.exit("no gsicheck found; build l10ntools in --aoo, pass
--solver, or use --no-gsicheck")
+ a.gsicheck, a.libdir = os.path.join(solver, "bin", "gsicheck"),
os.path.join(solver, "lib")
+
+ a.py = None
+ if a.po_repo:
+ a.py = toolkit_python(a, workdir)
+ a.pot = os.path.join(a.run, "pot")
+ with open(os.path.join(a.run, "oo2po.log"), "w") as log:
+ if log_run([a.py, "-m", "translate.convert.oo2po", "--pot",
a.template, a.pot], log):
+ sys.exit("oo2po failed, see %s/oo2po.log" % a.run)
+
+ if a.po_repo:
+ tree_po = {x.replace("-", "_") for x in langs}
+ extra = sorted(set(os.listdir(os.path.join(a.po_repo, "PO-Files"))) -
tree_po) if not a.langs else []
+ if extra:
+ print("PO files without a tree file, not integrated: %s" % "
".join(extra))
+
+ with concurrent.futures.ThreadPoolExecutor(a.jobs) as pool:
+ results = list(pool.map(lambda x: do_language(x, a), langs))
+
+ lines = ["| language | keyed from | moved | updated | added | left out |
translated before | after | status |",
+ "|---|---|---|---|---|---|---|---|---|"]
+ for r in results:
+ lines.append("| %s | %s | %s | %s | %s | %s | %s | %s | %s |" % (
+ r["lang"], r.get("keyed from", ""), r.get("moved", ""),
r.get("updated", ""), r.get("added", ""),
+ r.get("left out", ""), r.get("before", ""), r.get("after", ""),
r.get("note", r["status"])))
+ total = collections.Counter()
+ for r in results:
+ total["before"] += r.get("before", 0)
+ total["after"] += r.get("after", 0)
+ lines.append("")
+ lines.append("Translated entries whose key is in the template, all
languages: %d before, %d after "
+ "(%d template entries per language)." % (total["before"],
total["after"], len(a.en)))
+ report = "\n".join(lines) + "\n"
+ si.write(os.path.join(a.run, "report.md"), report)
+ print(report)
+
+ failed = [r["lang"] for r in results if r["status"] != "ok"]
+ changed = [r for r in results if r["status"] == "ok" and r.get("changed")]
+ if a.apply:
+ for r in changed:
+ shutil.copyfile(r["final"], os.path.join(a.l10n, r["lang"],
"localize.sdf"))
+ print("copied %d files into %s" % (len(changed), a.l10n))
+ else:
+ print("%d languages changed; rerun with --apply, or copy
langs/<lang>/final.sdf (rekeyed.sdf if there "
+ "is no final.sdf) over the tree file" % len(changed))
+ if failed:
+ print("FAILED: %s (see langs/<lang>/log.txt)" % " ".join(failed))
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
