This is an automated email from the ASF dual-hosted git repository.
jimjag pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/openoffice-devtools.git
The following commit(s) were added to refs/heads/main by this push:
new 13b73cc translation-scripts: rekey follows moved files; fix issues
found in review
13b73cc is described below
commit 13b73cc035a5d3c1f9d3991ac9f4e729e0eb76ed
Author: Jim Jagielski <[email protected]>
AuthorDate: Thu Nov 12 10:51:52 2026 -0500
translation-scripts: rekey follows moved files; fix issues found in review
Review found move chains that left two lines on one key, and gate listing
keys as left out that its last merge still held.
---
translation-scripts/integrate/README.md | 16 ++++++----
translation-scripts/integrate/sdf_integrate.py | 35 +++++++++++++++-------
.../integrate/update_translations.py | 20 ++++++++++++-
3 files changed, 55 insertions(+), 16 deletions(-)
diff --git a/translation-scripts/integrate/README.md
b/translation-scripts/integrate/README.md
index c6d7af5..a874c46 100644
--- a/translation-scripts/integrate/README.md
+++ b/translation-scripts/integrate/README.md
@@ -20,7 +20,10 @@ integration round do together, for every tree language or
for `--langs de,fr`:
update_translations.py --aoo ~/src/openoffice --apply # write the tree
files
- English template: `../scripts/en-US.sdf`, or `--extract` to make a fresh one
- from `--aoo` (Linux, configured and built; `localize -e` fails on macOS).
+ from `--aoo` (Linux, configured and fully built; `localize -e` fails on
+ macOS). An extract with under 90% of the committed file's entries is refused.
+ After `--extract --apply`, commit the new file as `../scripts/en-US.sdf`
+ (`--save-template` writes it there), since later runs re-key against it.
- PO files: cloned from apache/openoffice-translation into the work directory,
only the languages asked for, or `--po-repo` for an existing checkout.
- translate-toolkit: 2.2.5, the server's version, installed into a venv in the
@@ -29,9 +32,9 @@ integration round do together, for every tree language or for
`--langs de,fr`:
Per language it re-keys the tree file (the steps below), makes templates, runs
`fix_msgctxt.py`, `pomerge` and `po2oo`, and gates the merge on `gsicheck`. The
-tree is matched against every committed version of `../scripts/en-US.sdf` and
-the 2022 one in openoffice-translation; if it is already keyed to the current
-template, nothing moves.
+tree is matched against every committed version of `../scripts/en-US.sdf`, the
+first of which is the 2022 one, and the copy in openoffice-translation if there
+is one; if it is already keyed to the current template, nothing moves.
`--mode add-only` (the default) fills only entries the tree has no translation
for. `--mode update` also replaces tree text with newer PO text. Use it with
@@ -97,7 +100,10 @@ the source and the build shows English for them. Extract a
current `en-US.sdf`
(`localize -e -l en-US -f en-US.sdf` on Linux), then per language:
1. Move tree entries to the current keys. An entry moves only if its English
- is the same under the old and the new key; everything else is left alone.
+ is the same under the old and the new key: the same key without the lid,
+ else the only key in the file with that English, else, for a file that is
+ gone, the only key elsewhere with the same type, ids and English. Everything
+ else is left alone.
sdf_integrate.py rekey --lang de \
--tree extras/l10n/source/de/localize.sdf \
diff --git a/translation-scripts/integrate/sdf_integrate.py
b/translation-scripts/integrate/sdf_integrate.py
index df525c0..6c12dc2 100755
--- a/translation-scripts/integrate/sdf_integrate.py
+++ b/translation-scripts/integrate/sdf_integrate.py
@@ -187,7 +187,9 @@ def rekey(lang, tree_path, old_templates, template):
current one, holds most of its keys; if that is the current template,
nothing moves. A move needs the English to be the same under both keys:
first the same key without the help lid (oldref), else the only key in the
- same file with that English. Return (text, moves, stats)."""
+ same file with that English, else, if the file is gone from the template,
+ the only key elsewhere with the same type, ids and English.
+ Return (text, moves, stats)."""
en_new = load(template, "en-US")
own = set(key(c) for c in (l.rstrip("\r\n").split("\t") for l in
read_lines(tree_path))
if len(c) == 15 and c[9] == lang)
@@ -203,10 +205,13 @@ def rekey(lang, tree_path, old_templates, template):
return f.read(), {}, collections.Counter({"keyed to the current
template": len(own)})
by_file = collections.defaultdict(lambda: collections.defaultdict(list))
old_by_file = collections.defaultdict(collections.Counter)
+ elsewhere, old_elsewhere = collections.defaultdict(list),
collections.Counter()
for k, cols in en_new.items():
by_file[k[:2]][english(cols)].append(k)
+ elsewhere[(k[0],) + k[2:] + (english(cols),)].append(k)
for k, cols in en_old.items():
old_by_file[k[:2]][english(cols)] += 1
+ old_elsewhere[(k[0],) + k[2:] + (english(cols),)] += 1
lines = read_lines(tree_path)
# Some tree files carry an en-US line paired with the translation; gsicheck
@@ -234,10 +239,15 @@ def rekey(lang, tree_path, old_templates, template):
if lidless in en_new and english(en_new[lidless]) == e:
target = lidless
if target is None:
- same = by_file[k[:2]].get(e, [])
+ same = by_file.get(k[:2], {}).get(e, [])
if (len(same) == 1 and old_by_file[k[:2]][e] == 1
and english(en_old.get(same[0], [None] * 15)) != e):
target = same[0]
+ if target is None and k[:2] not in by_file:
+ moved = elsewhere.get((k[0],) + k[2:] + (e,), [])
+ if (len(moved) == 1 and old_elsewhere[(k[0],) + k[2:] + (e,)] == 1
+ and english(en_old.get(moved[0], [None] * 15)) != e):
+ target = moved[0]
if target is None:
stats["English changed or gone, left" if new is None else
"English changed under the same key, left"] += 1
@@ -245,13 +255,17 @@ def rekey(lang, tree_path, old_templates, template):
plan[k] = target
# A target must end up with one line: not a tree line that stays, and not
- # the target of another move.
- staying = set(k for k in tree_keys if k and k not in plan)
- hits = collections.Counter(plan.values())
- for k, t in list(plan.items()):
- if t in staying or hits[t] > 1:
+ # the target of another move. Dropping a move makes its line stay, which
+ # can block another move, so repeat until nothing changes.
+ while True:
+ staying = set(k for k in tree_keys if k and k not in plan)
+ hits = collections.Counter(plan.values())
+ drop = [k for k, t in plan.items() if t in staying or hits[t] > 1]
+ if not drop:
+ break
+ for k in drop:
del plan[k]
- stats["target already has a line, left"] += 1
+ stats["target already has a line, left"] += len(drop)
stats["moved"] = len(plan)
out = []
@@ -323,12 +337,13 @@ def gate(lang, tree, new, template, out, gsicheck,
libdir, rounds=4, add_only=Fa
Return (changed, added, left-out keys, new errors not tied to an entry)."""
baseline = run_gsicheck(gsicheck, libdir, tree, lang)
skip = []
- for _ in range(rounds):
+ for n in range(max(rounds, 1)):
text, changed, added = merge(lang, tree, new, template, skip,
add_only, history)
write(out, text)
new_errors = run_gsicheck(gsicheck, libdir, out, lang) - baseline
keyed = {uid_to_key(uid) for _, uid in new_errors if uid} - set(skip)
- if not keyed:
+ # a key only counts as left out once a later merge has left it out
+ if not keyed or n == max(rounds, 1) - 1:
break
skip += sorted(keyed)
return changed, added, skip, new_errors
diff --git a/translation-scripts/integrate/update_translations.py
b/translation-scripts/integrate/update_translations.py
index 7078238..91364c7 100755
--- a/translation-scripts/integrate/update_translations.py
+++ b/translation-scripts/integrate/update_translations.py
@@ -84,6 +84,10 @@ def extract_template(aoo, out):
rc = log_run(["bash", "-c", cmd], log)
if rc or not os.path.getsize(out):
sys.exit("localize -e failed, see %s.log" % out)
+ have, before = len(si.load(out, "en-US")), len(si.load(DEFAULT_TEMPLATE,
"en-US"))
+ # a partly built tree extracts only the modules it has
+ if have < 0.9 * before:
+ sys.exit("%s has %d entries, %s has %d; is the tree fully built?" %
(out, have, DEFAULT_TEMPLATE, before))
def old_templates(template, po_repo, extra, outdir):
@@ -234,7 +238,7 @@ def do_language(lang, ctx):
result["after"] = usable(final, lang, a.en)
result["final"] = final
result["changed"] = not filecmp.cmp(final, tree, shallow=False)
- except Exception as e: # report and go on with the other languages
+ except (Exception, SystemExit) as e: # sdf_integrate exits on errors;
report and go on
result["status"] = "failed: %s" % e
log.write("FAILED: %s\n" % e)
return result
@@ -245,6 +249,8 @@ def main():
p.add_argument("--aoo", required=True, help="AOO checkout (the directory
with main/ and extras/)")
p.add_argument("--template", default=DEFAULT_TEMPLATE,
help="en-US.sdf of the current source (default:
%(default)s)")
+ p.add_argument("--save-template", action="store_true",
+ help="with --extract, write the new en-US.sdf over the
default template")
p.add_argument("--extract", action="store_true",
help="extract en-US.sdf from --aoo with localize -e first
(Linux, built tree)")
p.add_argument("--old-template", action="append", default=[],
@@ -276,6 +282,8 @@ def main():
if a.history and a.mode != "update":
sys.exit("--history only applies to --mode update")
a.history = list(dict.fromkeys(a.history))
+ if a.save_template and not a.extract:
+ sys.exit("--save-template needs --extract")
workdir = os.path.abspath(a.workdir)
a.run = os.path.join(workdir,
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
os.makedirs(a.run)
@@ -299,6 +307,10 @@ def main():
langs = wanted
a.po_repo = get_po_repo(a, workdir, langs)
+ for rev in a.history:
+ if subprocess.run(["git", "-C", a.po_repo, "rev-parse", "--verify",
"-q", rev + "^{commit}"],
+ capture_output=True).returncode:
+ sys.exit("--history: %s is not a commit in %s" % (rev, a.po_repo))
if a.po_repo:
rev = subprocess.run(["git", "-C", a.po_repo, "log", "-1",
"--format=%h %cs"],
capture_output=True, text=True).stdout.strip()
@@ -357,6 +369,12 @@ def main():
for r in changed:
shutil.copyfile(r["final"], os.path.join(a.l10n, r["lang"],
"localize.sdf"))
print("copied %d files into %s" % (len(changed), a.l10n))
+ if a.save_template:
+ shutil.copyfile(a.template, DEFAULT_TEMPLATE)
+ print("wrote %s; commit it together with the tree files" %
os.path.realpath(DEFAULT_TEMPLATE))
+ elif a.extract and a.apply:
+ print("WARNING: the tree is now keyed to %s; commit that file as %s
(or rerun with --save-template), "
+ "or later runs will re-key against an older template" %
(a.template, os.path.realpath(DEFAULT_TEMPLATE)))
else:
print("%d languages changed; rerun with --apply, or copy
langs/<lang>/final.sdf (rekeyed.sdf if there "
"is no final.sdf) over the tree file" % len(changed))
