This is an automated email from the ASF dual-hosted git repository. jimjag pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/openoffice-devtools.git
commit 06069e93d62a40d91edf35a0f00a6d5fea31e054 Author: Jim Jagielski <[email protected]> AuthorDate: Thu Nov 12 09:20:24 2026 -0500 translation-scripts: re-key SDF and PO files to a newer en-US.sdf The help sources lost their oldref attributes, which were the help keys' lid, so translations keyed from the 2022 template no longer reach the build. --- translation-scripts/integrate/README.md | 34 +++++ translation-scripts/integrate/fix_msgctxt.py | 167 +++++++++++++++++++++++++ translation-scripts/integrate/sdf_integrate.py | 104 ++++++++++++++- 3 files changed, 302 insertions(+), 3 deletions(-) diff --git a/translation-scripts/integrate/README.md b/translation-scripts/integrate/README.md index ed85cf8..03b38bd 100644 --- a/translation-scripts/integrate/README.md +++ b/translation-scripts/integrate/README.md @@ -52,3 +52,37 @@ OpenOffice tree. No other dependencies. 4. Copy `de.merged.sdf` over the tree file and commit it, one commit per language. `merge --skip de.left-out.json` gives the same result without running `gsicheck` again. + +## When the English SDF is newer than the PO files + +The Help sources lost their `oldref` attributes in 2024-25. `oldref` was the +help key's lid, so tree entries keyed from the 2022 `en-US.sdf` no longer match +the source and the build shows English for them. Extract a current `en-US.sdf` +(`localize -e -l en-US -f en-US.sdf` on Linux), then per language: + +1. Move tree entries to the current keys. An entry moves only if its English + is the same under the old and the new key; everything else is left alone. + + sdf_integrate.py rekey --lang de \ + --tree extras/l10n/source/de/localize.sdf \ + --old-template en-US.2022.sdf --template en-US.sdf \ + --out de.rekeyed.sdf + +2. Make templates from the new `en-US.sdf` (`oo2po --pot en-US.sdf pot/`) and + re-key the PO files to them before `pomerge`. Without this, units whose + msgctxt holds the old lid come out fuzzy or empty, and `po2oo` writes + translations to locations whose English has changed. + + fix_msgctxt.py --template pot --input PO-Files/de --output de.fixed + pomerge --template=pot/ --input=de.fixed/ --output=de.merged/ + po2oo --template=en-US.sdf --skipsource --keeptimestamp \ + --language=de de.merged/ de.po2oo.sdf + + `fix_msgctxt.py` runs under the same Python as translate-toolkit. + +3. Gate as above, against the re-keyed file. `--add-only` takes PO text only + where the tree has no translation, for when the tree already holds an + earlier, checked integration of the same PO files. + + sdf_integrate.py gate --add-only --lang de --tree de.rekeyed.sdf \ + --new de.po2oo.sdf --template en-US.sdf --out de.merged.sdf ... diff --git a/translation-scripts/integrate/fix_msgctxt.py b/translation-scripts/integrate/fix_msgctxt.py new file mode 100755 index 0000000..dc956ce --- /dev/null +++ b/translation-scripts/integrate/fix_msgctxt.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +################################################################ +"""Re-key exported PO units to a new template before pomerge. + +oo2po uses the location as msgctxt for duplicate msgids, and Help locations +carry the old oldref value as lid (file#gid.lid.type.field). Once the source +lost its oldref attributes the template contexts no longer match, and pomerge +then marks those units fuzzy or leaves them empty. + +For each unit, locations are looked up in the template as they are and with +the lid removed. A location counts only if the template unit there has the +same English; the unit then takes that unit's msgctxt and locations. Other +locations are removed, since po2oo applies a translation to every location a +unit lists. Units left without a location are kept for pomerge to place by +source text, unless that would overwrite a unit already placed by location. + +usage: fix_msgctxt.py --template POTDIR --input PODIR --output OUTDIR +""" +from __future__ import print_function + +import argparse +import io +import os +import sys + +from translate.storage import factory + + +def norm(loc): + return loc.replace('%23', '#') + + +def candidates(loc): + loc = norm(loc) + yield loc + if '#' in loc: + fname, rest = loc.split('#', 1) + parts = rest.split('.') + # gid.lid.type.field -> gid.type.field + if len(parts) == 4: + yield fname + '#' + '.'.join([parts[0]] + parts[2:]) + + +def template_index(potpath): + index = {} + first = {} + for unit in factory.getobject(potpath).units: + if unit.isheader(): + continue + first.setdefault(unit.source, unit.getcontext()) + for loc in unit.getlocations(): + index.setdefault(norm(loc), (unit.getcontext(), unit.source)) + return index, first + + +def set_locations(unit, locs): + unit.sourcecomments = [] + for loc in locs: + unit.addlocation(loc) + + +def fix_store(store, templ, stats): + index, first = templ + keep = [] + unplaced = [] + claimed = set() + for unit in store.units: + if unit.isheader() or unit.isobsolete(): + keep.append(unit) + continue + stats['units'] += 1 + good, other, ctx = [], [], None + for loc in unit.getlocations(): + hit = None + for cand in candidates(loc): + if cand in index: + hit = cand + break + if hit is None: + if norm(loc) not in other: + other.append(norm(loc)) + elif index[hit][1] == unit.source: + if hit not in good: + good.append(hit) + if ctx is None: + ctx = index[hit][0] + claimed.add(index[hit][0] + u'\x04' + unit.source) + else: + stats['locations_removed'] += 1 + keep.append(unit) + if good: + if good != unit.getlocations(): + set_locations(unit, good) + if ctx != unit.getcontext(): + if ctx: + unit.setcontext(ctx) + else: + unit.msgctxt = [] + stats['rewritten'] += 1 + else: + if other != unit.getlocations(): + set_locations(unit, other) + stats['unplaced'] += 1 + unplaced.append(unit) + # pomerge falls back to the first unit with the same source; keep that + # from overwriting a unit that was already placed by its location + drop = set(id(u) for u in unplaced + if u.source in first + and first[u.source] + u'\x04' + u.source in claimed) + stats['shadowed_dropped'] += len(drop) + store.units = [u for u in keep if id(u) not in drop] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--template', required=True) + ap.add_argument('--input', required=True) + ap.add_argument('--output', required=True) + args = ap.parse_args() + + keys = ('files', 'no_template', 'units', 'rewritten', 'locations_removed', + 'unplaced', 'shadowed_dropped') + stats = dict((k, 0) for k in keys) + for root, _dirs, files in os.walk(args.input): + for name in sorted(files): + if not name.endswith('.po'): + continue + src = os.path.join(root, name) + rel = os.path.relpath(src, args.input) + dst = os.path.join(args.output, rel) + if not os.path.isdir(os.path.dirname(dst)): + os.makedirs(os.path.dirname(dst)) + stats['files'] += 1 + pot = os.path.join(args.template, rel[:-3] + '.pot') + store = factory.getobject(src) + if os.path.exists(pot): + fix_store(store, template_index(pot), stats) + else: + stats['no_template'] += 1 + with io.open(dst, 'wb') as out: + store.serialize(out) + for key in keys: + print('%s: %d' % (key, stats[key])) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/translation-scripts/integrate/sdf_integrate.py b/translation-scripts/integrate/sdf_integrate.py index fbd2a96..d7ce227 100755 --- a/translation-scripts/integrate/sdf_integrate.py +++ b/translation-scripts/integrate/sdf_integrate.py @@ -96,7 +96,7 @@ def combine(lang, template, base_path, newer_path, history_paths): return "".join(out), stats -def merge(lang, tree_path, new_path, template, skip=()): +def merge(lang, tree_path, new_path, template, skip=(), add_only=False): """Return (text, changed keys, added keys).""" new = load(new_path, lang) if not new: @@ -122,6 +122,8 @@ def merge(lang, tree_path, new_path, template, skip=()): if n and k not in skip: updated = list(cols) for i in TEXT_COLUMNS: + if add_only and is_translation(cols[i], e[i] if e else None): + continue if n[i] != cols[i] and is_translation(n[i], e[i] if e else None): updated[i] = n[i] if updated != cols: @@ -147,6 +149,86 @@ def merge(lang, tree_path, new_path, template, skip=()): return "".join(out), changed, added +def english(cols): + return tuple(cols[i] for i in TEXT_COLUMNS) + + +def with_key(cols, k): + row = list(cols) + row[0], row[1], row[3], row[4], row[5], row[6], row[7] = k + return row + + +def rekey(lang, tree_path, old_template, template): + """Move tree entries whose key changed to the key their English has now. + + A move needs the English to be the same under both keys: first the same + key without the help lid (oldref), else the only key in the same file with + that English. Return (text, moves, stats).""" + en_old, en_new = load(old_template, "en-US"), load(template, "en-US") + by_file = collections.defaultdict(lambda: collections.defaultdict(list)) + old_by_file = collections.defaultdict(collections.Counter) + for k, cols in en_new.items(): + by_file[k[:2]][english(cols)].append(k) + for k, cols in en_old.items(): + old_by_file[k[:2]][english(cols)] += 1 + + with open(tree_path, encoding="utf-8", newline="") as f: + lines = f.read().splitlines(keepends=True) + # Some tree files carry an en-US line paired with the translation; gsicheck + # needs the pair to keep one key, so both lines move together. + tree_keys = [] + for line in lines: + cols = line.rstrip("\r\n").split("\t") + ok = len(cols) == 15 and cols[9] in (lang, "en-US") + tree_keys.append(key(cols) if ok else None) + + stats = collections.Counter() + plan = {} + for k in dict.fromkeys(filter(None, tree_keys)): + old, new = en_old.get(k), en_new.get(k) + if new is not None and (old is None or english(old) == english(new)): + stats["key unchanged"] += 1 + continue + if old is None: + stats["key in neither template, left"] += 1 + continue + e = english(old) + target = None + if k[4]: + lidless = k[:4] + ("",) + k[5:] + if lidless in en_new and english(en_new[lidless]) == e: + target = lidless + if target is None: + same = by_file[k[:2]].get(e, []) + if (len(same) == 1 and old_by_file[k[:2]][e] == 1 + and english(en_old.get(same[0], [None] * 15)) != e): + target = same[0] + if target is None: + stats["English changed or gone, left" if new is None else + "English changed under the same key, left"] += 1 + continue + plan[k] = target + + # A target must end up with one line: not a tree line that stays, and not + # the target of another move. + staying = set(k for k in tree_keys if k and k not in plan) + hits = collections.Counter(plan.values()) + for k, t in list(plan.items()): + if t in staying or hits[t] > 1: + del plan[k] + stats["target already has a line, left"] += 1 + stats["moved"] = len(plan) + + out = [] + for line, k in zip(lines, tree_keys): + if k in plan: + body = line.rstrip("\r\n") + line = "\t".join(with_key(body.split("\t"), plan[k])) + line[len(body):] + out.append(line) + return "".join(out), plan, stats + + def run_gsicheck(gsicheck, libdir, path, lang): """Return a Counter of (message, UniqueID) for every error except header lines.""" if not os.access(gsicheck, os.X_OK): @@ -188,9 +270,15 @@ def cmd_combine(a): print(a.lang, dict(stats)) +def cmd_rekey(a): + text, plan, stats = rekey(a.lang, a.tree, a.old_template, a.template) + write(a.out, text) + print(a.lang, dict(stats)) + + def cmd_merge(a): skip = [tuple(k) for k in json.load(open(a.skip))] if a.skip else [] - text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip) + text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip, a.add_only) write(a.out, text) print("%s: %d entries updated, %d keys added" % (a.lang, len(changed), len(added))) @@ -199,7 +287,7 @@ def cmd_gate(a): baseline = run_gsicheck(a.gsicheck, a.libdir, a.tree, a.lang) skip = [] for _ in range(a.rounds): - text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip) + text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip, a.add_only) write(a.out, text) new_errors = run_gsicheck(a.gsicheck, a.libdir, a.out, a.lang) - baseline keyed = {uid_to_key(uid) for _, uid in new_errors if uid} - set(skip) @@ -232,6 +320,14 @@ def main(): c.add_argument("--out", required=True) c.set_defaults(func=cmd_combine) + r = sub.add_parser("rekey", help="move tree entries to the keys of a newer en-US.sdf") + r.add_argument("--lang", required=True) + r.add_argument("--tree", required=True, help="extras/l10n/source/<lang>/localize.sdf") + r.add_argument("--old-template", required=True, help="en-US.sdf the tree keys come from") + r.add_argument("--template", required=True, help="en-US.sdf extracted from the current source") + r.add_argument("--out", required=True) + r.set_defaults(func=cmd_rekey) + for name, func, helptext in (("merge", cmd_merge, "merge without checking"), ("gate", cmd_gate, "merge and gate on gsicheck")): m = sub.add_parser(name, help=helptext) @@ -240,6 +336,8 @@ def main(): m.add_argument("--new", required=True, help="po2oo (or combine) output") m.add_argument("--template", required=True, help="en-US.sdf used for po2oo") m.add_argument("--out", required=True) + m.add_argument("--add-only", action="store_true", + help="fill only entries the tree has no translation for") if name == "merge": m.add_argument("--skip", help="JSON list of keys to leave as they are") else:
