This is an automated email from the ASF dual-hosted git repository. imbajin pushed a commit to branch feat/oink-core-platform in repository https://gitbox.apache.org/repos/asf/hugegraph-doc.git
commit b772d2404325ad3e3a2353729ea100bf7dd907dd Author: dark <[email protected]> AuthorDate: Fri Sep 4 20:32:12 2026 +0800 fix(versioning): normalize archive robots - force noindex,follow on regular archive and print pages - retain noindex,nofollow on generated 404 documents - cover both prior robots states in the regression fixture --- scripts/test_versioning.py | 13 +++++++++++-- scripts/versioning.py | 9 +++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/test_versioning.py b/scripts/test_versioning.py index 3d750f3bc..e27982391 100644 --- a/scripts/test_versioning.py +++ b/scripts/test_versioning.py @@ -586,7 +586,7 @@ class VersionUrlTest(unittest.TestCase): def test_historical_pages_are_noindex_and_aliases_are_locale_aware(self) -> None: with tempfile.TemporaryDirectory() as temp_name: output = Path(temp_name) - for language_prefix in ("", "cn/"): + for index, language_prefix in enumerate(("", "cn/")): target = ( output / language_prefix @@ -594,10 +594,19 @@ class VersionUrlTest(unittest.TestCase): ) target.parent.mkdir(parents=True) target.write_text( - '<meta name="robots" content="index, follow">', + '<meta name="robots" content="index, follow">' + if index == 0 + else '<meta name="robots" content="noindex, nofollow">', encoding="utf-8", ) + error_page = output / "404.html" + error_page.write_text( + '<meta name="robots" content="noindex,nofollow">', encoding="utf-8" + ) self.assertEqual(versioning.mark_historical_pages_noindex(output), 2) + self.assertIn( + "noindex,nofollow", error_page.read_text(encoding="utf-8") + ) self.assertEqual( versioning.write_historical_route_aliases( output, ORIGIN, "versions/1.0" diff --git a/scripts/versioning.py b/scripts/versioning.py index 8b6485a2c..c35bd8038 100644 --- a/scripts/versioning.py +++ b/scripts/versioning.py @@ -98,9 +98,8 @@ HREFLANG_LINK_RE = re.compile( r"(?=[^>]*\bhreflang=)[^>]*>", re.IGNORECASE, ) -INDEX_FOLLOW_META_RE = re.compile( - r"<meta\b(?=[^>]*\bname=[\"']?robots(?:[\"'\s>]|$))" - r"(?=[^>]*\bcontent=[\"']?index\s*,?\s*follow(?:[\"'\s>]|$))[^>]*>", +ROBOTS_META_RE = re.compile( + r"<meta\b(?=[^>]*\bname=[\"']?robots(?:[\"'\s>]|$))[^>]*>", re.IGNORECASE, ) MARKDOWN_DESTINATION_RE = re.compile( @@ -2108,8 +2107,10 @@ def mark_historical_pages_noindex(output: pathlib.Path) -> int: """Apply the archive indexing policy to every rendered historical page.""" changed = 0 for path in sorted(output.rglob("*.html")): + if path.name == "404.html": + continue source = path.read_text(encoding="utf-8") - rendered, count = INDEX_FOLLOW_META_RE.subn( + rendered, count = ROBOTS_META_RE.subn( '<meta name="robots" content="noindex,follow">', source ) if count:
