This is an automated email from the ASF dual-hosted git repository. imbajin pushed a commit to branch cx-oink-1-migration in repository https://gitbox.apache.org/repos/asf/hugegraph-doc.git
commit 1c6bc5b58bad650244c081648ce34a3ba3b94772 Author: imbajin <[email protected]> AuthorDate: Mon Aug 31 07:04:54 2026 +0800 feat(site): add versioned OINK builds - resolve latest, 1.7, and 1.5 to exact commit SHAs - build isolated production and staging artifacts - validate URL, language, metadata, and license contracts - aggregate reviewed outputs before staging or production publish --- .github/workflows/hugo.yml | 212 +++++- dist/url-contract.json | 36 + layouts/404.html | 10 - scripts/test_versioning.py | 208 ++++++ scripts/versioning.py | 1721 ++++++++++++++++++++++++++++++++++++++++++++ versions.json | 30 + 6 files changed, 2169 insertions(+), 48 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index 638825c29..bad8ca95f 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -15,18 +15,69 @@ env: HUGO_CACHEDIR: /tmp/hugo_cache jobs: - # Keep this job id aligned with the required `deploy` status in .asf.yaml. - # Publishing is added only after the aggregate artifact is complete. - deploy: + prepare: + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + versions: ${{ steps.matrix.outputs.versions }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + + - name: Validate source and version tooling + run: | + bash dist/validate-links.sh + python3 -m unittest discover -s scripts -p 'test_*.py' -v + + - name: Resolve immutable version matrix + id: matrix + env: + LATEST_SHA: ${{ github.sha }} + run: | + python3 scripts/versioning.py prepare \ + --latest-sha "$LATEST_SHA" \ + --output resolved-versions.json + echo "versions=$(jq -c '.include' resolved-versions.json)" >> "$GITHUB_OUTPUT" + + - name: Upload resolved version manifest + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: resolved-versions-${{ github.sha }} + path: resolved-versions.json + if-no-files-found: error + + build: + needs: prepare runs-on: ubuntu-latest permissions: contents: read + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.prepare.outputs.versions) }} + site: + - name: production + origin: https://hugegraph.apache.org/ + - name: staging + origin: https://hugegraph-oink.staged.apache.org/ + name: Build ${{ matrix.site.name }} / ${{ matrix.version.id }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: go.mod @@ -42,47 +93,132 @@ jobs: uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ env.HUGO_CACHEDIR }} - key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ hashFiles('go.sum') }} + key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}-${{ hashFiles('go.sum') }} restore-keys: | - ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}- + ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}- - name: Verify pinned OINK module run: test "$(hugo mod graph)" = "github.com/apache/hugegraph-doc github.com/pgsty/[email protected]" - - name: Check source links - run: bash dist/validate-links.sh - - - name: Build strict production artifact - run: >- - hugo --cleanDestinationDir --gc --minify - --environment production - --printPathWarnings --printI18nWarnings - --panicOnWarning --logLevel info - - - name: Validate production artifact - run: python3 dist/validate-site-output.py public https://hugegraph.apache.org/ - - - name: Build and validate staging URL shape - run: >- - hugo --cleanDestinationDir --gc --minify - --destination public-staging - --baseURL https://hugegraph-oink.staged.apache.org/ - --environment production - --printPathWarnings --printI18nWarnings - --panicOnWarning --logLevel info - - - name: Validate staging artifact - run: >- - python3 dist/validate-site-output.py public-staging - https://hugegraph-oink.staged.apache.org/ - - - name: Add ASF deployment metadata - run: cp .asf.yaml public/.asf.yaml - - - name: Upload site artifact + - name: Build isolated version artifact + env: + OINK_PYTHON: python3 + run: | + python3 scripts/versioning.py build \ + --version "${{ matrix.version.id }}" \ + --sha "${{ matrix.version.sha }}" \ + --site-origin "${{ matrix.site.origin }}" \ + --output "${RUNNER_TEMP}/version-public" + python3 scripts/versioning.py validate \ + --version "${{ matrix.version.id }}" \ + --sha "${{ matrix.version.sha }}" \ + --site-origin "${{ matrix.site.origin }}" \ + --artifact "${RUNNER_TEMP}/version-public" + + - name: Upload isolated version artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ matrix.site.name }}-${{ matrix.version.id }} + path: ${{ runner.temp }}/version-public + include-hidden-files: true + if-no-files-found: error + + # Keep this job id aligned with the required `deploy` status in .asf.yaml. + # It produces deployable artifacts but does not publish untrusted PR code. + deploy: + if: always() + needs: [prepare, build] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Require every version build to succeed + env: + PREPARE_RESULT: ${{ needs.prepare.result }} + BUILD_RESULT: ${{ needs.build.result }} + run: | + test "$PREPARE_RESULT" = success + test "$BUILD_RESULT" = success + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + + - name: Download resolved version manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: resolved-versions-${{ github.sha }} + path: resolved + + - name: Download production versions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: production-* + path: version-artifacts + + - name: Download staging versions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: staging-* + path: version-artifacts + + - name: Aggregate production and staging sites + run: | + python3 scripts/versioning.py aggregate \ + --artifacts version-artifacts \ + --artifact-prefix production- \ + --resolved-manifest resolved/resolved-versions.json \ + --site-origin https://hugegraph.apache.org/ \ + --output "${RUNNER_TEMP}/public-production" + python3 scripts/versioning.py aggregate \ + --artifacts version-artifacts \ + --artifact-prefix staging- \ + --resolved-manifest resolved/resolved-versions.json \ + --site-origin https://hugegraph-oink.staged.apache.org/ \ + --asf-profile oink \ + --asf-whoami asf-staging-oink \ + --output "${RUNNER_TEMP}/public-staging" + + - name: Upload production aggregate + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: hugegraph-site-production-${{ github.sha }} + path: ${{ runner.temp }}/public-production + include-hidden-files: true + if-no-files-found: error + + - name: Upload ASF staging aggregate uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: hugegraph-site-${{ github.sha }} - path: public + name: hugegraph-site-staging-${{ github.sha }} + path: ${{ runner.temp }}/public-staging include-hidden-files: true if-no-files-found: error + + publish: + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: deploy + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download reviewed production aggregate + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: hugegraph-site-production-${{ github.sha }} + path: public-production + + - name: Publish clean aggregate to asf-site + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./public-production + publish_branch: asf-site + keep_files: false + force_orphan: false + commit_message: ${{ github.event.head_commit.message }} diff --git a/dist/url-contract.json b/dist/url-contract.json new file mode 100644 index 000000000..b7b568d13 --- /dev/null +++ b/dist/url-contract.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": 1, + "baseline": { + "origin": "https://hugegraph.apache.org/";, + "verifiedAt": "2026-08-31", + "defaultStatus": 200, + "stagingOrigin": "https://hugegraph-oink.staged.apache.org/";, + "stagingStatusBeforePublish": 404 + }, + "routes": [ + {"path": "/", "versions": ["latest", "1.7", "1.5"], "file": "index.html", "contains": ["Apache HugeGraph"]}, + {"path": "/cn/", "versions": ["latest", "1.7", "1.5"], "file": "cn/index.html", "contains": ["Apache HugeGraph"]}, + {"path": "/docs/", "versions": ["latest", "1.7", "1.5"], "file": "docs/index.html", "contains": ["Documentation"]}, + {"path": "/cn/docs/", "versions": ["latest", "1.7", "1.5"], "file": "cn/docs/index.html", "contains": ["Documentation"]}, + {"path": "/blog/", "versions": ["latest"], "file": "blog/index.html", "contains": ["Blog"]}, + {"path": "/cn/blog/", "versions": ["latest"], "file": "cn/blog/index.html", "contains": ["Blog"]}, + {"path": "/about/", "versions": ["latest"], "file": "about/index.html", "contains": ["Apache HugeGraph"]}, + {"path": "/cn/about/", "versions": ["latest"], "file": "cn/about/index.html", "contains": ["Apache HugeGraph"]}, + {"path": "/community/", "versions": ["latest"], "file": "community/index.html", "contains": ["Community"]}, + {"path": "/cn/community/", "versions": ["latest"], "file": "cn/community/index.html", "contains": ["社区"]}, + {"path": "/docs/download/download/", "versions": ["latest", "1.7", "1.5"], "file": "docs/download/download/index.html", "contains": ["Download"]}, + {"path": "/cn/docs/download/download/", "versions": ["latest", "1.7", "1.5"], "file": "cn/docs/download/download/index.html", "contains": ["下载"]}, + {"path": "/docs/quickstart/hugegraph/hugegraph-server/", "versions": ["latest", "1.7", "1.5"], "file": "docs/quickstart/hugegraph/hugegraph-server/index.html", "contains": ["HugeGraph-Server"]}, + {"path": "/cn/docs/quickstart/hugegraph/hugegraph-server/", "versions": ["latest", "1.7", "1.5"], "file": "cn/docs/quickstart/hugegraph/hugegraph-server/index.html", "contains": ["HugeGraph-Server"]}, + {"path": "/docs/config/config-guide/", "versions": ["latest", "1.7", "1.5"], "file": "docs/config/config-guide/index.html", "contains": ["GremlinServer"]}, + {"path": "/cn/docs/config/config-guide/", "versions": ["latest", "1.7", "1.5"], "file": "cn/docs/config/config-guide/index.html", "contains": ["GremlinServer"]}, + {"path": "/docs/clients/restful-api/graphs/", "versions": ["latest", "1.7", "1.5"], "file": "docs/clients/restful-api/graphs/index.html", "contains": ["REST"]}, + {"path": "/cn/docs/clients/restful-api/graphs/", "versions": ["latest", "1.7", "1.5"], "file": "cn/docs/clients/restful-api/graphs/index.html", "contains": ["REST"]}, + {"path": "/docs/guides/architectural/", "versions": ["latest", "1.7", "1.5"], "file": "docs/guides/architectural/index.html", "contains": ["HugeGraph"]}, + {"path": "/cn/docs/guides/architectural/", "versions": ["latest", "1.7", "1.5"], "file": "cn/docs/guides/architectural/index.html", "contains": ["HugeGraph"]}, + {"path": "/404.html", "versions": ["latest", "1.7", "1.5"], "file": "404.html", "contains": ["404"]}, + {"path": "/cn/404.html", "versions": ["latest", "1.7", "1.5"], "file": "cn/404.html", "contains": ["404"]}, + {"path": "/client-go/", "versions": ["latest"], "file": "client-go/index.html", "contains": ["go-import", "go-source"]}, + {"path": "/icons/logo.svg", "versions": ["latest", "1.7", "1.5"], "file": "icons/logo.svg", "contains": ["<svg"], "baselineStatus": 404, "migrationStatus": 200} + ] +} diff --git a/layouts/404.html b/layouts/404.html deleted file mode 100644 index 378b73675..000000000 --- a/layouts/404.html +++ /dev/null @@ -1,10 +0,0 @@ -{{ define "main"}} - <main id="main"> - <div> - <h1 id="title">Not found</h1> - <p>Oops! This page doesn't exist. Try going back to our <a href="{{ "/" | relURL }}">home page</a>.</p> - - <p>You can learn how to make a 404 page like this in <a href="https://gohugo.io/templates/404/";>Custom 404 Pages</a>.</p> - </div> - </main> -{{ end }} diff --git a/scripts/test_versioning.py b/scripts/test_versioning.py new file mode 100644 index 000000000..9d5d2fb85 --- /dev/null +++ b/scripts/test_versioning.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. + +import argparse +import json +import tempfile +import unittest +from pathlib import Path + +import versioning + + +ORIGIN = "https://hugegraph.apache.org/"; +STAGING_ORIGIN = "https://hugegraph-oink.staged.apache.org/"; +PUBLISH_PATH = "versions/1.7" +ALLOWED_PATHS = {"/docs", "/versions/1.7/docs", "/versions/1.5/docs"} + + +def rewrite(value: str) -> str: + return versioning.rewrite_internal_url( + value, + origin=ORIGIN, + publish_path=PUBLISH_PATH, + allowed_paths=ALLOWED_PATHS, + ) + + +class VersionUrlTest(unittest.TestCase): + def test_scopes_root_relative_urls(self) -> None: + self.assertEqual(rewrite("/docs/"), "/versions/1.7/docs/") + self.assertEqual( + rewrite("/cn/docs/config/?mode=all#backend"), + "/versions/1.7/cn/docs/config/?mode=all#backend", + ) + + def test_preserves_exact_absolute_version_destinations(self) -> None: + self.assertEqual(rewrite("https://hugegraph.apache.org/docs";), f"{ORIGIN}docs") + self.assertEqual( + rewrite("https://hugegraph.apache.org/versions/1.5/docs/";), + "https://hugegraph.apache.org/versions/1.5/docs/";, + ) + + def test_rewrites_production_origin_for_staging(self) -> None: + self.assertEqual( + versioning.rewrite_internal_url( + "https://hugegraph.apache.org/docs/config/";, + origin=STAGING_ORIGIN, + publish_path=PUBLISH_PATH, + allowed_paths=ALLOWED_PATHS, + ), + "https://hugegraph-oink.staged.apache.org/versions/1.7/docs/config/";, + ) + self.assertEqual( + versioning.rewrite_internal_url( + "https://hugegraph.apache.org/blog/";, + origin=STAGING_ORIGIN, + publish_path="", + allowed_paths=ALLOWED_PATHS, + ), + "https://hugegraph-oink.staged.apache.org/blog/";, + ) + + def test_rejects_non_selector_cross_version_url(self) -> None: + with self.assertRaises(SystemExit): + rewrite("/versions/1.5/docs/config/") + + def test_maps_known_historical_routes(self) -> None: + self.assertEqual( + rewrite("https://hugegraph.apache.org/versions/1.7/docs/introduction/";), + "https://hugegraph.apache.org/versions/1.7/docs/introduction/readme/";, + ) + self.assertEqual( + rewrite("/docs/quickstart/hugegraph-loader#usage"), + "/versions/1.7/docs/quickstart/toolchain/hugegraph-loader/#usage", + ) + + def test_markdown_rewrite_skips_fenced_code(self) -> None: + source = ( + "[Docs](/docs/)\n" + "```html\n" + '<a href="/docs/">example</a>\n' + "```\n" + '<a href="/blog/">Blog</a>\n' + ) + rendered, count = versioning.rewrite_text_urls(source, rewrite, markdown=True) + self.assertEqual(count, 2) + self.assertIn("[Docs](/versions/1.7/docs/)", rendered) + self.assertIn('<a href="/docs/">example</a>', rendered) + self.assertIn('<a href="https://hugegraph.apache.org/blog/";>Blog</a>', rendered) + + def test_rejects_artifact_from_unexpected_sha(self) -> None: + expected = { + "id": "1.7", + "name": "1.7", + "ref": "release-1.7.0", + "publishPath": "versions/1.7", + "archived": True, + "githubBranch": "release-1.7.0", + "sha": "a" * 40, + } + actual = dict(expected) + actual["sha"] = "b" * 40 + with self.assertRaises(SystemExit): + versioning.require_metadata_matches(expected, actual, Path(".version.json")) + + def test_validate_command_rejects_artifact_sha_drift(self) -> None: + with tempfile.TemporaryDirectory() as temp_name: + artifact = Path(temp_name) / "artifact" + artifact.mkdir() + entry = json.loads( + (versioning.ROOT / "versions.json").read_text(encoding="utf-8") + )["versions"][0] + metadata = dict(entry) + metadata.update({"sha": "b" * 40, "baseURL": ORIGIN}) + (artifact / ".version.json").write_text( + json.dumps(metadata), encoding="utf-8" + ) + args = argparse.Namespace( + manifest=versioning.ROOT / "versions.json", + version="latest", + sha="a" * 40, + site_origin=ORIGIN, + artifact=artifact, + ) + with self.assertRaises(SystemExit): + versioning.validate_artifact(args) + + def test_rejects_active_and_ambiguous_url_schemes(self) -> None: + for value in ( + "javascript:alert(1)", + "data:text/html,test", + "file:///etc/passwd", + "ftp://example.org/file";, + "//example.org/path", + ): + with self.subTest(value=value), self.assertRaises(SystemExit): + versioning.require_safe_url_scheme(value, "fixture.html") + self.assertFalse(versioning.require_safe_url_scheme("mailto:[email protected]";, "x")) + self.assertFalse(versioning.require_safe_url_scheme("tel:+1", "x")) + self.assertTrue(versioning.require_safe_url_scheme("/docs/", "x")) + + def test_rejects_resolved_manifest_drift(self) -> None: + with tempfile.TemporaryDirectory() as temp_name: + manifest = json.loads( + (versioning.ROOT / "versions.json").read_text(encoding="utf-8") + ) + for entry in manifest["versions"]: + entry["sha"] = "a" * 40 + manifest["versions"][1]["name"] = "unexpected" + path = Path(temp_name) / "resolved.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with self.assertRaises(SystemExit): + versioning.load_resolved_manifest(path) + + def test_aggregate_rejects_metadata_sha_before_copy(self) -> None: + with tempfile.TemporaryDirectory() as temp_name: + temp = Path(temp_name) + manifest = json.loads( + (versioning.ROOT / "versions.json").read_text(encoding="utf-8") + ) + for entry in manifest["versions"]: + entry["sha"] = "a" * 40 + resolved = temp / "resolved.json" + resolved.write_text(json.dumps(manifest), encoding="utf-8") + latest = temp / "artifacts/latest" + latest.mkdir(parents=True) + metadata = dict(manifest["versions"][0]) + metadata["sha"] = "b" * 40 + (latest / ".version.json").write_text( + json.dumps(metadata), encoding="utf-8" + ) + args = argparse.Namespace( + resolved_manifest=resolved, + artifacts=temp / "artifacts", + artifact_prefix="", + site_origin=ORIGIN, + output=temp / "aggregate", + asf_profile=None, + asf_whoami=None, + ) + with self.assertRaises(SystemExit): + versioning.aggregate(args) + + def test_output_cleanup_is_limited_to_temporary_descendants(self) -> None: + with tempfile.TemporaryDirectory() as temp_name: + temp = Path(temp_name) + output = temp / "output" + output.mkdir() + (output / "stale").write_text("stale", encoding="utf-8") + self.assertEqual( + versioning.prepare_output_directory(output, "fixture"), + output.resolve(), + ) + self.assertFalse(output.exists()) + symlink = temp / "symlink" + symlink.symlink_to(temp, target_is_directory=True) + with self.assertRaises(SystemExit): + versioning.prepare_output_directory(symlink, "fixture") + with self.assertRaises(SystemExit): + versioning.prepare_output_directory(versioning.ROOT, "fixture") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/versioning.py b/scripts/versioning.py new file mode 100644 index 000000000..f82aecdcb --- /dev/null +++ b/scripts/versioning.py @@ -0,0 +1,1721 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve, build, and aggregate centrally rendered HugeGraph versions.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import html.parser +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.parse +import xml.etree.ElementTree as ET +from typing import NoReturn + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +URL_CONTRACT = ROOT / "dist/url-contract.json" +CANONICAL_ORIGIN = "https://hugegraph.apache.org/"; +SHELL_FILES = ("go.mod", "go.sum", "hugo.yaml") +SHELL_DIRS = ("assets", "data", "i18n", "layouts") +SHELL_CONTENT = ( + "content/en/_index.md", + "content/cn/_index.md", + "content/en/about/_index.md", + "content/cn/about/_index.md", + "content/en/docs/SUMMARY.md", + "content/cn/docs/SUMMARY.md", +) +MENU_CONTENT = tuple( + f"content/{language}/{section}/_index.md" + for language in ("en", "cn") + for section in ("docs", "blog", "community") +) +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +REPOSITORY_URL = "https://github.com/apache/hugegraph-doc.git"; +VERSION_REFS = { + "latest": "master", + "1.7": "release-1.7.0", + "1.5": "release-1.5.0", +} +KNOWN_HISTORICAL_ROUTES = { + "/docs/introduction": "/docs/introduction/readme/", + "/cn/docs/introduction": "/cn/docs/introduction/readme/", + "/docs/quickstart/hugegraph-loader": "/docs/quickstart/toolchain/hugegraph-loader/", + "/cn/docs/quickstart/hugegraph-loader": "/cn/docs/quickstart/toolchain/hugegraph-loader/", +} +URL_ATTRIBUTE_RE = re.compile( + r"(?P<prefix>[\s<](?:href|src|action|poster|data-td-index-src|data-td-url|data-td-image-zoom)=)" + r"(?P<quote>[\"']?)(?P<url>[^\s\"'<>`]+)(?P=quote)", + re.IGNORECASE, +) +ACTION_MANIFEST_RE = re.compile( + r"(?P<open><script\b[^>]*\bid=[\"']?td-action-manifest[\"']?[^>]*>)" + r"(?P<body>.*?)" + r"(?P<close></script>)", + re.IGNORECASE | re.DOTALL, +) +MARKDOWN_DESTINATION_RE = re.compile( + r"(?P<open>\]\(\s*<?)(?P<url>(?:https?://[^\s)>]+|/[^\s)>]+))(?P<close>>?[^)]*\))" +) +HREFLANG_FALLBACKS = { + "cn/docs/changelog/hugegraph-0.12.0-release-notes/index.html": {"en-US": "/"}, + "community/maturity/index.html": {"zh-CN": "/cn/"}, +} + + +class DocumentParser(html.parser.HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.urls: list[tuple[str, str]] = [] + self.canonical: list[str] = [] + self.hreflang: list[tuple[str, str]] = [] + self.meta: list[dict[str, str]] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = {key.lower(): value or "" for key, value in attrs} + for attribute in ( + "href", + "src", + "action", + "poster", + "data-td-index-src", + "data-td-url", + "data-td-image-zoom", + ): + if values.get(attribute): + self.urls.append((attribute, values[attribute])) + if tag == "link" and values.get("rel", "").lower() == "canonical": + self.canonical.append(values.get("href", "")) + if ( + tag == "link" + and values.get("rel", "").lower() == "alternate" + and values.get("hreflang") + ): + self.hreflang.append((values["hreflang"], values.get("href", ""))) + if tag == "meta": + self.meta.append(values) + + +def refresh_target(parser: DocumentParser) -> str | None: + refresh = [ + item.get("content", "") + for item in parser.meta + if item.get("http-equiv", "").lower() == "refresh" + ] + if not refresh: + return None + if len(refresh) != 1: + fail(f"expected one refresh directive, found {len(refresh)}") + match = re.search(r"(?:^|;)\s*url\s*=\s*(.+)\s*$", refresh[0], re.IGNORECASE) + if not match: + fail(f"malformed refresh directive: {refresh[0]}") + return match.group(1).strip(" \"'") + + +def require_safe_url_scheme(value: str, source: str) -> bool: + """Reject active or ambiguous schemes; return whether target validation applies.""" + if value.startswith("//"): + fail(f"protocol-relative URL in {source}: {value}") + scheme = urllib.parse.urlsplit(value).scheme.lower() + if scheme in {"mailto", "tel"}: + return False + if scheme not in {"", "http", "https"}: + fail(f"forbidden URL scheme in {source}: {value}") + return True + + +def fail(message: str) -> NoReturn: + raise SystemExit(message) + + +def prepare_output_directory(path: pathlib.Path, label: str) -> pathlib.Path: + raw = path.expanduser() + if raw.is_symlink(): + fail(f"{label} must not be a symbolic link: {raw}") + output = raw.resolve() + allowed_roots = { + pathlib.Path(tempfile.gettempdir()).resolve(), + pathlib.Path("/tmp").resolve(), + } + runner_temp = os.environ.get("RUNNER_TEMP") + if runner_temp: + allowed_roots.add(pathlib.Path(runner_temp).resolve()) + if not any(root != output and root in output.parents for root in allowed_roots): + fail(f"{label} must be below a controlled temporary directory: {output}") + if output.exists(): + if output.is_symlink() or not output.is_dir(): + fail(f"{label} is not a removable directory: {output}") + shutil.rmtree(output) + output.parent.mkdir(parents=True, exist_ok=True) + return output + + +def run(command: list[str], *, cwd: pathlib.Path = ROOT) -> str: + result = subprocess.run( + command, + cwd=cwd, + check=True, + stdout=subprocess.PIPE, + stderr=None, + text=True, + ) + return result.stdout.strip() + + +def load_manifest(path: pathlib.Path) -> dict: + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schemaVersion") != 1: + fail("versions manifest schemaVersion must be 1") + versions = data.get("versions") + if not isinstance(versions, list) or not versions: + fail("versions manifest must contain a non-empty versions array") + ids: set[str] = set() + paths: set[str] = set() + for entry in versions: + required = {"id", "name", "ref", "publishPath", "archived", "githubBranch"} + if not isinstance(entry, dict) or not required.issubset(entry): + fail(f"invalid version entry: {entry!r}") + version_id = entry["id"] + expected_ref = VERSION_REFS.get(version_id) + if entry["ref"] != expected_ref: + fail(f"unexpected source ref for {version_id}: {entry['ref']}") + if entry["githubBranch"] != expected_ref: + fail(f"unexpected GitHub branch for {version_id}: {entry['githubBranch']}") + if entry["name"] != version_id: + fail(f"unexpected display name for {version_id}: {entry['name']}") + if entry["archived"] is not (version_id != "latest"): + fail(f"unexpected archive state for {version_id}: {entry['archived']}") + publish_path = entry["publishPath"].strip("/") + if version_id in ids or publish_path in paths: + fail(f"duplicate version id or publish path: {version_id}") + if version_id == "latest" and publish_path: + fail("latest must publish at the site root") + if version_id != "latest" and not publish_path.startswith("versions/"): + fail(f"historical version {version_id} must publish below versions/") + if ".." in pathlib.PurePosixPath(publish_path).parts: + fail(f"unsafe publish path for {version_id}: {publish_path}") + entry["publishPath"] = publish_path + ids.add(version_id) + paths.add(publish_path) + if [entry["id"] for entry in versions] != ["latest", "1.7", "1.5"]: + fail("version order must be latest, 1.7, 1.5") + if data.get("repository") != REPOSITORY_URL: + fail(f"versions manifest repository must be {REPOSITORY_URL}") + return data + + +def load_resolved_manifest(path: pathlib.Path) -> dict: + resolved = load_manifest(path) + expected = load_manifest(ROOT / "versions.json") + if resolved.get("repository") != expected.get("repository"): + fail("resolved manifest repository does not match versions.json") + fields = ("id", "name", "ref", "publishPath", "archived", "githubBranch") + for expected_entry, resolved_entry in zip( + expected["versions"], resolved["versions"] + ): + if any( + resolved_entry.get(field) != expected_entry.get(field) for field in fields + ): + fail( + f"resolved manifest entry drifted from versions.json: {resolved_entry!r}" + ) + if not SHA_RE.fullmatch(resolved_entry.get("sha", "")): + fail(f"resolved manifest has invalid SHA: {resolved_entry!r}") + return resolved + + +def require_metadata_matches(entry: dict, metadata: dict, source: pathlib.Path) -> None: + fields = ( + "id", + "name", + "ref", + "publishPath", + "archived", + "githubBranch", + "sha", + ) + mismatches = [field for field in fields if metadata.get(field) != entry.get(field)] + if mismatches: + fail(f"version metadata mismatch in {source}: {', '.join(mismatches)}") + + +def resolve_remote(repository: str, ref: str) -> str: + output = run(["git", "ls-remote", "--heads", repository, f"refs/heads/{ref}"]) + rows = [row.split() for row in output.splitlines() if row.strip()] + if len(rows) != 1 or len(rows[0]) != 2 or not SHA_RE.fullmatch(rows[0][0]): + fail(f"cannot resolve exactly one branch SHA for {ref}: {output!r}") + return rows[0][0] + + +def prepare(args: argparse.Namespace) -> None: + manifest = load_manifest(args.manifest) + resolved = [] + for entry in manifest["versions"]: + item = dict(entry) + if entry["id"] == "latest": + sha = run(["git", "rev-parse", f"{args.latest_sha}^{{commit}}"]) + elif args.local: + sha = run( + ["git", "rev-parse", f"refs/remotes/origin/{entry['ref']}^{{commit}}"] + ) + else: + sha = resolve_remote(manifest["repository"], entry["ref"]) + if not SHA_RE.fullmatch(sha): + fail(f"resolved value is not a commit SHA for {entry['id']}: {sha}") + item["sha"] = sha + resolved.append(item) + result = { + "schemaVersion": 1, + "repository": manifest["repository"], + "versions": resolved, + "include": resolved, + } + rendered = json.dumps( + result, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + if args.output: + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + + +def overlay_shell(assembly: pathlib.Path, *, historical: bool, origin: str) -> None: + for obsolete in ( + "config.toml", + "package.json", + "netlify.toml", + "deploy.sh", + ".nvmrc", + ): + path = assembly / obsolete + if path.exists() or path.is_symlink(): + path.unlink() + for obsolete_dir in (*SHELL_DIRS, "themes", "config"): + path = assembly / obsolete_dir + if path.exists(): + shutil.rmtree(path) + for name in SHELL_FILES: + shutil.copy2(ROOT / name, assembly / name) + for name in SHELL_DIRS: + source = ROOT / name + if source.exists(): + shutil.copytree(source, assembly / name, dirs_exist_ok=True) + license_source = ROOT / "static/licenses" + if license_source.exists(): + shutil.copytree( + license_source, assembly / "static/licenses", dirs_exist_ok=True + ) + client_go_target = assembly / "static/client-go" + if historical: + if client_go_target.exists(): + shutil.rmtree(client_go_target) + else: + client_go_source = ROOT / "static/client-go" + if client_go_source.exists(): + shutil.copytree(client_go_source, client_go_target, dirs_exist_ok=True) + for relative in SHELL_CONTENT: + if historical and not relative.endswith("docs/SUMMARY.md"): + continue + source = ROOT / relative + target = assembly / relative + target.parent.mkdir(parents=True, exist_ok=True) + legacy_html = target.with_suffix(".html") + if legacy_html.exists(): + legacy_html.unlink() + shutil.copy2(source, target) + for relative in MENU_CONTENT: + strip_menu_frontmatter(assembly / relative) + if historical: + prune_historical_content(assembly, origin) + + +def prune_historical_content(assembly: pathlib.Path, origin: str) -> None: + """Keep versioned Docs only; shared site surfaces continue to latest.""" + normalized_origin = origin.rstrip("/") + "/" + for language in ("en", "cn"): + language_root = assembly / f"content/{language}" + for path in sorted(language_root.iterdir()): + if path.name == "docs": + continue + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + + footer_path = assembly / f"data/footer/{language}.yaml" + footer = footer_path.read_text(encoding="utf-8") + prefix = "cn/" if language == "cn" else "" + replacements = { + f"url: /{prefix}blog/": ( + "url: '" + + urllib.parse.urljoin(normalized_origin, f"{prefix}blog/") + + "'" + ), + f"url: /{prefix}community/": ( + "url: '" + + urllib.parse.urljoin(normalized_origin, f"{prefix}community/") + + "'" + ), + } + for old, new in replacements.items(): + if footer.count(old) != 1: + fail(f"expected one shared footer route in {footer_path}: {old}") + footer = footer.replace(old, new) + footer_path.write_text(footer, encoding="utf-8") + + blog_ref = ( + '{{< ref path="/blog/hugegraph/toplingdb/' + f'toplingdb-quick-start.md" lang="{language}">}}}}' + ) + latest_blog = urllib.parse.urljoin( + normalized_origin, + f"{prefix}blog/hugegraph/toplingdb/toplingdb-quick-start/", + ) + for path in sorted((language_root / "docs").rglob("*.md")): + text = path.read_text(encoding="utf-8") + if blog_ref in text: + path.write_text(text.replace(blog_ref, latest_blog), encoding="utf-8") + + +def strip_menu_frontmatter(path: pathlib.Path) -> None: + if not path.is_file(): + return + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + if not lines or lines[0].strip() != "---": + fail(f"cannot remove menu from non-YAML front matter: {path}") + try: + closing_index = next( + index for index, line in enumerate(lines[1:], 1) if line.strip() == "---" + ) + except StopIteration: + fail(f"unterminated YAML front matter: {path}") + frontmatter = lines[1:closing_index] + output = [lines[0]] + index = 0 + while index < len(frontmatter): + line = frontmatter[index] + if re.fullmatch(r"menu:\s*", line): + index += 1 + while index < len(frontmatter): + candidate = frontmatter[index] + if candidate.strip() and candidate[:1] not in {" ", "\t"}: + break + index += 1 + continue + output.append(line) + index += 1 + output.append(lines[closing_index]) + output.extend(lines[closing_index + 1 :]) + path.write_text("".join(output), encoding="utf-8") + + +def base_url(origin: str, publish_path: str) -> str: + normalized_origin = origin.rstrip("/") + "/" + if not publish_path: + return normalized_origin + return urllib.parse.urljoin(normalized_origin, publish_path.rstrip("/") + "/") + + +def apply_known_legacy_fixes(assembly: pathlib.Path, version: str) -> int: + fixed = 0 + if version in {"1.7", "1.5"}: + for language in ("en", "cn"): + language_prefix = "/cn" if language == "cn" else "" + summary_path = assembly / f"content/{language}/docs/SUMMARY.md" + summary = summary_path.read_text(encoding="utf-8") + summary_count = summary.count("performance/api-performance") + if summary_count != 3: + fail( + f"expected 3 historical performance routes in {summary_path}, " + f"found {summary_count}" + ) + summary_path.write_text( + summary.replace( + "performance/api-performance", "performance/api-preformance" + ), + encoding="utf-8", + ) + fixed += summary_count + replacements = [] + if language == "cn": + replacements.append( + ( + "/cn/docs/quickstart/hugegraph-server", + "/cn/docs/quickstart/hugegraph/hugegraph-server", + ) + ) + replacements.extend( + [ + ( + "/docs/quickstart/hugegraph-server", + f"{language_prefix}/docs/quickstart/hugegraph/hugegraph-server", + ), + ( + "/clients/gremlin-console.html", + f"{language_prefix}/docs/clients/gremlin-console/", + ), + ( + "./hugegraph-style.xml", + "https://github.com/apache/hugegraph/blob/"; + f"release-{version}.0/style/checkstyle.xml", + ), + ] + ) + for path in sorted((assembly / f"content/{language}").rglob("*.md")): + text = path.read_text(encoding="utf-8") + rendered = text + for old, new in replacements: + count = rendered.count(old) + rendered = rendered.replace(old, new) + fixed += count + if rendered != text: + path.write_text(rendered, encoding="utf-8") + if fixed < 4: + fail(f"expected known historical route fixes for {version}, found {fixed}") + return fixed + + +def version_urls(manifest: dict, origin: str) -> list[dict]: + urls = [] + for entry in manifest["versions"]: + path = f"{entry['publishPath']}/docs/" if entry["publishPath"] else "docs/" + urls.append( + { + "version": entry["id"], + "name": entry["name"], + "url": urllib.parse.urljoin(origin.rstrip("/") + "/", path), + "pagelinks": False, + } + ) + return urls + + +def historical_language_menus(origin: str) -> dict: + normalized_origin = origin.rstrip("/") + "/" + result = {} + for language, labels in ( + ( + "en", + { + "docs": "Documentation", + "download": "Download", + "blog": "Blog", + "community": "Community", + }, + ), + ( + "cn", + {"docs": "文档", "download": "下载", "blog": "博客", "community": "社区"}, + ), + ): + language_prefix = "cn/" if language == "cn" else "" + result[language] = { + "menus": { + "main": [ + { + "identifier": "docs", + "name": labels["docs"], + "pageRef": "/docs", + "weight": 10, + }, + { + "identifier": "download", + "name": labels["download"], + "pageRef": "/docs/download/download", + "weight": 20, + }, + { + "identifier": "blog", + "name": labels["blog"], + "url": urllib.parse.urljoin( + normalized_origin, f"{language_prefix}blog/" + ), + "weight": 30, + }, + { + "identifier": "community", + "name": labels["community"], + "url": urllib.parse.urljoin( + normalized_origin, f"{language_prefix}community/" + ), + "weight": 40, + }, + { + "identifier": "github", + "name": "GitHub", + "url": "https://github.com/apache/hugegraph";, + "weight": 50, + }, + ] + } + } + return result + + +def allowed_version_paths(manifest: dict) -> set[str]: + result = {"/", "/cn", "/blog", "/cn/blog", "/community", "/cn/community"} + for entry in manifest["versions"]: + path = f"/{entry['publishPath']}/docs" if entry["publishPath"] else "/docs" + result.add(path.rstrip("/") or "/") + return result + + +def is_latest_shared_path(path: str) -> bool: + normalized = path.rstrip("/") or "/" + return normalized in {"/", "/cn"} or any( + normalized == shared or normalized.startswith(shared + "/") + for shared in ("/blog", "/cn/blog", "/community", "/cn/community") + ) + + +def rewrite_internal_url( + value: str, + *, + origin: str, + publish_path: str, + allowed_paths: set[str], +) -> str: + """Scope a same-site absolute/root URL to one historical artifact.""" + if not value or value.startswith(("#", "?")): + return value + if value.startswith("//"): + fail(f"protocol-relative URL is not allowed in a version artifact: {value}") + parsed = urllib.parse.urlsplit(value) + origin_parts = urllib.parse.urlsplit(origin.rstrip("/") + "/") + canonical_parts = urllib.parse.urlsplit(CANONICAL_ORIGIN) + absolute = bool(parsed.scheme or parsed.netloc) + if absolute: + if parsed.scheme not in {"http", "https"} or parsed.netloc not in { + origin_parts.netloc, + canonical_parts.netloc, + }: + return value + path = parsed.path or "/" + elif value.startswith("/"): + path = parsed.path or "/" + else: + return value + + if not publish_path: + if absolute and parsed.netloc != origin_parts.netloc: + return urllib.parse.urlunsplit( + ( + origin_parts.scheme, + origin_parts.netloc, + path, + parsed.query, + parsed.fragment, + ) + ) + return value + + normalized = path.rstrip("/") or "/" + prefix = "/" + publish_path.strip("/") + internal_path = ( + path[len(prefix) :] + if normalized == prefix or normalized.startswith(prefix + "/") + else path + ) + normalized_internal = internal_path.rstrip("/") or "/" + mapped_path = KNOWN_HISTORICAL_ROUTES.get(normalized_internal) + if mapped_path is not None: + scoped_path = prefix + mapped_path + if absolute: + return urllib.parse.urlunsplit( + ( + origin_parts.scheme, + origin_parts.netloc, + scoped_path, + parsed.query, + parsed.fragment, + ) + ) + return urllib.parse.urlunsplit( + ("", "", scoped_path, parsed.query, parsed.fragment) + ) + if normalized == prefix or normalized.startswith(prefix + "/"): + if absolute and parsed.scheme != origin_parts.scheme: + return urllib.parse.urlunsplit( + ( + origin_parts.scheme, + origin_parts.netloc, + path, + parsed.query, + parsed.fragment, + ) + ) + return value + if is_latest_shared_path(normalized) or (absolute and normalized in allowed_paths): + return urllib.parse.urlunsplit( + ( + origin_parts.scheme, + origin_parts.netloc, + path, + parsed.query, + parsed.fragment, + ) + ) + if normalized == "/versions" or normalized.startswith("/versions/"): + fail(f"non-selector cross-version URL is not allowed: {value}") + + scoped_path = prefix + (path if path.startswith("/") else "/" + path) + if absolute: + return urllib.parse.urlunsplit( + ( + origin_parts.scheme, + origin_parts.netloc, + scoped_path, + parsed.query, + parsed.fragment, + ) + ) + return urllib.parse.urlunsplit(("", "", scoped_path, parsed.query, parsed.fragment)) + + +def rewrite_json_urls(value, rewrite): + if isinstance(value, dict): + return {key: rewrite_json_urls(item, rewrite) for key, item in value.items()} + if isinstance(value, list): + return [rewrite_json_urls(item, rewrite) for item in value] + if isinstance(value, str): + return rewrite(value) + return value + + +def rewrite_text_urls(text: str, rewrite, *, markdown: bool) -> tuple[str, int]: + count = 0 + + def replace_attribute(match: re.Match) -> str: + nonlocal count + old = match.group("url") + new = rewrite(old) + count += old != new + return ( + f"{match.group('prefix')}{match.group('quote')}{new}{match.group('quote')}" + ) + + if markdown: + in_fence = False + lines = [] + for line in text.splitlines(keepends=True): + if re.match(r"^\s*(```|~~~)", line): + in_fence = not in_fence + lines.append(line) + continue + if not in_fence: + + def replace_destination(match: re.Match) -> str: + nonlocal count + old = match.group("url") + new = rewrite(old) + count += old != new + return f"{match.group('open')}{new}{match.group('close')}" + + line = URL_ATTRIBUTE_RE.sub(replace_attribute, line) + line = MARKDOWN_DESTINATION_RE.sub(replace_destination, line) + lines.append(line) + text = "".join(lines) + else: + text = URL_ATTRIBUTE_RE.sub(replace_attribute, text) + return text, count + + +def scope_version_artifact( + output: pathlib.Path, + manifest: dict, + entry: dict, + origin: str, +) -> dict: + """Repair URL fields Hugo cannot canonify, then return auditable counts.""" + if not entry["publishPath"] and origin.rstrip("/") == CANONICAL_ORIGIN.rstrip("/"): + return {"files": 0, "urls": 0, "manifests": 0, "searchRefs": 0} + allowed_paths = allowed_version_paths(manifest) + + def rewrite(value: str) -> str: + return rewrite_internal_url( + value, + origin=origin, + publish_path=entry["publishPath"], + allowed_paths=allowed_paths, + ) + + stats = {"files": 0, "urls": 0, "manifests": 0, "searchRefs": 0} + for path in sorted(output.rglob("*")): + if not path.is_file(): + continue + if path.match("offline-search-index.*.json"): + parts = path.name.split(".") + digest = hashlib.md5(path.read_bytes()).hexdigest() + if len(parts) != 4 or parts[2] != digest: + fail( + f"stale search index fingerprint before rewrite: {path.relative_to(output)}" + ) + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, list): + fail(f"unexpected search index shape: {path}") + for item in data: + if not isinstance(item, dict) or not isinstance(item.get("ref"), str): + fail(f"invalid search index entry: {path}") + old = item["ref"] + item["ref"] = rewrite(old) + stats["searchRefs"] += old != item["ref"] + path.write_text( + json.dumps(data, ensure_ascii=False, separators=(",", ":")), + encoding="utf-8", + ) + stats["files"] += 1 + continue + if path.suffix not in {".html", ".md", ".xml"}: + continue + original = path.read_text(encoding="utf-8") + rendered, changed = rewrite_text_urls( + original, + rewrite, + markdown=path.suffix == ".md", + ) + + def replace_manifest(match: re.Match) -> str: + data = json.loads(match.group("body")) + rewritten = rewrite_json_urls(data, rewrite) + stats["manifests"] += 1 + return ( + match.group("open") + + json.dumps(rewritten, ensure_ascii=False, separators=(",", ":")) + + match.group("close") + ) + + if path.suffix == ".html": + rendered = ACTION_MANIFEST_RE.sub(replace_manifest, rendered) + if rendered != original: + path.write_text(rendered, encoding="utf-8") + stats["files"] += 1 + stats["urls"] += changed + renamed_indexes = [] + for path in sorted(output.glob("offline-search-index.*.json")): + digest = hashlib.md5(path.read_bytes()).hexdigest() + parts = path.name.split(".") + if len(parts) != 4: + fail(f"unexpected search index filename: {path.name}") + new_name = f"offline-search-index.{parts[1]}.{digest}.json" + if path.name != new_name: + new_path = path.with_name(new_name) + path.rename(new_path) + renamed_indexes.append((path.name, new_name)) + for old_name, new_name in renamed_indexes: + references = 0 + for path in sorted(output.rglob("*")): + if not path.is_file() or path.suffix not in { + ".html", + ".md", + ".xml", + ".txt", + ".json", + }: + continue + text = path.read_text(encoding="utf-8") + count = text.count(old_name) + if count: + path.write_text(text.replace(old_name, new_name), encoding="utf-8") + references += count + if references == 0: + fail(f"renamed search index has no generated reference: {old_name}") + stats["searchFingerprints"] = len(renamed_indexes) + return stats + + +def write_historical_home_redirects(output: pathlib.Path, origin: str) -> int: + normalized_origin = origin.rstrip("/") + "/" + targets = ( + (output / "index.html", "en-US", normalized_origin, "Apache HugeGraph"), + ( + output / "cn/index.html", + "zh-CN", + urllib.parse.urljoin(normalized_origin, "cn/"), + "Apache HugeGraph", + ), + ) + for path, language, target, title in targets: + escaped_target = html.escape(target, quote=True) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "<!doctype html>\n" + f'<html lang="{language}"><head><meta charset="utf-8">\n' + '<meta name="robots" content="noindex,follow">\n' + f'<link rel="canonical" href="{escaped_target}">\n' + f'<meta http-equiv="refresh" content="0; url={escaped_target}">\n' + f"<title>{title}</title></head><body>\n" + f'<p><a href="{escaped_target}">Continue to {title}</a></p>\n' + "</body></html>\n", + encoding="utf-8", + ) + return len(targets) + + +def public_url_for_file( + path: pathlib.Path, + root: pathlib.Path, + origin: str, + publish_path: str, +) -> str: + relative = path.relative_to(root).as_posix() + if relative == "index.html": + relative = "" + elif relative.endswith("/index.html"): + relative = relative[: -len("index.html")] + base = base_url(origin, publish_path) + return urllib.parse.urljoin(base, relative) + + +def target_exists(root: pathlib.Path, relative: str) -> bool: + relative = urllib.parse.unquote(relative).lstrip("/") + parts = pathlib.PurePosixPath(relative).parts + if ".." in parts: + return False + candidate = root.joinpath(*parts) + return candidate.is_file() or (candidate / "index.html").is_file() + + +def iter_json_strings(value): + if isinstance(value, dict): + for item in value.values(): + yield from iter_json_strings(item) + elif isinstance(value, list): + for item in value: + yield from iter_json_strings(item) + elif isinstance(value, str): + yield value + + +def iter_json_url_fields(value): + if isinstance(value, dict): + for key, item in value.items(): + if key in {"baseURL", "url", "markdown"} and isinstance(item, str): + yield item + else: + yield from iter_json_url_fields(item) + elif isinstance(value, list): + for item in value: + yield from iter_json_url_fields(item) + + +def validate_artifact(args: argparse.Namespace) -> None: + manifest = load_manifest(args.manifest) + entry = next( + (item for item in manifest["versions"] if item["id"] == args.version), None + ) + if entry is None: + fail(f"unknown version {args.version}") + root = args.artifact.resolve() + metadata_path = root / ".version.json" + if not metadata_path.is_file(): + fail(f"missing version metadata: {metadata_path}") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + expected_base = base_url(args.site_origin, entry["publishPath"]) + expected_entry = dict(entry) + expected_entry["sha"] = args.sha + require_metadata_matches(expected_entry, metadata, metadata_path) + if metadata.get("baseURL") != expected_base: + fail(f"version metadata does not match {entry['id']} at {expected_base}") + if entry["archived"]: + for shared_path in ( + "about", + "blog", + "client-go", + "community", + "cn/about", + "cn/blog", + "cn/community", + ): + if (root / shared_path).exists(): + fail(f"historical artifact contains latest-only output: {shared_path}") + + origin_parts = urllib.parse.urlsplit(args.site_origin.rstrip("/") + "/") + canonical_parts = urllib.parse.urlsplit(CANONICAL_ORIGIN) + prefix = "/" + entry["publishPath"].strip("/") if entry["publishPath"] else "" + allowed_paths = allowed_version_paths(manifest) + current_docs = (prefix + "/docs").rstrip("/") or "/docs" + checked_urls = 0 + canonical_pages = 0 + manifest_pages = 0 + contract_data = json.loads(URL_CONTRACT.read_text(encoding="utf-8")) + if contract_data.get("schemaVersion") != 1 or not isinstance( + contract_data.get("routes"), list + ): + fail(f"invalid URL contract: {URL_CONTRACT}") + contract_routes = 0 + for route in contract_data["routes"]: + if entry["id"] not in route.get("versions", []): + continue + public_path = route.get("path") + file_path = route.get("file") + markers = route.get("contains") + if ( + not isinstance(public_path, str) + or not public_path.startswith("/") + or not isinstance(file_path, str) + or ".." in pathlib.PurePosixPath(file_path).parts + or not isinstance(markers, list) + or not all(isinstance(marker, str) and marker for marker in markers) + ): + fail(f"invalid URL contract route: {route!r}") + expected_file = ( + "index.html" + if public_path == "/" + else public_path.lstrip("/") + + ("index.html" if public_path.endswith("/") else "") + ) + if file_path != expected_file: + fail(f"URL contract path/file mismatch: {public_path} -> {file_path}") + target = root / file_path + if not target.is_file(): + fail(f"URL contract target missing for {entry['id']}: {public_path}") + body = target.read_text(encoding="utf-8") + for marker in markers: + if marker not in body: + fail( + f"URL contract marker missing for {entry['id']} {public_path}: {marker}" + ) + contract_routes += 1 + + def validate_url(value: str, source: pathlib.Path) -> None: + nonlocal checked_urls + if not value or value.startswith(("#", "?")): + return + source_name = source.relative_to(root).as_posix() + if not require_safe_url_scheme(value, source_name): + return + parsed = urllib.parse.urlsplit(value) + if ( + parsed.netloc == canonical_parts.netloc + and parsed.netloc != origin_parts.netloc + ): + fail(f"production-origin URL leaked into staging {source_name}: {value}") + if parsed.netloc and parsed.netloc != origin_parts.netloc: + return + if not parsed.netloc and not value.startswith("/"): + source_relative = source.relative_to(root).as_posix() + if source.suffix == ".xml" or source_relative.startswith( + ("_print/", "cn/_print/") + ): + return + value = urllib.parse.urljoin( + public_url_for_file( + source, root, args.site_origin, entry["publishPath"] + ), + value, + ) + parsed = urllib.parse.urlsplit(value) + if parsed.netloc and parsed.netloc != origin_parts.netloc: + return + if parsed.scheme and parsed.scheme != origin_parts.scheme: + fail( + f"same-site URL uses the wrong scheme in {source.relative_to(root)}: {value}" + ) + path = parsed.path or "/" + normalized = path.rstrip("/") or "/" + if ( + parsed.netloc + and (normalized in allowed_paths or is_latest_shared_path(normalized)) + and normalized != current_docs + ): + checked_urls += 1 + return + if prefix and not (normalized == prefix or normalized.startswith(prefix + "/")): + fail( + f"URL escapes version {entry['id']} in {source.relative_to(root)}: {value}" + ) + relative = path[len(prefix) :] if prefix else path + if not target_exists(root, relative): + fail(f"missing local target from {source.relative_to(root)}: {value}") + checked_urls += 1 + + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + if path.match("offline-search-index.*.json"): + parts = path.name.split(".") + digest = hashlib.md5(path.read_bytes()).hexdigest() + if len(parts) != 4 or parts[2] != digest: + fail(f"stale search index fingerprint: {path.relative_to(root)}") + data = json.loads(path.read_text(encoding="utf-8")) + for item in data: + ref = item.get("ref") if isinstance(item, dict) else None + if not isinstance(ref, str): + fail(f"invalid search entry in {path.relative_to(root)}") + validate_url(ref, path) + continue + if path.name == "navigation.json": + data = json.loads(path.read_text(encoding="utf-8")) + if data.get("schemaVersion") != 1: + fail(f"invalid navigation schema in {path.relative_to(root)}") + relative = path.relative_to(root).as_posix() + language = "cn" if relative.startswith("cn/") else "en" + expected_root_url = ( + urllib.parse.urljoin(expected_base, "cn/") + if language == "cn" + else expected_base + ) + if data.get("baseURL") != expected_base or data.get("language") != language: + fail(f"navigation baseURL/language mismatch in {relative}") + if data.get("root", {}).get("url") != expected_root_url: + fail(f"navigation root is not language-scoped in {relative}") + for value in iter_json_url_fields(data): + validate_url(value, path) + value_parts = urllib.parse.urlsplit(value) + if value == expected_base or ( + value_parts.netloc and value_parts.netloc != origin_parts.netloc + ): + continue + value_path = value_parts.path + language_prefix = prefix + "/cn/" + if language == "cn" and not value_path.startswith(language_prefix): + fail(f"navigation URL loses Chinese scope in {relative}: {value}") + if language == "en" and value_path.startswith(language_prefix): + fail( + f"navigation URL crosses into Chinese scope in {relative}: {value}" + ) + continue + if path.suffix not in {".html", ".md", ".xml", ".txt"}: + continue + text = path.read_text(encoding="utf-8") + if path.suffix not in {".md", ".txt"}: + for match in URL_ATTRIBUTE_RE.finditer(text): + validate_url(match.group("url"), path) + if path.suffix == ".xml": + try: + xml_root = ET.fromstring(text) + except ET.ParseError as error: + fail(f"invalid XML in {path.relative_to(root)}: {error}") + for node in xml_root.iter(): + if ( + node.tag.split("}")[-1] in {"loc", "link", "guid"} + and (node.text or "").strip() + ): + validate_url((node.text or "").strip(), path) + if path.suffix in {".md", ".txt"}: + in_fence = False + for line in text.splitlines(): + if re.match(r"^\s*(```|~~~)", line): + in_fence = not in_fence + continue + if not in_fence: + for match in URL_ATTRIBUTE_RE.finditer(line): + validate_url(match.group("url"), path) + for match in MARKDOWN_DESTINATION_RE.finditer(line): + validate_url(match.group("url"), path) + relative = path.relative_to(root).as_posix() + if path.name == "llms.txt": + is_chinese = relative.startswith("cn/") + for value in re.findall(r"https?://[^)\s]+", text): + value_path = urllib.parse.urlsplit(value).path + chinese_prefix = prefix + "/cn/" + shared_latest = is_latest_shared_path(value_path) + if is_chinese and not value_path.startswith(chinese_prefix): + if shared_latest and value_path.startswith("/cn/"): + continue + if value_path != prefix + "/index.md": + fail(f"Chinese llms index loses language scope: {value}") + if not is_chinese and value_path.startswith(chinese_prefix): + if value_path != prefix + "/cn/index.md": + fail(f"English llms index crosses language scope: {value}") + if ( + not is_chinese + and shared_latest + and value_path.startswith("/cn/") + ): + fail(f"English llms index crosses language scope: {value}") + if path.suffix != ".html": + continue + relative = path.relative_to(root).as_posix() + archive_exceptions = {"404.html", "cn/404.html", "client-go/index.html"} + canonical_exceptions = {"client-go/index.html"} + if relative in {"404.html", "cn/404.html"} and ( + "td-site-header" not in text or "td-print-view" in text + ): + fail(f"404 page does not use the interactive OINK shell: {relative}") + document = DocumentParser() + document.feed(text) + alias_target = refresh_target(document) + if alias_target: + validate_url(alias_target, path) + manifests = list(ACTION_MANIFEST_RE.finditer(text)) + if manifests: + if len(manifests) != 1: + fail(f"unexpected action manifest count in {path.relative_to(root)}") + action_data = json.loads(manifests[0].group("body")) + for value in iter_json_url_fields(action_data): + if value: + validate_url(value, path) + actions = { + item.get("id"): item + for item in action_data.get("actions", []) + if isinstance(item, dict) + } + for action_id in ("open_chatgpt", "open_claude"): + action = actions.get(action_id) + placements = action.get("placements", {}) if action else {} + if ( + not action + or action.get("available") is not False + or action.get("url") != "" + or placements.get("page") is not False + or placements.get("palette") is not False + ): + fail( + f"assistant action is externally enabled in " + f"{path.relative_to(root)}: {action_id}" + ) + switch = next( + ( + item + for item in action_data.get("actions", []) + if item.get("id") == "switch_version" + ), + None, + ) + expected_options = version_urls(manifest, args.site_origin) + if switch is None or [ + ( + item.get("id"), + item.get("title"), + str(item.get("url", "")).rstrip("/"), + item.get("active"), + ) + for item in switch.get("options", []) + ] != [ + ( + item["version"], + item["name"], + item["url"].rstrip("/"), + item["version"] == entry["id"], + ) + for item in expected_options + ]: + fail(f"version switch contract mismatch in {path.relative_to(root)}") + if ( + entry["archived"] + and relative not in archive_exceptions + and not relative.startswith(("_print/", "cn/_print/")) + and "td-page-notice--primary" not in text + ): + fail(f"archive notice missing in {path.relative_to(root)}") + manifest_pages += 1 + + for _attribute, value in document.urls: + validate_url(value, path) + hostname = (urllib.parse.urlsplit(value).hostname or "").lower() + if hostname in {"chatgpt.com", "claude.ai", "anthropic.com"}: + fail(f"assistant link is externally enabled in {relative}: {value}") + + canonical_tags = [ + tag + for tag in re.findall(r"<link\b[^>]*>", text, flags=re.IGNORECASE) + if re.search( + r"\brel=[\"']?canonical(?:[\"'\s>]|$)", tag, flags=re.IGNORECASE + ) + ] + if relative in canonical_exceptions: + if len(canonical_tags) > 1: + fail(f"too many canonicals on special page {relative}") + if canonical_tags: + match = URL_ATTRIBUTE_RE.search(canonical_tags[0]) + if match is None: + fail(f"cannot parse canonical in {relative}") + validate_url(match.group("url"), path) + canonical_pages += 1 + else: + if len(canonical_tags) != 1: + fail( + f"expected one canonical in {relative}, found {len(canonical_tags)}" + ) + match = URL_ATTRIBUTE_RE.search(canonical_tags[0]) + if match is None or not canonical_tags[0].lower().startswith("<link"): + fail(f"cannot parse canonical in {relative}") + validate_url(match.group("url"), path) + actual_canonical = match.group("url") + if alias_target: + expected_canonical = urllib.parse.urljoin(expected_base, alias_target) + elif "_print/" in relative: + parts = list(pathlib.PurePosixPath(relative).parts) + parts.remove("_print") + regular = pathlib.PurePosixPath(*parts).as_posix() + if regular.endswith("index.html"): + regular = regular[: -len("index.html")] + expected_canonical = urllib.parse.urljoin(expected_base, regular) + else: + expected_canonical = public_url_for_file( + path, root, args.site_origin, entry["publishPath"] + ) + if actual_canonical != expected_canonical: + fail( + f"canonical mismatch in {relative}: " + f"{actual_canonical} != {expected_canonical}" + ) + canonical_pages += 1 + + is_regular_page = ( + "_print/" not in relative + and relative not in archive_exceptions + and not alias_target + ) + if is_regular_page: + actual_hreflang = dict(document.hreflang) + if len(actual_hreflang) != len(document.hreflang): + fail(f"duplicate hreflang entries in {relative}") + current_path = public_url_for_file( + path, root, args.site_origin, entry["publishPath"] + ) + current_parts = urllib.parse.urlsplit(current_path) + language_base_path = prefix + "/cn/" + if current_parts.path.startswith(language_base_path): + english_path = prefix + current_parts.path[len(prefix + "/cn") :] + else: + english_path = current_parts.path + chinese_path = ( + prefix + "/cn/" + if english_path.rstrip("/") == prefix + else prefix + "/cn" + english_path[len(prefix) :] + ) + expected_hreflang = { + "en-US": urllib.parse.urlunsplit( + (origin_parts.scheme, origin_parts.netloc, english_path, "", "") + ), + "zh-CN": urllib.parse.urlunsplit( + (origin_parts.scheme, origin_parts.netloc, chinese_path, "", "") + ), + } + for language, fallback_path in HREFLANG_FALLBACKS.get(relative, {}).items(): + expected_hreflang[language] = urllib.parse.urljoin( + expected_base, fallback_path.lstrip("/") + ) + if actual_hreflang != expected_hreflang: + fail( + f"hreflang mismatch in {relative}: " + f"{actual_hreflang} != {expected_hreflang}" + ) + + if not entry["archived"]: + client_go = root / "client-go/index.html" + if not client_go.is_file(): + fail("missing client-go/index.html") + client_parser = DocumentParser() + client_parser.feed(client_go.read_text(encoding="utf-8")) + go_import = [ + item.get("content", "") + for item in client_parser.meta + if item.get("name") == "go-import" + ] + if go_import != [ + "hugegraph.apache.org/client-go git " + "https://github.com/apache/hugegraph-toolchain.git"; + ]: + fail("client-go go-import metadata changed") + expected_go_source = ( + "hugegraph.apache.org/client-go " + "https://github.com/apache/hugegraph-toolchain " + "https://github.com/apache/hugegraph-toolchain/tree/master/"; + "hugegraph-client-go{/dir} https://github.com/apache/hugegraph-toolchain/"; + "blob/master/hugegraph-client-go{/dir}/{file}#L{line}" + ) + go_source = [ + item.get("content", "") + for item in client_parser.meta + if item.get("name") == "go-source" + ] + if go_source != [expected_go_source]: + fail("client-go go-source metadata changed") + package_url = "https://pkg.go.dev/hugegraph.apache.org/client-go"; + if refresh_target(client_parser) != package_url: + fail("client-go refresh target changed") + if [url for attribute, url in client_parser.urls if attribute == "href"] != [ + package_url + ]: + fail("client-go visible redirect link changed") + + license_root = root / "licenses/oink" + for name in ("LICENSE", "NOTICE", "VENDOR.json"): + if not (license_root / name).is_file(): + fail(f"missing OINK license bundle file: {name}") + vendor = json.loads((license_root / "VENDOR.json").read_text(encoding="utf-8")) + for dependency in vendor.get("dependencies", []): + for name in dependency.get("licenseFiles", []): + if not (license_root / name).is_file(): + fail(f"missing OINK dependency license: {name}") + + if manifest_pages == 0 or canonical_pages == 0: + fail("artifact validation did not inspect rendered pages") + print( + f"validated {entry['id']}: {canonical_pages} canonical pages, " + f"{manifest_pages} action manifests, {checked_urls} internal URLs, " + f"{contract_routes} contract routes" + ) + + +def build(args: argparse.Namespace) -> None: + manifest = load_manifest(args.manifest) + entry = next( + (item for item in manifest["versions"] if item["id"] == args.version), None + ) + if entry is None: + fail(f"unknown version {args.version}") + if not SHA_RE.fullmatch(args.sha): + fail(f"invalid source SHA: {args.sha}") + try: + run(["git", "cat-file", "-e", f"{args.sha}^{{commit}}"]) + except subprocess.CalledProcessError: + run(["git", "fetch", "--no-tags", "origin", args.sha]) + + output = prepare_output_directory(args.output, "version output") + + with tempfile.TemporaryDirectory(prefix=f"hugegraph-{args.version}-") as temp_name: + assembly = pathlib.Path(temp_name) / "site" + subprocess.run( + [ + "git", + "clone", + "--quiet", + "--shared", + "--no-checkout", + str(ROOT), + str(assembly), + ], + cwd=ROOT, + check=True, + ) + subprocess.run( + ["git", "checkout", "--quiet", "--detach", args.sha], + cwd=assembly, + check=True, + ) + overlay_shell( + assembly, + historical=bool(entry["archived"]), + origin=args.site_origin, + ) + site_base = base_url(args.site_origin, entry["publishPath"]) + override = { + "baseURL": site_base, + "canonifyURLs": True, + "params": { + "version": entry["id"], + "version_menu": "Releases", + "version_menu_pagelinks": False, + "versions": version_urls(manifest, args.site_origin), + "archived_version": bool(entry["archived"]), + "url_latest_version": urllib.parse.urljoin( + args.site_origin.rstrip("/") + "/", "docs/" + ), + "github_repo": "https://github.com/apache/hugegraph-doc";, + "github_branch": entry["githubBranch"], + }, + } + if entry["archived"]: + override["languages"] = historical_language_menus(args.site_origin) + override_path = assembly / "version-config.json" + override_path.write_text( + json.dumps(override, ensure_ascii=False), encoding="utf-8" + ) + hugo = os.environ.get("HUGO_BIN", "hugo") + go = os.environ.get("GO_BIN", "go") + go_executable = shutil.which(go) + if go_executable is None: + fail(f"Go executable is unavailable: {go}") + module_result = subprocess.run( + [ + go_executable, + "mod", + "download", + "-json", + "github.com/pgsty/[email protected]", + ], + cwd=assembly, + check=True, + stdout=subprocess.PIPE, + text=True, + ) + module = json.loads(module_result.stdout) + if ( + module.get("Path") != "github.com/pgsty/oink" + or module.get("Version") != "v1.0.0" + or module.get("Sum") != "h1:E+WHFP9zSRT+5RKoIkWNp+ASRGS1BKG+rDEi9by/BjE=" + ): + fail(f"unexpected OINK module metadata: {module!r}") + migration_script = pathlib.Path(module["Dir"]) / "bin/migrations/oink06.py" + if not migration_script.is_file(): + fail(f"pinned OINK migration tool is absent: {migration_script}") + migration_report = assembly / ".oink06-migration.json" + migration_python = os.environ.get("OINK_PYTHON", sys.executable) + subprocess.run( + [ + migration_python, + str(migration_script), + "migrate", + "--site", + str(assembly), + "--paths", + "content", + "--write", + "--json", + str(migration_report), + "--quiet", + ], + cwd=assembly, + check=True, + ) + known_fixes = apply_known_legacy_fixes(assembly, entry["id"]) + subprocess.run( + [ + migration_python, + str(migration_script), + "check", + "--site", + str(assembly), + "--paths", + "content", + ], + cwd=assembly, + check=True, + ) + command = [ + hugo, + "--config", + "hugo.yaml,version-config.json", + "--destination", + str(output), + "--cleanDestinationDir", + "--gc", + "--minify", + "--environment", + "production", + "--printPathWarnings", + "--printI18nWarnings", + "--panicOnWarning", + "--logLevel", + "info", + ] + build_environment = os.environ.copy() + go_directory = str(pathlib.Path(go_executable).resolve().parent) + build_environment["PATH"] = ( + go_directory + os.pathsep + build_environment.get("PATH", "") + ) + subprocess.run(command, cwd=assembly, check=True, env=build_environment) + url_scoping = scope_version_artifact( + output, + manifest, + entry, + args.site_origin, + ) + url_scoping["historicalHomeRedirects"] = ( + write_historical_home_redirects(output, args.site_origin) + if entry["archived"] + else 0 + ) + metadata = dict(entry) + migration_data = json.loads(migration_report.read_text(encoding="utf-8")) + metadata.update( + { + "sha": args.sha, + "baseURL": site_base, + "migration": { + "files": len(migration_data.get("files", [])), + "changed": sum( + 1 + for item in migration_data.get("files", []) + if item.get("changed") + ), + "findings": sum( + len(item.get("findings", [])) + for item in migration_data.get("files", []) + ), + "knownFixes": known_fixes, + "residual": 0, + }, + "urlScoping": url_scoping, + } + ) + (output / ".version.json").write_text( + json.dumps(metadata, ensure_ascii=False, sort_keys=True, indent=2) + "\n", + encoding="utf-8", + ) + print(f"built {entry['id']}@{args.sha} -> {output} ({site_base})") + + +def sitemap_locations(path: pathlib.Path) -> list[str]: + return [ + node.text or "" for node in ET.parse(path).iter() if node.tag.endswith("loc") + ] + + +def write_aggregate_sitemap(output: pathlib.Path, origin: str, manifest: dict) -> None: + locations = [ + urllib.parse.urljoin(origin.rstrip("/") + "/", "en/sitemap.xml"), + urllib.parse.urljoin(origin.rstrip("/") + "/", "cn/sitemap.xml"), + ] + for entry in manifest["versions"]: + if entry["publishPath"]: + locations.append( + urllib.parse.urljoin( + origin.rstrip("/") + "/", entry["publishPath"] + "/sitemap.xml" + ) + ) + root = ET.Element( + "sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"; + ) + for location in locations: + item = ET.SubElement(root, "sitemap") + ET.SubElement(item, "loc").text = location + ET.indent(root) + ET.ElementTree(root).write( + output / "sitemap.xml", encoding="utf-8", xml_declaration=True + ) + + +def copy_without_collision( + source: pathlib.Path, destination: pathlib.Path, seen: set[str] +) -> None: + for path in sorted(source.rglob("*")): + if path.is_dir(): + continue + relative = path.relative_to(source).as_posix() + target_relative = (destination / relative).as_posix() + if target_relative in seen: + fail(f"aggregate path collision: {target_relative}") + seen.add(target_relative) + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + +def aggregate(args: argparse.Namespace) -> None: + manifest = load_resolved_manifest(args.resolved_manifest) + output = prepare_output_directory(args.output, "aggregate output") + output.mkdir(parents=True) + seen: set[str] = set() + resolved = [] + for entry in manifest["versions"]: + source = args.artifacts / f"{args.artifact_prefix}{entry['id']}" + metadata_path = source / ".version.json" + if not metadata_path.is_file(): + fail(f"missing version metadata: {metadata_path}") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + require_metadata_matches(entry, metadata, metadata_path) + validate_artifact( + argparse.Namespace( + manifest=args.resolved_manifest, + version=entry["id"], + sha=entry["sha"], + site_origin=args.site_origin, + artifact=source, + ) + ) + destination = output / entry["publishPath"] + destination.mkdir(parents=True, exist_ok=True) + copy_without_collision(source, destination, seen) + resolved.append(metadata) + write_aggregate_sitemap(output, args.site_origin, manifest) + expected_sitemaps = sitemap_locations(output / "sitemap.xml") + for location in expected_sitemaps: + parsed = urllib.parse.urlsplit(location) + origin_parts = urllib.parse.urlsplit(args.site_origin.rstrip("/") + "/") + if parsed.scheme != origin_parts.scheme or parsed.netloc != origin_parts.netloc: + fail(f"aggregate sitemap escapes the configured origin: {location}") + if not target_exists(output, parsed.path): + fail(f"aggregate sitemap target is missing: {location}") + asf_text = (ROOT / ".asf.yaml").read_text(encoding="utf-8") + if args.asf_profile or args.asf_whoami: + if not args.asf_profile or not args.asf_whoami: + fail("ASF staging profile and whoami must be set together") + old_staging = "staging:\n profile: ~\n whoami: asf-staging" + new_staging = ( + f"staging:\n profile: {args.asf_profile}\n whoami: {args.asf_whoami}" + ) + if asf_text.count(old_staging) != 1: + fail("cannot locate the exact ASF staging block") + asf_text = asf_text.replace(old_staging, new_staging) + (output / ".asf.yaml").write_text(asf_text, encoding="utf-8") + metadata_dir = output / "build-metadata" + metadata_dir.mkdir() + (metadata_dir / "versions.json").write_text( + json.dumps( + {"schemaVersion": 1, "versions": resolved}, + ensure_ascii=False, + sort_keys=True, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(f"aggregated {len(resolved)} versions and {len(seen)} files -> {output}") + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--manifest", type=pathlib.Path, default=ROOT / "versions.json") + commands = result.add_subparsers(dest="command", required=True) + + prepare_parser = commands.add_parser("prepare") + prepare_parser.add_argument("--local", action="store_true") + prepare_parser.add_argument("--latest-sha", default="HEAD") + prepare_parser.add_argument("--output", type=pathlib.Path) + prepare_parser.set_defaults(func=prepare) + + build_parser = commands.add_parser("build") + build_parser.add_argument("--version", required=True) + build_parser.add_argument("--sha", required=True) + build_parser.add_argument("--site-origin", required=True) + build_parser.add_argument("--output", type=pathlib.Path, required=True) + build_parser.set_defaults(func=build) + + validate_parser = commands.add_parser("validate") + validate_parser.add_argument("--version", required=True) + validate_parser.add_argument("--sha", required=True) + validate_parser.add_argument("--site-origin", required=True) + validate_parser.add_argument("--artifact", type=pathlib.Path, required=True) + validate_parser.set_defaults(func=validate_artifact) + + aggregate_parser = commands.add_parser("aggregate") + aggregate_parser.add_argument("--artifacts", type=pathlib.Path, required=True) + aggregate_parser.add_argument("--artifact-prefix", default="") + aggregate_parser.add_argument( + "--resolved-manifest", type=pathlib.Path, required=True + ) + aggregate_parser.add_argument("--site-origin", required=True) + aggregate_parser.add_argument("--output", type=pathlib.Path, required=True) + aggregate_parser.add_argument("--asf-profile") + aggregate_parser.add_argument("--asf-whoami") + aggregate_parser.set_defaults(func=aggregate) + return result + + +def main() -> None: + args = parser().parse_args() + args.manifest = args.manifest.resolve() + if hasattr(args, "resolved_manifest"): + args.resolved_manifest = args.resolved_manifest.resolve() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/versions.json b/versions.json new file mode 100644 index 000000000..0642789de --- /dev/null +++ b/versions.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "repository": "https://github.com/apache/hugegraph-doc.git";, + "versions": [ + { + "id": "latest", + "name": "latest", + "ref": "master", + "publishPath": "", + "archived": false, + "githubBranch": "master" + }, + { + "id": "1.7", + "name": "1.7", + "ref": "release-1.7.0", + "publishPath": "versions/1.7", + "archived": true, + "githubBranch": "release-1.7.0" + }, + { + "id": "1.5", + "name": "1.5", + "ref": "release-1.5.0", + "publishPath": "versions/1.5", + "archived": true, + "githubBranch": "release-1.5.0" + } + ] +}
