This is an automated email from the ASF dual-hosted git repository. imbajin pushed a commit to branch feat/oink-core-platform in repository https://gitbox.apache.org/repos/asf/hugegraph-doc.git
commit befe5272c856a01e6fb14f23f7654debaba26876 Author: dark <[email protected]> AuthorDate: Fri Sep 4 20:07:38 2026 +0800 feat(ci): secure versioned publication - map PR, master, and trusted dispatch events to fixed targets - derive direct Hugo version menus from versions.json - isolate Hugo caches and provision Community render dependencies - separate read-only deploy gates from write-enabled publishing --- .github/workflows/hugo.yml | 341 +++++++++++++++++++++++++++++++-------------- scripts/test_versioning.py | 18 +++ scripts/versioning.py | 99 ++++++++----- 3 files changed, 323 insertions(+), 135 deletions(-) diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index ecc3c4664..90a25a680 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -4,230 +4,367 @@ on: pull_request: push: branches: [master] + workflow_dispatch: + inputs: + operation: + description: Fixed deployment operation + required: true + type: choice + options: [staging-next, production-history-refresh] + scope: + description: Versions to build + required: true + default: latest + type: choice + options: [latest, full] + candidate_branch: + description: ASF repository branch used for latest + required: true + default: master + type: string + confirmation: + description: Fixed target confirmation from the runbook + required: true + type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.operation || 'automatic' }} cancel-in-progress: true env: HUGO_VERSION: 0.165.0 HUGO_ENVIRONMENT: production - HUGO_CACHEDIR: /tmp/hugo_cache + PRODUCTION_ORIGIN: https://hugegraph.apache.org/ + STAGING_ORIGIN: https://hugegraph-oink.staged.apache.org/ jobs: prepare: runs-on: ubuntu-latest - permissions: - contents: read + timeout-minutes: 10 + permissions: { contents: read } + env: + HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-prepare outputs: versions: ${{ steps.matrix.outputs.versions }} + selection: ${{ steps.plan.outputs.selection }} + site_origin: ${{ steps.plan.outputs.site_origin }} + historical_origin: ${{ steps.plan.outputs.historical_origin }} + artifact_prefix: ${{ steps.plan.outputs.artifact_prefix }} + publish_branch: ${{ steps.plan.outputs.publish_branch }} + latest_sha: ${{ steps.plan.outputs.latest_sha }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.13" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: go.mod + cache: true + - name: Setup Hugo Extended + uses: peaceiris/actions-hugo@2752ce1d29631191ea3f27c23495fa06139a5b78 # v3.2.1 + with: + hugo-version: ${{ env.HUGO_VERSION }} + extended: true + + - name: Resolve trusted event plan + id: plan + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + OPERATION: ${{ inputs.operation }} + SCOPE: ${{ inputs.scope }} + CANDIDATE_BRANCH: ${{ inputs.candidate_branch }} + CONFIRMATION: ${{ inputs.confirmation }} + run: | + set -euo pipefail + selection="latest,1.7,1.5,1.3,1.0" + site_origin="$PRODUCTION_ORIGIN" + historical_origin="$PRODUCTION_ORIGIN" + artifact_prefix="production" + publish_branch="" + latest_sha="$EVENT_SHA" + + if [[ "$EVENT_NAME" == workflow_dispatch ]]; then + test "$GITHUB_REF" = refs/heads/master + candidate="$CANDIDATE_BRANCH" + test -n "$candidate" + [[ "$candidate" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] + [[ "$candidate" != refs/* && "$candidate" != */ && "$candidate" != *//* ]] + [[ "$candidate" != *..* ]] + [[ ! "$candidate" =~ ^[0-9a-fA-F]{40}$ ]] + [[ "$candidate" != asf-site && "$candidate" != asf-staging-oink ]] + resolved="$(git ls-remote --heads origin "refs/heads/$candidate")" + test "$(wc -l <<<"$resolved" | tr -d ' ')" = 1 + latest_sha="$(awk '{print $1}' <<<"$resolved")" + [[ "$latest_sha" =~ ^[0-9a-f]{40}$ ]] + git fetch --no-tags origin "$latest_sha" + + case "$OPERATION" in + staging-next) + test "$CONFIRMATION" = "publish asf-staging-oink" + site_origin="$STAGING_ORIGIN" + artifact_prefix="staging" + publish_branch="asf-staging-oink" + if [[ "$SCOPE" == latest ]]; then + selection="latest" + else + test "$SCOPE" = full + historical_origin="$STAGING_ORIGIN" + fi + ;; + production-history-refresh) + test "$CONFIRMATION" = "publish asf-site" + test "$candidate" = master + test "$SCOPE" = full + publish_branch="asf-site" + ;; + *) exit 1 ;; + esac + elif [[ "$EVENT_NAME" == push ]]; then + test "$GITHUB_REF" = refs/heads/master + publish_branch="asf-site" + else + test "$EVENT_NAME" = pull_request + fi + + { + echo "selection=$selection" + echo "site_origin=$site_origin" + echo "historical_origin=$historical_origin" + echo "artifact_prefix=$artifact_prefix" + echo "publish_branch=$publish_branch" + echo "latest_sha=$latest_sha" + } >> "$GITHUB_OUTPUT" - name: Validate source and version tooling run: | bash dist/validate-links.sh - python3 -m unittest discover -s scripts -p 'test_*.py' -v - + PYTHONDONTWRITEBYTECODE=1 \ + python3 -m unittest discover -s scripts -p 'test_*.py' -v - name: Resolve immutable version matrix id: matrix env: - LATEST_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + LATEST_SHA: ${{ steps.plan.outputs.latest_sha }} + SELECTION: ${{ steps.plan.outputs.selection }} run: | python3 scripts/versioning.py prepare \ - --latest-sha "$LATEST_SHA" \ + --latest-sha "$LATEST_SHA" --select "$SELECTION" \ --output resolved-versions.json echo "versions=$(jq -c '.include' resolved-versions.json)" >> "$GITHUB_OUTPUT" - - name: Upload resolved version manifest uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: resolved-versions-${{ github.sha }} + name: resolved-versions-${{ github.run_id }}-${{ github.run_attempt }} path: resolved-versions.json + retention-days: 7 if-no-files-found: error build: needs: prepare runs-on: ubuntu-latest - permissions: - contents: read + timeout-minutes: 20 + permissions: { contents: read } + env: + HUGO_CACHEDIR: /tmp/hugo-cache-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.version.id }} strategy: fail-fast: false + max-parallel: 5 matrix: version: ${{ fromJSON(needs.prepare.outputs.versions) }} - site: - - name: production - origin: https://hugegraph.apache.org/ - - name: staging - origin: https://hugegraph-oink.staged.apache.org/ - name: Build ${{ matrix.site.name }} / ${{ matrix.version.id }} + name: Build ${{ matrix.version.id }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.13" - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: go.mod cache: true - - name: Setup Hugo Extended uses: peaceiris/actions-hugo@2752ce1d29631191ea3f27c23495fa06139a5b78 # v3.2.1 with: hugo-version: ${{ env.HUGO_VERSION }} extended: true - - name: Cache Hugo resources uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ env.HUGO_CACHEDIR }} key: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}-${{ hashFiles('go.sum') }} - restore-keys: | - ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}- - + restore-keys: ${{ runner.os }}-hugo-${{ env.HUGO_VERSION }}-${{ matrix.version.id }}- - name: Verify pinned OINK module run: | go mod verify - go list -m all test "$(go list -m -f '{{ .Path }}')" = "github.com/apache/hugegraph-doc" test "$(go list -m all | wc -l)" -eq 2 test "$(go list -m -f '{{ .Path }}@{{ .Version }}' github.com/pgsty/oink)" = "github.com/pgsty/[email protected]" test -z "$(go list -m -f '{{ with .Replace }}{{ .Path }}@{{ .Version }}{{ end }}' github.com/pgsty/oink)" - - name: Build isolated version artifact env: OINK_PYTHON: python3 + SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} + HISTORICAL_ORIGIN: ${{ needs.prepare.outputs.historical_origin }} run: | python3 scripts/versioning.py build \ - --version "${{ matrix.version.id }}" \ - --sha "${{ matrix.version.sha }}" \ - --site-origin "${{ matrix.site.origin }}" \ + --version "${{ matrix.version.id }}" --sha "${{ matrix.version.sha }}" \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --output "${RUNNER_TEMP}/version-public" python3 scripts/versioning.py validate \ - --version "${{ matrix.version.id }}" \ - --sha "${{ matrix.version.sha }}" \ - --site-origin "${{ matrix.site.origin }}" \ + --version "${{ matrix.version.id }}" --sha "${{ matrix.version.sha }}" \ + --site-origin "$SITE_ORIGIN" --historical-origin "$HISTORICAL_ORIGIN" \ --artifact "${RUNNER_TEMP}/version-public" - - name: Upload isolated version artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ${{ matrix.site.name }}-${{ matrix.version.id }} + name: ${{ needs.prepare.outputs.artifact_prefix }}-${{ matrix.version.id }}-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/version-public include-hidden-files: true + retention-days: 1 if-no-files-found: error - # Keep this job id aligned with the required `deploy` status in .asf.yaml. - # It produces deployable artifacts but does not publish untrusted PR code. - deploy: - if: always() + aggregate: needs: [prepare, build] runs-on: ubuntu-latest - permissions: - contents: read + timeout-minutes: 15 + permissions: { contents: read } steps: - - name: Require every version build to succeed - env: - PREPARE_RESULT: ${{ needs.prepare.result }} - BUILD_RESULT: ${{ needs.build.result }} - run: | - test "$PREPARE_RESULT" = success - test "$BUILD_RESULT" = success - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - fetch-depth: 0 persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.13" - - - name: Download resolved version manifest - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: resolved-versions-${{ github.sha }} + name: resolved-versions-${{ github.run_id }}-${{ github.run_attempt }} path: resolved - - - name: Download production versions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - pattern: production-* - path: version-artifacts - - - name: Download staging versions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: staging-* + pattern: ${{ needs.prepare.outputs.artifact_prefix }}-*-${{ github.run_id }}-${{ github.run_attempt }} path: version-artifacts - - - name: Aggregate production and staging sites + - name: Assemble publishable site + env: + PREFIX: ${{ needs.prepare.outputs.artifact_prefix }} + SITE_ORIGIN: ${{ needs.prepare.outputs.site_origin }} + SELECTION: ${{ needs.prepare.outputs.selection }} run: | + extra=() + if [[ "$PREFIX" == staging ]]; then + extra+=(--asf-profile oink --asf-whoami asf-staging-oink) + fi python3 scripts/versioning.py aggregate \ - --artifacts version-artifacts \ - --artifact-prefix production- \ - --resolved-manifest resolved/resolved-versions.json \ - --site-origin https://hugegraph.apache.org/ \ - --output "${RUNNER_TEMP}/public-production" - python3 scripts/versioning.py aggregate \ - --artifacts version-artifacts \ - --artifact-prefix staging- \ + --artifacts version-artifacts --artifact-prefix "$PREFIX-" \ + --artifact-suffix "-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ --resolved-manifest resolved/resolved-versions.json \ - --site-origin https://hugegraph-oink.staged.apache.org/ \ - --asf-profile oink \ - --asf-whoami asf-staging-oink \ - --output "${RUNNER_TEMP}/public-staging" - - - name: Upload production aggregate + --site-origin "$SITE_ORIGIN" --select "$SELECTION" \ + --output "${RUNNER_TEMP}/public-site" "${extra[@]}" + - name: Upload publishable aggregate uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: hugegraph-site-production-${{ github.sha }} - path: ${{ runner.temp }}/public-production + name: hugegraph-site-${{ needs.prepare.outputs.artifact_prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/public-site include-hidden-files: true + retention-days: ${{ github.event_name == 'pull_request' && 1 || 7 }} if-no-files-found: error - - name: Upload ASF staging aggregate + e2e: + needs: [prepare, aggregate] + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: { contents: read } + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "24" + cache: npm + cache-dependency-path: tests/e2e/package-lock.json + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: hugegraph-site-${{ needs.prepare.outputs.artifact_prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + path: public-site + - name: Install Chromium test workspace + working-directory: tests/e2e + run: | + npm ci + npx playwright install --with-deps chromium + - name: Run blocking Chromium contracts + working-directory: tests/e2e + env: + SITE_ROOT: ${{ github.workspace }}/public-site + EXPECTED_VERSIONS: ${{ needs.prepare.outputs.selection }} + run: npm run test:ci + - name: Upload E2E report + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: hugegraph-site-staging-${{ github.sha }} - path: ${{ runner.temp }}/public-staging - include-hidden-files: true - if-no-files-found: error + name: playwright-report-${{ github.run_id }}-${{ github.run_attempt }} + path: | + tests/e2e/playwright-report + tests/e2e/test-results + retention-days: 7 + if-no-files-found: warn + + # Required check name in .asf.yaml. It gates artifacts without write access. + deploy: + if: always() + needs: [prepare, build, aggregate, e2e] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: { contents: read } + steps: + - name: Require all blocking jobs to succeed + env: + PREPARE_RESULT: ${{ needs.prepare.result }} + BUILD_RESULT: ${{ needs.build.result }} + AGGREGATE_RESULT: ${{ needs.aggregate.result }} + E2E_RESULT: ${{ needs.e2e.result }} + run: | + test "$PREPARE_RESULT" = success + test "$BUILD_RESULT" = success + test "$AGGREGATE_RESULT" = success + test "$E2E_RESULT" = success publish: - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - needs: deploy + if: needs.prepare.outputs.publish_branch != '' + needs: [prepare, deploy] runs-on: ubuntu-latest - permissions: - contents: write + timeout-minutes: 10 + permissions: { contents: write } steps: - - name: Download reviewed production aggregate - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: hugegraph-site-production-${{ github.sha }} - path: public-production - - - name: Publish clean aggregate to asf-site + name: hugegraph-site-${{ needs.prepare.outputs.artifact_prefix }}-${{ github.run_id }}-${{ github.run_attempt }} + path: public-site + - name: Verify fixed publication target + env: + TARGET: ${{ needs.prepare.outputs.publish_branch }} + PREFIX: ${{ needs.prepare.outputs.artifact_prefix }} + run: | + case "$PREFIX:$TARGET" in + production:asf-site|staging:asf-staging-oink) ;; + *) exit 1 ;; + esac + test -f public-site/build-metadata/versions.json + - name: Publish clean aggregate uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./public-production - publish_branch: asf-site + publish_dir: ./public-site + publish_branch: ${{ needs.prepare.outputs.publish_branch }} keep_files: false force_orphan: false - commit_message: ${{ github.event.head_commit.message }} + commit_message: Deploy ${{ needs.prepare.outputs.artifact_prefix }} site from run ${{ github.run_id }} attempt ${{ github.run_attempt }} diff --git a/scripts/test_versioning.py b/scripts/test_versioning.py index f2c2685c6..7bbf58c83 100644 --- a/scripts/test_versioning.py +++ b/scripts/test_versioning.py @@ -223,6 +223,24 @@ class VersionUrlTest(unittest.TestCase): f"{ORIGIN}cn/docs/", ) + def test_direct_hugo_config_is_derived_from_five_version_manifest(self) -> None: + manifest = versioning.load_manifest(versioning.ROOT / "versions.json") + latest = manifest["versions"][0] + config = versioning.derived_version_config(manifest, latest, ORIGIN) + expected = ["latest", "1.7", "1.5", "1.3", "1.0"] + self.assertEqual( + [item["version"] for item in config["params"]["versions"]], expected + ) + for language in ("en", "cn"): + self.assertEqual( + [ + item["version"] + for item in config["languages"][language]["params"]["versions"] + ], + expected, + ) + self.assertNotIn("1.2", json.dumps(config)) + def test_write_error_documents_keeps_localized_404_status_targets(self) -> None: with tempfile.TemporaryDirectory() as temp_name: output = Path(temp_name) diff --git a/scripts/versioning.py b/scripts/versioning.py index 1799a4063..6581879d8 100644 --- a/scripts/versioning.py +++ b/scripts/versioning.py @@ -2738,6 +2738,63 @@ def validate_artifact(args: argparse.Namespace) -> None: ) +def derived_version_config( + manifest: dict, + entry: dict, + site_origin: str, + historical_origin: str | None = None, +) -> dict: + """Derive every Hugo version-menu value from versions.json.""" + override = { + "baseURL": base_url(site_origin, entry["publishPath"]), + "canonifyURLs": True, + "params": { + "version": entry["id"], + "version_menu": "Releases", + "version_menu_pagelinks": False, + "versions": version_urls( + manifest, site_origin, "en", historical_origin + ), + "archived_version": bool(entry["archived"]), + "url_latest_version": urllib.parse.urljoin( + site_origin.rstrip("/") + "/", "docs/" + ), + "github_repo": "https://github.com/apache/hugegraph-doc";, + "github_branch": entry["githubBranch"], + }, + } + language_overrides = ( + historical_language_menus(site_origin) + if entry["archived"] + else {"en": {}, "cn": {}} + ) + for language in ("en", "cn"): + language_overrides[language]["params"] = language_version_params( + manifest, + site_origin, + language, + historical_origin, + ) + override["languages"] = language_overrides + return override + + +def render_config(args: argparse.Namespace) -> None: + """Write the manifest-derived override used by direct Hugo commands.""" + manifest = load_manifest(args.manifest) + entry = next( + (item for item in manifest["versions"] if item["id"] == args.version), None + ) + if entry is None: + fail(f"unknown version {args.version}") + override = derived_version_config( + manifest, entry, args.site_origin, args.historical_origin + ) + rendered = json.dumps(override, ensure_ascii=False, sort_keys=True, indent=2) + "\n" + args.output.write_text(rendered, encoding="utf-8") + print(f"rendered Hugo config for {entry['id']} -> {args.output}") + + def build(args: argparse.Namespace) -> None: manifest = load_manifest(args.manifest) entry = next( @@ -2780,40 +2837,9 @@ def build(args: argparse.Namespace) -> None: origin=args.site_origin, ) site_base = base_url(args.site_origin, entry["publishPath"]) - override = { - "baseURL": site_base, - "canonifyURLs": True, - "params": { - "version": entry["id"], - "version_menu": "Releases", - "version_menu_pagelinks": False, - "versions": version_urls( - manifest, - args.site_origin, - "en", - args.historical_origin, - ), - "archived_version": bool(entry["archived"]), - "url_latest_version": urllib.parse.urljoin( - args.site_origin.rstrip("/") + "/", "docs/" - ), - "github_repo": "https://github.com/apache/hugegraph-doc";, - "github_branch": entry["githubBranch"], - }, - } - language_overrides = ( - historical_language_menus(args.site_origin) - if entry["archived"] - else {"en": {}, "cn": {}} + override = derived_version_config( + manifest, entry, args.site_origin, args.historical_origin ) - for language in ("en", "cn"): - language_overrides[language]["params"] = language_version_params( - manifest, - args.site_origin, - language, - args.historical_origin, - ) - override["languages"] = language_overrides override_path = assembly / "version-config.json" override_path.write_text( json.dumps(override, ensure_ascii=False), encoding="utf-8" @@ -3137,6 +3163,13 @@ def parser() -> argparse.ArgumentParser: prepare_parser.add_argument("--output", type=pathlib.Path) prepare_parser.set_defaults(func=prepare) + config_parser = commands.add_parser("config") + config_parser.add_argument("--version", default="latest") + config_parser.add_argument("--site-origin", default=CANONICAL_ORIGIN) + config_parser.add_argument("--historical-origin") + config_parser.add_argument("--output", type=pathlib.Path, required=True) + config_parser.set_defaults(func=render_config) + build_parser = commands.add_parser("build") build_parser.add_argument("--version", required=True) build_parser.add_argument("--sha", required=True)
