https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/69523
>From 6a7f298f403e0e454644f3d945242120f8b2b321 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 18 Oct 2023 04:56:28 -0700 Subject: [PATCH 1/4] workflows: Refactor release-tasks.yml * Split out the lit release job and the documentation build jobs into their own workflow files. This makes it possible to manually run these jobs via workflow_dispatch. * Improve tag/user validation and ensure it gets run for each release task. --- .github/workflows/release-binaries.yml | 51 ++++--- .github/workflows/release-documentation.yml | 84 +++++++++++ .github/workflows/release-doxygen.yml | 62 ++++++++ .github/workflows/release-lit.yml | 74 +++++++++ .github/workflows/release-tasks.yml | 140 ++++++------------ .../workflows/set-release-binary-outputs.sh | 18 +-- llvm/utils/release/github-upload-release.py | 22 ++- 7 files changed, 321 insertions(+), 130 deletions(-) create mode 100644 .github/workflows/release-documentation.yml create mode 100644 .github/workflows/release-doxygen.yml create mode 100644 .github/workflows/release-lit.yml diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 75d4f419ab1fdc..d8b732413d2bf6 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -1,20 +1,29 @@ name: Release Binaries on: - push: - tags: - - 'llvmorg-*' workflow_dispatch: inputs: + release-version: + description: 'Release Version' + required: true + type: string upload: description: 'Upload binaries to the release page' required: true - default: true + default: false type: boolean - tag: - description: 'Tag to build' + + workflow_call: + inputs: + release-version: + description: 'Release Version' required: true type: string + upload: + description: 'Upload binaries to the release page' + required: true + default: false + type: boolean permissions: contents: read # Default everything to read-only @@ -23,21 +32,26 @@ jobs: prepare: name: Prepare to build binaries runs-on: ubuntu-22.04 - if: github.repository == 'llvm/llvm-project' outputs: - release-version: ${{ steps.validate-tag.outputs.release-version }} - release: ${{ steps.validate-tag.outputs.release }} - build-dir: ${{ steps.validate-tag.outputs.build-dir }} - rc-flags: ${{ steps.validate-tag.outputs.rc-flags }} - ref: ${{ steps.validate-tag.outputs.ref }} - upload: ${{ steps.validate-tag.outputs.upload }} + release-version: ${{ steps.vars.outputs.release-version }} + release: ${{ steps.vars.outputs.release }} + build-dir: ${{ steps.vars.outputs.build-dir }} + rc-flags: ${{ steps.vars.outputs.rc-flags }} + ref: ${{ steps.vars.outputs.ref }} + upload: ${{ steps.vars.outputs.upload }} steps: - name: Checkout LLVM uses: actions/checkout@v4 - - name: Validate and parse tag - id: validate-tag + - name: Check Permissions + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions + + - name: Collect Variables + id: vars # In order for the test-release.sh script to run correctly, the LLVM # source needs to be at the following location relative to the build dir: # | X.Y.Z-rcN | ./rcN/llvm-project @@ -47,15 +61,12 @@ jobs: # | X.Y.Z-rcN | -rc N -test-asserts # | X.Y.Z | -final run: | - tag="${{ github.ref_name }}" - trimmed=$(echo ${{ inputs.tag }} | xargs) - [[ "$trimmed" != "" ]] && tag="$trimmed" if [ -n "${{ inputs.upload }}" ]; then upload="${{ inputs.upload }}" else - upload="true" + upload="false" fi - bash .github/workflows/set-release-binary-outputs.sh "${{ github.actor }}" "$tag" "$upload" + bash .github/workflows/set-release-binary-outputs.sh "${{ inputs.release-version }}" "$upload" # Try to get around the 6 hour timeout by first running a job to fill # the build cache. diff --git a/.github/workflows/release-documentation.yml b/.github/workflows/release-documentation.yml new file mode 100644 index 00000000000000..832570ed6fe93f --- /dev/null +++ b/.github/workflows/release-documentation.yml @@ -0,0 +1,84 @@ +name: Release Documentation + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + + workflow_call: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + +jobs: + release-documentation: + name: Build and Upload Release Documentation + runs-on: ubuntu-latest + env: + upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }} + steps: + - name: Checkout LLVM + uses: actions/checkout@v4 + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + doxygen \ + graphviz \ + python3-github \ + ninja-build \ + texlive-font-utils + pip3 install --user -r ./llvm/docs/requirements.txt + echo "UPLOAD SETTING: $upload" + + - name: Build Documentation + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-doxygen + + - name: Create Release Notes Artifact + uses: actions/upload-artifact@v3 + with: + name: release-notes + path: docs-build/html-export/ + + - name: Clone www-releases + if: env.upload + uses: actions/checkout@v4 + with: + repository: ${{ github.repository_owner }}/www-releases + ref: main + fetch-depth: 0 + path: www-releases + + - name: Upload Release Notes + if: env.upload + env: + WWW_RELEASES_TOKEN: ${{ secrets.WWW_RELEASES_TOKEN }} + run: | + mkdir -p ../www-releases/${{ inputs.release-version }} + mv ./docs-build/html-export/* ../www-releases/${{ inputs.release-version }} + cd ../www-releases + git add ${{ inputs.release-version }} + git config user.email "llvm...@llvm.org" + git config user.name "llvmbot" + git commit -a -m "Add ${{ inputs.release-version }} documentation" + git push "https://$www_releases_to...@github.com/${{ github.repository_owner }}/www-releases" main:main diff --git a/.github/workflows/release-doxygen.yml b/.github/workflows/release-doxygen.yml new file mode 100644 index 00000000000000..390bac02d475b7 --- /dev/null +++ b/.github/workflows/release-doxygen.yml @@ -0,0 +1,62 @@ +name: Release Doxygen + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + + workflow_call: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + upload: + description: 'Upload documentation' + required: false + type: boolean + +jobs: + release-doxygen: + name: Build and Upload Release Doxygen + runs-on: ubuntu-latest + permissions: + contents: write + env: + upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }} + steps: + - name: Checkout LLVM + uses: actions/checkout@v4 + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + doxygen \ + graphviz \ + python3-github \ + ninja-build \ + texlive-font-utils + pip3 install --user -r ./llvm/docs/requirements.txt + + - name: Build Doxygen + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + echo "UPLOAD SETTING: $upload" + ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-sphinx + + - name: Upload Doxygen + if: env.upload + run: | + ./llvm/utils/release/github-upload-release.py --token "$GITHUB_TOKEN" --release "${{ inputs.release-version }}" --user "${{ github.actor }}" upload --files ./*doxygen*.tar.xz diff --git a/.github/workflows/release-lit.yml b/.github/workflows/release-lit.yml new file mode 100644 index 00000000000000..feaca871960864 --- /dev/null +++ b/.github/workflows/release-lit.yml @@ -0,0 +1,74 @@ +name: Release Lit + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + + workflow_call: + inputs: + release-version: + description: 'Release Version' + required: true + type: string + +jobs: + release-lit: + name: Release Lit + runs-on: ubuntu-latest + steps: + - name: Checkout LLVM + uses: actions/checkout@v4 + with: + ref: "llvmorg-${{ inputs.release-version }}" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-setuptools python3-psutil python3-github + + - name: Check Permissions + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --user ${{ github.actor }} check-permissions + + - name: Setup Cpp + uses: aminya/setup-cpp@v1 + with: + compiler: llvm-16.0.6 + cmake: true + ninja: true + + - name: Test lit + run: | + mkdir build && cd build + export FILECHECK_OPTS='-dump-input-filter=all -vv -color' + cmake ../llvm -DCMAKE_BUILD_TYPE=Release -G Ninja + ninja -v -j $(nproc) check-lit + + - name: Package lit + run: | + cd llvm/utils/lit + # Remove 'dev' suffix from lit version. + sed -i 's/ + "dev"//g' lit/__init__.py + python3 setup.py sdist + + - name: Upload lit to test.pypi.org + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + packages-dir: llvm/utils/lit/dist/ + + - name: Upload lit to pypi.org + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }} + packages-dir: llvm/utils/lit/dist/ diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml index 85b720e323d1b1..4bc883623d4b1b 100644 --- a/.github/workflows/release-tasks.yml +++ b/.github/workflows/release-tasks.yml @@ -1,7 +1,7 @@ name: Release Task permissions: - contents: read + contents: write on: push: @@ -10,112 +10,70 @@ on: - 'llvmorg-*' jobs: - release-tasks: - permissions: - contents: write # To upload assets to release. + validate-tag: + name: Validate Tag runs-on: ubuntu-latest if: github.repository == 'llvm/llvm-project' + outputs: + release-version: ${{ steps.validate-tag.outputs.release-version }} steps: - name: Validate Tag id: validate-tag run: | - test "${{ github.actor }}" = "tstellar" || test "${{ github.actor }}" = "tru" echo "${{ github.ref_name }}" | grep -e '^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$' release_version=$(echo "${{ github.ref_name }}" | sed 's/llvmorg-//g') echo "release-version=$release_version" >> "$GITHUB_OUTPUT" - - name: Checkout LLVM - uses: actions/checkout@v4 - + release-create: + name: Create a New Release + runs-on: ubuntu-latest + needs: validate-tag + steps: - name: Install Dependencies run: | sudo apt-get update - sudo apt-get install -y \ - doxygen \ - graphviz \ - python3-github \ - ninja-build \ - texlive-font-utils - pip3 install --user -r ./llvm/docs/requirements.txt - - - name: Create Release - run: | - ./llvm/utils/release/./github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} create - - - name: Build Documentation - run: | - ./llvm/utils/release/build-docs.sh -release ${{ steps.validate-tag.outputs.release-version }} - ./llvm/utils/release/github-upload-release.py --token ${{ github.token }} --release ${{ steps.validate-tag.outputs.release-version }} upload --files ./*doxygen*.tar.xz - - - name: Create Release Notes Artifact - uses: actions/upload-artifact@v3 - with: - name: release-notes - path: docs-build/html-export/ - - - name: Clone www-releases - if: ${{ !contains(steps.validate-tag.outputs.release-version, 'rc') }} - uses: actions/checkout@v4 - with: - repository: ${{ github.repository_owner }}/www-releases - ref: main - fetch-depth: 0 - path: www-releases - - - name: Upload Release Notes - if: ${{ !contains(steps.validate-tag.outputs.release-version, 'rc') }} - run: | - mkdir -p ../www-releases/${{ steps.validate-tag.outputs.release-version }} - mv ./docs-build/html-export/* ../www-releases/${{ steps.validate-tag.outputs.release-version }} - cd ../www-releases - git add ${{ steps.validate-tag.outputs.release-version }} - git config user.email "llvm...@llvm.org" - git config user.name "llvmbot" - git commit -a -m "Add ${{ steps.validate-tag.outputs.release-version }} documentation" - git push https://${{ secrets.WWW_RELEASES_TOKEN }}@github.com/${{ github.repository_owner }}/www-releases main:main + sudo apt-get install python3-github - release-lit: - runs-on: ubuntu-latest - if: github.repository == 'llvm/llvm-project' - steps: - name: Checkout LLVM uses: actions/checkout@v4 - - name: Setup Cpp - uses: aminya/setup-cpp@v1 - with: - compiler: llvm-16.0.6 - cmake: true - ninja: true - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-setuptools python3-psutil - - - name: Test lit - run: | - mkdir build && cd build - export FILECHECK_OPTS='-dump-input-filter=all -vv -color' - cmake ../llvm -DCMAKE_BUILD_TYPE=Release -G Ninja - ninja -v -j $(nproc) check-lit - - - name: Package lit + - name: Create Release + env: + GITHUB_TOKEN: ${{ github.token }} run: | - cd llvm/utils/lit - # Remove 'dev' suffix from lit version. - sed -i 's/ + "dev"//g' lit/__init__.py - python3 setup.py sdist - - - name: Upload lit to test.pypi.org - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.LLVM_LIT_TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - packages-dir: llvm/utils/lit/dist/ + ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} create + release-documentation: + name: Build and Upload Release Documentation + needs: + - validate-tag + uses: ./.github/workflows/release-documentation.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + upload: true + + release-doxygen: + name: Build and Upload Release Doxygen + needs: + - validate-tag + - release-create + uses: ./.github/workflows/release-doxygen.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + upload: true - - name: Upload lit to pypi.org - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.LLVM_LIT_PYPI_API_TOKEN }} - packages-dir: llvm/utils/lit/dist/ + release-lit: + name: Release Lit + needs: validate-tag + uses: ./.github/workflows/release-lit.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + + release-binaries: + name: Build Release Binaries + needs: + - validate-tag + - release-create + uses: ./.github/workflows/release-binaries.yml + with: + release-version: ${{ needs.validate-tag.outputs.release-version }} + upload: true diff --git a/.github/workflows/set-release-binary-outputs.sh b/.github/workflows/set-release-binary-outputs.sh index 8a7944e7e55fa0..0fa23fe3df968c 100644 --- a/.github/workflows/set-release-binary-outputs.sh +++ b/.github/workflows/set-release-binary-outputs.sh @@ -8,21 +8,9 @@ if [ -z "$GITHUB_OUTPUT" ]; then echo "Writing output variables to $GITHUB_OUTPUT" fi -github_user=$1 -tag=$2 -upload=$3 - -if [[ "$github_user" != "tstellar" && "$github_user" != "tru" ]]; then - echo "ERROR: User not allowed: $github_user" - exit 1 -fi -pattern='^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$' -echo "$tag" | grep -e $pattern -if [ $? != 0 ]; then - echo "ERROR: Tag '$tag' doesn't match pattern: $pattern" - exit 1 -fi -release_version=`echo "$tag" | sed 's/llvmorg-//g'` +release_version=$1 +upload=$2 +tag="llvmorg-$release_version" release=`echo "$release_version" | sed 's/-.*//g'` build_dir=`echo "$release_version" | sed 's,^[^-]\+,final,' | sed 's,[^-]\+-rc\(.\+\),rc\1,'` rc_flags=`echo "$release_version" | sed 's,^[^-]\+,-final,' | sed 's,[^-]\+-rc\(.\+\),-rc \1 -test-asserts,' | sed 's,--,-,'` diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py index f1ad78120a0aa6..07eb554d97d092 100755 --- a/llvm/utils/release/github-upload-release.py +++ b/llvm/utils/release/github-upload-release.py @@ -30,8 +30,10 @@ import argparse import github +import sys from textwrap import dedent + def create_release(repo, release, tag=None, name=None, message=None): if not tag: tag = "llvmorg-{}".format(release) @@ -67,22 +69,34 @@ def upload_files(repo, release, files): parser = argparse.ArgumentParser() -parser.add_argument("command", type=str, choices=["create", "upload"]) +parser.add_argument("command", type=str, choices=["create", "upload", "check-permissions"]) # All args parser.add_argument("--token", type=str) parser.add_argument("--release", type=str) +parser.add_argument("--user", type=str) # Upload args parser.add_argument("--files", nargs="+", type=str) - args = parser.parse_args() github = github.Github(args.token) -llvm_repo = github.get_organization("llvm").get_repo("llvm-project") +llvm_org = github.get_organization("llvm") +llvm_repo = llvm_org.get_repo("llvm-project") + +if args.user: + # Validate that this user is allowed to modify releases. + user = github.get_user(args.user) + team = llvm_org.get_team_by_slug('llvm-release-managers') + if not team.has_in_members(user): + print("User {} is not a allowed to modify releases".format(args.user)) + sys.exit(1) +elif args.command == "check-permissions": + print("--user option required for check-permissions") + sys.exit(1) if args.command == "create": - create_release(llvm_repo, args.release) + create_release(llvm_repo, args.release, args.user) if args.command == "upload": upload_files(llvm_repo, args.release, args.files) >From d5b347a76ae944f23ec1abafb7c954fff2877598 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Thu, 2 Nov 2023 13:55:10 -0700 Subject: [PATCH 2/4] Fix python formatting errors --- llvm/utils/release/github-upload-release.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py index 07eb554d97d092..a8bb569d2fc999 100755 --- a/llvm/utils/release/github-upload-release.py +++ b/llvm/utils/release/github-upload-release.py @@ -69,7 +69,9 @@ def upload_files(repo, release, files): parser = argparse.ArgumentParser() -parser.add_argument("command", type=str, choices=["create", "upload", "check-permissions"]) +parser.add_argument( + "command", type=str, choices=["create", "upload", "check-permissions"] +) # All args parser.add_argument("--token", type=str) @@ -88,13 +90,13 @@ def upload_files(repo, release, files): if args.user: # Validate that this user is allowed to modify releases. user = github.get_user(args.user) - team = llvm_org.get_team_by_slug('llvm-release-managers') + team = llvm_org.get_team_by_slug("llvm-release-managers") if not team.has_in_members(user): print("User {} is not a allowed to modify releases".format(args.user)) sys.exit(1) elif args.command == "check-permissions": - print("--user option required for check-permissions") - sys.exit(1) + print("--user option required for check-permissions") + sys.exit(1) if args.command == "create": create_release(llvm_repo, args.release, args.user) >From f5fefe500967f7fb0e72798cfa35f49559d3380e Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Thu, 2 Nov 2023 13:55:10 -0700 Subject: [PATCH 3/4] Use pip caching and remove debug statements --- .github/workflows/release-documentation.yml | 7 ++++++- .github/workflows/release-doxygen.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-documentation.yml b/.github/workflows/release-documentation.yml index 832570ed6fe93f..55a3008196c107 100644 --- a/.github/workflows/release-documentation.yml +++ b/.github/workflows/release-documentation.yml @@ -36,6 +36,12 @@ jobs: - name: Checkout LLVM uses: actions/checkout@v4 + - name: Setup Python env + uses: actions/setup-python@v4 + with: + cache: 'pip' + cache-dependency-path: './llvm/docs/requirements.txt' + - name: Install Dependencies run: | sudo apt-get update @@ -46,7 +52,6 @@ jobs: ninja-build \ texlive-font-utils pip3 install --user -r ./llvm/docs/requirements.txt - echo "UPLOAD SETTING: $upload" - name: Build Documentation env: diff --git a/.github/workflows/release-doxygen.yml b/.github/workflows/release-doxygen.yml index 390bac02d475b7..856ede631b7d20 100644 --- a/.github/workflows/release-doxygen.yml +++ b/.github/workflows/release-doxygen.yml @@ -38,6 +38,12 @@ jobs: - name: Checkout LLVM uses: actions/checkout@v4 + - name: Setup Python env + uses: actions/setup-python@v4 + with: + cache: 'pip' + cache-dependency-path: './llvm/docs/requirements.txt' + - name: Install Dependencies run: | sudo apt-get update @@ -53,7 +59,6 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - echo "UPLOAD SETTING: $upload" ./llvm/utils/release/build-docs.sh -release "${{ inputs.release-version }}" -no-sphinx - name: Upload Doxygen >From 7699de239ade3551178ebd54d015f0714fa6a3f2 Mon Sep 17 00:00:00 2001 From: Tom Stellard <tstel...@redhat.com> Date: Wed, 29 Nov 2023 15:19:30 -0800 Subject: [PATCH 4/4] Don't install doxygen for sphinx builds --- .github/workflows/release-documentation.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release-documentation.yml b/.github/workflows/release-documentation.yml index 55a3008196c107..b98660744f9700 100644 --- a/.github/workflows/release-documentation.yml +++ b/.github/workflows/release-documentation.yml @@ -46,7 +46,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y \ - doxygen \ graphviz \ python3-github \ ninja-build \ _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits