This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-s3-bundle-refactor in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 10950ac192c83cc5e96c40ec579ecfc849b2cdea Author: tqchen <[email protected]> AuthorDate: Sat Apr 25 16:29:20 2026 +0000 [CI][REFACTOR] Data-driven artifact stashing: s3.py --bundle flag This PR moves the list of build artifacts out of the generated Jenkinsfile and into ci/jenkins/data.py as the single source of truth. Main changes: - ci/scripts/jenkins/s3.py gains a --bundle <name> flag (repeatable) that resolves bundle names from ci/jenkins/data.py files_to_stash at runtime; --items is preserved for back-compat - ci/jenkins/templates/utils/macros.j2 upload_artifacts macro is updated to emit --bundle <name> flags instead of inlining the file list as --items - Template callsites (cpu, gpu, arm) updated to pass bundles=[...] names - Generated .groovy files regenerated; no build/ paths appear in artifact upload blocks -- only bundle names --- ci/jenkins/generated/arm_jenkinsfile.groovy | 4 +-- ci/jenkins/generated/cpu_jenkinsfile.groovy | 4 +-- ci/jenkins/generated/gpu_jenkinsfile.groovy | 6 ++-- ci/jenkins/templates/arm_jenkinsfile.groovy.j2 | 2 +- ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 2 +- ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 4 +-- ci/jenkins/templates/utils/macros.j2 | 9 ++++- ci/scripts/jenkins/s3.py | 47 +++++++++++++++++++++++--- 8 files changed, 61 insertions(+), 17 deletions(-) diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index 17bddc9b2c..16e56f0ea2 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-02-09T16:32:44.108985 +// Generated at 2026-04-25T16:28:24.516990 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -496,7 +496,7 @@ def run_build(node_type) { cmake_build(ci_arm, 'build') make_cpp_tests(ci_arm, 'build') sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --items build/libtvm.so build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja", + script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --bundle tvm_lib --bundle cpptest", label: 'Upload artifacts to S3', ) }) diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index fb9edab77b..c59e2f38c8 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2025-08-24T16:41:22.367054 +// Generated at 2026-04-25T16:28:24.504356 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -496,7 +496,7 @@ def run_build(node_type) { cmake_build(ci_cpu, 'build') make_cpp_tests(ci_cpu, 'build') sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --items build/libtvm.so build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake build/libtvm_allvisible.so build/cpptest build/build.ninja build/CMakeFiles/rules.ninja", + script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --bundle tvm_lib --bundle tvm_allvisible --bundle cpptest", label: 'Upload artifacts to S3', ) }) diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 45f5604727..50ea15c2df 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-02-09T16:32:44.095534 +// Generated at 2026-04-25T16:28:24.538403 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -492,7 +492,7 @@ def run_build(node_type) { sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" cmake_build("${ci_gpu} --no-gpu", 'build') sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --items build/libtvm.so build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake build/libtvm_allvisible.so build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so", + script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --bundle tvm_lib --bundle tvm_allvisible --bundle tvm_lib_gpu_extra", label: 'Upload artifacts to S3', ) @@ -502,7 +502,7 @@ def run_build(node_type) { sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build" cmake_build("${ci_gpu} --no-gpu", 'build') sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2 --items build/libtvm.so build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake", + script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2 --bundle tvm_lib", label: 'Upload artifacts to S3', ) }) diff --git a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 index 35aa9bf250..2bddbab4c7 100644 --- a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 @@ -31,5 +31,5 @@ ) cmake_build(ci_arm, 'build') make_cpp_tests(ci_arm, 'build') - {{ m.upload_artifacts(tag='arm', filenames=tvm_lib + cpptest) }} + {{ m.upload_artifacts(tag='arm', bundles=["tvm_lib", "cpptest"]) }} {% endcall %} diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 index 367da73ebe..8edcd0fe22 100644 --- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 @@ -31,7 +31,7 @@ ) cmake_build(ci_cpu, 'build') make_cpp_tests(ci_cpu, 'build') - {{ m.upload_artifacts(tag='cpu', filenames=tvm_lib + tvm_allvisible + cpptest) }} + {{ m.upload_artifacts(tag='cpu', bundles=["tvm_lib", "tvm_allvisible", "cpptest"]) }} {% endcall %} {% set test_method_names = [] %} diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 index 2769ae2c5d..86575402ea 100644 --- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 @@ -27,13 +27,13 @@ ) %} sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" cmake_build("${ci_gpu} --no-gpu", 'build') - {{ m.upload_artifacts(tag='gpu', filenames=tvm_lib + tvm_allvisible + tvm_lib_gpu_extra) }} + {{ m.upload_artifacts(tag='gpu', bundles=["tvm_lib", "tvm_allvisible", "tvm_lib_gpu_extra"]) }} // compiler test sh "rm -rf build" sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build" cmake_build("${ci_gpu} --no-gpu", 'build') - {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib) }} + {{ m.upload_artifacts(tag='gpu2', bundles=["tvm_lib"]) }} {% endcall %} {% set test_method_names = [] %} diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2 index c96432840d..b1bd3679ac 100644 --- a/ci/jenkins/templates/utils/macros.j2 +++ b/ci/jenkins/templates/utils/macros.j2 @@ -166,12 +166,19 @@ test() }, {% endmacro %} -{% macro upload_artifacts(action, tag, filenames) %} +{% macro upload_artifacts(tag, bundles=none, filenames=none) %} +{% if bundles is not none %} +sh( + script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/{{ tag }}{% for b in bundles %} --bundle {{ b }}{% endfor %}", + label: 'Upload artifacts to S3', + ) +{% elif filenames is not none %} {% set items = ' '.join(filenames) %} sh( script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/{{ tag }} --items {{ items }}", label: 'Upload artifacts to S3', ) +{% endif %} {% endmacro %} {% macro download_artifacts(tag) %} diff --git a/ci/scripts/jenkins/s3.py b/ci/scripts/jenkins/s3.py index dd44490b1e..eb986dec99 100755 --- a/ci/scripts/jenkins/s3.py +++ b/ci/scripts/jenkins/s3.py @@ -17,6 +17,7 @@ # under the License. import argparse +import importlib.util import logging import re from enum import Enum @@ -24,6 +25,30 @@ from pathlib import Path from cmd_utils import REPO_ROOT, Sh, init_log +DATA_PY = REPO_ROOT / "ci" / "jenkins" / "data.py" + + +def load_files_to_stash(): + """Load the files_to_stash dict from ci/jenkins/data.py.""" + spec = importlib.util.spec_from_file_location("data", DATA_PY) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.files_to_stash + + +def resolve_bundles(bundle_names: list[str]) -> list[str]: + """Resolve a list of bundle names to a flat list of file paths.""" + files_to_stash = load_files_to_stash() + items = [] + for name in bundle_names: + if name not in files_to_stash: + known = list(files_to_stash.keys()) + logging.error(f"Unknown bundle '{name}'. Known bundles: {known}") + raise SystemExit(1) + items.extend(files_to_stash[name]) + return items + + RETRY_SCRIPT = REPO_ROOT / "ci" / "scripts" / "jenkins" / "retry.sh" S3_DOWNLOAD_REGEX = re.compile(r"download: s3://.* to (.*)") SH = Sh() @@ -93,8 +118,19 @@ if __name__ == "__main__": "--prefix", help="s3 bucket + tag (e.g. s3://tvm-ci-prod/PR-1234/cpu", required=True ) parser.add_argument("--items", help="files and folders to upload", nargs="+") + parser.add_argument( + "--bundle", + help="bundle name(s) from ci/jenkins/data.py files_to_stash (repeatable)", + action="append", + dest="bundles", + metavar="NAME", + ) args = parser.parse_args() + + if args.items is not None and args.bundles is not None: + parser.error("--items and --bundle are mutually exclusive") + logging.info(args) sh = Sh() @@ -115,17 +151,18 @@ if __name__ == "__main__": logging.error(f"Unsupported action: {args.action}") exit(1) - if args.items is None: + if args.bundles is not None: + items = resolve_bundles(args.bundles) + elif args.items is not None: + items = args.items + else: if args.action == "upload": - logging.error("Cannot upload without --items") + logging.error("Cannot upload without --items or --bundle") exit(1) else: # Download the whole prefix items = ["."] - else: - items = args.items - for item in items: if action == Action.DOWNLOAD: source = s3_path
