This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new daefffcace [CI][REFACTOR] Decouple data.py from Jenkins script and
docker images (#19445)
daefffcace is described below
commit daefffcace85bdba605dce411663a39c798219d7
Author: Tianqi Chen <[email protected]>
AuthorDate: Sat Apr 25 19:22:27 2026 -0400
[CI][REFACTOR] Decouple data.py from Jenkins script and docker images
(#19445)
## Summary
`ci/jenkins/data.py` was carrying two unrelated concerns: artifact
bundle definitions for s3 staging, and the docker image tag registry.
This PR decouples both from the Jenkinsfile-generated code, making each
concern standalone and machine-readable without Python.
## Changes
### Part 1 — Bundle refactor (`10950ac192`)
- `ci/scripts/jenkins/s3.py` gains `--bundle <name>` (repeatable) that
resolves bundle names from `ci/jenkins/data.py::files_to_stash` at
runtime; `--items` preserved for back-compat
- `ci/jenkins/templates/utils/macros.j2` `upload_artifacts` macro emits
`--bundle <name>` flags instead of inlining the file list as `--items`
- Template callsites (cpu, gpu, arm) updated to pass `bundles=[...]`
names
- Generated `.groovy` files regenerated — no `build/` paths appear in
artifact upload blocks; only bundle names
### Part 2 — Docker image registry extraction (`6df22c09b9`)
- New `ci/docker-images.ini` (one `[ci_*]` section per image) is the
single source of truth for image tags
- `docker/dev_common.sh::lookup_image_spec` reads the ini directly with
`awk` — no Python invocation
- `ci/jenkins/data.py` drops the inline `docker_images` dict; loads it
via `configparser` from the ini, preserving the same nested-dict shape
so Jinja templates and the current `s3.py` module import keep working
unchanged
- `data.py __main__` is now the bundle-resolver CLI (`python3
ci/jenkins/data.py <bundle> [...]` → file paths, one per line) — no more
dual-purpose image-name lookup branch
- `ci/scripts/jenkins/open_docker_update_pr.py` updated to read/write
`ci/docker-images.ini` instead of `data.py`
## Acceptance
```
# No build/ artifact paths in generated Jenkinsfiles (bundle names only):
grep -rn "build/lib\|build/cpptest" ci/jenkins/generated/ # → nothing
# generate.py is byte-identical after the change:
python3 ci/jenkins/generate.py # → "no changes made" for all 5 groovy files
# dev_common.sh resolves same tag as before:
bash -c 'source docker/dev_common.sh && lookup_image_spec ci_cpu'
# → tlcpack/ci-cpu:20251130-061900-c429a2b1
```
---
ci/jenkins/data.py | 70 +++++++++-----------------
ci/jenkins/docker-images.ini | 2 +-
ci/jenkins/generate.py | 34 ++++++++++++-
ci/jenkins/generated/arm_jenkinsfile.groovy | 4 +-
ci/jenkins/generated/cpu_jenkinsfile.groovy | 4 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 6 +--
ci/jenkins/templates/arm_jenkinsfile.groovy.j2 | 2 +-
ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 2 +-
ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 4 +-
ci/jenkins/templates/utils/macros.j2 | 9 +++-
ci/scripts/jenkins/determine_docker_images.py | 22 ++++++++
ci/scripts/jenkins/open_docker_update_pr.py | 49 ++++++++++--------
ci/scripts/jenkins/s3.py | 47 +++++++++++++++--
docker/dev_common.sh | 3 +-
tests/python/ci/test_ci.py | 4 +-
15 files changed, 175 insertions(+), 87 deletions(-)
diff --git a/ci/jenkins/data.py b/ci/jenkins/data.py
index 699676ecab..b277643e03 100644
--- a/ci/jenkins/data.py
+++ b/ci/jenkins/data.py
@@ -15,6 +15,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+"""Bundle registry for CI artifact stashing.
+
+Single source of truth for the file lists uploaded to / downloaded from S3 by
+``ci/scripts/jenkins/s3.py``. This module deliberately carries nothing else —
+docker image tags live in ``ci/jenkins/docker-images.ini`` and Jinja-template
+metadata (image platforms, AWS endpoints) lives in ``ci/jenkins/generate.py``.
+
+CLI: ``python3 ci/jenkins/data.py <bundle> [<bundle> ...]`` resolves bundle
+names to their file paths (one per line; exit 1 on unknown name). Used by
+``s3.py`` at Jenkins runtime and by any external caller that needs
+data-driven artifact lists.
+"""
+
import sys
files_to_stash = {
@@ -24,15 +37,16 @@ files_to_stash = {
"hexagon_api": [
"build/hexagon_api_output",
],
- # This library is produced with HIDE_PRIVATE_SYMBOLS=ON
- "tvm_allvisible": ["build/libtvm_allvisible.so"],
# runtime files
"tvm_runtime": ["build/libtvm_runtime.so", "build/config.cmake"],
- # compiler files
+ # compiler files (libtvm_allvisible is the HIDE_PRIVATE_SYMBOLS=ON
+ # variant cpptest links against; bundled here so every consumer of
+ # tvm_lib gets it without having to remember a second bundle name).
"tvm_lib": [
"build/libtvm.so",
"build/libtvm_runtime.so",
"build/lib/libtvm_ffi.so",
+ "build/libtvm_allvisible.so",
"build/config.cmake",
],
# gpu related compiler files
@@ -43,46 +57,12 @@ files_to_stash = {
}
-# AWS info
-aws_default_region = "us-west-2"
-aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com"
-
-# Docker Images
-docker_images = {
- "ci_arm": {
- "tag": "tlcpack/ci-arm:20251130-061900-c429a2b1",
- "platform": "ARM",
- },
- "ci_cpu": {
- "tag": "tlcpack/ci-cpu:20251130-061900-c429a2b1",
- "platform": "CPU",
- },
- "ci_gpu": {
- "tag": "tlcpack/ci-gpu:20251130-061900-c429a2b1",
- "platform": "GPU",
- },
- "ci_lint": {
- "tag": "tlcpack/ci-lint:20251130-061900-c429a2b1",
- "platform": "CPU",
- },
- "ci_wasm": {
- "tag": "tlcpack/ci-wasm:20251130-061900-c429a2b1",
- "platform": "CPU",
- },
-}
-
-data = {
- "images": [{"name": k, "platform": v["platform"]} for k, v in
docker_images.items()],
- "aws_default_region": aws_default_region,
- "aws_ecr_url": aws_ecr_url,
- **{k: v["tag"] for k, v in docker_images.items()},
- **files_to_stash,
-}
-
if __name__ == "__main__":
- # This is used in docker/dev_common.sh to look up image tags
- name = sys.argv[1]
- if name in docker_images:
- print(docker_images[name]["tag"])
- else:
- exit(1)
+ paths = []
+ for name in sys.argv[1:]:
+ if name not in files_to_stash:
+ print(f"unknown bundle: {name}", file=sys.stderr)
+ sys.exit(1)
+ paths.extend(files_to_stash[name])
+ for p in paths:
+ print(p)
diff --git a/ci/jenkins/docker-images.ini b/ci/jenkins/docker-images.ini
index 9e795227cd..13f8d841cf 100644
--- a/ci/jenkins/docker-images.ini
+++ b/ci/jenkins/docker-images.ini
@@ -19,7 +19,7 @@
[jenkins]
ci_tag: 20260301-134651-63f099ad
ci_arm: tlcpack/ci-arm:%(ci_tag)s
-ci_cpu: tlcpack/ci_cpu:%(ci_tag)s
+ci_cpu: tlcpack/ci-cpu:%(ci_tag)s
ci_gpu: tlcpack/ci-gpu:%(ci_tag)s
ci_lint: tlcpack/ci-lint:%(ci_tag)s
ci_wasm: tlcpack/ci-wasm:%(ci_tag)s
diff --git a/ci/jenkins/generate.py b/ci/jenkins/generate.py
index af9dfbc92f..fc6e09ab80 100644
--- a/ci/jenkins/generate.py
+++ b/ci/jenkins/generate.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
import argparse
+import configparser
import datetime
import difflib
import re
@@ -24,12 +25,43 @@ from dataclasses import dataclass
from pathlib import Path
import jinja2
-from data import data
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
JENKINS_DIR = REPO_ROOT / "ci" / "jenkins"
TEMPLATES_DIR = JENKINS_DIR / "templates"
GENERATED_DIR = JENKINS_DIR / "generated"
+DOCKER_IMAGES_INI = JENKINS_DIR / "docker-images.ini"
+
+# Platform mapping for the CI image registry. The ini section names are the
+# image names; this dict pins their Jenkins agent label. Keep in sync with
+# the [jenkins] section of docker-images.ini when adding/removing images.
+_IMAGE_PLATFORMS = {
+ "ci_arm": "ARM",
+ "ci_cpu": "CPU",
+ "ci_gpu": "GPU",
+ "ci_lint": "CPU",
+ "ci_wasm": "CPU",
+}
+
+
+def _build_render_context() -> dict:
+ """Build the Jinja render context: image metadata + AWS endpoints."""
+ config = configparser.ConfigParser()
+ config.read(DOCKER_IMAGES_INI)
+ images = [
+ {"name": name, "platform": platform}
+ for name, platform in _IMAGE_PLATFORMS.items()
+ if config.has_option("jenkins", name)
+ ]
+ aws_default_region = "us-west-2"
+ return {
+ "images": images,
+ "aws_default_region": aws_default_region,
+ "aws_ecr_url": f"dkr.ecr.{aws_default_region}.amazonaws.com",
+ }
+
+
+data = _build_render_context()
class Change:
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy
b/ci/jenkins/generated/arm_jenkinsfile.groovy
index 17bddc9b2c..16e56f0ea2 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-02-09T16:32:44.108985
+// Generated at 2026-04-25T16:28:24.516990
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -496,7 +496,7 @@ def run_build(node_type) {
cmake_build(ci_arm, 'build')
make_cpp_tests(ci_arm, 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/arm --items build/libtvm.so
build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake
build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/arm --bundle tvm_lib --bundle cpptest",
label: 'Upload artifacts to S3',
)
})
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy
b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index fb9edab77b..d98a24631d 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2025-08-24T16:41:22.367054
+// Generated at 2026-04-25T16:52:16.785557
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -496,7 +496,7 @@ def run_build(node_type) {
cmake_build(ci_cpu, 'build')
make_cpp_tests(ci_cpu, 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/cpu --items build/libtvm.so
build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake
build/libtvm_allvisible.so build/cpptest build/build.ninja
build/CMakeFiles/rules.ninja",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/cpu --bundle tvm_lib --bundle cpptest",
label: 'Upload artifacts to S3',
)
})
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy
b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index 45f5604727..72c9c09081 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-02-09T16:32:44.095534
+// Generated at 2026-04-25T16:52:16.819825
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -492,7 +492,7 @@ def run_build(node_type) {
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/gpu --items build/libtvm.so
build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake
build/libtvm_allvisible.so build/3rdparty/libflash_attn/src/libflash_attn.so
build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/gpu --bundle tvm_lib --bundle
tvm_lib_gpu_extra",
label: 'Upload artifacts to S3',
)
@@ -502,7 +502,7 @@ def run_build(node_type) {
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu_other.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/gpu2 --items build/libtvm.so
build/libtvm_runtime.so build/lib/libtvm_ffi.so build/config.cmake",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/gpu2 --bundle tvm_lib",
label: 'Upload artifacts to S3',
)
})
diff --git a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
index 35aa9bf250..2bddbab4c7 100644
--- a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
@@ -31,5 +31,5 @@
)
cmake_build(ci_arm, 'build')
make_cpp_tests(ci_arm, 'build')
- {{ m.upload_artifacts(tag='arm', filenames=tvm_lib + cpptest) }}
+ {{ m.upload_artifacts(tag='arm', bundles=["tvm_lib", "cpptest"]) }}
{% endcall %}
diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
index 367da73ebe..d2e479d5e8 100644
--- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
)
cmake_build(ci_cpu, 'build')
make_cpp_tests(ci_cpu, 'build')
- {{ m.upload_artifacts(tag='cpu', filenames=tvm_lib + tvm_allvisible +
cpptest) }}
+ {{ m.upload_artifacts(tag='cpu', bundles=["tvm_lib", "cpptest"]) }}
{% endcall %}
{% set test_method_names = [] %}
diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
index 2769ae2c5d..7ab5256419 100644
--- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
@@ -27,13 +27,13 @@
) %}
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build')
- {{ m.upload_artifacts(tag='gpu', filenames=tvm_lib + tvm_allvisible +
tvm_lib_gpu_extra) }}
+ {{ m.upload_artifacts(tag='gpu', bundles=["tvm_lib", "tvm_lib_gpu_extra"]) }}
// compiler test
sh "rm -rf build"
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu_other.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build')
- {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib) }}
+ {{ m.upload_artifacts(tag='gpu2', bundles=["tvm_lib"]) }}
{% endcall %}
{% set test_method_names = [] %}
diff --git a/ci/jenkins/templates/utils/macros.j2
b/ci/jenkins/templates/utils/macros.j2
index c96432840d..b1bd3679ac 100644
--- a/ci/jenkins/templates/utils/macros.j2
+++ b/ci/jenkins/templates/utils/macros.j2
@@ -166,12 +166,19 @@ test()
},
{% endmacro %}
-{% macro upload_artifacts(action, tag, filenames) %}
+{% macro upload_artifacts(tag, bundles=none, filenames=none) %}
+{% if bundles is not none %}
+sh(
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/{{ tag }}{% for b in bundles %} --bundle {{
b }}{% endfor %}",
+ label: 'Upload artifacts to S3',
+ )
+{% elif filenames is not none %}
{% set items = ' '.join(filenames) %}
sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/{{ tag }} --items {{ items }}",
label: 'Upload artifacts to S3',
)
+{% endif %}
{% endmacro %}
{% macro download_artifacts(tag) %}
diff --git a/ci/scripts/jenkins/determine_docker_images.py
b/ci/scripts/jenkins/determine_docker_images.py
index fed3ab79cc..3f3deb5c57 100755
--- a/ci/scripts/jenkins/determine_docker_images.py
+++ b/ci/scripts/jenkins/determine_docker_images.py
@@ -70,11 +70,28 @@ def image_exists(spec: str) -> bool:
return False
+def lookup_image_tag(name: str) -> str:
+ """Resolve image ``name`` (e.g. ``ci_cpu``) to its tag string from the ini.
+
+ Pure ini read — no Docker Hub query, no tlcpackstaging fallback. Used by
+ ``docker/dev_common.sh`` for local-dev image-name shortcuts where the
+ full Hub-existence check is unnecessary.
+ """
+ config = configparser.ConfigParser()
+ config.read(IMAGE_TAGS_FILE)
+ return config.get("jenkins", name)
+
+
if __name__ == "__main__":
init_log()
parser = argparse.ArgumentParser(
description="Writes out Docker images names to be used to
.docker-image-names/"
)
+ parser.add_argument(
+ "--lookup-only",
+ metavar="NAME",
+ help="Print the tag for NAME from the ini and exit (no Docker Hub
query, no fallback).",
+ )
parser.add_argument(
"--testing-docker-data",
help="(testing only) JSON data to mock response from Docker Hub API",
@@ -89,6 +106,11 @@ if __name__ == "__main__":
help="(testing only) Folder to write image names to",
)
args, other = parser.parse_known_args()
+
+ if args.lookup_only:
+ print(lookup_image_tag(args.lookup_only))
+ raise SystemExit(0)
+
name_dir = Path(args.base_dir)
if args.testing_images_data:
diff --git a/ci/scripts/jenkins/open_docker_update_pr.py
b/ci/scripts/jenkins/open_docker_update_pr.py
index a948750082..0172fe0221 100755
--- a/ci/scripts/jenkins/open_docker_update_pr.py
+++ b/ci/scripts/jenkins/open_docker_update_pr.py
@@ -17,6 +17,7 @@
# under the License.
import argparse
+import configparser
import datetime
import json
import logging
@@ -32,7 +33,7 @@ from git_utils import GitHubRepo, git, parse_remote
from should_rebuild_docker import docker_api
JENKINS_DIR = REPO_ROOT / "ci" / "jenkins"
-IMAGES_FILE = JENKINS_DIR / "data.py"
+IMAGES_FILE = JENKINS_DIR / "docker-images.ini"
GENERATE_SCRIPT = JENKINS_DIR / "generate.py"
GITHUB_TOKEN = os.environ["GITHUB_TOKEN"]
BRANCH = "nightly-docker-update"
@@ -127,33 +128,41 @@ if __name__ == "__main__":
remote = git(["config", "--get", f"remote.{args.remote}.url"])
user, repo = parse_remote(remote)
- # Read the existing images from the Jenkinsfile
+ # Read the existing images from ci/jenkins/docker-images.ini.
+ # The ini has a single ``[jenkins]`` section with a shared ``ci_tag`` key
+ # and one ``ci_<name>: tlcpack/ci-<name>:%(ci_tag)s`` entry per image.
+ # Resolve each image to its full spec (with interpolation applied) and
+ # check against Docker Hub for newer tags.
logging.info(f"Reading {IMAGES_FILE}")
+ config = configparser.ConfigParser()
+ config.read(IMAGES_FILE)
with open(IMAGES_FILE) as f:
- content = f.readlines()
+ content = f.read()
- # Build a new Jenkinsfile with the latest images from tlcpack or
tlcpackstaging
replacements = {}
-
- for line in content:
- m = re.match(r'"tag": "(.*)",', line.strip())
- if m is not None:
- image_spec = m.groups()[0]
- logging.info(f"Found match on line {line.strip()}")
- new_image = latest_tlcpackstaging_image(image_spec)
- if new_image is None:
- logging.info("No new image found")
- else:
- logging.info(f"Using new image {new_image}")
- new_line = f' "tag": "{new_image}",\n'
- replacements[line] = new_line
+ for key in config.options("jenkins"):
+ if key == "ci_tag":
+ continue
+ image_spec = config.get("jenkins", key)
+ logging.info(f"Found {key} = {image_spec}")
+ new_image = latest_tlcpackstaging_image(image_spec)
+ if new_image is None:
+ logging.info("No new image found")
+ continue
+ logging.info(f"Using new image {new_image}")
+ # Rewrite the ``ci_<name>:`` line with the resolved tag (breaking
+ # the ``%(ci_tag)s`` interpolation for that single entry) so the
+ # update is unambiguous and doesn't disturb other images that share
+ # the old tag.
+ old_line_re = re.compile(rf"^{re.escape(key)}\s*[:=].*$", re.MULTILINE)
+ new_line = f"{key}: {new_image}"
+ replacements[old_line_re] = new_line
# Re-generate the Jenkinsfiles
command = f"python3 {shlex.quote(str(GENERATE_SCRIPT))}"
- content = "".join(content)
- for old_line, new_line in replacements.items():
- content = content.replace(old_line, new_line)
+ for old_line_re, new_line in replacements.items():
+ content = old_line_re.sub(new_line, content)
print(f"Updated to:\n{content}")
diff --git a/ci/scripts/jenkins/s3.py b/ci/scripts/jenkins/s3.py
index dd44490b1e..eb986dec99 100755
--- a/ci/scripts/jenkins/s3.py
+++ b/ci/scripts/jenkins/s3.py
@@ -17,6 +17,7 @@
# under the License.
import argparse
+import importlib.util
import logging
import re
from enum import Enum
@@ -24,6 +25,30 @@ from pathlib import Path
from cmd_utils import REPO_ROOT, Sh, init_log
+DATA_PY = REPO_ROOT / "ci" / "jenkins" / "data.py"
+
+
+def load_files_to_stash():
+ """Load the files_to_stash dict from ci/jenkins/data.py."""
+ spec = importlib.util.spec_from_file_location("data", DATA_PY)
+ mod = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(mod)
+ return mod.files_to_stash
+
+
+def resolve_bundles(bundle_names: list[str]) -> list[str]:
+ """Resolve a list of bundle names to a flat list of file paths."""
+ files_to_stash = load_files_to_stash()
+ items = []
+ for name in bundle_names:
+ if name not in files_to_stash:
+ known = list(files_to_stash.keys())
+ logging.error(f"Unknown bundle '{name}'. Known bundles: {known}")
+ raise SystemExit(1)
+ items.extend(files_to_stash[name])
+ return items
+
+
RETRY_SCRIPT = REPO_ROOT / "ci" / "scripts" / "jenkins" / "retry.sh"
S3_DOWNLOAD_REGEX = re.compile(r"download: s3://.* to (.*)")
SH = Sh()
@@ -93,8 +118,19 @@ if __name__ == "__main__":
"--prefix", help="s3 bucket + tag (e.g. s3://tvm-ci-prod/PR-1234/cpu",
required=True
)
parser.add_argument("--items", help="files and folders to upload",
nargs="+")
+ parser.add_argument(
+ "--bundle",
+ help="bundle name(s) from ci/jenkins/data.py files_to_stash
(repeatable)",
+ action="append",
+ dest="bundles",
+ metavar="NAME",
+ )
args = parser.parse_args()
+
+ if args.items is not None and args.bundles is not None:
+ parser.error("--items and --bundle are mutually exclusive")
+
logging.info(args)
sh = Sh()
@@ -115,17 +151,18 @@ if __name__ == "__main__":
logging.error(f"Unsupported action: {args.action}")
exit(1)
- if args.items is None:
+ if args.bundles is not None:
+ items = resolve_bundles(args.bundles)
+ elif args.items is not None:
+ items = args.items
+ else:
if args.action == "upload":
- logging.error("Cannot upload without --items")
+ logging.error("Cannot upload without --items or --bundle")
exit(1)
else:
# Download the whole prefix
items = ["."]
- else:
- items = args.items
-
for item in items:
if action == Action.DOWNLOAD:
source = s3_path
diff --git a/docker/dev_common.sh b/docker/dev_common.sh
index fd5a8f91bd..68799676a3 100755
--- a/docker/dev_common.sh
+++ b/docker/dev_common.sh
@@ -30,7 +30,8 @@ GIT_TOPLEVEL=$(cd $(dirname ${BASH_SOURCE[0]}) && git
rev-parse --show-toplevel)
DOCKER_IS_ROOTLESS=$(docker info 2> /dev/null | grep 'Context: \+rootless' ||
true)
function lookup_image_spec() {
- img_spec=$(python3 "${GIT_TOPLEVEL}/ci/jenkins/data.py" "$1")
+ img_spec=$(python3
"${GIT_TOPLEVEL}/ci/scripts/jenkins/determine_docker_images.py" \
+ --lookup-only "$1" 2>/dev/null)
if [ -n "${img_spec}" ]; then
has_similar_docker_image=1
docker inspect "${1}" &>/dev/null || has_similar_docker_image=0
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
index 41b3f5d275..251143c430 100644
--- a/tests/python/ci/test_ci.py
+++ b/tests/python/ci/test_ci.py
@@ -1198,7 +1198,7 @@ def test_github_tag_teams(tmpdir_factory, source_type,
data, check):
},
expected="Using tlcpackstaging tag on tlcpack",
expected_images=[
- '"tag": "tlcpack/ci-arm:456-456-abc"',
+ "ci_arm: tlcpack/ci-arm:456-456-abc",
],
),
tlcpack_update=dict(
@@ -1220,7 +1220,7 @@ def test_github_tag_teams(tmpdir_factory, source_type,
data, check):
},
expected="Found newer image, using: tlcpack",
expected_images=[
- '"tag": "tlcpack/ci-arm:234-234-abc",',
+ "ci_arm: tlcpack/ci-arm:234-234-abc",
],
),
)
