This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-s3-bundle-refactor in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 6df22c09b9d7538877c956c75c2c17fe45ca4f04 Author: tqchen <[email protected]> AuthorDate: Sat Apr 25 16:37:25 2026 +0000 [CI][REFACTOR] Move docker image tags from ci/jenkins/data.py to ci/docker-images.ini This PR moves the docker image tag registry out of ci/jenkins/data.py and into a dedicated ci/docker-images.ini (one [section] per image), making it the single source of truth for both shell scripts and Jinja template rendering. Main changes: - New ci/docker-images.ini with section-per-image INI format; bash reads it directly with awk — no Python invocation needed - docker/dev_common.sh::lookup_image_spec rewritten to use awk on the ini; drop-in equivalent (same tag strings returned for all ci_* names) - ci/jenkins/data.py drops the inline docker_images dict; loads it via configparser from the ini, preserving the same nested-dict shape so existing template substitutions and the s3.py module import keep working unchanged - ci/jenkins/data.py __main__ block replaced with the bundle-resolver CLI: `python3 ci/jenkins/data.py <bundle> [...]` prints resolved file paths (one per line); exits 1 on unknown bundle name — no more image-name lookup branch - ci/scripts/jenkins/open_docker_update_pr.py updated to read/write ci/docker-images.ini instead of data.py (regex updated from `"tag": "..."` to `tag = ...` ini format) --- ci/docker-images.ini | 26 +++++++++++++++ ci/jenkins/data.py | 51 ++++++++++++++--------------- ci/scripts/jenkins/open_docker_update_pr.py | 13 ++++---- docker/dev_common.sh | 12 ++++++- 4 files changed, 68 insertions(+), 34 deletions(-) diff --git a/ci/docker-images.ini b/ci/docker-images.ini new file mode 100644 index 0000000000..d36b8b238e --- /dev/null +++ b/ci/docker-images.ini @@ -0,0 +1,26 @@ +; Docker image registry for TVM CI. +; This is the single source of truth for docker image tags used by dev_common.sh +; (via awk) and ci/jenkins/data.py (via configparser) for Jinja template rendering. +; +; To update image tags, edit the `tag` value in the appropriate section and +; re-run ci/jenkins/generate.py to regenerate the .groovy files. + +[ci_arm] +tag = tlcpack/ci-arm:20251130-061900-c429a2b1 +platform = ARM + +[ci_cpu] +tag = tlcpack/ci-cpu:20251130-061900-c429a2b1 +platform = CPU + +[ci_gpu] +tag = tlcpack/ci-gpu:20251130-061900-c429a2b1 +platform = GPU + +[ci_lint] +tag = tlcpack/ci-lint:20251130-061900-c429a2b1 +platform = CPU + +[ci_wasm] +tag = tlcpack/ci-wasm:20251130-061900-c429a2b1 +platform = CPU diff --git a/ci/jenkins/data.py b/ci/jenkins/data.py index 699676ecab..3f39660a5c 100644 --- a/ci/jenkins/data.py +++ b/ci/jenkins/data.py @@ -15,7 +15,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import configparser import sys +from pathlib import Path files_to_stash = { # Executables and build files needed to run c++ tests @@ -47,28 +49,14 @@ files_to_stash = { aws_default_region = "us-west-2" aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" -# Docker Images +# Docker image registry — loaded from ci/docker-images.ini. +# Shape preserved: {"ci_cpu": {"tag": "...", "platform": "..."}, ...} +_DOCKER_IMAGES_INI = Path(__file__).parent.parent / "docker-images.ini" +_ini = configparser.ConfigParser() +_ini.read(_DOCKER_IMAGES_INI) docker_images = { - "ci_arm": { - "tag": "tlcpack/ci-arm:20251130-061900-c429a2b1", - "platform": "ARM", - }, - "ci_cpu": { - "tag": "tlcpack/ci-cpu:20251130-061900-c429a2b1", - "platform": "CPU", - }, - "ci_gpu": { - "tag": "tlcpack/ci-gpu:20251130-061900-c429a2b1", - "platform": "GPU", - }, - "ci_lint": { - "tag": "tlcpack/ci-lint:20251130-061900-c429a2b1", - "platform": "CPU", - }, - "ci_wasm": { - "tag": "tlcpack/ci-wasm:20251130-061900-c429a2b1", - "platform": "CPU", - }, + section: {"tag": _ini.get(section, "tag"), "platform": _ini.get(section, "platform")} + for section in _ini.sections() } data = { @@ -80,9 +68,18 @@ data = { } if __name__ == "__main__": - # This is used in docker/dev_common.sh to look up image tags - name = sys.argv[1] - if name in docker_images: - print(docker_images[name]["tag"]) - else: - exit(1) + # Bundle-resolver CLI: resolve one or more bundle names to file paths. + # Used by ci/scripts/jenkins/s3.py and any external caller that needs + # data-driven artifact lists. + # + # python3 ci/jenkins/data.py tvm_lib cpptest + # + # Prints one resolved path per line; exits 1 on unknown bundle name. + paths = [] + for name in sys.argv[1:]: + if name not in files_to_stash: + print(f"unknown bundle: {name}", file=sys.stderr) + sys.exit(1) + paths.extend(files_to_stash[name]) + for p in paths: + print(p) diff --git a/ci/scripts/jenkins/open_docker_update_pr.py b/ci/scripts/jenkins/open_docker_update_pr.py index a948750082..2777e46909 100755 --- a/ci/scripts/jenkins/open_docker_update_pr.py +++ b/ci/scripts/jenkins/open_docker_update_pr.py @@ -32,7 +32,7 @@ from git_utils import GitHubRepo, git, parse_remote from should_rebuild_docker import docker_api JENKINS_DIR = REPO_ROOT / "ci" / "jenkins" -IMAGES_FILE = JENKINS_DIR / "data.py" +IMAGES_FILE = REPO_ROOT / "ci" / "docker-images.ini" GENERATE_SCRIPT = JENKINS_DIR / "generate.py" GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] BRANCH = "nightly-docker-update" @@ -127,25 +127,25 @@ if __name__ == "__main__": remote = git(["config", "--get", f"remote.{args.remote}.url"]) user, repo = parse_remote(remote) - # Read the existing images from the Jenkinsfile + # Read the existing images from ci/docker-images.ini logging.info(f"Reading {IMAGES_FILE}") with open(IMAGES_FILE) as f: content = f.readlines() - # Build a new Jenkinsfile with the latest images from tlcpack or tlcpackstaging + # Build updated ini with the latest images from tlcpack or tlcpackstaging replacements = {} for line in content: - m = re.match(r'"tag": "(.*)",', line.strip()) + m = re.match(r"tag\s*=\s*(.*)", line.strip()) if m is not None: - image_spec = m.groups()[0] + image_spec = m.groups()[0].strip() logging.info(f"Found match on line {line.strip()}") new_image = latest_tlcpackstaging_image(image_spec) if new_image is None: logging.info("No new image found") else: logging.info(f"Using new image {new_image}") - new_line = f' "tag": "{new_image}",\n' + new_line = f"tag = {new_image}\n" replacements[line] = new_line # Re-generate the Jenkinsfiles @@ -175,6 +175,7 @@ if __name__ == "__main__": logging.info("Creating git commit") git(["checkout", "-B", BRANCH]) git(["add", str(JENKINS_DIR.relative_to(REPO_ROOT))]) + git(["add", str(IMAGES_FILE.relative_to(REPO_ROOT))]) git(["config", "user.name", "tvm-bot"]) git(["config", "user.email", "[email protected]"]) git(["commit", "-m", message]) diff --git a/docker/dev_common.sh b/docker/dev_common.sh index fd5a8f91bd..9c8c3fb96c 100755 --- a/docker/dev_common.sh +++ b/docker/dev_common.sh @@ -30,7 +30,17 @@ GIT_TOPLEVEL=$(cd $(dirname ${BASH_SOURCE[0]}) && git rev-parse --show-toplevel) DOCKER_IS_ROOTLESS=$(docker info 2> /dev/null | grep 'Context: \+rootless' || true) function lookup_image_spec() { - img_spec=$(python3 "${GIT_TOPLEVEL}/ci/jenkins/data.py" "$1") + local name="$1" + local ini="${GIT_TOPLEVEL}/ci/docker-images.ini" + img_spec=$(awk -v section="[$name]" ' + $0 == section { in_section=1; next } + /^\[/ { in_section=0 } + in_section && /^tag *=/ { + sub(/^tag *= */, "") + print + exit + } + ' "$ini") if [ -n "${img_spec}" ]; then has_similar_docker_image=1 docker inspect "${1}" &>/dev/null || has_similar_docker_image=0
