This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tvm-simplify-jenkins-pytest-sharding in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 8c342a931ec8e1e9d64dace8a0c9678503bbd1aa Author: Tianqi Chen <[email protected]> AuthorDate: Sat Jul 4 23:45:42 2026 +0000 Simplify Jenkins pytest plumbing --- ci/scripts/jenkins/pytest_ids.py | 43 -------- ci/scripts/jenkins/pytest_wrapper.py | 137 ------------------------ conftest.py | 18 +--- docker/install/ubuntu_install_python_package.sh | 2 - tests/scripts/setup-pytest-env.sh | 14 +-- 5 files changed, 9 insertions(+), 205 deletions(-) diff --git a/ci/scripts/jenkins/pytest_ids.py b/ci/scripts/jenkins/pytest_ids.py deleted file mode 100755 index 548069e855..0000000000 --- a/ci/scripts/jenkins/pytest_ids.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import argparse -import io -from contextlib import redirect_stdout - -import pytest - - -class NodeidsCollector: - def pytest_collection_modifyitems(self, items): - self.nodeids = [item.nodeid for item in items] - - -def main(folder): - collector = NodeidsCollector() - f = io.StringIO() - with redirect_stdout(f): - pytest.main(["-qq", "--collect-only", folder], plugins=[collector]) - for nodeid in collector.nodeids: - print(nodeid) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="List pytest nodeids for a folder") - parser.add_argument("--folder", required=True, help="test folder to inspect") - args = parser.parse_args() - main(args.folder) diff --git a/ci/scripts/jenkins/pytest_wrapper.py b/ci/scripts/jenkins/pytest_wrapper.py deleted file mode 100755 index a667810293..0000000000 --- a/ci/scripts/jenkins/pytest_wrapper.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import argparse -import logging -import os -import textwrap -import urllib.parse -from pathlib import Path - -import junitparser -from cmd_utils import init_log - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent - - -def lstrip(s: str, prefix: str) -> str: - if s.startswith(prefix): - s = s[len(prefix) :] - return s - - -def classname_to_file(classname: str) -> str: - classname = lstrip(classname, "cython.") - classname = lstrip(classname, "ctypes.") - return classname.replace(".", "/") + ".py" - - -def failed_test_ids() -> list[str]: - FAILURE_TYPES = (junitparser.Failure, junitparser.Error) - junit_dir = REPO_ROOT / "build" / "pytest-results" - failed_node_ids = [] - for junit in junit_dir.glob("*.xml"): - xml = junitparser.JUnitXml.fromfile(str(junit)) - for suite in xml: - # handle suites - for case in suite: - if case.result is None: - logging.warn(f"Incorrectly formatted JUnit found, result was None on {case}") - continue - - if len(case.result) > 0 and isinstance(case.result[0], FAILURE_TYPES): - node_id = classname_to_file(case.classname) + "::" + case.name - failed_node_ids.append(node_id) - - return list(set(failed_node_ids)) - - -def repro_command(build_type: str, failed_node_ids: list[str]) -> str | None: - """ - Parse available JUnit XML files and output a command that users can run to - reproduce CI failures locally - """ - test_args = [f"--tests {node_id}" for node_id in failed_node_ids] - test_args_str = " ".join(test_args) - return f"python3 tests/scripts/ci.py {build_type} {test_args_str}" - - -def make_issue_url(failed_node_ids: list[str]) -> str: - names = [f"`{node_id}`" for node_id in failed_node_ids] - run_url = os.getenv("RUN_DISPLAY_URL", "<insert run URL>") - test_bullets = [f" - `{node_id}`" for node_id in failed_node_ids] - params = { - "labels": "test: flaky", - "title": "[Flaky Test] " + ", ".join(names), - "body": textwrap.dedent( - """ - These tests were found to be flaky (intermittently failing on `main` or failed in a PR with unrelated changes). See [the docs](https://github.com/apache/tvm/blob/main/docs/contribute/ci.rst#handling-flaky-failures) for details. - - ### Tests(s)\n - """ - ) - + "\n".join(test_bullets) - + f"\n\n### Jenkins Links\n\n - {run_url}", - } - return "https://github.com/apache/tvm/issues/new?"; + urllib.parse.urlencode(params) - - -def show_failure_help(failed_suites: list[str]) -> None: - failed_node_ids = failed_test_ids() - - if len(failed_node_ids) == 0: - return - - build_type = os.getenv("PLATFORM") - - if build_type is None: - raise RuntimeError("build type was None, cannot show command") - - repro = repro_command(build_type=build_type, failed_node_ids=failed_node_ids) - if repro is None: - print("No test failures detected") - return - - print(f"Report flaky test shortcut: {make_issue_url(failed_node_ids)}") - print("=============================== PYTEST FAILURES ================================") - print( - "These pytest suites failed to execute. The results can be found in the " - "Jenkins 'Tests' tab or by scrolling up through the raw logs here. " - "If there is no test listed below, the failure likely came from a segmentation " - "fault which you can find in the logs above.\n" - ) - if failed_suites is not None and len(failed_suites) > 0: - print("\n".join([f" - {suite}" for suite in failed_suites])) - print("") - - print("You can reproduce these specific failures locally with this command:\n") - print(textwrap.indent(repro, prefix=" ")) - print("") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Print information about a failed pytest run") - args, other = parser.parse_known_args() - init_log() - - try: - show_failure_help(failed_suites=other) - except Exception as e: - # This script shouldn't ever introduce failures since it's just there to - # add extra information, so ignore any errors - logging.exception(e) diff --git a/conftest.py b/conftest.py index dd32937620..8edf545cd7 100644 --- a/conftest.py +++ b/conftest.py @@ -25,15 +25,6 @@ IS_IN_CI = os.getenv("CI", "") == "true" REPO_ROOT = Path(__file__).resolve().parent -# These are long running tests (manually curated and extracted from CI logs) -# that should be allocated to test shards in a round-robin fashion. These are -# taken from the 20 (arbitrary number) of tests as from -# https://ci.tlcpack.ai/job/tvm/job/main/2907/testReport -_slowest_tests = [] -HARDCODED_ALLOCATIONS = {} -for idx, test in enumerate(_slowest_tests): - HARDCODED_ALLOCATIONS[test] = idx - # These rely on running on the same node to pass successfully FIXED_ALLOCATION_PREFIXES = { "tests/python/testing/test_tvm_testing_features.py": 0, @@ -52,13 +43,8 @@ def find_shard_index(nodeid: str, num_shards: int) -> int: ) return target_shard_idx - if nodeid in HARDCODED_ALLOCATIONS: - hash = HARDCODED_ALLOCATIONS[nodeid] - else: - hash = hashlib.md5(nodeid.encode()) - hash = int(hash.hexdigest(), 16) - - return hash % num_shards + node_hash = hashlib.md5(nodeid.encode()).hexdigest() + return int(node_hash, 16) % num_shards def pytest_collection_modifyitems(config, items): diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index d07f19ed3c..4445dfd83e 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -32,13 +32,11 @@ uv pip install --upgrade \ Pillow==12.1.1 \ "psutil~=7.0" \ "pytest~=8.3" \ - "pytest-profiling~=1.8" \ "pytest-xdist~=3.6" \ pytest-rerunfailures==16.1 \ "requests~=2.32" \ "scipy~=1.13" \ "Jinja2~=3.1" \ - junitparser==4.0.2 \ "six~=1.17" \ "tornado~=6.4" \ "ml_dtypes~=0.5" \ diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index d9e6f27a68..bac4cf6ccb 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -39,7 +39,10 @@ function cleanup() { set +x if [ "${#pytest_errors[@]}" -gt 0 ]; then echo "These pytest invocations failed, the results can be found in the Jenkins 'Tests' tab or by scrolling up through the raw logs here." - python3 ci/scripts/jenkins/pytest_wrapper.py "${pytest_errors[@]}" + echo "" + for error in "${pytest_errors[@]}"; do + echo " ${error}" + done exit 1 fi set -x @@ -47,12 +50,11 @@ function cleanup() { trap cleanup 0 function run_pytest() { - ffi_type="cython" set -e local test_suite_name="$1" shift extra_args=( "$@" ) - if [ -z "${ffi_type}" -o -z "${test_suite_name}" ]; then + if [ -z "${test_suite_name}" ]; then echo "error: run_pytest called incorrectly: run_pytest ${test_suite_name}" "${extra_args[@]}" echo "usage: run_pytest <TEST_SUITE_NAME> [pytest args...]" exit 2 @@ -74,7 +76,7 @@ function run_pytest() { fi fi - suite_name="${test_suite_name}-${current_shard}-${ffi_type}" + suite_name="${test_suite_name}-${current_shard}" DEFAULT_PARALLELISM=auto @@ -93,13 +95,11 @@ function run_pytest() { exit_code=0 set +e python3 -m pytest \ - -o "junit_suite_name=${suite_name}" \ "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \ - "--junit-prefix=${ffi_type}" \ "${extra_args[@]}" || exit_code=$? # Pytest will return error code -5 if no test is collected. if [ "$exit_code" -ne "0" ] && [ "$exit_code" -ne "5" ]; then - pytest_errors+=("${suite_name}: $@") + pytest_errors+=("${suite_name}: $*") fi # To avoid overwriting. set -e
