https://github.com/hnrklssn updated
https://github.com/llvm/llvm-project/pull/153821
>From b2bdc5bbac4260089097114f7bd5a968c9d6d761 Mon Sep 17 00:00:00 2001
From: "Henrik G. Olsson" <h_ols...@apple.com>
Date: Fri, 15 Aug 2025 14:56:11 +0200
Subject: [PATCH] Reland "[Utils] Add new --update-tests flag to llvm-lit"
This reverts commit e495231238b86ae2a3c7bb5f94634c19ca2af19a to reland
the --update-tests feature, originally landed in #108425.
---
clang/test/lit.cfg.py | 10 ++++++
llvm/docs/CommandGuide/lit.rst | 5 +++
llvm/test/lit.cfg.py | 10 ++++++
llvm/utils/lit/lit/LitConfig.py | 3 ++
llvm/utils/lit/lit/TestRunner.py | 12 +++++++
llvm/utils/lit/lit/cl_arguments.py | 6 ++++
llvm/utils/lit/lit/llvm/config.py | 5 +++
llvm/utils/lit/lit/main.py | 1 +
llvm/utils/update_any_test_checks.py | 54 ++++++++++++++++++++++++++--
9 files changed, 103 insertions(+), 3 deletions(-)
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 1957bb1715eb6..12e4d0e454270 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -410,3 +410,13 @@ def calculate_arch_features(arch_string):
# possibly be present in system and user configuration files, so disable
# default configs for the test runs.
config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ utilspath = os.path.join(config.llvm_src_root, "utils")
+ sys.path.append(utilspath)
+ from update_any_test_checks import utc_lit_plugin
+
+ lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index eb90e950a3770..15c249d8e6d31 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -399,6 +399,11 @@ ADDITIONAL OPTIONS
Show all features used in the test suite (in ``XFAIL``, ``UNSUPPORTED`` and
``REQUIRES``) and exit.
+.. option:: --update-tests
+
+ Pass failing tests to functions in the ``lit_config.test_updaters`` list to
+ check whether any of them know how to update the test to make it pass.
+
EXIT STATUS
-----------
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 8c2d1a454e8f9..bc240425d6d0e 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -715,3 +715,13 @@ def host_unwind_supports_jit():
if config.has_logf128:
config.available_features.add("has_logf128")
+
+if lit_config.update_tests:
+ import sys
+ import os
+
+ utilspath = os.path.join(config.llvm_src_root, "utils")
+ sys.path.append(utilspath)
+ from update_any_test_checks import utc_lit_plugin
+
+ lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index cb4aef6f72a87..df297b91be1b6 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -39,6 +39,7 @@ def __init__(
parallelism_groups={},
per_test_coverage=False,
gtest_sharding=True,
+ update_tests=False,
):
# The name of the test runner.
self.progname = progname
@@ -91,6 +92,8 @@ def __init__(
self.parallelism_groups = parallelism_groups
self.per_test_coverage = per_test_coverage
self.gtest_sharding = bool(gtest_sharding)
+ self.update_tests = update_tests
+ self.test_updaters = []
@property
def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index e7cd70766a3dd..f2c5c6d0dbe93 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1192,6 +1192,18 @@ def executeScriptInternal(
str(result.timeoutReached),
)
+ if litConfig.update_tests:
+ for test_updater in litConfig.test_updaters:
+ try:
+ update_output = test_updater(result, test)
+ except Exception as e:
+ out += f"Exception occurred in test updater: {e}"
+ continue
+ if update_output:
+ for line in update_output.splitlines():
+ out += f"# {line}\n"
+ break
+
return out, err, exitCode, timeoutInfo
diff --git a/llvm/utils/lit/lit/cl_arguments.py
b/llvm/utils/lit/lit/cl_arguments.py
index e88951520e660..8f9211ee3f538 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -230,6 +230,12 @@ def parse_args():
action="store_true",
help="Exit with status zero even if some tests fail",
)
+ execution_group.add_argument(
+ "--update-tests",
+ dest="update_tests",
+ action="store_true",
+ help="Try to update regression tests to reflect current behavior, if
possible",
+ )
execution_test_time_group = execution_group.add_mutually_exclusive_group()
execution_test_time_group.add_argument(
"--skip-test-time-recording",
diff --git a/llvm/utils/lit/lit/llvm/config.py
b/llvm/utils/lit/lit/llvm/config.py
index 649636d4bcf4c..44119ec8c0eca 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -64,12 +64,17 @@ def __init__(self, lit_config, config):
self.with_environment("_TAG_REDIR_ERR", "TXT")
self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG)
POSIX(ON)")
+ if lit_config.update_tests:
+ self.use_lit_shell = True
+
# Choose between lit's internal shell pipeline runner and a real shell.
# If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an
# override.
lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
if lit_shell_env:
self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
+ if not self.use_lit_shell and lit_config.update_tests:
+ print("note: --update-tests is not supported when using
external shell")
if not self.use_lit_shell:
features.add("shell")
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 9650a0e901173..5255e2c5e1b51 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -43,6 +43,7 @@ def main(builtin_params={}):
per_test_coverage=opts.per_test_coverage,
gtest_sharding=opts.gtest_sharding,
maxRetriesPerTest=opts.maxRetriesPerTest,
+ update_tests=opts.update_tests,
)
discovered_tests = lit.discovery.find_tests_for_inputs(
diff --git a/llvm/utils/update_any_test_checks.py
b/llvm/utils/update_any_test_checks.py
index e8eef1a46c504..76fe336593929 100755
--- a/llvm/utils/update_any_test_checks.py
+++ b/llvm/utils/update_any_test_checks.py
@@ -34,9 +34,12 @@ def find_utc_tool(search_path, utc_name):
return None
-def run_utc_tool(utc_name, utc_tool, testname):
+def run_utc_tool(utc_name, utc_tool, testname, environment):
result = subprocess.run(
- [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+ [utc_tool, testname],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=environment,
)
return (result.returncode, result.stdout, result.stderr)
@@ -60,6 +63,42 @@ def expand_listfile_args(arg_list):
return exp_arg_list
+def utc_lit_plugin(result, test):
+ testname = test.getFilePath()
+ if not testname:
+ return None
+
+ script_name = os.path.abspath(__file__)
+ utc_search_path = os.path.join(os.path.dirname(script_name),
os.path.pardir)
+
+ with open(testname, "r") as f:
+ header = f.readline().strip()
+
+ m = RE_ASSERTIONS.search(header)
+ if m is None:
+ return None
+
+ utc_name = m.group(1)
+ utc_tool = find_utc_tool([utc_search_path], utc_name)
+ if not utc_tool:
+ return f"update-utc-tests: {utc_name} not found"
+
+ return_code, stdout, stderr = run_utc_tool(
+ utc_name, utc_tool, testname, test.config.environment
+ )
+
+ stderr = stderr.decode(errors="replace")
+ if return_code != 0:
+ if stderr:
+ return f"update-utc-tests: {utc_name} exited with return code
{return_code}\n{stderr.rstrip()}"
+ return f"update-utc-tests: {utc_name} exited with return code
{return_code}"
+
+ stdout = stdout.decode(errors="replace")
+ if stdout:
+ return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}"
+ return f"update-utc-tests: updated {testname}"
+
+
def main():
from argparse import RawTextHelpFormatter
@@ -78,6 +117,11 @@ def main():
nargs="*",
help="Additional directories to scan for update_*_test_checks scripts",
)
+ parser.add_argument(
+ "--path",
+ help="""Additional directories to scan for executables invoked by the
update_*_test_checks scripts,
+separated by the platform path separator""",
+ )
parser.add_argument("tests", nargs="+")
config = parser.parse_args()
@@ -88,6 +132,10 @@ def main():
script_name = os.path.abspath(__file__)
utc_search_path.append(os.path.join(os.path.dirname(script_name),
os.path.pardir))
+ local_env = os.environ.copy()
+ if config.path:
+ local_env["PATH"] = config.path + os.pathsep + local_env["PATH"]
+
not_autogenerated = []
utc_tools = {}
have_error = False
@@ -117,7 +165,7 @@ def main():
continue
future = executor.submit(
- run_utc_tool, utc_name, utc_tools[utc_name], testname
+ run_utc_tool, utc_name, utc_tools[utc_name], testname,
local_env
)
jobs.append((testname, future))
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
