https://github.com/ashgti created https://github.com/llvm/llvm-project/pull/143797
This fixes the `dotest.py -# <num>` flag to work. Currently, if you try to use the flag you'll get an error on the second run of the suite because python unittest.TestSuite will remove the tests from the suite after they run, see https://github.com/python/cpython/blob/8d07562849f2d64475fe8f8af11934c9b06a7106/Lib/unittest/suite.py#L69. This means the second time the suite is invoked, there are no tests to run and it causes an error. To mitigate this, I updated the test loading operation to load the test `-#` count number of times. I also added a flag for `--failfast` to stop testing immediately at the first failure. This, combine with the `-#` flag is helpful for evaluating stability by doing `python3 <build-dir>/bin/lldb-dotest -# 10 --failfast <dir|file>` to repeatedly run the test and stop at the first failure. Additionally, I removed a stale flag that does not appear to be referenced anywhere in the dotest_args.py. >From 249d23695db85e78c803a867e6abaa067282e134 Mon Sep 17 00:00:00 2001 From: John Harrison <harj...@google.com> Date: Wed, 11 Jun 2025 15:15:57 -0700 Subject: [PATCH] [lldb] Improving dotest script flags. This fixes the `dotest.py -# <num>` flag to work. Currently, if you try to use the flag you'll get an error on the second run of the suite because python unittest.TestSuite will remove the tests from the suite after they run, see https://github.com/python/cpython/blob/8d07562849f2d64475fe8f8af11934c9b06a7106/Lib/unittest/suite.py#L69. This means the second time the suite is invoked, there are no tests to run and it causes an error. To mitigate this, I updated the test loading operation to load the test `-#` count number of times. Additionally, I also added a flag for `--failfast` to stop testing immediately at the first failure. This, combine with the `-#` flag is helpful for evaluating stability by doing `python3 <build-dir>/bin/lldb-dotest -# 10 --failfast <dir|file>` to repeatedly run the test and stop at the first failure. --- .../Python/lldbsuite/test/configuration.py | 3 ++ lldb/packages/Python/lldbsuite/test/dotest.py | 42 +++++++++---------- .../Python/lldbsuite/test/dotest_args.py | 12 +++--- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py index b2d91fd211477..5637cba5b88d8 100644 --- a/lldb/packages/Python/lldbsuite/test/configuration.py +++ b/lldb/packages/Python/lldbsuite/test/configuration.py @@ -21,6 +21,9 @@ # The test suite. suite = unittest.TestSuite() +# Stop at the first failure. +failfast = False + # The list of categories we said we care about categories_list = None # set to true if we are going to use categories for cherry-picking test cases diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py index d7f274ac4f60e..4480456a8c2ae 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest.py +++ b/lldb/packages/Python/lldbsuite/test/dotest.py @@ -407,6 +407,9 @@ def parseOptionsAndInitTestdirs(): if args.sharp: configuration.count = args.sharp + if args.failfast: + configuration.failfast = args.failfast + if sys.platform.startswith("win32"): os.environ["LLDB_DISABLE_CRASH_DIALOG"] = str(args.disable_crash_dialog) os.environ["LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE"] = str(True) @@ -665,17 +668,23 @@ def iter_filters(): filtered = True print("adding filter spec %s to module %s" % (filterspec, repr(module))) tests = unittest.defaultTestLoader.loadTestsFromName(filterspec, module) - configuration.suite.addTests(tests) + # Add the test -# times. + for _ in range(configuration.count): + configuration.suite.addTests(tests) # Forgo this module if the (base, filterspec) combo is invalid if configuration.filters and not filtered: return if not filtered: - # Add the entire file's worth of tests since we're not filtered. - # Also the fail-over case when the filterspec branch - # (base, filterspec) combo doesn't make sense. - configuration.suite.addTests(unittest.defaultTestLoader.loadTestsFromName(base)) + # Add the test -# times. + for _ in range(configuration.count): + # Add the entire file's worth of tests since we're not filtered. + # Also the fail-over case when the filterspec branch + # (base, filterspec) combo doesn't make sense. + configuration.suite.addTests( + unittest.defaultTestLoader.loadTestsFromName(base) + ) def visit(prefix, dir, names): @@ -1101,23 +1110,12 @@ def run_suite(): exitTestSuite(1) # Invoke the test runner. - if configuration.count == 1: - result = unittest.TextTestRunner( - stream=sys.stderr, - verbosity=configuration.verbose, - resultclass=test_result.LLDBTestResult, - ).run(configuration.suite) - else: - # We are invoking the same test suite more than once. In this case, - # mark __ignore_singleton__ flag as True so the signleton pattern is - # not enforced. - test_result.LLDBTestResult.__ignore_singleton__ = True - for i in range(configuration.count): - result = unittest.TextTestRunner( - stream=sys.stderr, - verbosity=configuration.verbose, - resultclass=test_result.LLDBTestResult, - ).run(configuration.suite) + result = unittest.TextTestRunner( + stream=sys.stderr, + verbosity=configuration.verbose, + resultclass=test_result.LLDBTestResult, + failfast=configuration.failed, + ).run(configuration.suite) configuration.failed = not result.wasSuccessful() diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py index e9c21388bc213..39bf1b28caa48 100644 --- a/lldb/packages/Python/lldbsuite/test/dotest_args.py +++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py @@ -204,12 +204,6 @@ def create_parser(): action="append", help='Run "setting set SETTING VALUE" before executing any test.', ) - group.add_argument( - "-y", - type=int, - metavar="count", - help="Specify the iteration count used to collect our benchmarks. An example is the number of times to do 'thread step-over' to measure stepping speed.", - ) group.add_argument( "-#", type=int, @@ -217,6 +211,12 @@ def create_parser(): dest="sharp", help="Repeat the test suite for a specified number of times", ) + group.add_argument( + "--failfast", + dest="failfast", + action="store_true", + help="Stop on first fail or error", + ) group.add_argument( "--channel", metavar="channel", _______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
