Noorul Islam K M <noo...@collab.net> writes: > Paul Burba <ptbu...@gmail.com> writes: > >> If someone with the requisite linux skills/hardware could tweak >> makefile.in so it can take advantage of the --milestone-filter option, >> well that would be fabulous. >> >> Paul >> >> On Thu, Feb 17, 2011 at 5:09 PM, <pbu...@apache.org> wrote: >>> Author: pburba >>> Date: Thu Feb 17 22:09:02 2011 >>> New Revision: 1071809 >>> >>> URL: http://svn.apache.org/viewvc?rev=1071809&view=rev >>> Log: >>> Add an option (--milestone-filter=REGEX) to the Python tests so we can list >>> a >>> subset of the tests based on their associated issues' target milestone. >>> >>> This option is currently only available to win-tests.py and >>> subversion/tests/cmdline/svntest/main.py. So it isn't quite as useful >>> on non-Windows platforms just yet. >>> >>> Now we can easily answer questions like, "What xfailing merge tests need to >>> be fixed before we can release 1.7?" >>> >>> C:\SVN\src-trunk>win-tests.py --list --milestone-filter="(1.7.*)|(---)" >>> --mode-filter xfail --log-to-stdout --test merge >>> Listing Debug configuration on local repository. >>> LISTING: merge_tests.py >>> Test # Mode Test Description >>> ------ ----- ---------------- >>> 64 XFAIL merge target with non inheritable mergeinfo >>> [#2970(blue-sky),#3642(1.7.0)] >>> 75 XFAIL merge added subtree [#1962(1.7-consider)] >>> >>> * build/run_tests.py >>> >>> (TestHarness.__init__): Add mode_filter argument. >>> >>> (TestHarness._run_c_test): Issue warning that --milestone-filter doesn't >>> work when listing C tests. >>> >>> (TestHarness._run_py_test): Accept --milestone-filter option. >>> >>> * subversion/tests/cmdline/svntest/main.py >>> >>> (global): Import xml and urllib. >>> >>> (TestSpawningThread.run_one): Support --milestone-filter option. >>> >>> (TestRunner.list): Add optional argument mapping issues to target >>> milestones. >>> >>> (TestRunner.get_issues): New. >>> >>> (_create_parser): Handle --milestone-filter. >>> >>> (get_target_milestones_for_issues): New. >>> >>> (execute_tests): Handle --milestone-filter. >>> >>> * win-tests.py >>> >>> (_usage_exit): Add --milestone-filter to usage text. >>> >>> (milestone_filter): New global variable. >>> >>> (global): Accept --milestone-filter as a valid option, pass it to >>> run_tests.TestHarness(). >>> >>> >>> Modified: >>> subversion/trunk/build/run_tests.py >>> subversion/trunk/subversion/tests/cmdline/svntest/main.py >>> subversion/trunk/win-tests.py >>> >>> Modified: subversion/trunk/build/run_tests.py >>> URL: >>> http://svn.apache.org/viewvc/subversion/trunk/build/run_tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff >>> ============================================================================== >>> --- subversion/trunk/build/run_tests.py (original) >>> +++ subversion/trunk/build/run_tests.py Thu Feb 17 22:09:02 2011 >>> @@ -79,7 +79,8 @@ class TestHarness: >>> server_minor_version=None, verbose=None, >>> cleanup=None, enable_sasl=None, parallel=None, >>> config_file=None, >>> fsfs_sharding=None, fsfs_packing=None, >>> - list_tests=None, svn_bin=None, mode_filter=None): >>> + list_tests=None, svn_bin=None, mode_filter=None, >>> + milestone_filter=None): >>> '''Construct a TestHarness instance. >>> >>> ABS_SRCDIR and ABS_BUILDDIR are the source and build directories. >>> @@ -91,8 +92,12 @@ class TestHarness: >>> HTTP_LIBRARY is the HTTP library for DAV-based communications. >>> SERVER_MINOR_VERSION is the minor version of the server being tested. >>> SVN_BIN is the path where the svn binaries are installed. >>> - mode_filter restricts the TestHarness to tests with the expected mode >>> - XFail, Skip, Pass, or All tests (default). >>> + MODE_FILTER restricts the TestHarness to tests with the expected mode >>> + XFail, Skip, Pass, or All tests (default). MILESTONE_FILTER is a >>> + string representation of a valid regular expression pattern; when used >>> + in conjunction with LIST_TESTS, the only tests that are listed are >>> + those with an associated issue in the tracker which has a target >>> + milestone that matches the regex. >>> ''' >>> self.srcdir = abs_srcdir >>> self.builddir = abs_builddir >>> @@ -114,6 +119,7 @@ class TestHarness: >>> if config_file is not None: >>> self.config_file = os.path.abspath(config_file) >>> self.list_tests = list_tests >>> + self.milestone_filter = milestone_filter >>> self.svn_bin = svn_bin >>> self.mode_filter = mode_filter >>> self.log = None >>> @@ -280,6 +286,8 @@ class TestHarness: >>> if not self.list_tests: >>> sys.stdout.write('.' * dot_count) >>> sys.stdout.flush() >>> + elif self.milestone_filter: >>> + print 'WARNING: --milestone-filter option does not currently work >>> with C tests' >>> >>> if os.access(progbase, os.X_OK): >>> progname = './' + progbase >>> @@ -349,6 +357,8 @@ class TestHarness: >>> svntest.main.options.server_minor_version = self.server_minor_version >>> if self.list_tests is not None: >>> svntest.main.options.list_tests = True >>> + if self.milestone_filter is not None: >>> + svntest.main.options.milestone_filter = self.milestone_filter >>> if self.svn_bin is not None: >>> svntest.main.options.svn_bin = self.svn_bin >>> if self.fsfs_sharding is not None: >>> >>> Modified: subversion/trunk/subversion/tests/cmdline/svntest/main.py >>> URL: >>> http://svn.apache.org/viewvc/subversion/trunk/subversion/tests/cmdline/svntest/main.py?rev=1071809&r1=1071808&r2=1071809&view=diff >>> ============================================================================== >>> --- subversion/trunk/subversion/tests/cmdline/svntest/main.py (original) >>> +++ subversion/trunk/subversion/tests/cmdline/svntest/main.py Thu Feb 17 >>> 22:09:02 2011 >>> @@ -34,6 +34,8 @@ import time # for time() >>> import traceback # for print_exc() >>> import threading >>> import optparse # for argument parsing >>> +import xml >>> +import urllib >>> >>> try: >>> # Python >=3.0 >>> @@ -1132,6 +1134,8 @@ class TestSpawningThread(threading.Threa >>> args.append('--server-minor-version=' + >>> str(options.server_minor_version)) >>> if options.mode_filter: >>> args.append('--mode-filter=' + options.mode_filter) >>> + if options.milestone_filter: >>> + args.append('--milestone-filter=' + options.milestone_filter) >>> >>> result, stdout_lines, stderr_lines = spawn_process(command, 0, 0, None, >>> *args) >>> @@ -1152,26 +1156,61 @@ class TestRunner: >>> self.pred = svntest.testcase.create_test_case(func) >>> self.index = index >>> >>> - def list(self): >>> + def list(self, milestones_dict=None): >>> + """Print test doc strings. MILESTONES_DICT is an optional mapping >>> + of issue numbers to target milestones.""" >>> if options.mode_filter.upper() == 'ALL' \ >>> or options.mode_filter.upper() == self.pred.list_mode().upper() \ >>> or (options.mode_filter.upper() == 'PASS' \ >>> and self.pred.list_mode() == ''): >>> + issues = [] >>> tail = '' >>> if self.pred.issues: >>> - tail += " [%s]" % ','.join(['#%d' % i for i in self.pred.issues]) >>> - if options.verbose and self.pred.inprogress: >>> - tail += " [[%s]]" % self.pred.inprogress >>> - else: >>> - print(" %3d %-5s %s%s" % (self.index, >>> - self.pred.list_mode(), >>> - self.pred.description, >>> - tail)) >>> + if not options.milestone_filter or milestones_dict is None: >>> + issues = self.pred.issues >>> + else: # Limit listing by requested target milestone(s). >>> + filter_issues = [] >>> + matches_filter = False >>> + >>> + # Get the milestones for all the issues associated with this >>> test. >>> + # If any one of them matches the MILESTONE_FILTER then we'll >>> print >>> + # them all. >>> + for issue in self.pred.issues: >>> + # A safe starting assumption. >>> + milestone = 'unknown' >>> + if milestones_dict: >>> + if milestones_dict.has_key(str(issue)): >>> + milestone = milestones_dict[str(issue)] >>> + >>> + filter_issues.append(str(issue) + '(' + milestone + ')') >>> + pattern = re.compile(options.milestone_filter) >>> + if pattern.match(milestone): >>> + matches_filter = True >>> + >>> + # Did at least one of the associated issues meet our filter? >>> + if matches_filter: >>> + issues = filter_issues >>> + >>> + tail += " [%s]" % ','.join(['#%s' % str(i) for i in issues]) >>> + >>> + # If there is no filter or this test made if through >>> + # the filter then print it! >>> + if options.milestone_filter is None or len(issues): >>> + if options.verbose and self.pred.inprogress: >>> + tail += " [[%s]]" % self.pred.inprogress >>> + else: >>> + print(" %3d %-5s %s%s" % (self.index, >>> + self.pred.list_mode(), >>> + self.pred.description, >>> + tail)) >>> sys.stdout.flush() >>> >>> def get_mode(self): >>> return self.pred.list_mode() >>> >>> + def get_issues(self): >>> + return self.pred.issues >>> + >>> def get_function_name(self): >>> return self.pred.get_function_name() >>> >>> @@ -1376,6 +1415,8 @@ def _create_parser(): >>> parser = optparse.OptionParser(usage=usage) >>> parser.add_option('-l', '--list', action='store_true', dest='list_tests', >>> help='Print test doc strings instead of running them') >>> + parser.add_option('--milestone-filter', action='store', >>> dest='milestone_filter', >>> + help='Limit --list to those with target milestone >>> specified') >>> parser.add_option('-v', '--verbose', action='store_true', dest='verbose', >>> help='Print binary command-lines (not with --quiet)') >>> parser.add_option('-q', '--quiet', action='store_true', >>> @@ -1470,6 +1511,47 @@ def run_tests(test_list, serial_only = F >>> >>> sys.exit(execute_tests(test_list, serial_only)) >>> >>> +def get_target_milestones_for_issues(issue_numbers): >>> + xml_url = "http://subversion.tigris.org/issues/xml.cgi?id="; >>> + issue_dict = {} >>> + >>> + if isinstance(issue_numbers, int): >>> + issue_numbers = [str(issue_numbers)] >>> + elif isinstance(issue_numbers, str): >>> + issue_numbers = [issue_numbers] >>> + >>> + if issue_numbers is None or len(issue_numbers) == 0: >>> + return issue_dict >>> + >>> + for num in issue_numbers: >>> + xml_url += str(num) + ',' >>> + issue_dict[str(num)] = 'unknown' >>> + >>> + try: >>> + # Parse the xml for ISSUE_NO from the issue tracker into a Document. >>> + issue_xml_f = urllib.urlopen(xml_url) >>> + except: >>> + print "WARNING: Unable to contact issue tracker; " \ >>> + "milestones defaulting to 'unknown'." >>> + return issue_dict >>> + >>> + try: >>> + xmldoc = xml.dom.minidom.parse(issue_xml_f) >>> + issue_xml_f.close() >>> + >>> + # Get the target milestone for each issue. >>> + issue_element = xmldoc.getElementsByTagName('issue') >>> + for i in issue_element: >>> + issue_id_element = i.getElementsByTagName('issue_id') >>> + issue_id = issue_id_element[0].childNodes[0].nodeValue >>> + milestone_element = i.getElementsByTagName('target_milestone') >>> + milestone = milestone_element[0].childNodes[0].nodeValue >>> + issue_dict[issue_id] = milestone >>> + except: >>> + print "ERROR: Unable to parse target milestones from issue tracker" >>> + raise >>> + >>> + return issue_dict >>> >>> # Main func. This is the "entry point" that all the test scripts call >>> # to run their list of tests. >>> @@ -1586,6 +1668,23 @@ def execute_tests(test_list, serial_only >>> testnums = list(range(1, len(test_list))) >>> >>> if options.list_tests: >>> + >>> + # If we want to list the target milestones, then get all the issues >>> + # associated with all the individual tests. >>> + milestones_dict = None >>> + if options.milestone_filter: >>> + issues_dict = {} >>> + for testnum in testnums: >>> + issues = TestRunner(test_list[testnum], testnum).get_issues() >>> + test_mode = TestRunner(test_list[testnum], >>> testnum).get_mode().upper() >>> + if issues: >>> + for issue in issues: >>> + if (options.mode_filter.upper() == 'ALL' or >>> + options.mode_filter.upper() == test_mode or >>> + (options.mode_filter.upper() == 'PASS' and test_mode == >>> '')): >>> + issues_dict[issue]=issue >>> + milestones_dict = >>> get_target_milestones_for_issues(issues_dict.keys()) >>> + >>> header = "Test # Mode Test Description\n" \ >>> "------ ----- ----------------" >>> printed_header = False >>> @@ -1597,7 +1696,7 @@ def execute_tests(test_list, serial_only >>> if not printed_header: >>> print header >>> printed_header = True >>> - TestRunner(test_list[testnum], testnum).list() >>> + TestRunner(test_list[testnum], testnum).list(milestones_dict) >>> # We are simply listing the tests so always exit with success. >>> return 0 >>> >>> >>> Modified: subversion/trunk/win-tests.py >>> URL: >>> http://svn.apache.org/viewvc/subversion/trunk/win-tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff >>> ============================================================================== >>> --- subversion/trunk/win-tests.py (original) >>> +++ subversion/trunk/win-tests.py Thu Feb 17 22:09:02 2011 >>> @@ -79,6 +79,10 @@ def _usage_exit(): >>> print(" --http-library : dav library to use, neon (default) or >>> serf") >>> print(" --javahl : Run the javahl tests instead of the >>> normal tests") >>> print(" --list : print test doc strings only") >>> + print(" --milestone-filter=RE : RE is a regular expression pattern >>> that (when") >>> + print(" used with --list) limits the tests >>> listed to") >>> + print(" those with an associated issue in the >>> tracker") >>> + print(" which has a target milestone that >>> matches RE.") >>> print(" --mode-filter=TYPE : limit tests to expected TYPE = XFAIL, >>> SKIP, PASS,") >>> print(" or 'ALL' (default)") >>> print(" --enable-sasl : enable Cyrus SASL authentication for") >>> @@ -123,7 +127,7 @@ opts, args = my_getopt(sys.argv[1:], 'hr >>> 'fsfs-packing', 'fsfs-sharding=', 'javahl', >>> 'list', 'enable-sasl', 'bin=', 'parallel', >>> 'config-file=', 'server-minor-version=', >>> - 'log-to-stdout', 'mode-filter=']) >>> + 'log-to-stdout', 'mode-filter=', >>> 'milestone-filter=']) >>> if len(args) > 1: >>> print('Warning: non-option arguments after the first one will be ignored') >>> >>> @@ -140,6 +144,7 @@ httpd_port = None >>> httpd_service = None >>> http_library = 'neon' >>> list_tests = None >>> +milestone_filter = None >>> test_javahl = None >>> enable_sasl = None >>> svn_bin = None >>> @@ -195,6 +200,8 @@ for opt, val in opts: >>> test_javahl = 1 >>> elif opt == '--list': >>> list_tests = 1 >>> + elif opt == '--milestone-filter': >>> + milestone_filter = val >>> elif opt == '--mode-filter': >>> mode_filter = val >>> elif opt == '--enable-sasl': >>> @@ -688,7 +695,8 @@ if not test_javahl: >>> server_minor_version, not quiet, >>> cleanup, enable_sasl, parallel, config_file, >>> fsfs_sharding, fsfs_packing, >>> - list_tests, svn_bin, mode_filter) >>> + list_tests, svn_bin, mode_filter, >>> + milestone_filter) >>> old_cwd = os.getcwd() >>> try: >>> os.chdir(abs_builddir) >>> >>> >>> > > Please find attached patch for Makefile.in to make the same work on > linux. I am not that proficient with make files. But still I think I did > it right. > > Log > [[[ > > Follow-up to r1071809. Allow 'make check' to take MILESTON_FILTER and > MODE_FILTER options. > > Now we can easily answer questions like, "What xfailing merge tests need to > be fixed before we can release 1.7?" > > $ make check MILESTONE_FILTER="(1.7.*)|(---)" MODE_FILTER=xfail > TESTS=subversion/tests/cmdline/merge_tests.py > > LISTING: merge_tests.py > Test # Mode Test Description > ------ ----- ---------------- > 64 XFAIL merge target with non inheritable mergeinfo > [#2970(blue-sky),#3642(1.7.0)] > 75 XFAIL merge added subtree [#1962(1.7-consider)] > > * Makefile.in > (check): Pass --list, --milestone-filter, --mode-filter and > --log-to-stdout to run_tests.py if MILESTONE_FILTER is set. > > * build/run_tests.py > (__doc__): Add --list, --milestone-filter and --mode-filter options to > usage doc. > > (main): Accept --list, --milestone-filter and --mode-filter as a valid > option, pass it to TestHarness() > > Patch by: Noorul Islam K M <noorul{_AT_}collab.net> > ]]] > > Index: Makefile.in > =================================================================== > --- Makefile.in (revision 1072234) > +++ Makefile.in (working copy) > @@ -473,6 +473,10 @@ > if test "$(LOG_TO_STDOUT)" != ""; then \ > flags="--log-to-stdout $$flags"; \ > fi; \ > + if test "$(MILESTONE_FILTER)" != ""; then \ > + flags="--list --milestone-filter=$(MILESTONE_FILTER) \ > + --mode-filter=$(MODE_FILTER) --log-to-stdout $$flags"; \ > + fi; \ > $(PYTHON) $(top_srcdir)/build/run_tests.py \ > --config-file $(top_srcdir)/subversion/tests/tests.conf \ > $$flags \ > Index: build/run_tests.py > =================================================================== > --- build/run_tests.py (revision 1072234) > +++ build/run_tests.py (working copy) > @@ -27,6 +27,7 @@ > [--verbose] [--log-to-stdout] [--cleanup] [--parallel] > [--url=<base-url>] [--http-library=<http-library>] > [--enable-sasl] > [--fs-type=<fs-type>] [--fsfs-packing] [--fsfs-sharding=<n>] > + [--list] [--milestone-filter=<regex>] [--mode-filter=<type>] > [--server-minor-version=<version>] > [--config-file=<file>] > <abs_srcdir> <abs_builddir> > @@ -522,7 +523,7 @@ > 'http-library=', 'server-minor-version=', > 'fsfs-packing', 'fsfs-sharding=', > 'enable-sasl', 'parallel', 'config-file=', > - 'log-to-stdout']) > + 'log-to-stdout', 'list', 'milestone-filter=', > 'mode-filter=']) > except getopt.GetoptError: > args = [] > > @@ -532,9 +533,9 @@ > > base_url, fs_type, verbose, cleanup, enable_sasl, http_library, \ > server_minor_version, fsfs_sharding, fsfs_packing, parallel, \ > - config_file, log_to_stdout = \ > + config_file, log_to_stdout, list_tests, mode_filter, milestone_filter= \ > None, None, None, None, None, None, None, None, None, None, > None, \ > - None > + None, None, None, None > for opt, val in opts: > if opt in ['-u', '--url']: > base_url = val > @@ -560,6 +561,12 @@ > config_file = val > elif opt in ['--log-to-stdout']: > log_to_stdout = 1 > + elif opt in ['--list']: > + list_tests = 1 > + elif opt in ['--milestone-filter']: > + milestone_filter = val > + elif opt in ['--mode-filter']: > + mode_filter = val > else: > raise getopt.GetoptError > > @@ -573,7 +580,8 @@ > th = TestHarness(args[0], args[1], logfile, faillogfile, > base_url, fs_type, http_library, server_minor_version, > verbose, cleanup, enable_sasl, parallel, config_file, > - fsfs_sharding, fsfs_packing) > + fsfs_sharding, fsfs_packing, list_tests, > + mode_filter=mode_filter, > milestone_filter=milestone_filter) > > failed = th.run(args[2:]) > if failed:
Pinging to get some attention to this thread. Thanks and Regards Noorul
