Paul Burba <ptbu...@gmail.com> writes:
> If someone with the requisite linux skills/hardware could tweak
> makefile.in so it can take advantage of the --milestone-filter option,
> well that would be fabulous.
>
> Paul
>
> On Thu, Feb 17, 2011 at 5:09 PM, <pbu...@apache.org> wrote:
>> Author: pburba
>> Date: Thu Feb 17 22:09:02 2011
>> New Revision: 1071809
>>
>> URL: http://svn.apache.org/viewvc?rev=1071809&view=rev
>> Log:
>> Add an option (--milestone-filter=REGEX) to the Python tests so we can list a
>> subset of the tests based on their associated issues' target milestone.
>>
>> This option is currently only available to win-tests.py and
>> subversion/tests/cmdline/svntest/main.py. So it isn't quite as useful
>> on non-Windows platforms just yet.
>>
>> Now we can easily answer questions like, "What xfailing merge tests need to
>> be fixed before we can release 1.7?"
>>
>> C:\SVN\src-trunk>win-tests.py --list --milestone-filter="(1.7.*)|(---)"
>> --mode-filter xfail --log-to-stdout --test merge
>> Listing Debug configuration on local repository.
>> LISTING: merge_tests.py
>> Test # Mode Test Description
>> ------ ----- ----------------
>> 64 XFAIL merge target with non inheritable mergeinfo
>> [#2970(blue-sky),#3642(1.7.0)]
>> 75 XFAIL merge added subtree [#1962(1.7-consider)]
>>
>> * build/run_tests.py
>>
>> (TestHarness.__init__): Add mode_filter argument.
>>
>> (TestHarness._run_c_test): Issue warning that --milestone-filter doesn't
>> work when listing C tests.
>>
>> (TestHarness._run_py_test): Accept --milestone-filter option.
>>
>> * subversion/tests/cmdline/svntest/main.py
>>
>> (global): Import xml and urllib.
>>
>> (TestSpawningThread.run_one): Support --milestone-filter option.
>>
>> (TestRunner.list): Add optional argument mapping issues to target
>> milestones.
>>
>> (TestRunner.get_issues): New.
>>
>> (_create_parser): Handle --milestone-filter.
>>
>> (get_target_milestones_for_issues): New.
>>
>> (execute_tests): Handle --milestone-filter.
>>
>> * win-tests.py
>>
>> (_usage_exit): Add --milestone-filter to usage text.
>>
>> (milestone_filter): New global variable.
>>
>> (global): Accept --milestone-filter as a valid option, pass it to
>> run_tests.TestHarness().
>>
>>
>> Modified:
>> subversion/trunk/build/run_tests.py
>> subversion/trunk/subversion/tests/cmdline/svntest/main.py
>> subversion/trunk/win-tests.py
>>
>> Modified: subversion/trunk/build/run_tests.py
>> URL:
>> http://svn.apache.org/viewvc/subversion/trunk/build/run_tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>> ==============================================================================
>> --- subversion/trunk/build/run_tests.py (original)
>> +++ subversion/trunk/build/run_tests.py Thu Feb 17 22:09:02 2011
>> @@ -79,7 +79,8 @@ class TestHarness:
>> server_minor_version=None, verbose=None,
>> cleanup=None, enable_sasl=None, parallel=None,
>> config_file=None,
>> fsfs_sharding=None, fsfs_packing=None,
>> - list_tests=None, svn_bin=None, mode_filter=None):
>> + list_tests=None, svn_bin=None, mode_filter=None,
>> + milestone_filter=None):
>> '''Construct a TestHarness instance.
>>
>> ABS_SRCDIR and ABS_BUILDDIR are the source and build directories.
>> @@ -91,8 +92,12 @@ class TestHarness:
>> HTTP_LIBRARY is the HTTP library for DAV-based communications.
>> SERVER_MINOR_VERSION is the minor version of the server being tested.
>> SVN_BIN is the path where the svn binaries are installed.
>> - mode_filter restricts the TestHarness to tests with the expected mode
>> - XFail, Skip, Pass, or All tests (default).
>> + MODE_FILTER restricts the TestHarness to tests with the expected mode
>> + XFail, Skip, Pass, or All tests (default). MILESTONE_FILTER is a
>> + string representation of a valid regular expression pattern; when used
>> + in conjunction with LIST_TESTS, the only tests that are listed are
>> + those with an associated issue in the tracker which has a target
>> + milestone that matches the regex.
>> '''
>> self.srcdir = abs_srcdir
>> self.builddir = abs_builddir
>> @@ -114,6 +119,7 @@ class TestHarness:
>> if config_file is not None:
>> self.config_file = os.path.abspath(config_file)
>> self.list_tests = list_tests
>> + self.milestone_filter = milestone_filter
>> self.svn_bin = svn_bin
>> self.mode_filter = mode_filter
>> self.log = None
>> @@ -280,6 +286,8 @@ class TestHarness:
>> if not self.list_tests:
>> sys.stdout.write('.' * dot_count)
>> sys.stdout.flush()
>> + elif self.milestone_filter:
>> + print 'WARNING: --milestone-filter option does not currently work
>> with C tests'
>>
>> if os.access(progbase, os.X_OK):
>> progname = './' + progbase
>> @@ -349,6 +357,8 @@ class TestHarness:
>> svntest.main.options.server_minor_version = self.server_minor_version
>> if self.list_tests is not None:
>> svntest.main.options.list_tests = True
>> + if self.milestone_filter is not None:
>> + svntest.main.options.milestone_filter = self.milestone_filter
>> if self.svn_bin is not None:
>> svntest.main.options.svn_bin = self.svn_bin
>> if self.fsfs_sharding is not None:
>>
>> Modified: subversion/trunk/subversion/tests/cmdline/svntest/main.py
>> URL:
>> http://svn.apache.org/viewvc/subversion/trunk/subversion/tests/cmdline/svntest/main.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>> ==============================================================================
>> --- subversion/trunk/subversion/tests/cmdline/svntest/main.py (original)
>> +++ subversion/trunk/subversion/tests/cmdline/svntest/main.py Thu Feb 17
>> 22:09:02 2011
>> @@ -34,6 +34,8 @@ import time # for time()
>> import traceback # for print_exc()
>> import threading
>> import optparse # for argument parsing
>> +import xml
>> +import urllib
>>
>> try:
>> # Python >=3.0
>> @@ -1132,6 +1134,8 @@ class TestSpawningThread(threading.Threa
>> args.append('--server-minor-version=' +
>> str(options.server_minor_version))
>> if options.mode_filter:
>> args.append('--mode-filter=' + options.mode_filter)
>> + if options.milestone_filter:
>> + args.append('--milestone-filter=' + options.milestone_filter)
>>
>> result, stdout_lines, stderr_lines = spawn_process(command, 0, 0, None,
>> *args)
>> @@ -1152,26 +1156,61 @@ class TestRunner:
>> self.pred = svntest.testcase.create_test_case(func)
>> self.index = index
>>
>> - def list(self):
>> + def list(self, milestones_dict=None):
>> + """Print test doc strings. MILESTONES_DICT is an optional mapping
>> + of issue numbers to target milestones."""
>> if options.mode_filter.upper() == 'ALL' \
>> or options.mode_filter.upper() == self.pred.list_mode().upper() \
>> or (options.mode_filter.upper() == 'PASS' \
>> and self.pred.list_mode() == ''):
>> + issues = []
>> tail = ''
>> if self.pred.issues:
>> - tail += " [%s]" % ','.join(['#%d' % i for i in self.pred.issues])
>> - if options.verbose and self.pred.inprogress:
>> - tail += " [[%s]]" % self.pred.inprogress
>> - else:
>> - print(" %3d %-5s %s%s" % (self.index,
>> - self.pred.list_mode(),
>> - self.pred.description,
>> - tail))
>> + if not options.milestone_filter or milestones_dict is None:
>> + issues = self.pred.issues
>> + else: # Limit listing by requested target milestone(s).
>> + filter_issues = []
>> + matches_filter = False
>> +
>> + # Get the milestones for all the issues associated with this test.
>> + # If any one of them matches the MILESTONE_FILTER then we'll print
>> + # them all.
>> + for issue in self.pred.issues:
>> + # A safe starting assumption.
>> + milestone = 'unknown'
>> + if milestones_dict:
>> + if milestones_dict.has_key(str(issue)):
>> + milestone = milestones_dict[str(issue)]
>> +
>> + filter_issues.append(str(issue) + '(' + milestone + ')')
>> + pattern = re.compile(options.milestone_filter)
>> + if pattern.match(milestone):
>> + matches_filter = True
>> +
>> + # Did at least one of the associated issues meet our filter?
>> + if matches_filter:
>> + issues = filter_issues
>> +
>> + tail += " [%s]" % ','.join(['#%s' % str(i) for i in issues])
>> +
>> + # If there is no filter or this test made if through
>> + # the filter then print it!
>> + if options.milestone_filter is None or len(issues):
>> + if options.verbose and self.pred.inprogress:
>> + tail += " [[%s]]" % self.pred.inprogress
>> + else:
>> + print(" %3d %-5s %s%s" % (self.index,
>> + self.pred.list_mode(),
>> + self.pred.description,
>> + tail))
>> sys.stdout.flush()
>>
>> def get_mode(self):
>> return self.pred.list_mode()
>>
>> + def get_issues(self):
>> + return self.pred.issues
>> +
>> def get_function_name(self):
>> return self.pred.get_function_name()
>>
>> @@ -1376,6 +1415,8 @@ def _create_parser():
>> parser = optparse.OptionParser(usage=usage)
>> parser.add_option('-l', '--list', action='store_true', dest='list_tests',
>> help='Print test doc strings instead of running them')
>> + parser.add_option('--milestone-filter', action='store',
>> dest='milestone_filter',
>> + help='Limit --list to those with target milestone
>> specified')
>> parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
>> help='Print binary command-lines (not with --quiet)')
>> parser.add_option('-q', '--quiet', action='store_true',
>> @@ -1470,6 +1511,47 @@ def run_tests(test_list, serial_only = F
>>
>> sys.exit(execute_tests(test_list, serial_only))
>>
>> +def get_target_milestones_for_issues(issue_numbers):
>> + xml_url = "http://subversion.tigris.org/issues/xml.cgi?id=";
>> + issue_dict = {}
>> +
>> + if isinstance(issue_numbers, int):
>> + issue_numbers = [str(issue_numbers)]
>> + elif isinstance(issue_numbers, str):
>> + issue_numbers = [issue_numbers]
>> +
>> + if issue_numbers is None or len(issue_numbers) == 0:
>> + return issue_dict
>> +
>> + for num in issue_numbers:
>> + xml_url += str(num) + ','
>> + issue_dict[str(num)] = 'unknown'
>> +
>> + try:
>> + # Parse the xml for ISSUE_NO from the issue tracker into a Document.
>> + issue_xml_f = urllib.urlopen(xml_url)
>> + except:
>> + print "WARNING: Unable to contact issue tracker; " \
>> + "milestones defaulting to 'unknown'."
>> + return issue_dict
>> +
>> + try:
>> + xmldoc = xml.dom.minidom.parse(issue_xml_f)
>> + issue_xml_f.close()
>> +
>> + # Get the target milestone for each issue.
>> + issue_element = xmldoc.getElementsByTagName('issue')
>> + for i in issue_element:
>> + issue_id_element = i.getElementsByTagName('issue_id')
>> + issue_id = issue_id_element[0].childNodes[0].nodeValue
>> + milestone_element = i.getElementsByTagName('target_milestone')
>> + milestone = milestone_element[0].childNodes[0].nodeValue
>> + issue_dict[issue_id] = milestone
>> + except:
>> + print "ERROR: Unable to parse target milestones from issue tracker"
>> + raise
>> +
>> + return issue_dict
>>
>> # Main func. This is the "entry point" that all the test scripts call
>> # to run their list of tests.
>> @@ -1586,6 +1668,23 @@ def execute_tests(test_list, serial_only
>> testnums = list(range(1, len(test_list)))
>>
>> if options.list_tests:
>> +
>> + # If we want to list the target milestones, then get all the issues
>> + # associated with all the individual tests.
>> + milestones_dict = None
>> + if options.milestone_filter:
>> + issues_dict = {}
>> + for testnum in testnums:
>> + issues = TestRunner(test_list[testnum], testnum).get_issues()
>> + test_mode = TestRunner(test_list[testnum],
>> testnum).get_mode().upper()
>> + if issues:
>> + for issue in issues:
>> + if (options.mode_filter.upper() == 'ALL' or
>> + options.mode_filter.upper() == test_mode or
>> + (options.mode_filter.upper() == 'PASS' and test_mode ==
>> '')):
>> + issues_dict[issue]=issue
>> + milestones_dict = get_target_milestones_for_issues(issues_dict.keys())
>> +
>> header = "Test # Mode Test Description\n" \
>> "------ ----- ----------------"
>> printed_header = False
>> @@ -1597,7 +1696,7 @@ def execute_tests(test_list, serial_only
>> if not printed_header:
>> print header
>> printed_header = True
>> - TestRunner(test_list[testnum], testnum).list()
>> + TestRunner(test_list[testnum], testnum).list(milestones_dict)
>> # We are simply listing the tests so always exit with success.
>> return 0
>>
>>
>> Modified: subversion/trunk/win-tests.py
>> URL:
>> http://svn.apache.org/viewvc/subversion/trunk/win-tests.py?rev=1071809&r1=1071808&r2=1071809&view=diff
>> ==============================================================================
>> --- subversion/trunk/win-tests.py (original)
>> +++ subversion/trunk/win-tests.py Thu Feb 17 22:09:02 2011
>> @@ -79,6 +79,10 @@ def _usage_exit():
>> print(" --http-library : dav library to use, neon (default) or
>> serf")
>> print(" --javahl : Run the javahl tests instead of the
>> normal tests")
>> print(" --list : print test doc strings only")
>> + print(" --milestone-filter=RE : RE is a regular expression pattern that
>> (when")
>> + print(" used with --list) limits the tests
>> listed to")
>> + print(" those with an associated issue in the
>> tracker")
>> + print(" which has a target milestone that
>> matches RE.")
>> print(" --mode-filter=TYPE : limit tests to expected TYPE = XFAIL,
>> SKIP, PASS,")
>> print(" or 'ALL' (default)")
>> print(" --enable-sasl : enable Cyrus SASL authentication for")
>> @@ -123,7 +127,7 @@ opts, args = my_getopt(sys.argv[1:], 'hr
>> 'fsfs-packing', 'fsfs-sharding=', 'javahl',
>> 'list', 'enable-sasl', 'bin=', 'parallel',
>> 'config-file=', 'server-minor-version=',
>> - 'log-to-stdout', 'mode-filter='])
>> + 'log-to-stdout', 'mode-filter=',
>> 'milestone-filter='])
>> if len(args) > 1:
>> print('Warning: non-option arguments after the first one will be ignored')
>>
>> @@ -140,6 +144,7 @@ httpd_port = None
>> httpd_service = None
>> http_library = 'neon'
>> list_tests = None
>> +milestone_filter = None
>> test_javahl = None
>> enable_sasl = None
>> svn_bin = None
>> @@ -195,6 +200,8 @@ for opt, val in opts:
>> test_javahl = 1
>> elif opt == '--list':
>> list_tests = 1
>> + elif opt == '--milestone-filter':
>> + milestone_filter = val
>> elif opt == '--mode-filter':
>> mode_filter = val
>> elif opt == '--enable-sasl':
>> @@ -688,7 +695,8 @@ if not test_javahl:
>> server_minor_version, not quiet,
>> cleanup, enable_sasl, parallel, config_file,
>> fsfs_sharding, fsfs_packing,
>> - list_tests, svn_bin, mode_filter)
>> + list_tests, svn_bin, mode_filter,
>> + milestone_filter)
>> old_cwd = os.getcwd()
>> try:
>> os.chdir(abs_builddir)
>>
>>
>>
Please find attached patch for Makefile.in to make the same work on
linux. I am not that proficient with make files. But still I think I did
it right.
Log
[[[
Follow-up to r1071809. Allow 'make check' to take MILESTON_FILTER and
MODE_FILTER options.
Now we can easily answer questions like, "What xfailing merge tests need to
be fixed before we can release 1.7?"
$ make check MILESTONE_FILTER="(1.7.*)|(---)" MODE_FILTER=xfail
TESTS=subversion/tests/cmdline/merge_tests.py
LISTING: merge_tests.py
Test # Mode Test Description
------ ----- ----------------
64 XFAIL merge target with non inheritable mergeinfo
[#2970(blue-sky),#3642(1.7.0)]
75 XFAIL merge added subtree [#1962(1.7-consider)]
* Makefile.in
(check): Pass --list, --milestone-filter, --mode-filter and
--log-to-stdout to run_tests.py if MILESTONE_FILTER is set.
* build/run_tests.py
(__doc__): Add --list, --milestone-filter and --mode-filter options to
usage doc.
(main): Accept --list, --milestone-filter and --mode-filter as a valid
option, pass it to TestHarness()
Patch by: Noorul Islam K M <noorul{_AT_}collab.net>
]]]
Index: Makefile.in
===================================================================
--- Makefile.in (revision 1072234)
+++ Makefile.in (working copy)
@@ -473,6 +473,10 @@
if test "$(LOG_TO_STDOUT)" != ""; then \
flags="--log-to-stdout $$flags"; \
fi; \
+ if test "$(MILESTONE_FILTER)" != ""; then \
+ flags="--list --milestone-filter=$(MILESTONE_FILTER) \
+ --mode-filter=$(MODE_FILTER) --log-to-stdout $$flags"; \
+ fi; \
$(PYTHON) $(top_srcdir)/build/run_tests.py \
--config-file $(top_srcdir)/subversion/tests/tests.conf \
$$flags \
Index: build/run_tests.py
===================================================================
--- build/run_tests.py (revision 1072234)
+++ build/run_tests.py (working copy)
@@ -27,6 +27,7 @@
[--verbose] [--log-to-stdout] [--cleanup] [--parallel]
[--url=<base-url>] [--http-library=<http-library>] [--enable-sasl]
[--fs-type=<fs-type>] [--fsfs-packing] [--fsfs-sharding=<n>]
+ [--list] [--milestone-filter=<regex>] [--mode-filter=<type>]
[--server-minor-version=<version>]
[--config-file=<file>]
<abs_srcdir> <abs_builddir>
@@ -522,7 +523,7 @@
'http-library=', 'server-minor-version=',
'fsfs-packing', 'fsfs-sharding=',
'enable-sasl', 'parallel', 'config-file=',
- 'log-to-stdout'])
+ 'log-to-stdout', 'list', 'milestone-filter=',
'mode-filter='])
except getopt.GetoptError:
args = []
@@ -532,9 +533,9 @@
base_url, fs_type, verbose, cleanup, enable_sasl, http_library, \
server_minor_version, fsfs_sharding, fsfs_packing, parallel, \
- config_file, log_to_stdout = \
+ config_file, log_to_stdout, list_tests, mode_filter, milestone_filter= \
None, None, None, None, None, None, None, None, None, None, None, \
- None
+ None, None, None, None
for opt, val in opts:
if opt in ['-u', '--url']:
base_url = val
@@ -560,6 +561,12 @@
config_file = val
elif opt in ['--log-to-stdout']:
log_to_stdout = 1
+ elif opt in ['--list']:
+ list_tests = 1
+ elif opt in ['--milestone-filter']:
+ milestone_filter = val
+ elif opt in ['--mode-filter']:
+ mode_filter = val
else:
raise getopt.GetoptError
@@ -573,7 +580,8 @@
th = TestHarness(args[0], args[1], logfile, faillogfile,
base_url, fs_type, http_library, server_minor_version,
verbose, cleanup, enable_sasl, parallel, config_file,
- fsfs_sharding, fsfs_packing)
+ fsfs_sharding, fsfs_packing, list_tests,
+ mode_filter=mode_filter, milestone_filter=milestone_filter)
failed = th.run(args[2:])
if failed:
