tfiala updated this revision to Diff 57087.
tfiala added a comment.
Adjust last patch to include full context.
http://reviews.llvm.org/D20193
Files:
packages/Python/lldbsuite/test/dotest.py
packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park
packages/Python/lldbsuite/test_event/event_builder.py
packages/Python/lldbsuite/test_event/formatter/pickled.py
packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py
packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py
packages/Python/lldbsuite/test_event/test/src/event_collector.py
Index: packages/Python/lldbsuite/test_event/test/src/event_collector.py
===================================================================
--- /dev/null
+++ packages/Python/lldbsuite/test_event/test/src/event_collector.py
@@ -0,0 +1,85 @@
+from __future__ import absolute_import
+from __future__ import print_function
+
+import os
+import subprocess
+import sys
+import tempfile
+
+# noinspection PyUnresolvedReferences
+from six.moves import cPickle
+
+
+def path_to_dotest_py():
+ return os.path.join(
+ os.path.dirname(__file__),
+ os.path.pardir,
+ os.path.pardir,
+ os.path.pardir,
+ os.path.pardir,
+ os.path.pardir,
+ os.path.pardir,
+ "test",
+ "dotest.py")
+
+
+def _make_pickled_events_filename():
+ with tempfile.NamedTemporaryFile(
+ prefix="lldb_test_event_pickled_event_output",
+ delete=False) as temp_file:
+ return temp_file.name
+
+
+def _collect_events_with_command(command, events_filename):
+ # Run the single test with dotest.py, outputting
+ # the raw pickled events to a temp file.
+ with open(os.devnull, 'w') as dev_null_file:
+ subprocess.call(
+ command,
+ stdout=dev_null_file,
+ stderr=dev_null_file)
+
+ # Unpickle the events
+ events = []
+ if os.path.exists(events_filename):
+ with open(events_filename, "rb") as events_file:
+ while True:
+ try:
+ # print("reading event")
+ event = cPickle.load(events_file)
+ # print("read event: {}".format(event))
+ if event:
+ events.append(event)
+ except EOFError:
+ # This is okay.
+ break
+ os.remove(events_filename)
+ return events
+
+
+def collect_events_whole_file(test_filename):
+ events_filename = _make_pickled_events_filename()
+ command = [
+ sys.executable,
+ path_to_dotest_py(),
+ "--inferior",
+ "--results-formatter=lldbsuite.test_event.formatter.pickled.RawPickledFormatter",
+ "--results-file={}".format(events_filename),
+ "-p", os.path.basename(test_filename),
+ os.path.dirname(test_filename)
+ ]
+ return _collect_events_with_command(command, events_filename)
+
+
+def collect_events_for_directory_with_filter(test_filename, filter_desc):
+ events_filename = _make_pickled_events_filename()
+ command = [
+ sys.executable,
+ path_to_dotest_py(),
+ "--inferior",
+ "--results-formatter=lldbsuite.test_event.formatter.pickled.RawPickledFormatter",
+ "--results-file={}".format(events_filename),
+ "-f", filter_desc,
+ os.path.dirname(test_filename)
+ ]
+ return _collect_events_with_command(command, events_filename)
Index: packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py
===================================================================
--- /dev/null
+++ packages/Python/lldbsuite/test_event/test/src/TestCatchInvalidDecorator.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+Tests that the event system reports issues during decorator
+handling as errors.
+"""
+# System-provided imports
+import os
+import unittest
+
+# Local-provided imports
+import event_collector
+
+
+class TestCatchInvalidDecorator(unittest.TestCase):
+
+ TEST_DIR = os.path.join(
+ os.path.dirname(__file__),
+ os.path.pardir,
+ "resources",
+ "invalid_decorator")
+
+ def test_with_whole_file(self):
+ """
+ Test that a non-existent decorator generates a test-event error
+ when running all tests in the file.
+ """
+ # Determine the test case file we're using.
+ test_file = os.path.join(self.TEST_DIR, "TestInvalidDecorator.py")
+
+ # Collect all test events generated for this file.
+ error_results = _filter_error_results(
+ event_collector.collect_events_whole_file(test_file))
+
+ self.assertGreater(
+ len(error_results),
+ 0,
+ "At least one job or test error result should have been returned")
+
+ def test_with_function_filter(self):
+ """
+ Test that a non-existent decorator generates a test-event error
+ when running a filtered test.
+ """
+ # Collect all test events generated during running of tests
+ # in a given directory using a test name filter. Internally,
+ # this runs through a different code path that needs to be
+ # set up to catch exceptions.
+ error_results = _filter_error_results(
+ event_collector.collect_events_for_directory_with_filter(
+ self.TEST_DIR,
+ "NonExistentDecoratorTestCase.test"))
+
+ self.assertGreater(
+ len(error_results),
+ 0,
+ "At least one job or test error result should have been returned")
+
+
+def _filter_error_results(events):
+ # Filter out job result events.
+ return [
+ event
+ for event in events
+ if event.get("event", None) in ["job_result", "test_result"] and
+ event.get("status", None) == "error"
+ ]
+
+
+if __name__ == "__main__":
+ unittest.main()
Index: packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py
===================================================================
--- /dev/null
+++ packages/Python/lldbsuite/test_event/test/resources/invalid_decorator/TestInvalidDecorator.py
@@ -0,0 +1,13 @@
+from __future__ import print_function
+from lldbsuite.test import lldbtest
+from lldbsuite.test import decorators
+
+
+class NonExistentDecoratorTestCase(lldbtest.TestBase):
+
+ mydir = lldbtest.TestBase.compute_mydir(__file__)
+
+ @decorators.nonExistentDecorator(bugnumber="yt/1300")
+ def test(self):
+ """Verify non-existent decorators are picked up by test runner."""
+ pass
Index: packages/Python/lldbsuite/test_event/formatter/pickled.py
===================================================================
--- packages/Python/lldbsuite/test_event/formatter/pickled.py
+++ packages/Python/lldbsuite/test_event/formatter/pickled.py
@@ -33,6 +33,7 @@
def __init__(self, out_file, options):
super(RawPickledFormatter, self).__init__(out_file, options)
self.pid = os.getpid()
+ self.use_send = out_file is not None and "send" in dir(out_file)
def handle_event(self, test_event):
super(RawPickledFormatter, self).handle_event(test_event)
@@ -50,8 +51,14 @@
# Tack on the pid.
test_event["pid"] = self.pid
- # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes}
- import struct
- msg = cPickle.dumps(test_event)
- packet = struct.pack("!I%ds" % len(msg), len(msg), msg)
- self.out_file.send(packet)
+ # If sending over a socket, we need to format this
+ # in a way that can be collected properly on the other
+ # end.
+ if self.use_send:
+ # Send it as {serialized_length_of_serialized_bytes}{serialized_bytes}
+ import struct
+ msg = cPickle.dumps(test_event)
+ packet = struct.pack("!I%ds" % len(msg), len(msg), msg)
+ self.out_file.send(packet)
+ else:
+ cPickle.dump(test_event, self.out_file)
Index: packages/Python/lldbsuite/test_event/event_builder.py
===================================================================
--- packages/Python/lldbsuite/test_event/event_builder.py
+++ packages/Python/lldbsuite/test_event/event_builder.py
@@ -321,6 +321,19 @@
return event
@staticmethod
+ def event_for_job_test_add_error(test_filename, exception, backtrace):
+ event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT)
+ event["status"] = EventBuilder.STATUS_ERROR
+ if test_filename is not None:
+ event["test_filename"] = EventBuilder._assert_is_python_sourcefile(test_filename)
+ if exception is not None and "__class__" in dir(exception):
+ event["issue_class"] = exception.__class__
+ event["issue_message"] = exception
+ if backtrace is not None:
+ event["issue_backtrace"] = backtrace
+ return event
+
+ @staticmethod
def event_for_job_exceptional_exit(
pid, worker_index, exception_code, exception_description,
test_filename, command_line):
Index: packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park
===================================================================
--- /dev/null
+++ packages/Python/lldbsuite/test/issue_verification/TestInvalidDecorator.py.park
@@ -0,0 +1,13 @@
+from __future__ import print_function
+from lldbsuite.test import lldbtest
+from lldbsuite.test import decorators
+
+
+class NonExistentDecoratorTestCase(lldbtest.TestBase):
+
+ mydir = lldbtest.TestBase.compute_mydir(__file__)
+
+ @decorators.nonExistentDecorator(bugnumber="yt/1300")
+ def test(self):
+ """Verify non-existent decorators are picked up by test runner."""
+ pass
Index: packages/Python/lldbsuite/test/dotest.py
===================================================================
--- packages/Python/lldbsuite/test/dotest.py
+++ packages/Python/lldbsuite/test/dotest.py
@@ -676,73 +676,98 @@
# This is to locate the lldb.py module. Insert it right after sys.path[0].
sys.path[1:1] = [lldbPythonDir]
+
+def visit_file(dir, name):
+ # Try to match the regexp pattern, if specified.
+ if configuration.regexp:
+ import re
+ if not re.search(configuration.regexp, name):
+ # We didn't match the regex, we're done.
+ return
+
+ # We found a match for our test. Add it to the suite.
+
+ # Update the sys.path first.
+ if not sys.path.count(dir):
+ sys.path.insert(0, dir)
+ base = os.path.splitext(name)[0]
+
+ # Thoroughly check the filterspec against the base module and admit
+ # the (base, filterspec) combination only when it makes sense.
+ filterspec = None
+ for filterspec in configuration.filters:
+ # Optimistically set the flag to True.
+ filtered = True
+ module = __import__(base)
+ parts = filterspec.split('.')
+ obj = module
+ for part in parts:
+ try:
+ parent, obj = obj, getattr(obj, part)
+ except AttributeError:
+ # The filterspec has failed.
+ filtered = False
+ break
+
+ # If filtered, we have a good filterspec. Add it.
+ if filtered:
+ # print("adding filter spec %s to module %s" % (filterspec, module))
+ configuration.suite.addTests(
+ unittest2.defaultTestLoader.loadTestsFromName(filterspec, module))
+ continue
+
+ # Forgo this module if the (base, filterspec) combo is invalid
+ if configuration.filters and not filtered:
+ return
+
+ if not filterspec or not filtered:
+ # Add the entire file's worth of tests since we're not filtered.
+ # Also the fail-over case when the filterspec branch
+ # (base, filterspec) combo doesn't make sense.
+ configuration.suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base))
+
+
def visit(prefix, dir, names):
"""Visitor function for os.path.walk(path, visit, arg)."""
dir_components = set(dir.split(os.sep))
excluded_components = set(['.svn', '.git'])
if dir_components.intersection(excluded_components):
- #print("Detected an excluded dir component: %s" % dir)
return
- for name in names:
- if '.py' == os.path.splitext(name)[1] and name.startswith(prefix):
+ # Gather all the Python test file names that follow the Test*.py pattern.
+ python_test_files = [
+ name
+ for name in names
+ if name.endswith('.py') and name.startswith(prefix)]
+ # Visit all the python test files.
+ for name in python_test_files:
+ try:
+ # Ensure we error out if we have multiple tests with the same
+ # base name.
+ # Future improvement: find all the places where we work with base
+ # names and convert to full paths. We have directory structure
+ # to disambiguate these, so we shouldn't need this constraint.
if name in configuration.all_tests:
raise Exception("Found multiple tests with the name %s" % name)
configuration.all_tests.add(name)
- # Try to match the regexp pattern, if specified.
- if configuration.regexp:
- import re
- if re.search(configuration.regexp, name):
- #print("Filename: '%s' matches pattern: '%s'" % (name, regexp))
- pass
- else:
- #print("Filename: '%s' does not match pattern: '%s'" % (name, regexp))
- continue
-
- # We found a match for our test. Add it to the suite.
-
- # Update the sys.path first.
- if not sys.path.count(dir):
- sys.path.insert(0, dir)
- base = os.path.splitext(name)[0]
-
- # Thoroughly check the filterspec against the base module and admit
- # the (base, filterspec) combination only when it makes sense.
- filterspec = None
- for filterspec in configuration.filters:
- # Optimistically set the flag to True.
- filtered = True
- module = __import__(base)
- parts = filterspec.split('.')
- obj = module
- for part in parts:
- try:
- parent, obj = obj, getattr(obj, part)
- except AttributeError:
- # The filterspec has failed.
- filtered = False
- break
-
- # If filtered, we have a good filterspec. Add it.
- if filtered:
- #print("adding filter spec %s to module %s" % (filterspec, module))
- configuration.suite.addTests(
- unittest2.defaultTestLoader.loadTestsFromName(filterspec, module))
- continue
-
- # Forgo this module if the (base, filterspec) combo is invalid
- if configuration.filters and not filtered:
- continue
-
- # Add either the filtered test case(s) (which is done before) or the entire test class.
- if not filterspec or not filtered:
- # A simple case of just the module name. Also the failover case
- # from the filterspec branch when the (base, filterspec) combo
- # doesn't make sense.
- configuration.suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base))
+ # Run the relevant tests in the python file.
+ visit_file(dir, name)
+ except Exception as ex:
+ # Convert this exception to a test event error for the file.
+ test_filename = os.path.abspath(os.path.join(dir, name))
+ if configuration.results_formatter_object is not None:
+ # Grab the backtrace for the exception.
+ import traceback
+ backtrace = traceback.format_exc()
+
+ # Generate the test event.
+ configuration.results_formatter_object.handle_event(
+ EventBuilder.event_for_job_test_add_error(
+ test_filename, ex, backtrace))
+ raise
def disabledynamics():
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
