Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg (101817 => 101818)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg 2011-12-02 18:00:43 UTC (rev 101817)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg 2011-12-02 18:01:31 UTC (rev 101818)
@@ -58,6 +58,35 @@
WithProperties = properties.WithProperties
+class TestWithFailureCount(shell.Test):
+ failedTestsFormatString = "%d tests failed"
+
+ def countFailures(self, cmd):
+ return 0
+
+ def commandComplete(self, cmd):
+ shell.Test.commandComplete(self, cmd)
+ self.failedTestCount = self.countFailures(cmd)
+
+ def evaluateCommand(self, cmd):
+ if self.failedTestCount:
+ return FAILURE
+
+ if cmd.rc != 0:
+ return FAILURE
+
+ return SUCCESS
+
+ def getText(self, cmd, results):
+ return self.getText2(cmd, results)
+
+ def getText2(self, cmd, results):
+ if results != SUCCESS and self.failedTestCount:
+ return [self.failedTestsFormatString % self.failedTestCount]
+
+ return [self.name]
+
+
class ConfigureBuild(buildstep.BuildStep):
name = "configure build"
description = ["configuring build"]
@@ -411,11 +440,13 @@
logText = cmd.logs['stdio'].getText()
self._parseNewRunWebKitTestsOutput(logText)
-class RunUnitTests(shell.Test):
+
+class RunUnitTests(TestWithFailureCount):
name = "run-api-tests"
description = ["unit tests running"]
descriptionDone = ["unit-tests"]
command = ["perl", "./Tools/Scripts/run-api-tests", WithProperties("--%(configuration)s"), "--verbose"]
+ failedTestsFormatString = "%d unit tests failed or timed out"
def start(self):
platform = self.getProperty('platform')
@@ -425,36 +456,21 @@
self.setCommand(self.command + ['--chromium'])
return shell.Test.start(self)
-
-class TestWithFailureCount(shell.Test):
- failedTestsFormatString = "%d tests failed"
-
def countFailures(self, cmd):
- return 0
+ log_text = cmd.logs['stdio'].getText()
+ count = 0
- def commandComplete(self, cmd):
- shell.Test.commandComplete(self, cmd)
- self.failedTestCount = self.countFailures(cmd)
+ split = re.split(r'^Tests that timed out:$', log_text, flags=re.MULTILINE)
+ if len(split) > 1:
+ count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
- def evaluateCommand(self, cmd):
- if self.failedTestCount:
- return FAILURE
+ split = re.split(r'^Tests that failed:$', split[0], flags=re.MULTILINE)
+ if len(split) > 1:
+ count += len(re.findall(r'^\s+\S+$', split[1], flags=re.MULTILINE))
- if cmd.rc != 0:
- return FAILURE
+ return count
- return SUCCESS
- def getText(self, cmd, results):
- return self.getText2(cmd, results)
-
- def getText2(self, cmd, results):
- if results != SUCCESS and self.failedTestCount:
- return [self.failedTestsFormatString % self.failedTestCount]
-
- return [self.name]
-
-
class RunPythonTests(TestWithFailureCount):
name = "webkitpy-test"
description = ["python-tests running"]
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py (101817 => 101818)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py 2011-12-02 18:00:43 UTC (rev 101817)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py 2011-12-02 18:01:31 UTC (rev 101818)
@@ -64,6 +64,152 @@
self.assertEquals(run_webkit_tests.incorrectLayoutLines, expected_incorrect_lines)
+class StubStdio(object):
+ def __init__(self, stdio):
+ self._stdio = stdio
+
+ def getText(self):
+ return self._stdio
+
+
+class StubRemoteCommand(object):
+ def __init__(self, rc, stdio):
+ self.rc = rc
+ self.logs = {'stdio': StubStdio(stdio)}
+
+
+class RunUnitTestsTest(unittest.TestCase):
+ def assertFailures(self, expected_failure_count, stdio):
+ if expected_failure_count:
+ rc = 1
+ expected_results = FAILURE
+ expected_text = '{0} unit tests failed or timed out'.format(expected_failure_count)
+ else:
+ rc = 0
+ expected_results = SUCCESS
+ expected_text = 'run-api-tests'
+
+ cmd = StubRemoteCommand(rc, stdio)
+ step = RunUnitTests()
+ step.commandComplete(cmd)
+ actual_results = step.evaluateCommand(cmd)
+ actual_failure_count = step.failedTestCount
+ actual_text = step.getText(cmd, actual_results)[0]
+
+ self.assertEqual(expected_results, actual_results)
+ self.assertEqual(expected_failure_count, actual_failure_count)
+ self.assertEqual(expected_text, actual_text)
+
+ def test_no_failures_or_timeouts(self):
+ self.assertFailures(0, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[ PASSED ] 1 test.
+""")
+
+ def test_one_failure(self):
+ self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[ PASSED ] 1 test.
+Tests that failed:
+ WebKit2.WebKit2.CanHandleRequest
+""")
+
+ def test_multiple_failures(self):
+ self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[ PASSED ] 1 test.
+Tests that failed:
+ WebKit2.WebKit2.CanHandleRequest
+ WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
+ WebKit2.WebKit2.HitTestResultNodeHandle
+ WebKit2.WebKit2.InjectedBundleBasic
+""")
+
+ def test_one_timeout(self):
+ self.assertFailures(1, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[ PASSED ] 1 test.
+Tests that timed out:
+ WebKit2.WebKit2.CanHandleRequest
+""")
+
+ def test_multiple_timeouts(self):
+ self.assertFailures(4, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[ PASSED ] 1 test.
+Tests that timed out:
+ WebKit2.WebKit2.CanHandleRequest
+ WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
+ WebKit2.WebKit2.HitTestResultNodeHandle
+ WebKit2.WebKit2.InjectedBundleBasic
+""")
+
+ def test_multiple_failures_and_timeouts(self):
+ self.assertFailures(8, """Note: Google Test filter = WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[==========] Running 1 test from 1 test case.
+[----------] Global test environment set-up.
+[----------] 1 test from WebViewDestructionWithHostWindow
+[ RUN ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose
+[ OK ] WebViewDestructionWithHostWindow.DestroyViewWindowWithoutClose (127 ms)
+[----------] 1 test from WebViewDestructionWithHostWindow (127 ms total)
+
+[----------] Global test environment tear-down
+[==========] 1 test from 1 test case ran. (127 ms total)
+[ PASSED ] 1 test.
+Tests that failed:
+ WebKit2.WebKit2.CanHandleRequest
+ WebKit2.WebKit2.DocumentStartUserScriptAlertCrashTest
+ WebKit2.WebKit2.HitTestResultNodeHandle
+Tests that timed out:
+ WebKit2.WebKit2.InjectedBundleBasic
+ WebKit2.WebKit2.LoadCanceledNoServerRedirectCallback
+ WebKit2.WebKit2.MouseMoveAfterCrash
+ WebKit2.WebKit2.ResponsivenessTimerDoesntFireEarly
+ WebKit2.WebKit2.WebArchive
+""")
+
+
+
# FIXME: We should run this file as part of test-webkitpy.
# Unfortunately test-webkitpy currently requires that unittests
# be located in a directory with a valid module name.
Modified: trunk/Tools/ChangeLog (101817 => 101818)
--- trunk/Tools/ChangeLog 2011-12-02 18:00:43 UTC (rev 101817)
+++ trunk/Tools/ChangeLog 2011-12-02 18:01:31 UTC (rev 101818)
@@ -1,5 +1,40 @@
2011-12-02 Adam Roben <[email protected]>
+ Teach build.webkit.org to display how many unit tests failed or timed out
+
+ Fixes <http://webkit.org/b/73659> It's hard to tell how many unit tests are failing on
+ build.webkit.org
+
+ We now display something like "5 unit tests failed or timed out". Eventually we might want
+ to split out how many failure vs. timeouts there were.
+
+ Reviewed by Darin Adler.
+
+ * BuildSlaveSupport/build.webkit.org-config/master.cfg:
+ (TestWithFailureCount): Moved this class up to the top of the file so it is before any other
+ classes that may want to subclass it.
+ (RunUnitTests): Changed to inherit from TestWithFailureCount.
+ (RunUnitTests.countFailures): Added. Counts the number of tests following the "Tests that
+ timed out:" and "Tests that failed:" lines, if present.
+
+ * BuildSlaveSupport/build.webkit.org-config/mastercfg_unittest.py:
+ (StubStdio):
+ (StubRemoteCommand):
+ Added these two stub classes to mimic buildbot's RemoteCommand and log classes.
+
+ (RunUnitTestsTest.assertFailures): Helper method to check that we interpreted the results of
+ the test run correctly.
+
+ (RunUnitTestsTest.test_no_failures_or_timeouts):
+ (RunUnitTestsTest.test_one_failure):
+ (RunUnitTestsTest.test_multiple_failures):
+ (RunUnitTestsTest.test_one_timeout):
+ (RunUnitTestsTest.test_multiple_timeouts):
+ (RunUnitTestsTest.test_multiple_failures_and_timeouts):
+ Test various cases.
+
+2011-12-02 Adam Roben <[email protected]>
+
Teach prepare-ChangeLog to treat master.cfg as a Python file
Fixes <http://webkit.org/b/73658> prepare-ChangeLog doesn't show modified classes/functions
