Modified: trunk/Tools/ChangeLog (116551 => 116552)
--- trunk/Tools/ChangeLog 2012-05-09 19:48:51 UTC (rev 116551)
+++ trunk/Tools/ChangeLog 2012-05-09 19:59:44 UTC (rev 116552)
@@ -1,3 +1,28 @@
+2012-05-09 Dirk Pranke <[email protected]>
+
+ nrwt: --skip-failing-tests should also skip flaky tests
+ https://bugs.webkit.org/show_bug.cgi?id=85941
+
+ Reviewed by Ojan Vafai.
+
+ This changes --skip-failing-tests so that we skip tests that
+ are believed to pass sometimes as well as tests that are
+ believed to always fail.
+
+ In addition, in testing this I discovered that we had lousy
+ test coverage of the details of retrying test failures and
+ --clobber-old-results, so I wrote some more tests.
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager.prepare_lists_and_print_output):
+ * Scripts/webkitpy/layout_tests/port/test.py:
+ (unit_test_list):
+ (TestPort.__init__):
+ (TestDriver.run_test):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (MainTest.test_skip_failing_tests):
+ (MainTest.test_retrying_and_flaky_tests):
+
2012-05-09 Christophe Dumez <[email protected]>
[EFL][DRT] EFL's DRT needs to support LayoutTestController.dumpIconChanges()
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (116551 => 116552)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-05-09 19:48:51 UTC (rev 116551)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-05-09 19:59:44 UTC (rev 116552)
@@ -478,10 +478,10 @@
skipped = skipped.union(self._http_tests())
if num_all_test_files > 1 and not self._options.force:
- skipped = skipped.union(self._expectations.get_tests_with_result_type(test_expectations.SKIP))
+ skipped.update(self._expectations.get_tests_with_result_type(test_expectations.SKIP))
if self._options.skip_failing_tests:
- failing = self._expectations.get_tests_with_result_type(test_expectations.FAIL)
- self._test_files -= failing
+ skipped.update(self._expectations.get_tests_with_result_type(test_expectations.FAIL))
+ skipped.update(self._expectations.get_tests_with_result_type(test_expectations.FLAKY))
self._test_files -= skipped
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py (116551 => 116552)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2012-05-09 19:48:51 UTC (rev 116551)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2012-05-09 19:59:44 UTC (rev 116552)
@@ -136,6 +136,7 @@
expected_text="foo\r\r\r\n", actual_text="foo\n")
tests.add('failures/expected/text.html', actual_text='text_fail-png')
tests.add('failures/expected/skip_text.html', actual_text='text diff')
+ tests.add('failures/flaky/text.html')
tests.add('failures/unexpected/missing_text.html', expected_text=None)
tests.add('failures/unexpected/missing_image.html', expected_image=None)
tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
@@ -334,6 +335,7 @@
port_name = port_name or 'test-mac-leopard'
Port.__init__(self, host, port_name, **kwargs)
self._tests = unit_test_list()
+ self._flakes = set()
self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
self._results_directory = None
@@ -513,6 +515,11 @@
audio = None
actual_text = test.actual_text
+
+ if 'flaky' in test_name and not test_name in self._port._flakes:
+ self._port._flakes.add(test_name)
+ actual_text = 'flaky text failure'
+
if actual_text and test_args and test_name == 'passes/args.html':
actual_text = actual_text + ' ' + ' '.join(test_args)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (116551 => 116552)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-05-09 19:48:51 UTC (rev 116551)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-05-09 19:59:44 UTC (rev 116552)
@@ -460,10 +460,16 @@
self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
def test_skip_failing_tests(self):
- batches = get_tests_run(['--skip-failing-tests'])
+ # This tests that we skip both known failing and known flaky tests. Because there are
+ # no known flaky tests in the default test_expectations, we add additional expectations.
+ host = MockHost()
+ host.filesystem.write_text_file('/tmp/overrides.txt', 'BUGX : passes/image.html = IMAGE PASS\n')
+
+ batches = get_tests_run(['--skip-failing-tests', '--additional-expectations', '/tmp/overrides.txt'], host=host)
has_passes_text = False
for batch in batches:
self.assertFalse('failures/expected/text.html' in batch)
+ self.assertFalse('passes/image.html' in batch)
has_passes_text = has_passes_text or ('passes/text.html' in batch)
self.assertTrue(has_passes_text)
@@ -718,11 +724,28 @@
tests_included=True, host=host)
self.assertEqual(user.opened_urls, [path.abspath_to_uri('/tmp/cwd/foo/results.html')])
- def test_retries_directory(self):
+ def test_retrying_and_flaky_tests(self):
host = MockHost()
- res, out, err, user = logging_run(tests_included=True, host=host)
+ res, out, err, _ = logging_run(['failures/flaky'], tests_included=True, host=host)
+ self.assertEquals(res, 0)
+ self.assertTrue('Retrying' in err.getvalue())
+ self.assertTrue('Unexpected flakiness' in out.getvalue())
+ self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/tests_run0.txt'))
+ self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt'))
+ # Now we test that --clobber-old-results does remove the old entries and the old retries,
+ # and that we don't retry again.
+ res, out, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host)
+ self.assertEquals(res, 1)
+ self.assertTrue('Clobbering old results' in err.getvalue())
+ self.assertTrue('flaky/text.html' in err.getvalue())
+ self.assertTrue('Unexpected text diff' in out.getvalue())
+ self.assertFalse('Unexpected flakiness' in out.getvalue())
+ self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
+ self.assertFalse(host.filesystem.exists('retries'))
+
+
# These next tests test that we run the tests in ascending alphabetical
# order per directory. HTTP tests are sharded separately from other tests,
# so we have to test both.
