- Revision
- 101907
- Author
- [email protected]
- Date
- 2011-12-02 22:06:56 -0800 (Fri, 02 Dec 2011)
Log Message
[NRWT] reftest asserts intermittently on the Qt-WK2 bot
https://bugs.webkit.org/show_bug.cgi?id=73453
Reviewed by Ryosuke Niwa.
We used to assert if a ref test didn't generate pixel results. Instead, just claim it is a failing test.
The verbose output will include the failure reason.
* Scripts/webkitpy/layout_tests/controllers/single_test_runner.py: Don't assert, return FailureReftestNoImagesGenerated instead.
* Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
(write_test_result):
* Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py: Remove stray letter t.
* Scripts/webkitpy/layout_tests/models/test_failures.py: Add a new failure type: FailureReftestNoImagesGenerated
(determine_result_type):
(FailureReftestNoImagesGenerated.__init__):
(FailureReftestNoImagesGenerated):
(FailureReftestNoImagesGenerated.message):
* Scripts/webkitpy/layout_tests/port/test.py: Add a test case for integration tests.
* Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py: Update test counts.
(MainTest.test_run_singly_actually_runs_tests):
(MainTest.test_unexpected_failures):
(EndToEndTest.test_end_to_end):
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (101906 => 101907)
--- trunk/Tools/ChangeLog 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/ChangeLog 2011-12-03 06:06:56 UTC (rev 101907)
@@ -1,3 +1,28 @@
+2011-12-02 Tony Chang <[email protected]>
+
+ [NRWT] reftest asserts intermittently on the Qt-WK2 bot
+ https://bugs.webkit.org/show_bug.cgi?id=73453
+
+ Reviewed by Ryosuke Niwa.
+
+ We used to assert if a ref test didn't generate pixel results. Instead, just claim it is a failing test.
+ The verbose output will include the failure reason.
+
+ * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py: Don't assert, return FailureReftestNoImagesGenerated instead.
+ * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
+ (write_test_result):
+ * Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py: Remove stray letter t.
+ * Scripts/webkitpy/layout_tests/models/test_failures.py: Add a new failure type: FailureReftestNoImagesGenerated
+ (determine_result_type):
+ (FailureReftestNoImagesGenerated.__init__):
+ (FailureReftestNoImagesGenerated):
+ (FailureReftestNoImagesGenerated.message):
+ * Scripts/webkitpy/layout_tests/port/test.py: Add a test case for integration tests.
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py: Update test counts.
+ (MainTest.test_run_singly_actually_runs_tests):
+ (MainTest.test_unexpected_failures):
+ (EndToEndTest.test_end_to_end):
+
2011-12-02 Ryosuke Niwa <[email protected]>
ChangeLog should be able to parse annotated changelogs
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (101906 => 101907)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2011-12-03 06:06:56 UTC (rev 101907)
@@ -296,9 +296,9 @@
if failures:
return TestResult(self._test_name, failures, total_test_time, has_stderr)
- assert(driver_output1.image_hash or driver_output2.image_hash)
-
- if mismatch:
+ if not driver_output1.image_hash and not driver_output2.image_hash:
+ failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
+ elif mismatch:
if driver_output1.image_hash == driver_output2.image_hash:
failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
elif driver_output1.image_hash != driver_output2.image_hash:
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py (101906 => 101907)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2011-12-03 06:06:56 UTC (rev 101907)
@@ -80,7 +80,7 @@
writer.write_image_files(driver_output.image, expected_image=None)
writer.copy_file(failure.reference_filename)
else:
- assert isinstance(failure, (test_failures.FailureTimeout,))
+ assert isinstance(failure, (test_failures.FailureTimeout, test_failures.FailureReftestNoImagesGenerated))
class TestResultWriter(object):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py (101906 => 101907)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py 2011-12-03 06:06:56 UTC (rev 101907)
@@ -34,7 +34,7 @@
class TestResultWriterTest(unittest.TestCase):
- def test_reftest_diff_imaget(self):
+ def test_reftest_diff_image(self):
"""A write_test_result should call port.diff_image with tolerance=0 in case of FailureReftestMismatch."""
used_tolerance_values = []
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py (101906 => 101907)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2011-12-03 06:06:56 UTC (rev 101907)
@@ -59,8 +59,7 @@
is_text_failure = FailureTextMismatch in failure_types
is_image_failure = (FailureImageHashIncorrect in failure_types or
FailureImageHashMismatch in failure_types)
- is_reftest_failure = (FailureReftestMismatch in failure_types or
- FailureReftestMismatchDidNotOccur in failure_types)
+ is_reftest_failure = set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
is_audio_failure = (FailureAudioMismatch in failure_types)
if is_text_failure and is_image_failure:
return test_expectations.IMAGE_PLUS_TEXT
@@ -205,6 +204,17 @@
return "Mismatch with the reference did not occur"
+class FailureReftestNoImagesGenerated(TestFailure):
+ """Both the reftest and the -expected html file didn't generate pixel results."""
+
+ def __init__(self, reference_filename=None):
+ self.reference_filename = reference_filename
+
+ @staticmethod
+ def message():
+ return "Reftest didn't generate pixel results."
+
+
class FailureMissingAudio(TestFailure):
"""Actual result image was missing."""
@@ -227,4 +237,4 @@
FailureTextMismatch, FailureMissingImageHash,
FailureMissingImage, FailureImageHashMismatch,
FailureImageHashIncorrect, FailureReftestMismatch,
- FailureReftestMismatchDidNotOccur)
+ FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py (101906 => 101907)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-12-03 06:06:56 UTC (rev 101907)
@@ -186,6 +186,8 @@
tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
+ tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
+ tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
# FIXME: Add a reftest which crashes.
tests.add('reftests/foo/test.html')
tests.add('reftests/foo/test-ref.html')
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (101906 => 101907)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-12-03 05:56:31 UTC (rev 101906)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-12-03 06:06:56 UTC (rev 101907)
@@ -187,7 +187,7 @@
# Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
# FIXME: It's nice to have a routine in port/test.py that returns this number.
-unexpected_tests_count = 11
+unexpected_tests_count = 12
class MainTest(unittest.TestCase):
@@ -383,7 +383,7 @@
def test_run_singly_actually_runs_tests(self):
res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
- self.assertEquals(res, 7)
+ self.assertEquals(res, 8)
def test_single_file(self):
# FIXME: We should consider replacing more of the get_tests_run()-style tests
