Diff
Modified: trunk/Tools/ChangeLog (133379 => 133380)
--- trunk/Tools/ChangeLog 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/ChangeLog 2012-11-03 00:48:53 UTC (rev 133380)
@@ -1,5 +1,51 @@
2012-11-02 Dirk Pranke <dpra...@chromium.org>
+ nrwt prints an awkward result message for missing results
+ https://bugs.webkit.org/show_bug.cgi?id=100915
+
+ Reviewed by Ojan Vafai.
+
+ Completely rework how we print out the description of
+ each tests's result - we now will print either "passed" or "failed",
+ followed by " unexpectedly" if we weren't expecting the result,
+ followed by a short description of each type of failure that happened
+ (in parentheses). Also reworked the description of each type of
+ failure slightly to be clearer.
+
+ Also rework the tests to make sure we have coverage of all of the
+ messages. Examples of the new output are posted as an attachment to
+ the bug.
+
+ * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+ (TestExpectations):
+ * Scripts/webkitpy/layout_tests/models/test_failures.py:
+ (FailureTimeout.message):
+ (FailureMissingResult.message):
+ (FailureTextMismatch.message):
+ (FailureMissingImageHash.message):
+ (FailureMissingImage.message):
+ (FailureImageHashMismatch.message):
+ (FailureImageHashIncorrect.message):
+ (FailureReftestMismatch.message):
+ (FailureReftestMismatchDidNotOccur.message):
+ (FailureReftestNoImagesGenerated.message):
+ (FailureMissingAudio.message):
+ (FailureAudioMismatch.message):
+ (FailureEarlyExit.message):
+ * Scripts/webkitpy/layout_tests/port/test.py:
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (MainTest.test_run_singly_actually_runs_tests):
+ (MainTest.test_missing_and_unexpected_results):
+ * Scripts/webkitpy/layout_tests/views/printing.py:
+ (Printer._print_result_summary_entry):
+ (Printer.print_finished_test):
+ (Printer._result_message):
+ (Printer._print_unexpected_results):
+ * Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py:
+ * Scripts/webkitpy/tool/commands/queues_unittest.py:
+
+2012-11-02 Dirk Pranke <dpra...@chromium.org>
+
webkitpy: clean up logging in common.system.autoinstall
https://bugs.webkit.org/show_bug.cgi?id=101090
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -758,16 +758,16 @@
'missing': MISSING}
# (aggregated by category, pass/fail/skip, type)
- EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped', ''),
- PASS: ('passes', 'passed', ''),
- FAIL: ('failures', 'failed', ''),
- IMAGE: ('image-only failures', 'failed', ' (image diff)'),
- TEXT: ('text-only failures', 'failed', ' (text diff)'),
- IMAGE_PLUS_TEXT: ('image and text failures', 'failed', ' (image and text diff)'),
- AUDIO: ('audio failures', 'failed', ' (audio diff)'),
- CRASH: ('crashes', 'crashed', ''),
- TIMEOUT: ('timeouts', 'timed out', ''),
- MISSING: ('no expected results found', 'no expected result found', '')}
+ EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
+ PASS: 'passes',
+ FAIL: 'failures',
+ IMAGE: 'image-only failures',
+ TEXT: 'text-only failures',
+ IMAGE_PLUS_TEXT: 'image and text failures',
+ AUDIO: 'audio failures',
+ CRASH: 'crashes',
+ TIMEOUT: 'timeouts',
+ MISSING: 'missing results'}
EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, FAIL, IMAGE, SKIP)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -110,20 +110,18 @@
class FailureTimeout(TestFailure):
- """Test timed out. We also want to restart DumpRenderTree if this happens."""
def __init__(self, is_reftest=False):
super(FailureTimeout, self).__init__()
self.is_reftest = is_reftest
def message(self):
- return "Test timed out"
+ return "test timed out"
def driver_needs_restart(self):
return True
class FailureCrash(TestFailure):
- """DumpRenderTree/WebKitTestRunner crashed."""
def __init__(self, is_reftest=False, process_name='DumpRenderTree', pid=None):
super(FailureCrash, self).__init__()
self.process_name = process_name
@@ -132,7 +130,7 @@
def message(self):
if self.pid:
- return "%s (pid %d) crashed" % (self.process_name, self.pid)
+ return "%s crashed [pid=%d]" % (self.process_name, self.pid)
return self.process_name + " crashed"
def driver_needs_restart(self):
@@ -140,101 +138,79 @@
class FailureMissingResult(TestFailure):
- """Expected result was missing."""
-
def message(self):
- return "No expected results found"
+ return "-expected.txt was missing"
class FailureTextMismatch(TestFailure):
- """Text diff output failed."""
-
def message(self):
- return "Text diff mismatch"
+ return "text diff"
-
class FailureMissingImageHash(TestFailure):
- """Actual result hash was missing."""
-
def message(self):
- return "No expected image hash found"
+ return "-expected.png was missing an embedded checksum"
class FailureMissingImage(TestFailure):
- """Actual result image was missing."""
-
def message(self):
- return "No expected image found"
+ return "-expected.png was missing"
class FailureImageHashMismatch(TestFailure):
- """Image hashes didn't match."""
def __init__(self, diff_percent=0):
super(FailureImageHashMismatch, self).__init__()
self.diff_percent = diff_percent
def message(self):
- return "Image mismatch"
+ return "image diff"
class FailureImageHashIncorrect(TestFailure):
- """Actual result hash is incorrect."""
-
def message(self):
- return "Images match, expected image hash incorrect. "
+ return "-expected.png embedded checksum is incorrect"
class FailureReftestMismatch(TestFailure):
- """The result didn't match the reference rendering."""
-
def __init__(self, reference_filename=None):
super(FailureReftestMismatch, self).__init__()
self.reference_filename = reference_filename
self.diff_percent = None
def message(self):
- return "Mismatch with reference"
+ return "reference mismatch"
class FailureReftestMismatchDidNotOccur(TestFailure):
- """Unexpected match between the result and the reference rendering."""
-
def __init__(self, reference_filename=None):
super(FailureReftestMismatchDidNotOccur, self).__init__()
self.reference_filename = reference_filename
def message(self):
- return "Mismatch with the reference did not occur"
+ return "reference mismatch didn't happen"
class FailureReftestNoImagesGenerated(TestFailure):
- """Both the reftest and the -expected html file didn't generate pixel results."""
-
def __init__(self, reference_filename=None):
super(FailureReftestNoImagesGenerated, self).__init__()
self.reference_filename = reference_filename
def message(self):
- return "Reftest didn't generate pixel results."
+ return "reference didn't generate pixel results."
class FailureMissingAudio(TestFailure):
- """Actual result image was missing."""
-
def message(self):
- return "No expected audio found"
+ return "expected audio result was missing"
class FailureAudioMismatch(TestFailure):
- """Audio files didn't match."""
-
def message(self):
- return "Audio mismatch"
+ return "audio mismatch"
class FailureEarlyExit(TestFailure):
def message(self):
- return "Skipped due to early exit"
+ return "skipped due to early exit"
# Convenient collection of all failure classes for anything that might
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures_unittest.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures_unittest.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures_unittest.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -70,4 +70,4 @@
def test_crashes(self):
self.assertEquals(FailureCrash().message(), 'DumpRenderTree crashed')
- self.assertEquals(FailureCrash(process_name='foo', pid=1234).message(), 'foo (pid 1234) crashed')
+ self.assertEquals(FailureCrash(process_name='foo', pid=1234).message(), 'foo crashed [pid=1234]')
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -64,7 +64,6 @@
self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
self.expected_text = self.actual_text
- self.expected_checksum = self.actual_checksum
self.expected_image = self.actual_image
self.actual_audio = None
@@ -117,16 +116,15 @@
actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
actual_text=None, expected_text=None,
actual_image=None, expected_image=None,
- actual_checksum=None, expected_checksum=None)
+ actual_checksum=None)
tests.add('failures/expected/keyboard.html', keyboard=True)
tests.add('failures/expected/missing_check.html',
- expected_checksum=None,
- expected_image=None)
+ expected_image='missing_check-png')
tests.add('failures/expected/missing_image.html', expected_image=None)
tests.add('failures/expected/missing_audio.html', expected_audio=None,
actual_text=None, expected_text=None,
actual_image=None, expected_image=None,
- actual_checksum=None, expected_checksum=None)
+ actual_checksum=None)
tests.add('failures/expected/missing_text.html', expected_text=None)
tests.add('failures/expected/newlines_leading.html',
expected_text="\nfoo\n", actual_text="foo\n")
@@ -138,6 +136,7 @@
tests.add('failures/expected/skip_text.html', actual_text='text diff')
tests.add('failures/flaky/text.html')
tests.add('failures/unexpected/missing_text.html', expected_text=None)
+ tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
tests.add('failures/unexpected/missing_image.html', expected_image=None)
tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
RenderView at (0,0) size 800x600
@@ -152,12 +151,18 @@
error="mock-std-error-output")
tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
error="mock-std-error-output")
+ tests.add('failures/unexpected/pass.html')
+ tests.add('failures/unexpected/text-checksum.html',
+ actual_text='text-checksum_fail-txt',
+ actual_checksum='text-checksum_fail-checksum')
tests.add('failures/unexpected/text-image-checksum.html',
actual_text='text-image-checksum_fail-txt',
+ actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
actual_checksum='text-image-checksum_fail-checksum')
tests.add('failures/unexpected/checksum-with-matching-image.html',
actual_checksum='text-image-checksum_fail-checksum')
tests.add('failures/unexpected/skip_pass.html')
+ tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
tests.add('failures/unexpected/timeout.html', timeout=True)
tests.add('http/tests/passes/text.html')
tests.add('http/tests/passes/image.html')
@@ -169,10 +174,9 @@
actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
actual_text=None, expected_text=None,
actual_image=None, expected_image=None,
- actual_checksum=None, expected_checksum=None)
+ actual_checksum=None)
tests.add('passes/platform_image.html')
tests.add('passes/checksum_in_image.html',
- expected_checksum=None,
expected_image='tEXtchecksum\x00checksum_in_image-checksum')
tests.add('passes/skipped/skip.html')
@@ -281,6 +285,7 @@
Bug(test) failures/expected/hang.html [ WontFix ]
Bug(test) failures/expected/keyboard.html [ WontFix ]
Bug(test) failures/expected/exception.html [ WontFix ]
+Bug(test) failures/unexpected/pass.html [ Failure ]
Bug(test) passes/skipped/skip.html [ Skip ]
""")
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -176,7 +176,8 @@
# Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
# FIXME: It's nice to have a routine in port/test.py that returns this number.
-unexpected_tests_count = 14
+unexpected_failures = 12
+unexpected_tests_count = unexpected_failures + 4
class StreamTestingMixin(object):
@@ -502,7 +503,7 @@
def test_run_singly_actually_runs_tests(self):
res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
- self.assertEquals(res, 10)
+ self.assertEquals(res, unexpected_failures)
def test_single_file(self):
# FIXME: We should consider replacing more of the get_tests_run()-style tests
@@ -567,7 +568,7 @@
file_list = host.filesystem.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEquals(res, 1)
- expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
+ expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","image_diff_percent":1},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
self.assertTrue(json_string.find('"num_regressions":1') != -1)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -299,7 +299,7 @@
desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result]
if not_passing and len(results):
pct = len(results) * 100.0 / not_passing
- self._print_for_bot(" %5d %-24s (%4.1f%%)" % (len(results), desc[0], pct))
+ self._print_for_bot(" %5d %-24s (%4.1f%%)" % (len(results), desc, pct))
def _print_one_line_summary(self, total, expected, unexpected):
incomplete = total - expected - unexpected
@@ -357,29 +357,34 @@
def print_finished_test(self, result, expected, exp_str, got_str):
self.num_completed += 1
test_name = result.test_name
+
+ result_message = self._result_message(result.type, result.failures, expected, self._options.verbose)
+
if self._options.details:
self._print_test_trace(result, exp_str, got_str)
elif (self._options.verbose and not self._options.debug_rwt_logging) or not expected:
- desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result.type]
- suffix = ' ' + desc[1]
- if not expected:
- suffix += ' unexpectedly' + desc[2]
- self.writeln(self._test_status_line(test_name, suffix))
+ self.writeln(self._test_status_line(test_name, result_message))
elif self.num_completed == self.num_tests:
self._meter.write_update('')
else:
- desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result.type]
- suffix = ' ' + desc[1]
if test_name == self._running_tests[0]:
- self._completed_tests.insert(0, [test_name, suffix])
+ self._completed_tests.insert(0, [test_name, result_message])
else:
- self._completed_tests.append([test_name, suffix])
+ self._completed_tests.append([test_name, result_message])
- for test_name, suffix in self._completed_tests:
- self._meter.write_throttled_update(self._test_status_line(test_name, suffix))
+ for test_name, result_message in self._completed_tests:
+ self._meter.write_throttled_update(self._test_status_line(test_name, result_message))
self._completed_tests = []
self._running_tests.remove(test_name)
+ def _result_message(self, result_type, failures, expected, verbose):
+ exp_string = ' unexpectedly' if not expected else ''
+ if result_type == test_expectations.PASS:
+ return ' passed%s' % exp_string
+ else:
+ return ' failed%s (%s)' % (exp_string, ', '.join(failure.message() for failure in failures))
+
+
def _print_test_trace(self, result, exp_str, got_str):
test_name = result.test_name
self._print_default(self._test_status_line(test_name, ''))
@@ -448,7 +453,7 @@
descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
for key, tests in flaky.iteritems():
result = TestExpectations.EXPECTATIONS[key.lower()]
- self._print_for_bot("Unexpected flakiness: %s (%d)" % (descriptions[result][0], len(tests)))
+ self._print_for_bot("Unexpected flakiness: %s (%d)" % (descriptions[result], len(tests)))
tests.sort()
for test in tests:
@@ -466,10 +471,10 @@
descriptions = TestExpectations.EXPECTATION_DESCRIPTIONS
for key, tests in regressions.iteritems():
result = TestExpectations.EXPECTATIONS[key.lower()]
- self._print_for_bot("Regressions: Unexpected %s : (%d)" % (descriptions[result][0], len(tests)))
+ self._print_for_bot("Regressions: Unexpected %s (%d)" % (descriptions[result], len(tests)))
tests.sort()
for test in tests:
- self._print_for_bot(" %s [ %s ] " % (test, TestExpectationParser._inverted_expectation_tokens[key]))
+ self._print_for_bot(" %s [ %s ]" % (test, TestExpectationParser._inverted_expectation_tokens[key]))
self._print_for_bot("")
if len(unexpected_results['tests']) and self._options.debug_rwt_logging:
Modified: trunk/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -113,7 +113,7 @@
foo/bar.html was authored by aba...@webkit.org.
http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html
-The dummy-queue just saw foo/bar.html flake (Text diff mismatch) while processing attachment 10000 on bug 50000.
+The dummy-queue just saw foo/bar.html flake (text diff) while processing attachment 10000 on bug 50000.
Bot: mock-bot-id Port: MockPort Platform: MockPlatform 1.0
The bots will update this with information from each new failure.
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py (133379 => 133380)
--- trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py 2012-11-03 00:23:10 UTC (rev 133379)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py 2012-11-03 00:48:53 UTC (rev 133380)
@@ -403,14 +403,14 @@
queue = TestCommitQueue(MockTool())
expected_stderr = """MOCK bug comment: bug_id=50002, cc=None
--- Begin comment ---
-The commit-queue just saw foo/bar.html flake (Text diff mismatch) while processing attachment 10000 on bug 50000.
+The commit-queue just saw foo/bar.html flake (text diff) while processing attachment 10000 on bug 50000.
Port: MockPort Platform: MockPlatform 1.0
--- End comment ---
MOCK add_attachment_to_bug: bug_id=50002, description=Failure diff from bot filename=failure.diff mimetype=None
MOCK bug comment: bug_id=50002, cc=None
--- Begin comment ---
-The commit-queue just saw bar/baz.html flake (Text diff mismatch) while processing attachment 10000 on bug 50000.
+The commit-queue just saw bar/baz.html flake (text diff) while processing attachment 10000 on bug 50000.
Port: MockPort Platform: MockPlatform 1.0
--- End comment ---
