Diff
Modified: trunk/Tools/ChangeLog (101726 => 101727)
--- trunk/Tools/ChangeLog 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/ChangeLog 2011-12-02 01:46:21 UTC (rev 101727)
@@ -1,3 +1,46 @@
+2011-12-01 Ryosuke Niwa <[email protected]>
+
+ Parse reftest.list and extract types of ref tests
+ https://bugs.webkit.org/show_bug.cgi?id=66837
+
+ Reviewed by Dirk Pranke.
+
+ Add support for reftest.list to base port.
+
+ * Scripts/webkitpy/common/find_files.py:
+ (find):
+ * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+ (SingleTestRunner.__init__):
+ * Scripts/webkitpy/layout_tests/port/base.py:
+ (Port.__init__): Initialize self._reftest_list. It's a dictionary mapping from a test directory
+ to a dictionary of {test path: ("==" or "!=", reference file path)}
+ (Port._get_reftest_list): Added; calls test_file.parse_reftest_list to fill self._reftest_list.
+ (Port._reference_file_for): Added; obtains the reference file name given a test name.
+ (Port.is_reftest): Added; Calls _reference_file_for.
+ (Port.reftest_expected_filename): Calls _reference_file_for.
+ (Port.reftest_expected_mismatch_filename): Ditto.
+ (Port.find_test_files):
+ (is_reference_html_file): Treat any file that starts with ref- or notref- or ends with
+ -expected, -expected-mismach, -ref, or -notref as a reference file.
+ (_is_test_file):
+ (_parse_reftest_list): Added.
+ * Scripts/webkitpy/layout_tests/port/base_unittest.py:
+ (PortTest.test_is_test_file):
+ (PortTest.test_parse_reftest_list):
+ * Scripts/webkitpy/layout_tests/port/dryrun.py:
+ (DryrunDriver.run_test):
+ * Scripts/webkitpy/layout_tests/port/test.py:
+ (unit_test_filesystem.add_test_file):
+ (unit_test_filesystem.add_file):
+ (unit_test_filesystem):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (get_tests_run.RecordingTestDriver.run_test):
+ (MainTest.test_unexpected_failures):
+ (MainTest.test_missing_and_unexpected_results):
+ (EndToEndTest.test_end_to_end):
+ * Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py:
+ (Rebaseliner._compile_rebaselining_tests):
+
2011-12-01 Sheriff Bot <[email protected]>
Unreviewed, rolling out r101711.
Modified: trunk/Tools/Scripts/webkitpy/common/find_files.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/common/find_files.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/common/find_files.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -62,7 +62,6 @@
everything under the base_dir.
"""
- global _skipped_directories
paths = paths or ['*']
skipped_directories = skipped_directories or set(['.svn', '_svn'])
return _normalized_find(filesystem, _normalize(filesystem, base_dir, paths), skipped_directories, file_filter)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -68,17 +68,15 @@
return
reftest_expected_filename = port.reftest_expected_filename(self._test_name)
- if fs.exists(reftest_expected_filename):
+ if reftest_expected_filename and fs.exists(reftest_expected_filename):
self._is_reftest = True
self._reference_filename = reftest_expected_filename
reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._test_name)
- if fs.exists(reftest_expected_mismatch_filename):
+ if reftest_expected_mismatch_filename and fs.exists(reftest_expected_mismatch_filename):
if self._is_reftest:
- _log.error('It is not allowed that one test file has both'
- ' expected.html file and expected-mismatch.html file'
- ' at the same time. Please remove either %s or %s.',
- reftest_expected_filename, reftest_expected_mismatch_filename)
+ _log.error('One test file cannot have both match and mismatch references. Please remove either %s or %s',
+ reftest_expected_filename, reftest_expected_mismatch_filename)
else:
self._is_reftest = True
self._is_mismatch_reftest = True
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -36,6 +36,7 @@
import difflib
import errno
import os
+import re
from webkitpy.common.memoized import memoized
@@ -147,6 +148,7 @@
self.set_option_default('configuration', self.default_configuration())
self._test_configuration = None
+ self._reftest_list = {}
self._multiprocessing_is_available = (multiprocessing is not None)
self._results_directory = None
@@ -424,13 +426,40 @@
text = self._filesystem.read_binary_file(baseline_path)
return text.replace("\r\n", "\n")
+ def _get_reftest_list(self, test_name):
+ dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
+ if dirname not in self._reftest_list:
+ self._reftest_list[dirname] = _parse_reftest_list(self._filesystem, dirname)
+ return self._reftest_list[dirname]
+
+ def _reference_file_for(self, test_name, expectation):
+ reftest_list = self._get_reftest_list(test_name)
+ if not reftest_list:
+ if expectation == '==':
+ return self.expected_filename(test_name, '.html')
+ else:
+ return self.expected_filename(test_name, '-mismatch.html')
+
+ filename = self._filesystem.join(self.layout_tests_dir(), test_name)
+ if filename not in reftest_list or reftest_list[filename][0] != expectation:
+ return None
+ return reftest_list[filename][1]
+
+ def is_reftest(self, test_name):
+ reftest_list = self._get_reftest_list(test_name)
+ if not reftest_list:
+ has_expected = self._filesystem.exists(self.expected_filename(test_name, '.html'))
+ return has_expected or self._filesystem.exists(self.expected_filename(test_name, '-mismatch.html'))
+ filename = self._filesystem.join(self.layout_tests_dir(), test_name)
+ return filename in reftest_list
+
def reftest_expected_filename(self, test_name):
"""Return the filename of reference we expect the test matches."""
- return self.expected_filename(test_name, '.html')
+ return self._reference_file_for(test_name, '==')
def reftest_expected_mismatch_filename(self, test_name):
"""Return the filename of reference we don't expect the test matches."""
- return self.expected_filename(test_name, '-mismatch.html')
+ return self._reference_file_for(test_name, '!=')
def test_to_uri(self, test_name):
"""Convert a test name to a URI."""
@@ -466,7 +495,7 @@
def find_test_files(self, paths):
# When collecting test cases, skip these directories
- skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])
+ skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests', 'reference', 'reftest'])
return find_files.find(self.filesystem, self.layout_tests_dir(), paths, skipped_directories, _is_test_file)
def test_dirs(self):
@@ -1002,12 +1031,17 @@
# When collecting test cases, we include any file with these extensions.
-_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
- '.htm', '.php', '.svg', '.mht'])
+_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', '.htm', '.php', '.svg', '.mht'])
-def is_reference_html_file(filename):
- return filename.endswith('-expected.html') or filename.endswith('-expected-mismatch.html')
+def is_reference_html_file(filesystem, dirname, filename):
+ if filename.startswith('ref-') or filename.endswith('notref-'):
+ return True
+ filename_wihout_ext, unused = filesystem.splitext(filename)
+ for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
+ if filename_wihout_ext.endswith(suffix):
+ return True
+ return False
def _has_supported_extension(filesystem, filename):
@@ -1017,4 +1051,21 @@
def _is_test_file(filesystem, dirname, filename):
- return _has_supported_extension(filesystem, filename) and not is_reference_html_file(filename)
+ return _has_supported_extension(filesystem, filename) and not is_reference_html_file(filesystem, dirname, filename)
+
+
+def _parse_reftest_list(filesystem, test_dirpath):
+ reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
+ if not filesystem.isfile(reftest_list_path):
+ return None
+ reftest_list_file = filesystem.read_text_file(reftest_list_path)
+
+ parsed_list = dict()
+ for line in reftest_list_file.split('\n'):
+ line = re.sub('#.+$', '', line)
+ split_line = line.split()
+ if len(split_line) < 3:
+ continue
+ expectation_type, test_file, ref_file = split_line
+ parsed_list[filesystem.join(test_dirpath, test_file)] = (expectation_type, filesystem.join(test_dirpath, ref_file))
+ return parsed_list
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -43,6 +43,7 @@
from webkitpy.layout_tests.port import Port, Driver, DriverOutput
from webkitpy.layout_tests.port.base import _is_test_file
+from webkitpy.layout_tests.port.base import _parse_reftest_list
from webkitpy.layout_tests.port.test import TestPort
import config
@@ -306,11 +307,30 @@
filesystem = MockFileSystem()
self.assertTrue(_is_test_file(filesystem, '', 'foo.html'))
self.assertTrue(_is_test_file(filesystem, '', 'foo.shtml'))
+ self.assertTrue(_is_test_file(filesystem, '', 'test-ref-test.html'))
self.assertFalse(_is_test_file(filesystem, '', 'foo.png'))
self.assertFalse(_is_test_file(filesystem, '', 'foo-expected.html'))
self.assertFalse(_is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
+ self.assertFalse(_is_test_file(filesystem, '', 'foo-ref.html'))
+ self.assertFalse(_is_test_file(filesystem, '', 'foo-notref.html'))
+ self.assertFalse(_is_test_file(filesystem, '', 'foo-notref.xht'))
+ self.assertFalse(_is_test_file(filesystem, '', 'foo-ref.xhtml'))
+ self.assertFalse(_is_test_file(filesystem, '', 'ref-foo.html'))
+ self.assertFalse(_is_test_file(filesystem, '', 'notref-foo.xhr'))
+ def test_parse_reftest_list(self):
+ port = TestPort()
+ port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
+ "",
+ "# some comment",
+ "!= test-2.html test-notref.html # more comments",
+ "== test-3.html test-ref.html"])
+ reftest_list = _parse_reftest_list(port.host.filesystem, 'bar')
+ self.assertEqual(reftest_list, {'bar/test.html': ('==', 'bar/test-ref.html'),
+ 'bar/test-2.html': ('!=', 'bar/test-notref.html'),
+ 'bar/test-3.html': ('==', 'bar/test-ref.html')})
+
class VirtualTest(unittest.TestCase):
"""Tests that various methods expected to be virtual are."""
def assertVirtual(self, method, *args, **kwargs):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -112,9 +112,7 @@
def run_test(self, driver_input):
start_time = time.time()
fs = self._port._filesystem
- if (fs.exists(self._port.reftest_expected_filename(driver_input.test_name)) or
- fs.exists(self._port.reftest_expected_mismatch_filename(driver_input.test_name)) or
- driver_input.test_name.endswith('-expected.html')):
+ if (self._port.is_reftest(driver_input.test_name) or driver_input.test_name.endswith('-expected.html')):
text = 'test-text'
image = 'test-image'
checksum = 'test-checksum'
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -187,7 +187,14 @@
tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
# FIXME: Add a reftest which crashes.
+ tests.add('reftests/foo/test.html')
+ tests.add('reftests/foo/test-ref.html')
+ # The following files shouldn't be treated as reftests
+ tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
+ tests.add('reftests/foo/reference/bar/common.html')
+ tests.add('reftests/foo/reftest/bar/shared.html')
+
tests.add('websocket/tests/passes/text.html')
# For --no-http tests, test that platform specific HTTP tests are properly skipped.
@@ -213,23 +220,25 @@
test_list = unit_test_list()
files = files or {}
- def add_file(files, test, suffix, contents):
+ def add_test_file(files, test, suffix, contents):
dirname = test.name[0:test.name.rfind('/')]
base = test.base
- path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
- files[path] = contents
+ add_file(files, dirname + '/' + base + suffix, contents)
+ def add_file(files, file_name, contents):
+ files[LAYOUT_TEST_DIR + '/' + file_name] = contents
+
# Add each test and the expected output, if any.
for test in test_list.tests.values():
- add_file(files, test, '.html', '')
+ add_test_file(files, test, '.html', '')
if test.is_reftest:
continue
if test.actual_audio:
- add_file(files, test, '-expected.wav', test.expected_audio)
+ add_test_file(files, test, '-expected.wav', test.expected_audio)
continue
- add_file(files, test, '-expected.txt', test.expected_text)
- add_file(files, test, '-expected.png', test.expected_image)
+ add_test_file(files, test, '-expected.txt', test.expected_text)
+ add_test_file(files, test, '-expected.png', test.expected_image)
# Add the test_expectations file.
@@ -255,6 +264,10 @@
WONTFIX SKIP : failures/expected/exception.html = CRASH
"""
+ add_file(files, 'reftests/foo/reftest.list', """
+== test.html test-ref.html
+""")
+
# FIXME: This test was only being ignored because of missing a leading '/'.
# Fixing the typo causes several tests to assert, so disabling the test entirely.
# Add in a file should be ignored by port.find_test_files().
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -36,8 +36,8 @@
import codecs
import itertools
import logging
-import os
import Queue
+import re
import sys
import thread
import time
@@ -166,7 +166,9 @@
# In case of reftest, one test calls the driver's run_test() twice.
# We should not add a reference html used by reftests to tests unless include_reference_html parameter
# is explicitly given.
- if include_reference_html or not is_reference_html_file(test_input.test_name):
+ filesystem = self._port.host.filesystem
+ dirname, filename = filesystem.split(test_name)
+ if include_reference_html or not is_reference_html_file(filesystem, dirname, filename):
self._current_test_batch.append(test_name)
return TestDriver.run_test(self, test_input)
@@ -426,7 +428,7 @@
# Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
# FIXME: It's nice to have a routine in port/test.py that returns this number.
- unexpected_tests_count = 7
+ unexpected_tests_count = 8
self.assertEqual(res, unexpected_tests_count)
self.assertFalse(out.empty())
@@ -448,10 +450,24 @@
expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
json_string = fs.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
- self.assertTrue(json_string.find('"num_regression":1') == -1)
- self.assertTrue(json_string.find('"num_flaky":1') == -1)
+ self.assertTrue(json_string.find('"num_regressions":1') != -1)
+ self.assertTrue(json_string.find('"num_flaky":0') != -1)
self.assertTrue(json_string.find('"num_missing":1') != -1)
+ def test_missing_and_unexpected_results(self):
+ # Test that we update expectations in place. If the expectation
+ # is missing, update the expected generic location.
+ fs = unit_test_filesystem()
+ res, out, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, filesystem=fs, record_results=True)
+ file_list = fs.written_files.keys()
+ file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+ self.assertEquals(res, 1)
+ json_string = fs.read_text_file('/tmp/layout-test-results/full_results.json')
+ self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
+ self.assertTrue(json_string.find('"num_regressions":1') != -1)
+ self.assertTrue(json_string.find('"num_flaky":0') != -1)
+ self.assertTrue(json_string.find('"num_missing":1') != -1)
+
def test_missing_and_unexpected_results_with_custom_exit_code(self):
# Test that we update expectations in place. If the expectation
# is missing, update the expected generic location.
@@ -762,8 +778,11 @@
fs = unit_test_filesystem()
res, out, err, user = logging_run(record_results=True, tests_included=True, filesystem=fs)
- # Seven tests should fail, so the return code should be 7.
- self.assertEquals(res, 7)
+ # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
+ # FIXME: It's nice to have a routine in port/test.py that returns this number.
+ unexpected_tests_count = 8
+
+ self.assertEquals(res, unexpected_tests_count)
results = self.parse_full_results(fs.files['/tmp/layout-test-results/full_results.json'])
# Check to ensure we're passing back image diff %age correctly.
Modified: trunk/Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py (101726 => 101727)
--- trunk/Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py 2011-12-02 01:32:32 UTC (rev 101726)
+++ trunk/Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py 2011-12-02 01:46:21 UTC (rev 101727)
@@ -251,9 +251,7 @@
fs = self._target_port._filesystem
for test in self._rebaselining_tests:
- test_abspath = self._target_port.abspath_for_test(test)
- if (fs.exists(self._target_port.reftest_expected_filename(test_abspath)) or
- fs.exists(self._target_port.reftest_expected_mismatch_filename(test_abspath))):
+ if self._target_port.is_reftest(test):
_log.error('%s seems to be a reftest. We can not rebase for reftests.', test)
self._rebaselining_tests = set()
return False
