Modified: trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js (91619 => 91620)
--- trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js 2011-07-22 23:32:15 UTC (rev 91619)
+++ trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js 2011-07-22 23:33:05 UTC (rev 91620)
@@ -466,7 +466,7 @@
// Append JSON script elements.
var g_resultsByBuilder = {};
-var g_expectationsByTest = {};
+var g_expectations = {};
function ADD_RESULTS(builds)
{
for (var builderName in builds) {
@@ -484,17 +484,23 @@
'&testtype=' + g_currentState.testType + '&name=';
}
-// Only webkit tests have a sense of test expectations.
-var g_waitingOnExpectations = isLayoutTestResults() && !isTreeMap();
-if (g_waitingOnExpectations) {
- function ADD_EXPECTATIONS(expectations)
- {
+// FIXME: Make the dashboard understand different ports' expectations files.
+var CHROMIUM_EXPECTATIONS_URL = 'http://svn.webkit.org/repository/webkit/trunk/LayoutTests/platform/chromium/test_expectations.txt';
+
+function requestExpectationsFile()
+{
+ request(CHROMIUM_EXPECTATIONS_URL, function(xhr) {
g_waitingOnExpectations = false;
- g_expectationsByTest = expectations;
+ g_expectations = xhr.responseText;
handleResourceLoad();
- }
+ },
+ function() {
+ console.error('Could not load expectations file from ' + CHROMIUM_EXPECTATIONS_URL);
+ });
}
+var g_waitingOnExpectations = isLayoutTestResults() && !isTreeMap();
+
function isTreeMap()
{
return endsWith(window.location.pathname, 'treemap.html');
@@ -532,10 +538,8 @@
for (var builderName in g_builders)
appendJSONScriptElementFor(builderName);
- // Grab expectations file from the fastest builder, which is Linux release
- // right now. Could be changed to any other builder if needed.
if (g_waitingOnExpectations)
- appendScript(pathToBuilderResultsFile(g_expectationsBuilder) + 'expectations.json');
+ requestExpectationsFile();
}
var g_hasDoneInitialPageGeneration = false;
Modified: trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html (91619 => 91620)
--- trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html 2011-07-22 23:32:15 UTC (rev 91619)
+++ trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html 2011-07-22 23:33:05 UTC (rev 91620)
@@ -180,10 +180,6 @@
border: 1px solid lightgray;
height: 0px;
}
-.different-platform {
- color: gray;
- font-size: 10px;
-}
#passing-tests {
-webkit-column-count: 3;
-webkit-column-gap: 25px;
@@ -788,48 +784,45 @@
resultsObject.isWontFixSkip = stringContains(expectations.modifiers, 'WONTFIX') || stringContains(expectations.modifiers, 'SKIP');
}
-function addTestToAllExpectations(test, expectations)
+function addTestToAllExpectations(test, expectations, modifiers)
{
if (!g_allExpectations[test])
g_allExpectations[test] = {};
- for (var j = 0; j < expectations.length; j++) {
- var modifiers = expectations[j].modifiers.split(' ');
- var allPlatforms = [];
- var allBuildTypes = [];
- modifiers.forEach(function(modifier) {
- if (modifier in BUILD_TYPES) {
- allBuildTypes.push(modifier);
- return;
- }
-
- if (PLATFORMS.indexOf(modifier) != -1) {
- allPlatforms.push(modifier);
- return;
- }
-
- if (modifier in PLATFORM_UNIONS) {
- PLATFORM_UNIONS[modifier].forEach(function(platform) {
- allPlatforms.push(platform);
- });
- }
- })
+ var allPlatforms = [];
+ var allBuildTypes = [];
+ modifiers.split(' ').forEach(function(modifier) {
+ if (modifier in BUILD_TYPES) {
+ allBuildTypes.push(modifier);
+ return;
+ }
- if (!allPlatforms.length)
- allPlatforms = PLATFORMS;
-
- if (!allBuildTypes.length)
- allBuildTypes = Object.keys(BUILD_TYPES);
+ if (PLATFORMS.indexOf(modifier) != -1) {
+ allPlatforms.push(modifier);
+ return;
+ }
- allPlatforms.forEach(function(platform) {
- if (!g_allExpectations[test][platform])
- g_allExpectations[test][platform] = {};
+ if (modifier in PLATFORM_UNIONS) {
+ PLATFORM_UNIONS[modifier].forEach(function(platform) {
+ allPlatforms.push(platform);
+ });
+ }
+ })
+
+ if (!allPlatforms.length)
+ allPlatforms = PLATFORMS;
+
+ if (!allBuildTypes.length)
+ allBuildTypes = Object.keys(BUILD_TYPES);
+
+ allPlatforms.forEach(function(platform) {
+ if (!g_allExpectations[test][platform])
+ g_allExpectations[test][platform] = {};
- allBuildTypes.forEach(function(buildType) {
- g_allExpectations[test][platform][buildType] = expectations[j];
- });
- });
- }
+ allBuildTypes.forEach(function(buildType) {
+ g_allExpectations[test][platform][buildType] = {modifiers: modifiers, expectations: expectations};
+ });
+ });
}
// Data structure to hold the processed expectations.
@@ -843,6 +836,33 @@
// buildType, then ALL.
var g_allExpectations;
+function parsedExpectations()
+{
+ var expectations = [];
+ var lines = g_expectations.split('\n');
+ lines.forEach(function(line) {
+ line = trimString(line);
+ if (!line || startsWith(line, '//'))
+ return;
+
+ // FIXME: Make this robust against not having modifiers and/or expectations.
+ // Right now, run-webkit-tests doesn't allow such lines, but it may in the future.
+ var match = line.match(/([^:]+)*:([^=]+)=(.*)/);
+ if (!match) {
+ console.error('Line could not be parsed: ' + line);
+ return;
+ }
+
+ // FIXME: Should we include line number and comment lines here?
+ expectations.push({
+ modifiers: trimString(match[1]),
+ path: trimString(match[2]),
+ expectations: trimString(match[3])
+ });
+ });
+ return expectations;
+}
+
function processExpectations()
{
if (g_allExpectations)
@@ -850,9 +870,7 @@
g_allExpectations = {};
- var expectationsArray = [];
- for (var path in g_expectationsByTest)
- expectationsArray.push({path: path, expectations: g_expectationsByTest[path]});
+ var expectationsArray = parsedExpectations();
// Sort the array to hit more specific paths last. More specific
// paths (e.g. foo/bar/baz.html) override entries for less-specific ones (e.g. foo/bar).
@@ -861,23 +879,24 @@
var allTests = getAllTests();
for (var i = 0; i < expectationsArray.length; i++) {
var path = expectationsArray[i].path;
+ var modifiers = expectationsArray[i].modifiers;
var expectations = expectationsArray[i].expectations;
var pathMatchesAnyTest = false;
if (allTests[path]) {
pathMatchesAnyTest = true;
- addTestToAllExpectations(path, expectations);
+ addTestToAllExpectations(path, expectations, modifiers);
} else {
for (var test in allTests) {
if (startsWith(test, path)) {
pathMatchesAnyTest = true;
- addTestToAllExpectations(test, expectations);
+ addTestToAllExpectations(test, expectations, modifiers);
}
}
}
if (!pathMatchesAnyTest)
- addTestToAllExpectations(path, expectations);
+ addTestToAllExpectations(path, expectations, modifiers);
}
}
@@ -1810,12 +1829,6 @@
html += htmlForSingleTestRow(testResults[j]);
html = htmlForTestTable(html);
} else {
- if (g_expectationsByTest[test]) {
- for (var i = 0; i < g_expectationsByTest[test].length; i++) {
- html += '<div>' + g_expectationsByTest[test][i].modifiers + ' | ' +
- g_expectationsByTest[test][i].expectations + '</div>';
- }
- }
html += '<div class="not-found">Test not found. Either it does ' +
'not exist, is skipped or passes on all platforms.</div>';
}
Modified: trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js (91619 => 91620)
--- trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js 2011-07-22 23:32:15 UTC (rev 91619)
+++ trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js 2011-07-22 23:33:05 UTC (rev 91620)
@@ -42,7 +42,7 @@
{
allExpectations = null;
allTests = null;
- g_expectationsByTest = {};
+ g_expectations = '';
g_resultsByBuilder = {};
g_builders = {};
g_allExpectations = null;
@@ -98,7 +98,7 @@
var expectationsArray = [
{'modifiers': 'RELEASE', 'expectations': 'FAIL'}
];
- g_expectationsByTest[test] = expectationsArray;
+ g_expectations = 'RELEASE : ' + test + ' = FAIL';
runExpectationsTest(builder, test, 'FAIL', 'RELEASE');
}
@@ -110,7 +110,8 @@
{'modifiers': 'RELEASE', 'expectations': 'FAIL'},
{'modifiers': 'DEBUG', 'expectations': 'CRASH'}
];
- g_expectationsByTest[test] = expectationsArray;
+ g_expectations = 'RELEASE : ' + test + ' = FAIL\n' +
+ 'DEBUG : ' + test + ' = CRASH';
runExpectationsTest(builder, test, 'FAIL', 'RELEASE');
}
@@ -122,20 +123,18 @@
{'modifiers': 'RELEASE', 'expectations': 'FAIL'},
{'modifiers': 'DEBUG', 'expectations': 'CRASH'}
];
- g_expectationsByTest[test] = expectationsArray;
+ g_expectations = 'RELEASE : ' + test + ' = FAIL\n' +
+ 'DEBUG : ' + test + ' = CRASH';
runExpectationsTest(builder, test, 'CRASH', 'DEBUG');
}
function testOverrideJustBuildType()
{
var test = 'bar/1.html';
- g_expectationsByTest['bar'] = [
- {'modifiers': 'WONTFIX', 'expectations': 'FAIL PASS TIMEOUT'}
- ];
- g_expectationsByTest[test] = [
- {'modifiers': 'WONTFIX MAC', 'expectations': 'FAIL'},
- {'modifiers': 'LINUX DEBUG', 'expectations': 'CRASH'},
- ];
+ g_expectations = 'WONTFIX : bar = FAIL PASS TIMEOUT\n' +
+ 'WONTFIX MAC : ' + test + ' = FAIL\n' +
+ 'LINUX DEBUG : ' + test + ' = CRASH';
+
runExpectationsTest('Webkit Win', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
runExpectationsTest('Webkit Win (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
runExpectationsTest('Webkit Linux', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
@@ -272,27 +271,15 @@
}
}
- g_expectationsByTest = {
- 'foo': [
- {'modifiers': '', 'expectations': 'FAIL PASS CRASH'}
- ],
- 'foo/test1.html': [
- {'modifiers': 'RELEASE BUGFOO', 'expectations': 'FAIL'},
- {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
- ],
- 'foo/test2.html': [
- {'modifiers': '', 'expectations': 'FAIL'},
- {'modifiers': 'LINUX DEBUG', 'expectations': 'CRASH'}
- ],
- 'test1.html': [
- {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
- {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
- ],
- 'http/tests/appcache/interrupted-update.html': [
- {'modifiers': 'WIN7', 'expectations': 'TIMEOUT'},
- {'modifiers': 'MAC LINUX XP VISTA', 'expectations': 'FAIL'}
- ]
- }
+ g_expectations = 'BUG123 : foo = FAIL PASS CRASH\n' +
+ 'RELEASE BUGFOO : foo/test1.html = FAIL\n' +
+ 'DEBUG : foo/test1.html = CRASH\n' +
+ 'BUG456 : foo/test2.html = FAIL\n' +
+ 'LINUX DEBUG : foo/test2.html = CRASH\n' +
+ 'RELEASE : test1.html = FAIL\n' +
+ 'DEBUG : test1.html = CRASH\n' +
+ 'WIN7 : http/tests/appcache/interrupted-update.html = TIMEOUT\n' +
+ 'MAC LINUX XP VISTA : http/tests/appcache/interrupted-update.html = FAIL\n';
processExpectations();
@@ -303,16 +290,16 @@
assertEquals(JSON.stringify(expectations), '{"modifiers":"RELEASE BUGFOO","expectations":"FAIL"}');
var expectations = getExpectations('foo/test2.html', 'LUCID', 'RELEASE');
- assertEquals(JSON.stringify(expectations), '{"modifiers":"","expectations":"FAIL"}');
+ assertEquals(JSON.stringify(expectations), '{"modifiers":"BUG456","expectations":"FAIL"}');
var expectations = getExpectations('foo/test2.html', 'LEOPARD', 'DEBUG');
- assertEquals(JSON.stringify(expectations), '{"modifiers":"","expectations":"FAIL"}');
+ assertEquals(JSON.stringify(expectations), '{"modifiers":"BUG456","expectations":"FAIL"}');
var expectations = getExpectations('foo/test2.html', 'LUCID', 'DEBUG');
assertEquals(JSON.stringify(expectations), '{"modifiers":"LINUX DEBUG","expectations":"CRASH"}');
var expectations = getExpectations('foo/test3.html', 'LUCID', 'DEBUG');
- assertEquals(JSON.stringify(expectations), '{"modifiers":"","expectations":"FAIL PASS CRASH"}');
+ assertEquals(JSON.stringify(expectations), '{"modifiers":"BUG123","expectations":"FAIL PASS CRASH"}');
var expectations = getExpectations('test1.html', 'XP', 'DEBUG');
assertEquals(JSON.stringify(expectations), '{"modifiers":"DEBUG","expectations":"CRASH"}');
