[webkit-changes] [91212] trunk/Tools

Mon, 18 Jul 2011 14:51:28 -0700

Title: [91212] trunk/Tools
Revision
91212
Author
[email protected]
Date
2011-07-18 14:51:18 -0700 (Mon, 18 Jul 2011)

Log Message

update the flakiness dashboard to understand the new platforms/formats in test_expectations
https://bugs.webkit.org/show_bug.cgi?id=64743

Reviewed by Adam Barth.

The test expectations format changed a long time ago and the flakiness dashboard
was never updated to match.

* TestResultServer/static-dashboards/dashboard_base.js:
* TestResultServer/static-dashboards/flakiness_dashboard.html:
* TestResultServer/static-dashboards/flakiness_dashboard_tests.js:
* TestResultServer/static-dashboards/timeline_explorer.html:

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (91211 => 91212)


--- trunk/Tools/ChangeLog	2011-07-18 21:27:10 UTC (rev 91211)
+++ trunk/Tools/ChangeLog	2011-07-18 21:51:18 UTC (rev 91212)
@@ -1,3 +1,18 @@
+2011-07-18  Ojan Vafai  <[email protected]>
+
+        update the flakiness dashboard to understand the new platforms/formats in test_expectations
+        https://bugs.webkit.org/show_bug.cgi?id=64743
+
+        Reviewed by Adam Barth.
+
+        The test expectations format changed a long time ago and the flakiness dashboard
+        was never updated to match.
+
+        * TestResultServer/static-dashboards/dashboard_base.js:
+        * TestResultServer/static-dashboards/flakiness_dashboard.html:
+        * TestResultServer/static-dashboards/flakiness_dashboard_tests.js:
+        * TestResultServer/static-dashboards/timeline_explorer.html:
+
 2011-07-18  Eric Seidel  <[email protected]>
 
         webkit-patch apply-attachment does not work when not called from the root of the checkout

Modified: trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js (91211 => 91212)


--- trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js	2011-07-18 21:27:10 UTC (rev 91211)
+++ trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js	2011-07-18 21:51:18 UTC (rev 91212)
@@ -904,17 +904,14 @@
 // @param {Object} results results for the current builder
 // @return Object with these properties:
 //     - testNames: array mapping test index to test names.
-//     - resultsByBuild: array of builds, for each build a (sparse) array of test
-//                                         results by test index.
-//     - flakyTests: array with the boolean value true at test indices that are
-//                                 considered flaky (more than one single-build failure).
-//     - flakyDeltasByBuild: array of builds, for each build a count of flaky
-//                                                 test results by expectation, as well as a total.
-function testResultsByBuild(builderResults)
+//     - resultsByBuild: array of builds, for each build a (sparse) array of test results by test index.
+//     - flakyTests: array with the boolean value true at test indices that are considered flaky (more than one single-build failure).
+//     - flakyDeltasByBuild: array of builds, for each build a count of flaky test results by expectation, as well as a total.
+function decompressResults(builderResults)
 {
     var builderTestResults = builderResults[TESTS_KEY];
     var buildCount = builderResults[FIXABLE_COUNTS_KEY].length;
-    var testResultsByBuild = new Array(buildCount);
+    var resultsByBuild = new Array(buildCount);
     var flakyDeltasByBuild = new Array(buildCount);
 
     // Pre-sizing the test result arrays for each build saves us ~250ms
@@ -922,8 +919,8 @@
     for (var testName in builderTestResults)
         testCount++;
     for (var i = 0; i < buildCount; i++) {
-        testResultsByBuild[i] = new Array(testCount);
-        testResultsByBuild[i][testCount - 1] = undefined;
+        resultsByBuild[i] = new Array(testCount);
+        resultsByBuild[i][testCount - 1] = undefined;
         flakyDeltasByBuild[i] = {};
     }
 
@@ -948,7 +945,7 @@
                 oneBuildFailureCount++;
 
             for (var j = 0; j < count; j++) {
-                testResultsByBuild[currentBuildIndex++][testIndex] = value;
+                resultsByBuild[currentBuildIndex++][testIndex] = value;
                 if (currentBuildIndex == buildCount)
                     break;
             }
@@ -991,7 +988,7 @@
 
     return {
         testNames: testNames,
-        resultsByBuild: testResultsByBuild,
+        resultsByBuild: resultsByBuild,
         flakyTests: flakyTests,
         flakyDeltasByBuild: flakyDeltasByBuild
     };

Modified: trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html (91211 => 91212)


--- trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html	2011-07-18 21:27:10 UTC (rev 91211)
+++ trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html	2011-07-18 21:51:18 UTC (rev 91212)
@@ -319,24 +319,32 @@
 var TEST_URL_BASE_PATH_TRAC = 'http://trac.webkit.org/browser/trunk/LayoutTests/';
 var TEST_URL_BASE_PATH = "http://svn.webkit.org/repository/webkit/trunk/LayoutTests/";
 var TEST_RESULTS_BASE_PATH = 'http://build.chromium.org/f/chromium/layout_test_results/';
+
+// FIXME: Make this an array.
 var PLATFORMS = {
-'MAC': 'MAC',
-'LINUX': 'LINUX',
-'WIN': 'WIN',
-'WIN-XP': 'WIN-XP',
-'WIN-VISTA': 'WIN-VISTA'
+    'SNOWLEOPARD': 'SNOWLEOPARD',
+    'LEOPARD': 'LEOPARD',
+    'XP': 'XP',
+    'VISTA': 'VISTA',
+    'WIN7': 'WIN7',
+    'LUCID': 'LUCID'
 };
+
+// FIXME: Make the g_allExpectations data structure explicitly list every platform instead of having a fallbacks concept.
 var PLATFORM_FALLBACKS = {
-'WIN': 'ALL',
-'WIN-XP': 'WIN',
-'WIN-VISTA': 'WIN',
-'LINUX': 'ALL',
-'MAC': 'ALL'
+    'WIN': 'ALL',
+    'XP': 'WIN',
+    'VISTA': 'WIN',
+    'MAC': 'ALL',
+    'SNOWLEOPARD': 'MAC',
+    'LEOPARD': 'MAC',
+    'LINUX': 'ALL',
+    'LUCID': 'LINUX'
 };
 var BUILD_TYPES = {'DEBUG': 'DBG', 'RELEASE': 'RELEASE'};
 var MIN_SECONDS_FOR_SLOW_TEST = 4;
 var MIN_SECONDS_FOR_SLOW_TEST_DEBUG = 2 * MIN_SECONDS_FOR_SLOW_TEST;
-var FAIL_RESULTS = ['IMAGE', 'IMAGE+TEXT', 'TEXT', 'SIMPLIFIED', 'MISSING'];
+var FAIL_RESULTS = ['IMAGE', 'IMAGE+TEXT', 'TEXT', 'MISSING'];
 var CHUNK_SIZE = 25;
 var MAX_RESULTS = 1500;
 
@@ -525,8 +533,27 @@
         // If the build name does not contain a platform
         // or build type, assume Windows Release.
         var currentBuildUppercase = builderName.toUpperCase();
-        var platform = matchingElement(currentBuildUppercase, PLATFORMS) || 'WIN';
-        var buildType = matchingElement(currentBuildUppercase, BUILD_TYPES) || 'RELEASE';
+        
+        var platform;
+        if (stringContains(currentBuildUppercase, 'MAC')) {
+            if (stringContains(currentBuildUppercase, '10.5'))
+                platform = 'LEOPARD';
+            else
+                platform = 'SNOWLEOPARD';
+        } else if (stringContains(currentBuildUppercase, 'WIN7'))
+            platform = 'WIN7';
+        else if (stringContains(currentBuildUppercase, 'VISTA'))
+            platform = 'VISTA';
+        else if (stringContains(currentBuildUppercase, 'WIN'))
+            platform = 'XP';
+        else if (stringContains(currentBuildUppercase, 'LINUX'))
+            platform = 'LUCID';
+        else {
+            platform = '';
+            console.log('Could not resolve platform for builder: ' + builderName);
+        }
+        
+        var buildType = stringContains(currentBuildUppercase, 'DBG') ? 'DEBUG' : 'RELEASE';
         g_perBuilderPlatformAndBuildType[builderName] = {platform: platform, buildType: buildType};
     }
     return g_perBuilderPlatformAndBuildType[builderName];
@@ -733,7 +760,7 @@
 
     var platformObject;
     while (platform && !(platformObject = testObject[platform]))
-      platform = PLATFORM_FALLBACKS[platform];
+        platform = PLATFORM_FALLBACKS[platform];
 
     if (platformObject) {
         if (platformObject[buildType])
@@ -836,16 +863,14 @@
         addFallbacks(function(platformKey) {
             addFallbacks(function(buildTypeKey) {
                 // Setting the ALL key overrides any previously seen expectations.
-                if (platformKey == ALL) {
-                    // FIXME: This is broken. PLATFORMS is an object, not an array.
-                    // Figure out if we should just be deleting this or fixing it.
-                    for (var i = 0; i < PLATFORMS.length; i++) {
-                      if (PLATFORMS[i] in g_allExpectations[test]) {
-                        // Setting the ALL key overrides any previously seen
-                        // expectations.
-                        if (buildTypeKey == ALL)
-                          g_allExpectations[test][PLATFORMS[i]] = {};
-                      }
+                if (platformKey == ALL && g_allExpectations[test]) {
+                    for (var platform in PLATFORM_FALLBACKS) {
+                          if (platform in g_allExpectations[test]) {
+                              // Setting the ALL key overrides any previously seen
+                              // expectations.
+                              if (buildTypeKey == ALL)
+                                  g_allExpectations[test][platform] = {};
+                          }
                     }
                 } else if (buildTypeKey == ALL && g_allExpectations[test])
                     g_allExpectations[test][platformKey] = {}
@@ -859,7 +884,7 @@
 
                 testHolder[platformKey][buildTypeKey] = expectations[j];
             }, modifiers, BUILD_TYPES);
-        }, modifiers, PLATFORMS);
+        }, modifiers, PLATFORM_FALLBACKS);
     }
 }
 
@@ -986,7 +1011,7 @@
         var rawResults = rawTest.results;
         resultsForTest.rawResults = rawResults;
 
-        // FIXME: Switch to testResultsByBuild
+        // FIXME: Switch to resultsByBuild
         var times = resultsForTest.rawTimes;
         var numTimesSeen = 0;
         var numResultsSeen = 0;
@@ -1689,6 +1714,7 @@
         var results = g_testToResultsMap[test];
         for (var i = 0; i < results.length; i++) {
             var thisResult = results[i];
+            
             if (!thisResult.missing && !thisResult.extra)
                 continue;

Modified: trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js (91211 => 91212)


--- trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js	2011-07-18 21:27:10 UTC (rev 91211)
+++ trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js	2011-07-18 21:51:18 UTC (rev 91212)
@@ -33,7 +33,7 @@
 // A test just consists of calling runExpectationsTest with the appropriate
 // arguments.
 
-// FIXME: Add tests for processMissingAndExtraExpectations
+// FIXME: move this over to using qunit
 
 // Clears out the global objects modified or used by processExpectations and
 // populateExpectationsData. A bit gross since it's digging into implementation
@@ -45,6 +45,8 @@
     g_expectationsByTest = {};
     g_resultsByBuilder = {};
     g_builders = {};
+    g_allExpectations = null;
+    g_allTests = null;
 }
 
 // Processes the expectations for a test and asserts that the final expectations
@@ -70,21 +72,28 @@
     var resultsForTest = createResultsObjectForTest(test, builder);
     populateExpectationsData(resultsForTest);
 
-    assertEquals(resultsForTest, resultsForTest.expectations, expectations);
-    assertEquals(resultsForTest, resultsForTest.modifiers, modifiers);
+    var message = 'Builder: ' + resultsForTest.builder + ' test: ' + resultsForTest.test;
+    assertEquals(resultsForTest.expectations, expectations, message);
+    assertEquals(resultsForTest.modifiers, modifiers, message);
 }
 
-function assertEquals(resultsForTest, actual, expected)
+function assertEquals(actual, expected, message)
 {
-    if (expected !== actual)
-        throw Error('Builder: ' + resultsForTest.builder + ' test: ' + resultsForTest.test + ' got: ' + actual + ' expected: ' + expected);
+    if (expected !== actual) {
+        if (message)
+            message += ' ';
+        else
+            message = '';
+
+        throw Error(message + 'got: ' + actual + ' expected: ' + expected);
+    }
 }
 
 function throwError(resultsForTests, actual, expected) {}
 
 function testReleaseFail()
 {
-    var builder = 'Webkit';
+    var builder = 'Webkit Win';
     var test = 'foo/1.html';
     var expectationsArray = [
         {'modifiers': 'RELEASE', 'expectations': 'FAIL'}
@@ -95,7 +104,7 @@
 
 function testReleaseFailDebugCrashReleaseBuilder()
 {
-    var builder = 'Webkit';
+    var builder = 'Webkit Win';
     var test = 'foo/1.html';
     var expectationsArray = [
         {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
@@ -107,7 +116,7 @@
 
 function testReleaseFailDebugCrashDebugBuilder()
 {
-    var builder = 'Webkit(dbg)';
+    var builder = 'Webkit Win (dbg)';
     var test = 'foo/1.html';
     var expectationsArray = [
         {'modifiers': 'RELEASE', 'expectations': 'FAIL'},
@@ -127,14 +136,81 @@
         {'modifiers': 'WONTFIX MAC', 'expectations': 'FAIL'},
         {'modifiers': 'LINUX DEBUG', 'expectations': 'CRASH'},
     ];
-    runExpectationsTest('Webkit', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
-    runExpectationsTest('Webkit (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
+    runExpectationsTest('Webkit Win', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
+    runExpectationsTest('Webkit Win (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
     runExpectationsTest('Webkit Linux', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
     runExpectationsTest('Webkit Linux (dbg)(3)', test, 'CRASH', 'LINUX DEBUG');
     runExpectationsTest('Webkit Mac10.5', test, 'FAIL', 'WONTFIX MAC');
     runExpectationsTest('Webkit Mac10.5 (dbg)(3)', test, 'FAIL', 'WONTFIX MAC');
 }
 
+function testPlatformAndBuildType()
+{
+    var runPlatformAndBuildTypeTest = function(builder, expectedPlatform, expectedBuildType) {
+        g_perBuilderPlatformAndBuildType = {};
+        buildInfo = platformAndBuildType(builder);
+        var message = 'Builder: ' + builder;
+        assertEquals(buildInfo.platform, expectedPlatform, message);
+        assertEquals(buildInfo.buildType, expectedBuildType, message);
+    }
+    runPlatformAndBuildTypeTest('Webkit Win (deps)', 'XP', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Win (deps)(dbg)(1)', 'XP', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Win (deps)(dbg)(2)', 'XP', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Linux (deps)', 'LUCID', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Linux (deps)(dbg)(1)', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Linux (deps)(dbg)(2)', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6 (deps)', 'SNOWLEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6 (deps)(dbg)(1)', 'SNOWLEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6 (deps)(dbg)(2)', 'SNOWLEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Win', 'XP', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Vista', 'VISTA', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Win7', 'WIN7', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Win (dbg)(1)', 'XP', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Win (dbg)(2)', 'XP', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Linux', 'LUCID', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Linux 32', 'LUCID', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(1)', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(2)', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.5', 'LEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(1)', 'LEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(2)', 'LEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6', 'SNOWLEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6 (dbg)', 'SNOWLEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Chromium Win Release (Tests)', 'XP', 'RELEASE');
+    runPlatformAndBuildTypeTest('Chromium Linux Release (Tests)', 'LUCID', 'RELEASE');
+    runPlatformAndBuildTypeTest('Chromium Mac Release (Tests)', 'SNOWLEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Win - GPU', 'XP', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Vista - GPU', 'VISTA', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Win7 - GPU', 'WIN7', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Win (dbg)(1) - GPU', 'XP', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Win (dbg)(2) - GPU', 'XP', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Linux - GPU', 'LUCID', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Linux 32 - GPU', 'LUCID', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(1) - GPU', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(2) - GPU', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.5 - GPU', 'LEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(1) - GPU', 'LEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(2) - GPU', 'LEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6 - GPU', 'SNOWLEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('Webkit Mac10.6 (dbg) - GPU', 'SNOWLEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Win7 Tests - GPU', 'WIN7', 'RELEASE');
+    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(1) - GPU', 'WIN7', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(2) - GPU', 'WIN7', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Win7 x64 Tests (dbg)(1) - GPU', 'WIN7', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Win7 x64 Tests (dbg)(2) - GPU', 'WIN7', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Vista Tests (dbg)(1) - GPU', 'VISTA', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Vista Tests (dbg)(2) - GPU', 'VISTA', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Vista x64 Tests (dbg) - GPU', 'VISTA', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Mac 10.6 Tests - GPU', 'SNOWLEOPARD', 'RELEASE');
+    runPlatformAndBuildTypeTest('GPU Mac 10.6 Tests (dbg) - GPU', 'SNOWLEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Mac 10.5 Tests (dbg) - GPU', 'LEOPARD', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Linux Tests (dbg)(1) - GPU', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Linux Tests (dbg)(2) - GPU', 'LUCID', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(1) - GPU', 'WIN7', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(2) - GPU', 'WIN7', 'DEBUG');
+    runPlatformAndBuildTypeTest('GPU Linux Tests x64 - GPU', 'LUCID', 'RELEASE');
+}
+
 function runTests()
 {
     document.body.innerHTML = '<pre id=unittest-results></pre>';

Modified: trunk/Tools/TestResultServer/static-dashboards/timeline_explorer.html (91211 => 91212)


--- trunk/Tools/TestResultServer/static-dashboards/timeline_explorer.html	2011-07-18 21:27:10 UTC (rev 91211)
+++ trunk/Tools/TestResultServer/static-dashboards/timeline_explorer.html	2011-07-18 21:51:18 UTC (rev 91212)
@@ -145,7 +145,7 @@
 function initCurrentBuilderTestResults()
 {
     var startTime = Date.now();
-    g_currentBuilderTestResults = testResultsByBuild(g_resultsByBuilder[g_currentState.builder]);
+    g_currentBuilderTestResults = decompressResults(g_resultsByBuilder[g_currentState.builder]);
     console.log( 'Time to get test results by build: ' + (Date.now() - startTime));
 }
 

_______________________________________________
webkit-changes mailing list
[email protected]
http://lists.webkit.org/mailman/listinfo.cgi/webkit-changes

Reply via email to