Diff
Modified: trunk/Tools/ChangeLog (90651 => 90652)
--- trunk/Tools/ChangeLog 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/ChangeLog 2011-07-08 19:26:32 UTC (rev 90652)
@@ -1,3 +1,27 @@
+2011-07-08 Adam Barth <[email protected]>
+
+ Teach garden-o-matic how to display test results
+ https://bugs.webkit.org/show_bug.cgi?id=64141
+
+ Reviewed by Ojan Vafai.
+
+ This patch includes basic infrastructure for probing build.chromium.org
+ for test results. We only handle text and image tests, not anything
+ complicated like reftests. Also, we're using the revision/build
+ independent results store on the server, so we're avoiding that
+ complication for now.
+
+ It's slightly hacky that we need to probe the server to see what kinds
+ of results exist. A better solution would be to add CORS support to
+ the server or to use the local server to help.
+
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/index.html:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
+
2011-07-08 Dirk Pranke <[email protected]>
REGRESSION(90419) NRWT's httpd locking is broken for --child-processes=1
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -2,11 +2,27 @@
(function(){
+base.endsWith = function(string, suffix)
+{
+ if (suffix.length > string.length)
+ return false;
+ var expectedIndex = string.length - suffix.length;
+ return string.lastIndexOf(suffix) == expectedIndex;
+};
+
base.joinPath = function(parent, child)
{
if (parent.length == 0)
return child;
return parent + '/' + child;
+};
+
+base.trimExtension = function(url)
+{
+ var index = url.lastIndexOf('.');
+ if (index == -1)
+ return url;
+ return url.substr(0, index);
}
base.filterTree = function(tree, isLeaf, predicate)
@@ -29,6 +45,23 @@
walkSubtree(tree, '');
return filteredTree;
-}
+};
+base.probe = function(url, options)
+{
+ var scriptElement = document.createElement('script');
+ scriptElement.addEventListener('load', function() {
+ $(scriptElement).detach();
+ if (options.success)
+ options.success.call();
+ }, false);
+ scriptElement.addEventListener('error', function() {
+ $(scriptElement).detach();
+ if (options.error)
+ options.error.call();
+ }, false);
+ scriptElement.src = ""
+ document.head.appendChild(scriptElement);
+};
+
})();
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -5,6 +5,27 @@
equals(value, "path/to/test.html");
});
+test("endsWith", 9, function() {
+ ok(base.endsWith("xyz", ""));
+ ok(base.endsWith("xyz", "z"));
+ ok(base.endsWith("xyz", "yz"));
+ ok(base.endsWith("xyz", "xyz"));
+ ok(!base.endsWith("xyz", "wxyz"));
+ ok(!base.endsWith("xyz", "gwxyz"));
+ ok(base.endsWith("", ""));
+ ok(!base.endsWith("", "z"));
+ ok(!base.endsWith("xyxy", "yx"));
+});
+
+test("trimExtension", 6, function() {
+ equals(base.trimExtension("xyz"), "xyz");
+ equals(base.trimExtension("xy.z"), "xy");
+ equals(base.trimExtension("x.yz"), "x");
+ equals(base.trimExtension("x.y.z"), "x.y");
+ equals(base.trimExtension(".xyz"), "");
+ equals(base.trimExtension(""), "");
+});
+
test("joinPath with empty parent", 1, function() {
var value = base.joinPath("", "test.html");
equals(value, "test.html");
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/index.html (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/index.html 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/index.html 2011-07-08 19:26:32 UTC (rev 90652)
@@ -40,10 +40,14 @@
.failures .builders {
color: #888;
}
-.failures .builderName, .failures .actual {
+.builder .builderName, .builder .actual {
float: left;
width: 200px;
}
+.results iframe, .results img {
+ width: 400px;
+ height: 300px;
+}
</style>
<link rel="icon" id="favicon" type="image/png" href=""
</head>
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -17,20 +17,33 @@
{
var faviconURL = 'favicon-' + (hasFailures ? 'red' : 'green') + '.png';
$('#favicon').attr('href', faviconURL);
-};
+}
function fetchResults(onsuccess)
{
results.fetchResultsByBuilder(config.builders, function(resultsByBuilder) {
- var unexpectedFailures = ui.resultsByTest(results.unexpectedFailuresByTest(resultsByBuilder));
+ var unexpectedFailures = ui.summarizeResultsByTest(results.unexpectedFailuresByTest(resultsByBuilder));
$('.failures').append(unexpectedFailures);
onsuccess();
});
setIconState($('.failures').length);
}
+function expandResults()
+{
+ var resultsSummary = $(this);
+ var testName = $('.testName', resultsSummary).text();
+ $('.builderName', resultsSummary).each(function() {
+ var builderName = $(this).text();
+ results.fetchResultsURLs(builderName, testName, function(resultURLs) {
+ resultsSummary.append(ui.results(resultURLs));
+ });
+ });
+}
+
$('.hide').live('click', hide);
$('.quit').live('click', quit);
+$('.results-summary .test').live('click', expandResults);
$(document).ready(function() {
fetchResults(function() {
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -7,6 +7,26 @@
var kResultsName = 'full_results.json';
var kMasterName = 'ChromiumWebkit';
+var kLayoutTestResultsServer = 'http://build.chromium.org/f/chromium/layout_test_results/';
+var kLayoutTestResultsPath = '/results/layout-test-results/';
+
+var kPossibleSuffixList = [
+ '-expected.png',
+ '-actual.png',
+ '-diff.png',
+ '-expected.txt',
+ '-actual.txt',
+ '-diff.txt',
+ // FIXME: Add support for these result types.
+ // '-wdiff.html',
+ // '-pretty-diff.html',
+ // '-expected.html',
+ // '-expected-mismatch.html',
+ // '-expected.wav',
+ // '-actual.wav',
+ // ... and possibly more.
+];
+
var PASS = 'PASS';
var TIMEOUT = 'TIMEOUT';
var TEXT = 'TEXT';
@@ -16,6 +36,17 @@
var kFailingResults = [TIMEOUT, TEXT, CRASH, IMAGE, IMAGE_TEXT];
+// Kinds of results.
+results.kActualKind = 'actual';
+results.kExpectedKind = 'expected';
+results.kDiffKind = 'diff';
+results.kUnknownKind = 'unknown';
+
+// Types of tests.
+results.kImageType = 'image'
+results.kTextType = 'text'
+// FIXME: There are more types of tests.
+
function isFailure(result)
{
return kFailingResults.indexOf(result) != -1;
@@ -89,8 +120,77 @@
return unexpectedFailures;
};
-function resultsURL(builderName, name)
+function resultsDirectoryForBuilder(builderName)
{
+ return builderName.replace(/[ .()]/g, '_');
+}
+
+function resultsDirectoryURL(builderName)
+{
+ return kLayoutTestResultsServer + resultsDirectoryForBuilder(builderName) + kLayoutTestResultsPath;
+}
+
+results.resultKind = function(url)
+{
+ if (/-actual\.[a-z]+$/.test(url))
+ return results.kActualKind;
+ else if (/-expected\.[a-z]+$/.test(url))
+ return results.kExpectedKind;
+ else if (/diff\.[a-z]+$/.test(url))
+ return results.kDiffKind;
+ return results.kUnknownKind;
+}
+
+results.resultType = function(url)
+{
+ if (/\.png$/.test(url))
+ return results.kImageType;
+ return results.kTextType;
+}
+
+function sortResultURLsBySuffix(urls)
+{
+ var sortedURLs = [];
+ $.each(kPossibleSuffixList, function(i, suffix) {
+ $.each(urls, function(j, url) {
+ if (!base.endsWith(url, suffix))
+ return;
+ sortedURLs.push(url);
+ });
+ });
+ if (sortedURLs.length != urls.length)
+ throw "sortResultURLsBySuffix failed to return the same number of URLs."
+ return sortedURLs;
+}
+
+results.fetchResultsURLs = function(builderName, testName, callback)
+{
+ var stem = resultsDirectoryURL(builderName);
+ var testNameStem = base.trimExtension(testName);
+
+ var resultURLs = [];
+ var requestsInFlight = kPossibleSuffixList.length;
+
+ function checkComplete()
+ {
+ if (--requestsInFlight == 0)
+ callback(sortResultURLsBySuffix(resultURLs));
+ }
+
+ $.each(kPossibleSuffixList, function(index, suffix) {
+ var url = "" + testNameStem + suffix;
+ base.probe(url, {
+ success: function() {
+ resultURLs.push(url);
+ checkComplete();
+ },
+ error: checkComplete,
+ });
+ });
+};
+
+function resultsSummaryURL(builderName, name)
+{
return kTestResultsServer + 'testfile' +
'?builder=' + builderName +
'&master=' + kMasterName +
@@ -101,7 +201,7 @@
results.fetchResultsForBuilder = function(builderName, onsuccess)
{
$.ajax({
- url: resultsURL(builderName, kResultsName),
+ url: resultsSummaryURL(builderName, kResultsName),
dataType: 'jsonp',
success: function(data) {
onsuccess(new results.BuilderResults(data));
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -62,3 +62,74 @@
}
});
});
+
+test("resultKind", 12, function() {
+ equals(results.resultKind("http://example.com/foo-actual.txt"), "actual");
+ equals(results.resultKind("http://example.com/foo-expected.txt"), "expected");
+ equals(results.resultKind("http://example.com/foo-diff.txt"), "diff");
+ equals(results.resultKind("http://example.com/foo.bar-actual.txt"), "actual");
+ equals(results.resultKind("http://example.com/foo.bar-expected.txt"), "expected");
+ equals(results.resultKind("http://example.com/foo.bar-diff.txt"), "diff");
+ equals(results.resultKind("http://example.com/foo-actual.png"), "actual");
+ equals(results.resultKind("http://example.com/foo-expected.png"), "expected");
+ equals(results.resultKind("http://example.com/foo-diff.png"), "diff");
+ equals(results.resultKind("http://example.com/foo-pretty-diff.html"), "diff");
+ equals(results.resultKind("http://example.com/foo-wdiff.html"), "diff");
+ equals(results.resultKind("http://example.com/foo-xyz.html"), "unknown");
+});
+
+test("resultType", 12, function() {
+ equals(results.resultType("http://example.com/foo-actual.txt"), "text");
+ equals(results.resultType("http://example.com/foo-expected.txt"), "text");
+ equals(results.resultType("http://example.com/foo-diff.txt"), "text");
+ equals(results.resultType("http://example.com/foo.bar-actual.txt"), "text");
+ equals(results.resultType("http://example.com/foo.bar-expected.txt"), "text");
+ equals(results.resultType("http://example.com/foo.bar-diff.txt"), "text");
+ equals(results.resultType("http://example.com/foo-actual.png"), "image");
+ equals(results.resultType("http://example.com/foo-expected.png"), "image");
+ equals(results.resultType("http://example.com/foo-diff.png"), "image");
+ equals(results.resultType("http://example.com/foo-pretty-diff.html"), "text");
+ equals(results.resultType("http://example.com/foo-wdiff.html"), "text");
+ equals(results.resultType("http://example.com/foo.xyz"), "text");
+});
+
+test("fetchResultsURLs", 3, function() {
+ var realBase = window.base;
+
+ var pendingCallbacks = {};
+ window.base = {};
+ base.probe = function(url, options) {
+ pendingCallbacks[url] = options;
+ };
+ base.endsWith = realBase.endsWith;
+ base.trimExtension = realBase.trimExtension;
+
+ results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", function(resultURLs) {
+ deepEqual(resultURLs, [
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
+ ]);
+ });
+
+ var probedURLs = [];
+ for (var url in pendingCallbacks) {
+ probedURLs.push(url);
+ if (realBase.endsWith(url, '.txt'))
+ pendingCallbacks[url].success.call();
+ else
+ pendingCallbacks[url].error.call();
+ }
+
+ deepEqual(probedURLs, [
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.png",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.png",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.png",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
+ "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
+ ]);
+
+ window.base = realBase;
+ equal(window.base, realBase, "Failed to restore real base!");
+});
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -2,9 +2,9 @@
(function () {
-ui.resultsByTest = function(resultsByTest)
+ui.summarizeResultsByTest = function(resultsByTest)
{
- var block = $('<div class="results"></div>');
+ var block = $('<div class="results-summary"></div>');
$.each(resultsByTest, function(testName, resultNodesByBuilder) {
var testBlock = $('<div class="test"><div class="testName"></div><div class="builders"></div></div>');
block.append(testBlock);
@@ -21,4 +21,16 @@
return block;
};
+ui.results = function(resultsURLs)
+{
+ var block = $('<div class="results"></div>');
+ $.each(resultsURLs, function(index, resultURL) {
+ var kind = results.resultKind(resultURL);
+ var type = results.resultType(resultURL);
+ var fragment = type == results.kImageType ? '<img>' : '<iframe></iframe>';
+ block.append($(fragment).attr('src', resultURL).addClass(kind))
+ });
+ return block;
+};
+
})();
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js (90651 => 90652)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js 2011-07-08 19:21:41 UTC (rev 90651)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js 2011-07-08 19:26:32 UTC (rev 90652)
@@ -19,9 +19,9 @@
}
}
-test("BuilderResults.resultsByTest", 1, function() {
- var resultsByTest = ui.resultsByTest(kExampleResultsByTest);
- equal(resultsByTest.html(),
+test("summarizeResultsByTest", 1, function() {
+ var resultsSummary = ui.summarizeResultsByTest(kExampleResultsByTest);
+ equal(resultsSummary.html(),
'<div class="test">' +
'<div class="testName">scrollbars/custom-scrollbar-with-incomplete-style.html</div>' +
'<div class="builders">' +
@@ -36,3 +36,21 @@
'</div>' +
'</div>');
});
+
+test("results", 1, function() {
+ var testResults = ui.results([
+ 'http://example.com/layout-test-results/foo-bar-expected.txt',
+ 'http://example.com/layout-test-results/foo-bar-actual.txt',
+ 'http://example.com/layout-test-results/foo-bar-diff.txt',
+ 'http://example.com/layout-test-results/foo-bar-expected.png',
+ 'http://example.com/layout-test-results/foo-bar-actual.png',
+ 'http://example.com/layout-test-results/foo-bar-diff.png',
+ ]);
+ equal(testResults.html(),
+ '<iframe src="" class="expected"></iframe>' +
+ '<iframe src="" class="actual"></iframe>' +
+ '<iframe src="" class="diff"></iframe>' +
+ '<img src="" class="expected">' +
+ '<img src="" class="actual">' +
+ '<img src="" class="diff">');
+});
