[SCM] WebKit Debian packaging branch, debian/experimental, updated. debian/1.3.8-1-1049-g2e11a8e
eric at webkit.org
eric at webkit.org
Fri Jan 21 14:53:08 UTC 2011
The following commit has been merged in the debian/experimental branch:
commit 8a665105ad9e197b0b13428006db9577a20fdf36
Author: eric at webkit.org <eric at webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Date: Mon Jan 3 20:59:10 2011 +0000
2011-01-03 Eric Seidel <eric at webkit.org>
Reviewed by Adam Barth.
Move LayoutTestResults over to new-run-webkit-tests TestResult architecture
https://bugs.webkit.org/show_bug.cgi?id=51802
I'm not the biggest fan of the test_failures or test_results classes,
but it's better to have one shared set of classes, than separate ones
for new vs. old run-webkit-test result handling.
This moves the ORWT results class "LayoutTestResults" over to using
TestResult and TestFailure classes, making it easy for us to
make all our sheriff-bot and other webkitpy code NRWT ready.
This also makes it a trivial patch to generate results.json information
from ORWT results.html files (for flaky test analysis, etc.) as well
as making it a one-liner to report test failure types when the
commit-queue sees flaky tests.
This patch tried not to add new functionality, but only to replace
the guts of LayoutTestResults, while adding unit tests and hoping
not to break anything.
I also moved callers which assumed User.prompt* were static/class methods
to using them as instance methods (since we'll eventually want to make them such).
In the process of re-writing things, I broke the rebaseline command, so I wrote
a unit test to catch my breakage were I do do so again in the future.
* Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py:
* Scripts/webkitpy/common/net/layouttestresults.py:
* Scripts/webkitpy/common/net/layouttestresults_unittest.py:
* Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
* Scripts/webkitpy/layout_tests/layout_package/test_results.py:
* Scripts/webkitpy/tool/commands/queries.py:
* Scripts/webkitpy/tool/commands/rebaseline.py:
* Scripts/webkitpy/tool/commands/rebaseline_unittest.py:
* Scripts/webkitpy/tool/mocktool.py:
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@74927 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index 4f98a10..9c234cf 100644
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,43 @@
+2011-01-03 Eric Seidel <eric at webkit.org>
+
+ Reviewed by Adam Barth.
+
+ Move LayoutTestResults over to new-run-webkit-tests TestResult architecture
+ https://bugs.webkit.org/show_bug.cgi?id=51802
+
+ I'm not the biggest fan of the test_failures or test_results classes,
+ but it's better to have one shared set of classes, than separate ones
+ for new vs. old run-webkit-test result handling.
+
+ This moves the ORWT results class "LayoutTestResults" over to using
+ TestResult and TestFailure classes, making it easy for us to
+ make all our sheriff-bot and other webkitpy code NRWT ready.
+
+ This also makes it a trivial patch to generate results.json information
+ from ORWT results.html files (for flaky test analysis, etc.) as well
+ as making it a one-liner to report test failure types when the
+ commit-queue sees flaky tests.
+
+ This patch tried not to add new functionality, but only to replace
+ the guts of LayoutTestResults, while adding unit tests and hoping
+ not to break anything.
+
+ I also moved callers which assumed User.prompt* were static/class methods
+ to using them as instance methods (since we'll eventually want to make them such).
+
+ In the process of re-writing things, I broke the rebaseline command, so I wrote
+ a unit test to catch my breakage were I do do so again in the future.
+
+ * Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py:
+ * Scripts/webkitpy/common/net/layouttestresults.py:
+ * Scripts/webkitpy/common/net/layouttestresults_unittest.py:
+ * Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
+ * Scripts/webkitpy/layout_tests/layout_package/test_results.py:
+ * Scripts/webkitpy/tool/commands/queries.py:
+ * Scripts/webkitpy/tool/commands/rebaseline.py:
+ * Scripts/webkitpy/tool/commands/rebaseline_unittest.py:
+ * Scripts/webkitpy/tool/mocktool.py:
+
2011-01-03 Adam Barth <abarth at webkit.org>
Reviewed by Simon Fraser.
diff --git a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py
index 8499c9d..f544647 100644
--- a/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py
+++ b/Tools/Scripts/webkitpy/common/net/buildbot/buildbot_unittest.py
@@ -30,10 +30,16 @@ import unittest
from webkitpy.common.net.layouttestresults import LayoutTestResults
from webkitpy.common.net.buildbot import BuildBot, Builder, Build
+from webkitpy.layout_tests.layout_package import test_results
+from webkitpy.layout_tests.layout_package import test_failures
from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
class BuilderTest(unittest.TestCase):
+ def _mock_test_result(self, testname):
+ failures = [test_failures.FailureTextMismatch()]
+ return test_results.TestResult(testname, failures, test_run_time=None, total_time_for_all_diffs=None, time_for_diffs=None)
+
def _install_fetch_build(self, failure):
def _mock_fetch_build(build_number):
build = Build(
@@ -42,8 +48,8 @@ class BuilderTest(unittest.TestCase):
revision=build_number + 1000,
is_green=build_number < 4
)
- parsed_results = {LayoutTestResults.fail_key: failure(build_number)}
- build._layout_test_results = LayoutTestResults(parsed_results)
+ results = [self._mock_test_result(testname) for testname in failure(build_number)]
+ build._layout_test_results = LayoutTestResults(results)
return build
self.builder._fetch_build = _mock_fetch_build
diff --git a/Tools/Scripts/webkitpy/common/net/layouttestresults.py b/Tools/Scripts/webkitpy/common/net/layouttestresults.py
index 15e95ce..fa8c39d 100644
--- a/Tools/Scripts/webkitpy/common/net/layouttestresults.py
+++ b/Tools/Scripts/webkitpy/common/net/layouttestresults.py
@@ -27,8 +27,11 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# A module for parsing results.html files generated by old-run-webkit-tests
+# This class is one big hack and only needs to exist until we transition to new-run-webkit-tests.
from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer
+from webkitpy.layout_tests.layout_package import test_results
+from webkitpy.layout_tests.layout_package import test_failures
# FIXME: This should be unified with all the layout test results code in the layout_tests package
@@ -57,36 +60,79 @@ class LayoutTestResults(object):
]
@classmethod
+ def _failures_from_fail_row(self, row):
+ # The first anchor should have already been fetched by the caller,
+ # we look at all remaining anchors in this row, and guess what type
+ # of new-run-webkit-test failures they equate to.
+ failures = set()
+ for anchor in row.findAll("a"):
+ anchor_text = anchor.string
+ if anchor_text in ["expected image", "image diffs"] or anchor_text.contains('%'):
+ failures.add(test_failures.FailureImageHashMismatch())
+ elif anchor_text in ["expected", "actual", "diff", "pretty diff"]:
+ failures.add(test_failures.FailureTextMismatch())
+ else:
+ log("Unhandled link text in results.html parsing: %s. Please file a bug against webkitpy." % anchor_text)
+ # FIXME: Its possible the row contained no links due to ORWT brokeness.
+ # We should probably assume some type of failure anyway.
+
+ @classmethod
+ def _failures_from_row(cls, row, table_title):
+ if table_title == cls.fail_key:
+ return cls._failures_from_fail_row(row)
+ if table_title == cls.crash_key:
+ return [test_failures.FailureCrash()]
+ if table_title == cls.timeout_key:
+ return [test_failures.FailureTimeout()]
+ if table_title == cls.missing_key:
+ return [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()]
+ return None
+
+ @classmethod
+ def _test_result_from_row(cls, row, table_title):
+ test_name = unicode(row.find("a").string)
+ failures = cls._failures_from_row(row, table_title)
+ # TestResult is a class designed to work with new-run-webkit-tests.
+ # old-run-webkit-tests does not save quite enough information in results.html for us to parse.
+ # FIXME: It's unclear if test_name should include LayoutTests or not.
+ return test_results.TestResult(test_name, failures, test_run_time=None, total_time_for_all_diffs=None, time_for_diffs=None)
+
+ @classmethod
+ def _parse_results_table(cls, table):
+ table_title = unicode(table.findPreviousSibling("p").string)
+ if table_title not in cls.expected_keys:
+ # This Exception should only ever be hit if run-webkit-tests changes its results.html format.
+ raise Exception("Unhandled title: %s" % table_title)
+ # Ignore stderr failures. Everyone ignores them anyway.
+ if table_title == cls.stderr_key:
+ return []
+ # FIXME: We might end with two TestResults object for the same test if it appears in more than one row.
+ return [cls._test_result_from_row(row, table_title) for row in table.findAll("tr")]
+
+ @classmethod
def _parse_results_html(cls, page):
- if not page:
- return None
- parsed_results = {}
tables = BeautifulSoup(page).findAll("table")
- for table in tables:
- table_title = unicode(table.findPreviousSibling("p").string)
- if table_title not in cls.expected_keys:
- # This Exception should only ever be hit if run-webkit-tests changes its results.html format.
- raise Exception("Unhandled title: %s" % table_title)
- # We might want to translate table titles into identifiers before storing.
- parsed_results[table_title] = [unicode(row.find("a").string) for row in table.findAll("tr")]
-
- return parsed_results
+ return sum([cls._parse_results_table(table) for table in tables], [])
@classmethod
def results_from_string(cls, string):
- parsed_results = cls._parse_results_html(string)
- if not parsed_results:
+ if not string:
+ return None
+ test_results = cls._parse_results_html(string)
+ if not test_results:
return None
- return cls(parsed_results)
+ return cls(test_results)
- def __init__(self, parsed_results):
- self._parsed_results = parsed_results
+ def __init__(self, test_results):
+ self._test_results = test_results
- def parsed_results(self):
- return self._parsed_results
+ def test_results(self):
+ return self._test_results
- def results_matching_keys(self, result_keys):
- return sorted(sum([tests for key, tests in self._parsed_results.items() if key in result_keys], []))
+ def tests_matching_failure_types(self, failure_types):
+ return [result.filename for result in self._test_results if result.has_failure_matching_types(failure_types)]
def failing_tests(self):
- return self.results_matching_keys([self.fail_key, self.crash_key, self.timeout_key])
+ # These should match the "fail", "crash", and "timeout" keys.
+ failure_types = [test_failures.FailureTextMismatch, test_failures.FailureImageHashMismatch, test_failures.FailureCrash, test_failures.FailureTimeout]
+ return self.tests_matching_failure_types(failure_types)
diff --git a/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py b/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py
index 8490eae..e0a35e8 100644
--- a/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py
+++ b/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py
@@ -29,7 +29,8 @@
import unittest
from webkitpy.common.net.layouttestresults import LayoutTestResults
-
+from webkitpy.layout_tests.layout_package import test_results
+from webkitpy.layout_tests.layout_package import test_failures
class LayoutTestResultsTest(unittest.TestCase):
_example_results_html = """
@@ -57,18 +58,13 @@ class LayoutTestResultsTest(unittest.TestCase):
</html>
"""
- _expected_layout_test_results = {
- 'Tests that had stderr output:': [
- 'accessibility/aria-activedescendant-crash.html',
- ],
- 'Tests that had no expected results (probably new):': [
- 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html',
- ],
- }
-
def test_parse_layout_test_results(self):
+ failures = [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()]
+ testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html'
+ expected_results = [test_results.TestResult(testname, failures, test_run_time=None, total_time_for_all_diffs=None, time_for_diffs=None)]
+
results = LayoutTestResults._parse_results_html(self._example_results_html)
- self.assertEqual(self._expected_layout_test_results, results)
+ self.assertEqual(expected_results, results)
def test_results_from_string(self):
self.assertEqual(LayoutTestResults.results_from_string(None), None)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py
index 6d55761..d537d50 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py
@@ -35,6 +35,9 @@ import test_expectations
import cPickle
+# FIXME: This is backwards. Each TestFailure subclass should know what
+# test_expectation type it corresponds too. Then this method just
+# collects them all from the failure list and returns the worst one.
def determine_result_type(failure_list):
"""Takes a set of test_failures and returns which result type best fits
the list of failures. "Best fits" means we use the worst type of failure.
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py
index 2417fb7..8630437 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py
@@ -57,5 +57,11 @@ class TestResult(object):
def __ne__(self, other):
return not (self == other)
+ def has_failure_matching_types(self, types):
+ for failure in self.failures:
+ if type(failure) in types:
+ return True
+ return False
+
def dumps(self):
return cPickle.dumps(self)
diff --git a/Tools/Scripts/webkitpy/tool/commands/queries.py b/Tools/Scripts/webkitpy/tool/commands/queries.py
index f04f384..733751e 100644
--- a/Tools/Scripts/webkitpy/tool/commands/queries.py
+++ b/Tools/Scripts/webkitpy/tool/commands/queries.py
@@ -272,7 +272,7 @@ class FailureReason(AbstractDeclarativeCommand):
print "%s failing" % (pluralize("builder", len(red_statuses)))
builder_choices = [status["name"] for status in red_statuses]
# We could offer an "All" choice here.
- chosen_name = User.prompt_with_list("Which builder to diagnose:", builder_choices)
+ chosen_name = self._tool.user.prompt_with_list("Which builder to diagnose:", builder_choices)
# FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object.
for status in red_statuses:
if status["name"] == chosen_name:
@@ -345,7 +345,7 @@ class FindFlakyTests(AbstractDeclarativeCommand):
def _builder_to_analyze(self):
statuses = self._tool.buildbot.builder_statuses()
choices = [status["name"] for status in statuses]
- chosen_name = User.prompt_with_list("Which builder to analyze:", choices)
+ chosen_name = self._tool.user.prompt_with_list("Which builder to analyze:", choices)
for status in statuses:
if status["name"] == chosen_name:
return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"])
diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py
index 8c4b997..34a398a 100644
--- a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py
+++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py
@@ -34,6 +34,7 @@ import urllib
from webkitpy.common.net.buildbot import BuildBot
from webkitpy.common.net.layouttestresults import LayoutTestResults
from webkitpy.common.system.user import User
+from webkitpy.layout_tests.layout_package import test_failures
from webkitpy.layout_tests.port import factory
from webkitpy.tool.grammar import pluralize
from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
@@ -88,7 +89,7 @@ class Rebaseline(AbstractDeclarativeCommand):
shutil.move(downloaded_file, local_file)
def _tests_to_update(self, build):
- failing_tests = build.layout_test_results().results_matching_keys([LayoutTestResults.fail_key])
+ failing_tests = build.layout_test_results().tests_matching_failure_types([test_failures.FailureTextMismatch])
return self._tool.user.prompt_with_list("Which test(s) to rebaseline:", failing_tests, can_choose_multiple=True)
def _results_url_for_test(self, build, test):
diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py
index d6582a7..79e4cf4 100644
--- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py
+++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py
@@ -28,7 +28,19 @@
import unittest
-from webkitpy.tool.commands.rebaseline import BuilderToPort
+from webkitpy.common.system.outputcapture import OutputCapture
+from webkitpy.thirdparty.mock import Mock
+from webkitpy.tool.commands.rebaseline import BuilderToPort, Rebaseline
+from webkitpy.tool.mocktool import MockTool
+
+
+class RebaselineTest(unittest.TestCase):
+ # This just makes sure the code runs without exceptions.
+ def test_tests_to_update(self):
+ command = Rebaseline()
+ command.bind_to_tool(MockTool())
+ build = Mock()
+ OutputCapture().assert_outputs(self, command._tests_to_update, [build])
class BuilderToPortTest(unittest.TestCase):
diff --git a/Tools/Scripts/webkitpy/tool/mocktool.py b/Tools/Scripts/webkitpy/tool/mocktool.py
index 30a4bc3..eb7c248 100644
--- a/Tools/Scripts/webkitpy/tool/mocktool.py
+++ b/Tools/Scripts/webkitpy/tool/mocktool.py
@@ -540,10 +540,14 @@ class MockCheckout(object):
class MockUser(object):
- @staticmethod
- def prompt(message, repeat=1, raw_input=raw_input):
+ @classmethod
+ def prompt(cls, message, repeat=1, raw_input=raw_input):
return "Mock user response"
+ @classmethod
+ def prompt_with_list(cls, list_title, list_items, can_choose_multiple=False, raw_input=raw_input):
+ pass
+
def edit(self, files):
pass
--
WebKit Debian packaging
More information about the Pkg-webkit-commits
mailing list