[SCM] WebKit Debian packaging branch, debian/experimental, updated. upstream/1.3.3-9427-gc2be6fc
victorw at chromium.org
victorw at chromium.org
Wed Dec 22 11:45:22 UTC 2010
The following commit has been merged in the debian/experimental branch:
commit 2c46abc912981888be21596e715d572355793680
Author: victorw at chromium.org <victorw at chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Date: Thu Aug 5 21:24:00 2010 +0000
2010-08-05 Victor Wang <victorw at chromium.org>
Reviewed by Ojan Vafai.
Add option to generate/upload incremental json results to test result server.
Also refactor the json results generator unittest code to test
incremental and aggregated json results.
https://bugs.webkit.org/show_bug.cgi?id=43519
* Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
* Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
* Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py:
* Scripts/webkitpy/layout_tests/run_webkit_tests.py:
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@64786 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/WebKitTools/ChangeLog b/WebKitTools/ChangeLog
index 656a076..256cbca 100644
--- a/WebKitTools/ChangeLog
+++ b/WebKitTools/ChangeLog
@@ -1,3 +1,18 @@
+2010-08-05 Victor Wang <victorw at chromium.org>
+
+ Reviewed by Ojan Vafai.
+
+ Add option to generate/upload incremental json results to test result server.
+ Also refactor the json results generator unittest code to test
+ incremental and aggregated json results.
+
+ https://bugs.webkit.org/show_bug.cgi?id=43519
+
+ * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
+ * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
+ * Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py:
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+
2010-08-05 Jian Li <jianli at chromium.org>
Reviewed by David Levin.
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
index c0525ea..6c36c93 100644
--- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
@@ -56,7 +56,8 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase
def __init__(self, port, builder_name, build_name, build_number,
results_file_base_path, builder_base_url,
- test_timings, expectations, result_summary, all_tests):
+ test_timings, expectations, result_summary, all_tests,
+ generate_incremental_results=False):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
@@ -66,7 +67,8 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase
"""
super(JSONLayoutResultsGenerator, self).__init__(
builder_name, build_name, build_number, results_file_base_path,
- builder_base_url, {}, port.test_repository_paths())
+ builder_base_url, {}, port.test_repository_paths(),
+ generate_incremental_results)
self._port = port
self._expectations = expectations
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
index 595fc2b..d331527 100644
--- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
@@ -82,10 +82,12 @@ class JSONResultsGeneratorBase(object):
ALL_FIXABLE_COUNT = "allFixableCount"
RESULTS_FILENAME = "results.json"
+ INCREMENTAL_RESULTS_FILENAME = "incremental_results.json"
def __init__(self, builder_name, build_name, build_number,
results_file_base_path, builder_base_url,
- test_results_map, svn_repositories=None):
+ test_results_map, svn_repositories=None,
+ generate_incremental_results=False):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
@@ -108,41 +110,59 @@ class JSONResultsGeneratorBase(object):
self._builder_base_url = builder_base_url
self._results_file_path = os.path.join(results_file_base_path,
self.RESULTS_FILENAME)
+ self._incremental_results_file_path = os.path.join(
+ results_file_base_path, self.INCREMENTAL_RESULTS_FILENAME)
self._test_results_map = test_results_map
self._test_results = test_results_map.values()
+ self._generate_incremental_results = generate_incremental_results
self._svn_repositories = svn_repositories
if not self._svn_repositories:
self._svn_repositories = {}
self._json = None
+ self._archived_results = None
def generate_json_output(self):
"""Generates the JSON output file."""
+
+ # Generate the JSON output file that has full results.
+ # FIXME: stop writing out the full results file once all bots use
+ # incremental results.
if not self._json:
self._json = self.get_json()
if self._json:
- # Specify separators in order to get compact encoding.
- json_data = simplejson.dumps(self._json, separators=(',', ':'))
- json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX
+ self._generate_json_file(self._json, self._results_file_path)
- results_file = codecs.open(self._results_file_path, "w", "utf-8")
- results_file.write(json_string)
- results_file.close()
+ # Generate the JSON output file that only has incremental results.
+ if self._generate_incremental_results:
+ json = self.get_json(incremental=True)
+ if json:
+ self._generate_json_file(
+ json, self._incremental_results_file_path)
- def get_json(self):
+ def get_json(self, incremental=False):
"""Gets the results for the results.json file."""
- if self._json:
- return self._json
-
- results_json, error = self._get_archived_json_results()
- if error:
- # If there was an error don't write a results.json
- # file at all as it would lose all the information on the bot.
- _log.error("Archive directory is inaccessible. Not modifying "
- "or clobbering the results.json file: " + str(error))
- return None
+ if incremental:
+ results_json = {}
+ else:
+ if self._json:
+ return self._json
+
+ if not self._archived_results:
+ self._archived_results, error = \
+ self._get_archived_json_results()
+ if error:
+ # If there was an error don't write a results.json
+ # file at all as it would lose all the information on the
+ # bot.
+ _log.error("Archive directory is inaccessible. Not "
+ "modifying or clobbering the results.json "
+ "file: " + str(error))
+ return None
+
+ results_json = self._archived_results
builder_name = self._builder_name
if results_json and builder_name not in results_json:
@@ -168,8 +188,19 @@ class JSONResultsGeneratorBase(object):
for test in all_failing_tests:
self._insert_test_time_and_result(test, tests)
- self._json = results_json
- return self._json
+ return results_json
+
+ def set_archived_results(self, archived_results):
+ self._archived_results = archived_results
+
+ def _generate_json_file(self, json, file_path):
+ # Specify separators in order to get compact encoding.
+ json_data = simplejson.dumps(json, separators=(',', ':'))
+ json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX
+
+ results_file = codecs.open(file_path, "w", "utf-8")
+ results_file.write(json_string)
+ results_file.close()
def _get_test_timing(self, test_name):
"""Returns test timing data (elapsed time) in second
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
index 0a60cc7..785cc1c 100644
--- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
@@ -45,6 +45,13 @@ class JSONGeneratorTest(unittest.TestCase):
self.builder_name = 'DUMMY_BUILDER_NAME'
self.build_name = 'DUMMY_BUILD_NAME'
self.build_number = 'DUMMY_BUILDER_NUMBER'
+ self._json = None
+ self._num_runs = 0
+ self._tests_list = set([])
+ self._test_timings = {}
+ self._failed_tests = {}
+ self._passed_tests = set([])
+ self._skipped_tests = set([])
def _test_json_generation(self, passed_tests, failed_tests, skipped_tests):
# Make sure we have sets (rather than lists).
@@ -52,12 +59,13 @@ class JSONGeneratorTest(unittest.TestCase):
skipped_tests = set(skipped_tests)
tests_list = passed_tests | set(failed_tests.keys())
test_timings = {}
+ i = 0
for test in tests_list:
- test_timings[test] = float(random.randint(1, 10))
+ test_timings[test] = float(self._num_runs * 100 + i)
+ i += 1
port_obj = port.get(None)
- # Generate a JSON file.
generator = json_results_generator.JSONResultsGenerator(port_obj,
self.builder_name, self.build_name, self.build_number,
'',
@@ -68,8 +76,39 @@ class JSONGeneratorTest(unittest.TestCase):
skipped_tests,
tests_list)
- json = generator.get_json()
-
+ # Test incremental json results
+ incremental_json = generator.get_json(incremental=True)
+ self._verify_json_results(
+ tests_list,
+ test_timings,
+ passed_tests,
+ failed_tests,
+ skipped_tests,
+ incremental_json,
+ 1)
+
+ # Test aggregated json results
+ generator.set_archived_results(self._json)
+ json = generator.get_json(incremental=False)
+ self._json = json
+ self._num_runs += 1
+ self._tests_list |= tests_list
+ self._test_timings.update(test_timings)
+ self._failed_tests.update(failed_tests)
+ self._passed_tests |= passed_tests
+ self._skipped_tests |= skipped_tests
+ self._verify_json_results(
+ self._tests_list,
+ self._test_timings,
+ self._passed_tests,
+ self._failed_tests,
+ self._skipped_tests,
+ self._json,
+ self._num_runs)
+
+ def _verify_json_results(self, tests_list, test_timings,
+ passed_tests, failed_tests,
+ skipped_tests, json, num_runs):
# Aliasing to a short name for better access to its constants.
JRG = json_results_generator.JSONResultsGenerator
@@ -79,11 +118,18 @@ class JSONGeneratorTest(unittest.TestCase):
buildinfo = json[self.builder_name]
self.assertTrue(JRG.FIXABLE in buildinfo)
self.assertTrue(JRG.TESTS in buildinfo)
- self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == 1)
+ self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == num_runs)
self.assertTrue(buildinfo[JRG.BUILD_NUMBERS][0] == self.build_number)
- if tests_list or skipped_tests:
- fixable = buildinfo[JRG.FIXABLE][0]
+ if tests_list or skipped_tests:
+ fixable = {}
+ for fixable_items in buildinfo[JRG.FIXABLE]:
+ for (type, count) in fixable_items.iteritems():
+ if type in fixable:
+ fixable[type] = fixable[type] + count
+ else:
+ fixable[type] = count
+
if passed_tests:
self.assertTrue(fixable[JRG.PASS_RESULT] == len(passed_tests))
else:
@@ -100,26 +146,35 @@ class JSONGeneratorTest(unittest.TestCase):
for test_name, failure in failed_tests.iteritems():
self.assertTrue(test_name in tests)
test = tests[test_name]
- self.assertTrue(test[JRG.RESULTS][0][0] == 1)
- self.assertTrue(test[JRG.RESULTS][0][1] == JRG.FAIL_RESULT)
- self.assertTrue(test[JRG.TIMES][0][0] == 1)
- self.assertTrue(test[JRG.TIMES][0][1] ==
- int(test_timings[test_name]))
+
+ failed = 0
+ for result in test[JRG.RESULTS]:
+ if result[1] == JRG.FAIL_RESULT:
+ failed = result[0]
+ self.assertTrue(failed == 1)
+
+ timing_count = 0
+ for timings in test[JRG.TIMES]:
+ if timings[1] == test_timings[test_name]:
+ timing_count = timings[0]
+ self.assertTrue(timing_count == 1)
fixable_count = len(skipped_tests) + len(failed_tests.keys())
if skipped_tests or failed_tests:
- self.assertTrue(buildinfo[JRG.FIXABLE_COUNT][0] == fixable_count)
+ self.assertTrue(sum(buildinfo[JRG.FIXABLE_COUNT]) == fixable_count)
def test_json_generation(self):
reason = test_expectations.TEXT
self._test_json_generation([], {}, [])
- self._test_json_generation(['A', 'B'], {}, [])
- self._test_json_generation([], {'A': reason, 'B': reason}, [])
- self._test_json_generation([], {}, ['A', 'B'])
- self._test_json_generation(['A'], {'B': reason, 'C': reason}, [])
- self._test_json_generation([], {'A': reason, 'B': reason}, ['C', 'D'])
- self._test_json_generation(['A', 'B', 'C'], {'D': reason}, ['E', 'F'])
+ self._test_json_generation(['A1', 'B1'], {}, [])
+ self._test_json_generation([], {'A2': reason, 'B2': reason}, [])
+ self._test_json_generation([], {}, ['A3', 'B3'])
+ self._test_json_generation(['A4'], {'B4': reason, 'C4': reason}, [])
+ self._test_json_generation(
+ [], {'A5': reason, 'B5': reason}, ['C5', 'D5'])
+ self._test_json_generation(
+ ['A6', 'B6', 'C6'], {'D6': reason}, ['E6', 'F6'])
if __name__ == '__main__':
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
index 41aab62..57383fb 100755
--- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -889,7 +889,8 @@ class TestRunner:
self._port, self._options.builder_name, self._options.build_name,
self._options.build_number, self._options.results_directory,
BUILDER_BASE_URL, individual_test_timings,
- self._expectations, result_summary, self._test_files_list)
+ self._expectations, result_summary, self._test_files_list,
+ self._options.upload_incremental_results)
_log.debug("Finished writing JSON files.")
@@ -900,8 +901,12 @@ class TestRunner:
_log.info("Uploading JSON files for builder: %s",
self._options.builder_name)
- attrs = [('builder', self._options.builder_name)]
- json_files = ["expectations.json", "results.json"]
+ attrs = [("builder", self._options.builder_name)]
+ json_files = ["expectations.json"]
+ if self._options.upload_incremental_results:
+ json_files.append("incremental_results.json")
+ else:
+ json_files.append("results.json")
files = [(file, os.path.join(self._options.results_directory, file))
for file in json_files]
@@ -1656,6 +1661,10 @@ def parse_args(args=None):
optparse.make_option("--test-results-server", default="",
help=("If specified, upload results json files to this appengine "
"server.")),
+ optparse.make_option("--upload-incremental-results",
+ action="store_true",
+ default=False,
+ help="If true, upload incremental json results to server."),
]
option_list = (configuration_options + print_options +
--
WebKit Debian packaging
More information about the Pkg-webkit-commits
mailing list