[SCM] WebKit Debian packaging branch, debian/experimental, updated. upstream/1.3.3-9427-gc2be6fc
bulach at chromium.org
bulach at chromium.org
Wed Dec 22 11:55:59 UTC 2010
The following commit has been merged in the debian/experimental branch:
commit 2d734674b1d952620cabc441f1e2728df73a5e5f
Author: bulach at chromium.org <bulach at chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Date: Wed Aug 11 18:31:00 2010 +0000
2010-08-11 Marcus Bulach <bulach at chromium.org>
Reviewed by Eric Seidel.
Check in a script to list redundant test outputs.
https://bugs.webkit.org/show_bug.cgi?id=37630
If e.g. platform/mac-leopard is missing an expected test output, we
fall back on platform/mac. This means it's possible to grow redundant
test outputs, where we have the same expected data in both a platform
directory and another platform it falls back on.
(original patch by Evan Marting <evan at chromium.org> https://bugs.webkit.org/attachment.cgi?id=53398)
* Scripts/deduplicate-tests: Added.
* Scripts/webkitpy/layout_tests/deduplicate_tests.py: Added.
* Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py: Added.
git-svn-id: http://svn.webkit.org/repository/webkit/trunk@65166 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/WebKitTools/ChangeLog b/WebKitTools/ChangeLog
index 0aa140d..cf70775 100644
--- a/WebKitTools/ChangeLog
+++ b/WebKitTools/ChangeLog
@@ -1,3 +1,20 @@
+2010-08-11 Marcus Bulach <bulach at chromium.org>
+
+ Reviewed by Eric Seidel.
+
+ Check in a script to list redundant test outputs.
+ https://bugs.webkit.org/show_bug.cgi?id=37630
+
+ If e.g. platform/mac-leopard is missing an expected test output, we
+ fall back on platform/mac. This means it's possible to grow redundant
+ test outputs, where we have the same expected data in both a platform
+ directory and another platform it falls back on.
+ (original patch by Evan Marting <evan at chromium.org> https://bugs.webkit.org/attachment.cgi?id=53398)
+
+ * Scripts/deduplicate-tests: Added.
+ * Scripts/webkitpy/layout_tests/deduplicate_tests.py: Added.
+ * Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py: Added.
+
2010-08-11 Antonio Gomes <tonikitoo at webkit.org>
Reviewed by Ariya Hidayat.
diff --git a/WebKitTools/Scripts/deduplicate-tests b/WebKitTools/Scripts/deduplicate-tests
new file mode 100644
index 0000000..11ba7c2
--- /dev/null
+++ b/WebKitTools/Scripts/deduplicate-tests
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""deduplicate-tests -- print test results duplicated between platforms.
+
+If platform/mac-leopard is missing an expected test output, we fall back on
+platform/mac. This means it's possible to grow redundant test outputs,
+where we have the same expected data in both a platform directory and another
+platform it falls back on.
+
+This command dumps out all such files. You can use it like this:
+ deduplicate-tests --verbose # print out the duplicated files
+ deduplicate-tests | xargs git rm # delete them
+"""
+
+
+import optparse
+import webkitpy.layout_tests.deduplicate_tests as deduplicate_tests
+
+
+def parse_args():
+ """Provides a default set of command line args.
+
+ Returns a tuple of options, args from optparse"""
+
+ configuration_options = [
+ optparse.make_option("-v", "--verbose", dest="verbose",
+ action="store_true", default=False,
+ help="Verbose output."),
+ optparse.make_option("-g", "--glob", dest="glob_pattern",
+ default="*-expected*",
+ help="Specify the glob to filter the files, defaults to *-expected*."),
+ ]
+
+ option_list = (configuration_options)
+ option_parser = optparse.OptionParser(option_list=option_list)
+
+ options, _ = option_parser.parse_args()
+
+ return options
+
+
+def run(options):
+ if options.verbose:
+ format = ("* %(test)s\n"
+ "\tredundantly on %(platform)s and %(fallback)s\n"
+ "\tconsider deleting %(path)s")
+ else:
+ format = "%(path)s"
+
+ for dupe in deduplicate_tests.deduplicate(options.glob_pattern):
+ print(format % dupe)
+
+
+def main():
+ options = parse_args()
+ run(options)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/deduplicate_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
new file mode 100644
index 0000000..7a9cdae
--- /dev/null
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""deduplicate_tests -- lists duplicated between platforms.
+
+If platform/mac-leopard is missing an expected test output, we fall back on
+platform/mac. This means it's possible to grow redundant test outputs,
+where we have the same expected data in both a platform directory and another
+platform it falls back on.
+"""
+
+import collections
+import fnmatch
+import os
+import subprocess
+import sys
+import re
+import webkitpy.common.system.executive as executive
+import webkitpy.common.system.logutils as logutils
+import webkitpy.layout_tests.port.factory as port_factory
+
+_log = logutils.get_logger(__file__)
+logutils.configure_logging()
+
+_BASE_PLATFORM = 'base'
+
+
+def port_fallbacks():
+ """Get the port fallback information.
+ Returns:
+ A dictionary mapping platform name to a list of other platforms to fall
+ back on. All platforms fall back on 'base'.
+ """
+ fallbacks = {_BASE_PLATFORM: []}
+ for port_name in os.listdir(os.path.join('LayoutTests', 'platform')):
+ try:
+ platforms = port_factory.get(port_name).baseline_search_path()
+ except NotImplementedError:
+ _log.error("'%s' lacks baseline_search_path(), please fix." % port_name)
+ fallbacks[port_name] = [_BASE_PLATFORM]
+ continue
+ fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:]
+ fallbacks[port_name].append(_BASE_PLATFORM)
+ return fallbacks
+
+
+def parse_git_output(git_output, glob_pattern):
+ """Parses the output of git ls-tree and filters based on glob_pattern.
+ Args:
+ git_output: result of git ls-tree -r HEAD LayoutTests.
+ glob_pattern: a pattern to filter the files.
+ Returns:
+ A dictionary mapping (test name, hash of content) => [paths]
+ """
+ hashes = collections.defaultdict(set)
+ for line in git_output.split('\n'):
+ if not line:
+ break
+ attrs, path = line.strip().split('\t')
+ if not fnmatch.fnmatch(path, glob_pattern):
+ continue
+ path = path[len('LayoutTests/'):]
+ match = re.match(r'^(platform/.*?/)?(.*)', path)
+ test = match.group(2)
+ _, _, hash = attrs.split(' ')
+ hashes[(test, hash)].add(path)
+ return hashes
+
+
+def cluster_file_hashes(glob_pattern):
+ """Get the hashes of all the test expectations in the tree.
+ We cheat and use git's hashes.
+ Args:
+ glob_pattern: a pattern to filter the files.
+ Returns:
+ A dictionary mapping (test name, hash of content) => [paths]
+ """
+
+ # A map of file hash => set of all files with that hash.
+ hashes = collections.defaultdict(set)
+
+ # Fill in the map.
+ cmd = ('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests')
+ try:
+ git_output = executive.Executive().run_command(cmd)
+ except OSError, e:
+ if e.errno == 2: # No such file or directory.
+ _log.error("Error: 'No such file' when running git.")
+ _log.error("This script requires git.")
+ sys.exit(1)
+ raise e
+ return parse_git_output(git_output, glob_pattern)
+
+
+def extract_platforms(paths):
+ """Extracts the platforms from a list of paths matching ^platform/(.*?)/.
+ Args:
+ paths: a list of paths.
+ Returns:
+ A dictionary containing all platforms from paths.
+ """
+ platforms = {}
+ for path in paths:
+ match = re.match(r'^platform/(.*?)/', path)
+ if match:
+ platform = match.group(1)
+ else:
+ platform = _BASE_PLATFORM
+ platforms[platform] = path
+ return platforms
+
+
+def find_dups(hashes, port_fallbacks):
+ """Yields info about redundant test expectations.
+ Args:
+ hashes: a list of hashes as returned by cluster_file_hashes.
+ port_fallbacks: a list of fallback information as returned by get_port_fallbacks.
+ Returns:
+ a tuple containing (test, platform, fallback, platforms)
+ """
+ for (test, hash), cluster in hashes.items():
+ if len(cluster) < 2:
+ continue # Common case: only one file with that hash.
+
+ # Compute the list of platforms we have this particular hash for.
+ platforms = extract_platforms(cluster)
+ if len(platforms) == 1:
+ continue
+
+ # See if any of the platforms are redundant with each other.
+ for platform in platforms.keys():
+ for fallback in port_fallbacks[platform]:
+ if fallback in platforms.keys():
+ yield test, platform, fallback, platforms[platform]
+
+
+def deduplicate(glob_pattern):
+ """Traverses LayoutTests and returns information about duplicated files.
+ Args:
+ glob pattern to filter the files in LayoutTests.
+ Returns:
+ a dictionary containing test, path, platform and fallback.
+ """
+ fallbacks = port_fallbacks()
+ hashes = cluster_file_hashes(glob_pattern)
+ return [{'test': test, 'path': path, 'platform': platform, 'fallback': fallback}
+ for test, platform, fallback, path in find_dups(hashes, fallbacks)]
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py
new file mode 100644
index 0000000..878331a
--- /dev/null
+++ b/WebKitTools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for deduplicate_tests.py."""
+
+import deduplicate_tests
+import os
+import unittest
+
+
+class MockExecutive(object):
+ last_run_command = []
+ response = ''
+
+ class Executive(object):
+ def run_command(self,
+ args,
+ cwd=None,
+ input=None,
+ error_handler=None,
+ return_exit_code=False,
+ return_stderr=True,
+ decode_output=True):
+ MockExecutive.last_run_command += [args]
+ return MockExecutive.response
+
+
+class ListDuplicatesTest(unittest.TestCase):
+ def setUp(self):
+ MockExecutive.last_run_command = []
+ MockExecutive.response = ''
+ deduplicate_tests.executive = MockExecutive
+
+ def test_parse_git_output(self):
+ git_output = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+ hashes = deduplicate_tests.parse_git_output(git_output, '*')
+ expected = {('mac/foo-expected.txt', '5053240b3353f6eb39f7cb00259785f16d121df2'): set(['mac/foo-expected.txt']),
+ ('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png']),
+ ('foo-expected.txt', '4303df5389ca87cae83dd3236b8dd84e16606517'): set(['platform/mac/foo-expected.txt']),
+ ('foo-expected.txt', 'd6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/foo-expected.txt', 'platform/chromium-win/foo-expected.txt']),
+ ('foo-expected.txt', 'a004548d107ecc4e1ea08019daf0a14e8634a1ff'): set(['platform/chromium/foo-expected.txt'])}
+ self.assertEquals(expected, hashes)
+
+ hashes = deduplicate_tests.parse_git_output(git_output, '*.png')
+ expected = {('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png'])}
+ self.assertEquals(expected, hashes)
+
+ def test_extract_platforms(self):
+ self.assertEquals({'foo': 'platform/foo/bar',
+ 'zoo': 'platform/zoo/com'},
+ deduplicate_tests.extract_platforms(['platform/foo/bar', 'platform/zoo/com']))
+ self.assertEquals({'foo': 'platform/foo/bar',
+ deduplicate_tests._BASE_PLATFORM: 'what/'},
+ deduplicate_tests.extract_platforms(['platform/foo/bar', 'what/']))
+
+ def test_unique(self):
+ MockExecutive.response = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob abcd0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+ result = deduplicate_tests.deduplicate('*')
+ self.assertEquals(1, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(0, len(result))
+
+ def test_duplicates(self):
+ MockExecutive.response = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+
+ result = deduplicate_tests.deduplicate('*')
+ self.assertEquals(1, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(2, len(result))
+ self.assertEquals({'test': 'animage.png',
+ 'path': 'platform/chromium-linux/animage.png',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[0])
+ self.assertEquals({'test': 'foo-expected.txt',
+ 'path': 'platform/chromium-linux/foo-expected.txt',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[1])
+
+ result = deduplicate_tests.deduplicate('*.txt')
+ self.assertEquals(2, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(1, len(result))
+ self.assertEquals({'test': 'foo-expected.txt',
+ 'path': 'platform/chromium-linux/foo-expected.txt',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[0])
+
+ result = deduplicate_tests.deduplicate('*.png')
+ self.assertEquals(3, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(1, len(result))
+ self.assertEquals({'test': 'animage.png',
+ 'path': 'platform/chromium-linux/animage.png',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[0])
--
WebKit Debian packaging
More information about the Pkg-webkit-commits
mailing list