[Reproducible-commits] [diffoscope] 05/05: Allow .get_reverse() on Futures

Joachim Breitner nomeata at moszumanska.debian.org
Thu Dec 3 12:19:19 UTC 2015


This is an automated email from the git hooks/post-receive script.

nomeata pushed a commit to branch pu/parallel2
in repository diffoscope.

commit ed03918f21f96693b3f119b5d198f92aee8bd629
Author: Joachim Breitner <mail at joachim-breitner.de>
Date:   Thu Dec 3 13:12:50 2015 +0100

    Allow .get_reverse() on Futures
    
    by introducing the FutureDifference object, which defers the
    .get_reverse call until result() is going to be called..
---
 diffoscope/comparators/utils.py    |  4 ++--
 diffoscope/difference.py           | 35 +++++++++++++++++++++++++++++++++--
 tests/comparators/test_epub.py     |  6 +++---
 tests/comparators/test_gzip.py     |  8 ++++----
 tests/comparators/test_ipk.py      |  6 +++---
 tests/comparators/test_iso9660.py  |  6 +++---
 tests/comparators/test_rpm.py      |  6 +++---
 tests/comparators/test_squashfs.py |  8 ++++----
 tests/comparators/test_tar.py      |  8 ++++----
 tests/comparators/test_utils.py    |  8 ++++----
 tests/comparators/test_xz.py       |  8 ++++----
 tests/comparators/test_zip.py      |  6 +++---
 12 files changed, 70 insertions(+), 39 deletions(-)

diff --git a/diffoscope/comparators/utils.py b/diffoscope/comparators/utils.py
index 9c06f7a..995eebe 100644
--- a/diffoscope/comparators/utils.py
+++ b/diffoscope/comparators/utils.py
@@ -31,7 +31,7 @@ from threading import Thread
 import diffoscope.comparators
 from diffoscope.comparators.binary import File, NonExistingFile
 from diffoscope.config import Config
-from diffoscope.difference import Difference
+from diffoscope.difference import FutureDifference, Difference
 from diffoscope import logger, tool_required, get_temporary_directory
 
 
@@ -194,7 +194,7 @@ class Container(object, metaclass=ABCMeta):
                 yield NonExistingFile('/dev/null', other_file), other_file, NO_NOTIFICATION
 
     def compare(self, other, source=None):
-        return [Config.general.executor.submit(diffoscope.comparators.compare_files_with_notification, *args) for args in self.comparisons(other)]
+        return [FutureDifference(Config.general.executor.submit(diffoscope.comparators.compare_files_with_notification, *args)) for args in self.comparisons(other)]
 
 
 class ArchiveMember(File):
diff --git a/diffoscope/difference.py b/diffoscope/difference.py
index 6728718..4f80307 100644
--- a/diffoscope/difference.py
+++ b/diffoscope/difference.py
@@ -271,6 +271,37 @@ def diff(feeder1, feeder2):
         with fd_from_feeder(feeder2, end_nl_q2) as fd2:
             return run_diff(fd1, fd2, end_nl_q1, end_nl_q2)
 
+class FutureDifference(object):
+    """
+    We usually produce Futures in place of Differences, for parallelization.
+    These are never looked at until all of them are produced. Then
+    finish_threads is called, which replaces them by their .result().
+
+    There is, however, one operation that we do on Differences before:
+    get_reverse(). So this object allows a way to defer that to reverse time.
+    """
+
+    def __init__(self, future):
+        self._is_reversed = False
+        assert (isinstance(future, Future) or isinstance(future, FutureDifference))
+        self._future = future
+        self._difference = None
+
+    def result(self,*args):
+        if self._difference is None:
+            self._difference = self._future.result(*args)
+
+            if self._is_reversed:
+                self._difference = self._difference.get_reverse()
+
+        return self._difference
+
+    def get_reverse(self):
+        reversed_future = FutureDifference(self)
+        reversed_future._is_reverse = True
+        return reversed_future
+
+
 
 class Difference(object):
     def __init__(self, path1, path2, source=None, notification=None, comment=None):
@@ -386,7 +417,7 @@ class Difference(object):
     def finish_threads(self):
         finished_details = []
         for detail in self._details:
-            if isinstance(detail, Future):
+            if isinstance(detail, FutureDifference):
                 detail = detail.result()
             if not detail:
                 continue
@@ -416,7 +447,7 @@ class Difference(object):
 
     @property
     def unified_diff(self):
-        #if isinstance(self._unified_diff, Future):
+        #if isinstance(self._unified_diff, FutureDifference):
         #    self._unified_diff = self._unified_diff.result()
         return self._unified_diff
 
diff --git a/tests/comparators/test_epub.py b/tests/comparators/test_epub.py
index fe58f4b..f081882 100644
--- a/tests/comparators/test_epub.py
+++ b/tests/comparators/test_epub.py
@@ -40,12 +40,12 @@ def test_identification(epub1):
     assert isinstance(epub1, ZipFile)
 
 def test_no_differences(epub1):
-    difference = epub1.compare(epub1)
+    difference = epub1.synchronized_compare(epub1)
     assert not difference
 
 @pytest.fixture
 def differences(epub1, epub2):
-    return epub1.compare(epub2).details
+    return epub1.synchronized_compare(epub2).details
 
 @pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
 def test_differences(differences):
@@ -63,6 +63,6 @@ def test_differences(differences):
 @pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
 def test_compare_non_existing(monkeypatch, epub1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = epub1.compare(NonExistingFile('/nonexisting', epub1))
+    difference = epub1.synchronized_compare(NonExistingFile('/nonexisting', epub1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_gzip.py b/tests/comparators/test_gzip.py
index b6aab9b..5c5af14 100644
--- a/tests/comparators/test_gzip.py
+++ b/tests/comparators/test_gzip.py
@@ -40,12 +40,12 @@ def test_identification(gzip1):
     assert isinstance(gzip1, GzipFile)
 
 def test_no_differences(gzip1):
-    difference = gzip1.compare(gzip1)
+    difference = gzip1.synchronized_compare(gzip1)
     assert not difference
 
 @pytest.fixture
 def differences(gzip1, gzip2):
-    return gzip1.compare(gzip2).details
+    return gzip1.synchronized_compare(gzip2).details
 
 def test_metadata(differences):
     assert differences[0].source1 == 'metadata'
@@ -64,7 +64,7 @@ def test_content_source_without_extension(tmpdir):
     shutil.copy(TEST_FILE2_PATH, path2)
     gzip1 = specialize(FilesystemFile(path1))
     gzip2 = specialize(FilesystemFile(path2))
-    difference = gzip1.compare(gzip2).details
+    difference = gzip1.synchronized_compare(gzip2).details
     assert difference[1].source1 == 'test1-content'
     assert difference[1].source2 == 'test2-content'
 
@@ -74,6 +74,6 @@ def test_content_diff(differences):
 
 def test_compare_non_existing(monkeypatch, gzip1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = gzip1.compare(NonExistingFile('/nonexisting', gzip1))
+    difference = gzip1.synchronized_compare(NonExistingFile('/nonexisting', gzip1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_ipk.py b/tests/comparators/test_ipk.py
index 276fe56..6b1a4f9 100644
--- a/tests/comparators/test_ipk.py
+++ b/tests/comparators/test_ipk.py
@@ -39,12 +39,12 @@ def test_identification(ipk1):
     assert isinstance(ipk1, IpkFile)
 
 def test_no_differences(ipk1):
-    difference = ipk1.compare(ipk1)
+    difference = ipk1.synchronized_compare(ipk1)
     assert not difference
 
 @pytest.fixture
 def differences(ipk1, ipk2):
-    return ipk1.compare(ipk2).details
+    return ipk1.synchronized_compare(ipk2).details
 
 def test_metadata(differences):
     assert differences[0].source1 == 'metadata'
@@ -57,6 +57,6 @@ def test_compressed_files(differences):
 
 def test_compare_non_existing(monkeypatch, ipk1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = ipk1.compare(NonExistingFile('/nonexisting', ipk1))
+    difference = ipk1.synchronized_compare(NonExistingFile('/nonexisting', ipk1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_iso9660.py b/tests/comparators/test_iso9660.py
index 07dcb12..bab5948 100644
--- a/tests/comparators/test_iso9660.py
+++ b/tests/comparators/test_iso9660.py
@@ -40,12 +40,12 @@ def test_identification(iso1):
     assert isinstance(iso1, Iso9660File)
 
 def test_no_differences(iso1):
-    difference = iso1.compare(iso1)
+    difference = iso1.synchronized_compare(iso1)
     assert not difference
 
 @pytest.fixture
 def differences(iso1, iso2):
-    return iso1.compare(iso2).details
+    return iso1.synchronized_compare(iso2).details
 
 @pytest.mark.skipif(tool_missing('isoinfo'), reason='missing isoinfo')
 def test_iso9660_content(differences):
@@ -73,6 +73,6 @@ def test_compressed_files(differences):
 @pytest.mark.skipif(tool_missing('isoinfo'), reason='missing isoinfo')
 def test_compare_non_existing(monkeypatch, iso1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = iso1.compare(NonExistingFile('/nonexisting', iso1))
+    difference = iso1.synchronized_compare(NonExistingFile('/nonexisting', iso1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_rpm.py b/tests/comparators/test_rpm.py
index 501ce85..10b8015 100644
--- a/tests/comparators/test_rpm.py
+++ b/tests/comparators/test_rpm.py
@@ -46,12 +46,12 @@ def test_identification(rpm1):
 
 @pytest.mark.skipif(miss_rpm_module, reason='rpm module is not installed')
 def test_no_differences(rpm1):
-    difference = rpm1.compare(rpm1)
+    difference = rpm1.synchronized_compare(rpm1)
     assert not difference
 
 @pytest.fixture
 def differences(rpm1, rpm2):
-    return rpm1.compare(rpm2).details
+    return rpm1.synchronized_compare(rpm2).details
 
 @pytest.mark.skipif(miss_rpm_module, reason='rpm module is not installed')
 @pytest.mark.skipif(tool_missing('rpm2cpio'), reason='missing rpm2cpio')
@@ -80,6 +80,6 @@ def test_content(differences):
 @pytest.mark.skipif(tool_missing('rpm2cpio'), reason='missing rpm2cpio')
 def test_compare_non_existing(monkeypatch, rpm1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = rpm1.compare(NonExistingFile('/nonexisting', rpm1))
+    difference = rpm1.synchronized_compare(NonExistingFile('/nonexisting', rpm1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_squashfs.py b/tests/comparators/test_squashfs.py
index c735c0d..cadcaf7 100644
--- a/tests/comparators/test_squashfs.py
+++ b/tests/comparators/test_squashfs.py
@@ -41,17 +41,17 @@ def test_identification(squashfs1):
     assert isinstance(squashfs1, SquashfsFile)
 
 def test_no_differences(squashfs1):
-    difference = squashfs1.compare(squashfs1)
+    difference = squashfs1.synchronized_compare(squashfs1)
     assert not difference
 
 def test_no_warnings(capfd, squashfs1, squashfs2):
-    _ = squashfs1.compare(squashfs2)
+    _ = squashfs1.synchronized_compare(squashfs2)
     _, err = capfd.readouterr()
     assert err == ''
 
 @pytest.fixture
 def differences(squashfs1, squashfs2):
-    return squashfs1.compare(squashfs2).details
+    return squashfs1.synchronized_compare(squashfs2).details
 
 @pytest.mark.skipif(tool_missing('unsquashfs'), reason='missing unsquashfs')
 def test_superblock(differences):
@@ -81,6 +81,6 @@ def test_compressed_files(differences):
 @pytest.mark.skipif(tool_missing('unsquashfs'), reason='missing unsquashfs')
 def test_compare_non_existing(monkeypatch, squashfs1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = squashfs1.compare(NonExistingFile('/nonexisting', squashfs1))
+    difference = squashfs1.synchronized_compare(NonExistingFile('/nonexisting', squashfs1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_tar.py b/tests/comparators/test_tar.py
index e432cce..f14c7c4 100644
--- a/tests/comparators/test_tar.py
+++ b/tests/comparators/test_tar.py
@@ -39,12 +39,12 @@ def test_identification(tar1):
     assert isinstance(tar1, TarFile)
 
 def test_no_differences(tar1):
-    difference = tar1.compare(tar1)
+    difference = tar1.synchronized_compare(tar1)
     assert not difference
 
 @pytest.fixture
 def differences(tar1, tar2):
-    return tar1.compare(tar2).details
+    return tar1.synchronized_compare(tar2).details
 
 def test_listing(differences):
     expected_diff = open(os.path.join(os.path.dirname(__file__), '../data/tar_listing_expected_diff')).read()
@@ -65,7 +65,7 @@ def test_text_file(differences):
 
 def test_compare_non_existing(monkeypatch, tar1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = tar1.compare(NonExistingFile('/nonexisting', tar1))
+    difference = tar1.synchronized_compare(NonExistingFile('/nonexisting', tar1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
 
@@ -79,4 +79,4 @@ def test_no_permissions_dir_in_tarball(monkeypatch, no_permissions_tar):
     # We want to make sure OSError is not raised.
     # Comparing with non-existing file makes it easy to make sure all files are unpacked
     monkeypatch.setattr(Config, 'new_file', True)
-    no_permissions_tar.compare(NonExistingFile('/nonexistent', no_permissions_tar))
+    no_permissions_tar.synchronized_compare(NonExistingFile('/nonexistent', no_permissions_tar))
diff --git a/tests/comparators/test_utils.py b/tests/comparators/test_utils.py
index 010e654..76eaf9b 100644
--- a/tests/comparators/test_utils.py
+++ b/tests/comparators/test_utils.py
@@ -46,7 +46,7 @@ def fuzzy_tar3():
 
 @pytest.mark.skipif(miss_tlsh, reason='tlsh is missing')
 def test_fuzzy_matching(fuzzy_tar1, fuzzy_tar2):
-    differences = fuzzy_tar1.compare(fuzzy_tar2).details
+    differences = fuzzy_tar1.synchronized_compare(fuzzy_tar2).details
     expected_diff = codecs.open(os.path.join(os.path.dirname(__file__), '../data/text_iso8859_expected_diff'), encoding='utf-8').read()
     assert differences[1].source1 == './matching'
     assert differences[1].source2 == './fuzzy'
@@ -55,7 +55,7 @@ def test_fuzzy_matching(fuzzy_tar1, fuzzy_tar2):
 
 @pytest.mark.skipif(miss_tlsh, reason='tlsh is missing')
 def test_fuzzy_matching_only_once(fuzzy_tar1, fuzzy_tar3):
-    differences = fuzzy_tar1.compare(fuzzy_tar3).details
+    differences = fuzzy_tar1.synchronized_compare(fuzzy_tar3).details
     assert len(differences) == 2
     expected_diff = codecs.open(os.path.join(os.path.dirname(__file__), '../data/text_iso8859_expected_diff'), encoding='utf-8').read()
 
@@ -70,7 +70,7 @@ def fuzzy_tar_in_tar2():
 @pytest.mark.skipif(miss_tlsh, reason='tlsh is missing')
 def test_no_fuzzy_matching(monkeypatch, fuzzy_tar_in_tar1, fuzzy_tar_in_tar2):
     monkeypatch.setattr(Config, 'fuzzy_threshold', 0)
-    difference = fuzzy_tar_in_tar1.compare(fuzzy_tar_in_tar2)
+    difference = fuzzy_tar_in_tar1.synchronized_compare(fuzzy_tar_in_tar2)
     assert len(difference.details) == 1
     assert difference.details[0].source1 == 'tar --full-time -tvf {}'
 
@@ -78,7 +78,7 @@ def test_no_fuzzy_matching(monkeypatch, fuzzy_tar_in_tar1, fuzzy_tar_in_tar2):
 def test_no_fuzzy_matching_new_file(monkeypatch, fuzzy_tar_in_tar1, fuzzy_tar_in_tar2):
     monkeypatch.setattr(Config, 'fuzzy_threshold', 0)
     monkeypatch.setattr(Config, 'new_file', True)
-    difference = fuzzy_tar_in_tar1.compare(fuzzy_tar_in_tar2)
+    difference = fuzzy_tar_in_tar1.synchronized_compare(fuzzy_tar_in_tar2)
     assert len(difference.details) == 3
     assert difference.details[1].source2 == '/dev/null'
     assert difference.details[2].source1 == '/dev/null'
diff --git a/tests/comparators/test_xz.py b/tests/comparators/test_xz.py
index 494cc33..8fb80b7 100644
--- a/tests/comparators/test_xz.py
+++ b/tests/comparators/test_xz.py
@@ -41,12 +41,12 @@ def test_identification(xz1):
     assert isinstance(xz1, XzFile)
 
 def test_no_differences(xz1):
-    difference = xz1.compare(xz1)
+    difference = xz1.synchronized_compare(xz1)
     assert not difference
 
 @pytest.fixture
 def differences(xz1, xz2):
-    return xz1.compare(xz2).details
+    return xz1.synchronized_compare(xz2).details
 
 @pytest.mark.skipif(tool_missing('xz'), reason='missing xz')
 def test_content_source(differences):
@@ -61,7 +61,7 @@ def test_content_source_without_extension(tmpdir):
     shutil.copy(TEST_FILE2_PATH, path2)
     xz1 = specialize(FilesystemFile(path1))
     xz2 = specialize(FilesystemFile(path2))
-    difference = xz1.compare(xz2).details
+    difference = xz1.synchronized_compare(xz2).details
     assert difference[0].source1 == 'test1-content'
     assert difference[0].source2 == 'test2-content'
 
@@ -73,6 +73,6 @@ def test_content_diff(differences):
 @pytest.mark.skipif(tool_missing('xz'), reason='missing xz')
 def test_compare_non_existing(monkeypatch, xz1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = xz1.compare(NonExistingFile('/nonexisting', xz1))
+    difference = xz1.synchronized_compare(NonExistingFile('/nonexisting', xz1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_zip.py b/tests/comparators/test_zip.py
index 3396037..216f285 100644
--- a/tests/comparators/test_zip.py
+++ b/tests/comparators/test_zip.py
@@ -40,12 +40,12 @@ def test_identification(zip1):
     assert isinstance(zip1, ZipFile)
 
 def test_no_differences(zip1):
-    difference = zip1.compare(zip1)
+    difference = zip1.synchronized_compare(zip1)
     assert not difference
 
 @pytest.fixture
 def differences(zip1, zip2):
-    return zip1.compare(zip2).details
+    return zip1.synchronized_compare(zip2).details
 
 @pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
 def test_metadata(differences):
@@ -62,6 +62,6 @@ def test_compressed_files(differences):
 @pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
 def test_compare_non_existing(monkeypatch, zip1):
     monkeypatch.setattr(Config.general, 'new_file', True)
-    difference = zip1.compare(NonExistingFile('/nonexisting', zip1))
+    difference = zip1.synchronized_compare(NonExistingFile('/nonexisting', zip1))
     assert difference.source2 == '/nonexisting'
     assert difference.details[-1].source2 == '/dev/null'

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git



More information about the Reproducible-commits mailing list