[Reproducible-commits] [diffoscope] 05/05: Allow .get_reverse() on Futures
Joachim Breitner
nomeata at moszumanska.debian.org
Thu Dec 3 12:19:19 UTC 2015
This is an automated email from the git hooks/post-receive script.
nomeata pushed a commit to branch pu/parallel2
in repository diffoscope.
commit ed03918f21f96693b3f119b5d198f92aee8bd629
Author: Joachim Breitner <mail at joachim-breitner.de>
Date: Thu Dec 3 13:12:50 2015 +0100
Allow .get_reverse() on Futures
by introducing the FutureDifference object, which defers the
.get_reverse call until result() is going to be called..
---
diffoscope/comparators/utils.py | 4 ++--
diffoscope/difference.py | 35 +++++++++++++++++++++++++++++++++--
tests/comparators/test_epub.py | 6 +++---
tests/comparators/test_gzip.py | 8 ++++----
tests/comparators/test_ipk.py | 6 +++---
tests/comparators/test_iso9660.py | 6 +++---
tests/comparators/test_rpm.py | 6 +++---
tests/comparators/test_squashfs.py | 8 ++++----
tests/comparators/test_tar.py | 8 ++++----
tests/comparators/test_utils.py | 8 ++++----
tests/comparators/test_xz.py | 8 ++++----
tests/comparators/test_zip.py | 6 +++---
12 files changed, 70 insertions(+), 39 deletions(-)
diff --git a/diffoscope/comparators/utils.py b/diffoscope/comparators/utils.py
index 9c06f7a..995eebe 100644
--- a/diffoscope/comparators/utils.py
+++ b/diffoscope/comparators/utils.py
@@ -31,7 +31,7 @@ from threading import Thread
import diffoscope.comparators
from diffoscope.comparators.binary import File, NonExistingFile
from diffoscope.config import Config
-from diffoscope.difference import Difference
+from diffoscope.difference import FutureDifference, Difference
from diffoscope import logger, tool_required, get_temporary_directory
@@ -194,7 +194,7 @@ class Container(object, metaclass=ABCMeta):
yield NonExistingFile('/dev/null', other_file), other_file, NO_NOTIFICATION
def compare(self, other, source=None):
- return [Config.general.executor.submit(diffoscope.comparators.compare_files_with_notification, *args) for args in self.comparisons(other)]
+ return [FutureDifference(Config.general.executor.submit(diffoscope.comparators.compare_files_with_notification, *args)) for args in self.comparisons(other)]
class ArchiveMember(File):
diff --git a/diffoscope/difference.py b/diffoscope/difference.py
index 6728718..4f80307 100644
--- a/diffoscope/difference.py
+++ b/diffoscope/difference.py
@@ -271,6 +271,37 @@ def diff(feeder1, feeder2):
with fd_from_feeder(feeder2, end_nl_q2) as fd2:
return run_diff(fd1, fd2, end_nl_q1, end_nl_q2)
+class FutureDifference(object):
+ """
+ We usually produce Futures in place of Differences, for parallelization.
+ These are never looked at until all of them are produced. Then
+ finish_threads is called, which replaces them by their .result().
+
+ There is, however, one operation that we do on Differences before:
+ get_reverse(). So this object allows a way to defer that to reverse time.
+ """
+
+ def __init__(self, future):
+ self._is_reversed = False
+ assert (isinstance(future, Future) or isinstance(future, FutureDifference))
+ self._future = future
+ self._difference = None
+
+ def result(self,*args):
+ if self._difference is None:
+ self._difference = self._future.result(*args)
+
+ if self._is_reversed:
+ self._difference = self._difference.get_reverse()
+
+ return self._difference
+
+ def get_reverse(self):
+ reversed_future = FutureDifference(self)
+ reversed_future._is_reverse = True
+ return reversed_future
+
+
class Difference(object):
def __init__(self, path1, path2, source=None, notification=None, comment=None):
@@ -386,7 +417,7 @@ class Difference(object):
def finish_threads(self):
finished_details = []
for detail in self._details:
- if isinstance(detail, Future):
+ if isinstance(detail, FutureDifference):
detail = detail.result()
if not detail:
continue
@@ -416,7 +447,7 @@ class Difference(object):
@property
def unified_diff(self):
- #if isinstance(self._unified_diff, Future):
+ #if isinstance(self._unified_diff, FutureDifference):
# self._unified_diff = self._unified_diff.result()
return self._unified_diff
diff --git a/tests/comparators/test_epub.py b/tests/comparators/test_epub.py
index fe58f4b..f081882 100644
--- a/tests/comparators/test_epub.py
+++ b/tests/comparators/test_epub.py
@@ -40,12 +40,12 @@ def test_identification(epub1):
assert isinstance(epub1, ZipFile)
def test_no_differences(epub1):
- difference = epub1.compare(epub1)
+ difference = epub1.synchronized_compare(epub1)
assert not difference
@pytest.fixture
def differences(epub1, epub2):
- return epub1.compare(epub2).details
+ return epub1.synchronized_compare(epub2).details
@pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
def test_differences(differences):
@@ -63,6 +63,6 @@ def test_differences(differences):
@pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
def test_compare_non_existing(monkeypatch, epub1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = epub1.compare(NonExistingFile('/nonexisting', epub1))
+ difference = epub1.synchronized_compare(NonExistingFile('/nonexisting', epub1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_gzip.py b/tests/comparators/test_gzip.py
index b6aab9b..5c5af14 100644
--- a/tests/comparators/test_gzip.py
+++ b/tests/comparators/test_gzip.py
@@ -40,12 +40,12 @@ def test_identification(gzip1):
assert isinstance(gzip1, GzipFile)
def test_no_differences(gzip1):
- difference = gzip1.compare(gzip1)
+ difference = gzip1.synchronized_compare(gzip1)
assert not difference
@pytest.fixture
def differences(gzip1, gzip2):
- return gzip1.compare(gzip2).details
+ return gzip1.synchronized_compare(gzip2).details
def test_metadata(differences):
assert differences[0].source1 == 'metadata'
@@ -64,7 +64,7 @@ def test_content_source_without_extension(tmpdir):
shutil.copy(TEST_FILE2_PATH, path2)
gzip1 = specialize(FilesystemFile(path1))
gzip2 = specialize(FilesystemFile(path2))
- difference = gzip1.compare(gzip2).details
+ difference = gzip1.synchronized_compare(gzip2).details
assert difference[1].source1 == 'test1-content'
assert difference[1].source2 == 'test2-content'
@@ -74,6 +74,6 @@ def test_content_diff(differences):
def test_compare_non_existing(monkeypatch, gzip1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = gzip1.compare(NonExistingFile('/nonexisting', gzip1))
+ difference = gzip1.synchronized_compare(NonExistingFile('/nonexisting', gzip1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_ipk.py b/tests/comparators/test_ipk.py
index 276fe56..6b1a4f9 100644
--- a/tests/comparators/test_ipk.py
+++ b/tests/comparators/test_ipk.py
@@ -39,12 +39,12 @@ def test_identification(ipk1):
assert isinstance(ipk1, IpkFile)
def test_no_differences(ipk1):
- difference = ipk1.compare(ipk1)
+ difference = ipk1.synchronized_compare(ipk1)
assert not difference
@pytest.fixture
def differences(ipk1, ipk2):
- return ipk1.compare(ipk2).details
+ return ipk1.synchronized_compare(ipk2).details
def test_metadata(differences):
assert differences[0].source1 == 'metadata'
@@ -57,6 +57,6 @@ def test_compressed_files(differences):
def test_compare_non_existing(monkeypatch, ipk1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = ipk1.compare(NonExistingFile('/nonexisting', ipk1))
+ difference = ipk1.synchronized_compare(NonExistingFile('/nonexisting', ipk1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_iso9660.py b/tests/comparators/test_iso9660.py
index 07dcb12..bab5948 100644
--- a/tests/comparators/test_iso9660.py
+++ b/tests/comparators/test_iso9660.py
@@ -40,12 +40,12 @@ def test_identification(iso1):
assert isinstance(iso1, Iso9660File)
def test_no_differences(iso1):
- difference = iso1.compare(iso1)
+ difference = iso1.synchronized_compare(iso1)
assert not difference
@pytest.fixture
def differences(iso1, iso2):
- return iso1.compare(iso2).details
+ return iso1.synchronized_compare(iso2).details
@pytest.mark.skipif(tool_missing('isoinfo'), reason='missing isoinfo')
def test_iso9660_content(differences):
@@ -73,6 +73,6 @@ def test_compressed_files(differences):
@pytest.mark.skipif(tool_missing('isoinfo'), reason='missing isoinfo')
def test_compare_non_existing(monkeypatch, iso1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = iso1.compare(NonExistingFile('/nonexisting', iso1))
+ difference = iso1.synchronized_compare(NonExistingFile('/nonexisting', iso1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_rpm.py b/tests/comparators/test_rpm.py
index 501ce85..10b8015 100644
--- a/tests/comparators/test_rpm.py
+++ b/tests/comparators/test_rpm.py
@@ -46,12 +46,12 @@ def test_identification(rpm1):
@pytest.mark.skipif(miss_rpm_module, reason='rpm module is not installed')
def test_no_differences(rpm1):
- difference = rpm1.compare(rpm1)
+ difference = rpm1.synchronized_compare(rpm1)
assert not difference
@pytest.fixture
def differences(rpm1, rpm2):
- return rpm1.compare(rpm2).details
+ return rpm1.synchronized_compare(rpm2).details
@pytest.mark.skipif(miss_rpm_module, reason='rpm module is not installed')
@pytest.mark.skipif(tool_missing('rpm2cpio'), reason='missing rpm2cpio')
@@ -80,6 +80,6 @@ def test_content(differences):
@pytest.mark.skipif(tool_missing('rpm2cpio'), reason='missing rpm2cpio')
def test_compare_non_existing(monkeypatch, rpm1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = rpm1.compare(NonExistingFile('/nonexisting', rpm1))
+ difference = rpm1.synchronized_compare(NonExistingFile('/nonexisting', rpm1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_squashfs.py b/tests/comparators/test_squashfs.py
index c735c0d..cadcaf7 100644
--- a/tests/comparators/test_squashfs.py
+++ b/tests/comparators/test_squashfs.py
@@ -41,17 +41,17 @@ def test_identification(squashfs1):
assert isinstance(squashfs1, SquashfsFile)
def test_no_differences(squashfs1):
- difference = squashfs1.compare(squashfs1)
+ difference = squashfs1.synchronized_compare(squashfs1)
assert not difference
def test_no_warnings(capfd, squashfs1, squashfs2):
- _ = squashfs1.compare(squashfs2)
+ _ = squashfs1.synchronized_compare(squashfs2)
_, err = capfd.readouterr()
assert err == ''
@pytest.fixture
def differences(squashfs1, squashfs2):
- return squashfs1.compare(squashfs2).details
+ return squashfs1.synchronized_compare(squashfs2).details
@pytest.mark.skipif(tool_missing('unsquashfs'), reason='missing unsquashfs')
def test_superblock(differences):
@@ -81,6 +81,6 @@ def test_compressed_files(differences):
@pytest.mark.skipif(tool_missing('unsquashfs'), reason='missing unsquashfs')
def test_compare_non_existing(monkeypatch, squashfs1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = squashfs1.compare(NonExistingFile('/nonexisting', squashfs1))
+ difference = squashfs1.synchronized_compare(NonExistingFile('/nonexisting', squashfs1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_tar.py b/tests/comparators/test_tar.py
index e432cce..f14c7c4 100644
--- a/tests/comparators/test_tar.py
+++ b/tests/comparators/test_tar.py
@@ -39,12 +39,12 @@ def test_identification(tar1):
assert isinstance(tar1, TarFile)
def test_no_differences(tar1):
- difference = tar1.compare(tar1)
+ difference = tar1.synchronized_compare(tar1)
assert not difference
@pytest.fixture
def differences(tar1, tar2):
- return tar1.compare(tar2).details
+ return tar1.synchronized_compare(tar2).details
def test_listing(differences):
expected_diff = open(os.path.join(os.path.dirname(__file__), '../data/tar_listing_expected_diff')).read()
@@ -65,7 +65,7 @@ def test_text_file(differences):
def test_compare_non_existing(monkeypatch, tar1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = tar1.compare(NonExistingFile('/nonexisting', tar1))
+ difference = tar1.synchronized_compare(NonExistingFile('/nonexisting', tar1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
@@ -79,4 +79,4 @@ def test_no_permissions_dir_in_tarball(monkeypatch, no_permissions_tar):
# We want to make sure OSError is not raised.
# Comparing with non-existing file makes it easy to make sure all files are unpacked
monkeypatch.setattr(Config, 'new_file', True)
- no_permissions_tar.compare(NonExistingFile('/nonexistent', no_permissions_tar))
+ no_permissions_tar.synchronized_compare(NonExistingFile('/nonexistent', no_permissions_tar))
diff --git a/tests/comparators/test_utils.py b/tests/comparators/test_utils.py
index 010e654..76eaf9b 100644
--- a/tests/comparators/test_utils.py
+++ b/tests/comparators/test_utils.py
@@ -46,7 +46,7 @@ def fuzzy_tar3():
@pytest.mark.skipif(miss_tlsh, reason='tlsh is missing')
def test_fuzzy_matching(fuzzy_tar1, fuzzy_tar2):
- differences = fuzzy_tar1.compare(fuzzy_tar2).details
+ differences = fuzzy_tar1.synchronized_compare(fuzzy_tar2).details
expected_diff = codecs.open(os.path.join(os.path.dirname(__file__), '../data/text_iso8859_expected_diff'), encoding='utf-8').read()
assert differences[1].source1 == './matching'
assert differences[1].source2 == './fuzzy'
@@ -55,7 +55,7 @@ def test_fuzzy_matching(fuzzy_tar1, fuzzy_tar2):
@pytest.mark.skipif(miss_tlsh, reason='tlsh is missing')
def test_fuzzy_matching_only_once(fuzzy_tar1, fuzzy_tar3):
- differences = fuzzy_tar1.compare(fuzzy_tar3).details
+ differences = fuzzy_tar1.synchronized_compare(fuzzy_tar3).details
assert len(differences) == 2
expected_diff = codecs.open(os.path.join(os.path.dirname(__file__), '../data/text_iso8859_expected_diff'), encoding='utf-8').read()
@@ -70,7 +70,7 @@ def fuzzy_tar_in_tar2():
@pytest.mark.skipif(miss_tlsh, reason='tlsh is missing')
def test_no_fuzzy_matching(monkeypatch, fuzzy_tar_in_tar1, fuzzy_tar_in_tar2):
monkeypatch.setattr(Config, 'fuzzy_threshold', 0)
- difference = fuzzy_tar_in_tar1.compare(fuzzy_tar_in_tar2)
+ difference = fuzzy_tar_in_tar1.synchronized_compare(fuzzy_tar_in_tar2)
assert len(difference.details) == 1
assert difference.details[0].source1 == 'tar --full-time -tvf {}'
@@ -78,7 +78,7 @@ def test_no_fuzzy_matching(monkeypatch, fuzzy_tar_in_tar1, fuzzy_tar_in_tar2):
def test_no_fuzzy_matching_new_file(monkeypatch, fuzzy_tar_in_tar1, fuzzy_tar_in_tar2):
monkeypatch.setattr(Config, 'fuzzy_threshold', 0)
monkeypatch.setattr(Config, 'new_file', True)
- difference = fuzzy_tar_in_tar1.compare(fuzzy_tar_in_tar2)
+ difference = fuzzy_tar_in_tar1.synchronized_compare(fuzzy_tar_in_tar2)
assert len(difference.details) == 3
assert difference.details[1].source2 == '/dev/null'
assert difference.details[2].source1 == '/dev/null'
diff --git a/tests/comparators/test_xz.py b/tests/comparators/test_xz.py
index 494cc33..8fb80b7 100644
--- a/tests/comparators/test_xz.py
+++ b/tests/comparators/test_xz.py
@@ -41,12 +41,12 @@ def test_identification(xz1):
assert isinstance(xz1, XzFile)
def test_no_differences(xz1):
- difference = xz1.compare(xz1)
+ difference = xz1.synchronized_compare(xz1)
assert not difference
@pytest.fixture
def differences(xz1, xz2):
- return xz1.compare(xz2).details
+ return xz1.synchronized_compare(xz2).details
@pytest.mark.skipif(tool_missing('xz'), reason='missing xz')
def test_content_source(differences):
@@ -61,7 +61,7 @@ def test_content_source_without_extension(tmpdir):
shutil.copy(TEST_FILE2_PATH, path2)
xz1 = specialize(FilesystemFile(path1))
xz2 = specialize(FilesystemFile(path2))
- difference = xz1.compare(xz2).details
+ difference = xz1.synchronized_compare(xz2).details
assert difference[0].source1 == 'test1-content'
assert difference[0].source2 == 'test2-content'
@@ -73,6 +73,6 @@ def test_content_diff(differences):
@pytest.mark.skipif(tool_missing('xz'), reason='missing xz')
def test_compare_non_existing(monkeypatch, xz1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = xz1.compare(NonExistingFile('/nonexisting', xz1))
+ difference = xz1.synchronized_compare(NonExistingFile('/nonexisting', xz1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_zip.py b/tests/comparators/test_zip.py
index 3396037..216f285 100644
--- a/tests/comparators/test_zip.py
+++ b/tests/comparators/test_zip.py
@@ -40,12 +40,12 @@ def test_identification(zip1):
assert isinstance(zip1, ZipFile)
def test_no_differences(zip1):
- difference = zip1.compare(zip1)
+ difference = zip1.synchronized_compare(zip1)
assert not difference
@pytest.fixture
def differences(zip1, zip2):
- return zip1.compare(zip2).details
+ return zip1.synchronized_compare(zip2).details
@pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
def test_metadata(differences):
@@ -62,6 +62,6 @@ def test_compressed_files(differences):
@pytest.mark.skipif(tool_missing('zipinfo'), reason='missing zip')
def test_compare_non_existing(monkeypatch, zip1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = zip1.compare(NonExistingFile('/nonexisting', zip1))
+ difference = zip1.synchronized_compare(NonExistingFile('/nonexisting', zip1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the Reproducible-commits
mailing list