[Reproducible-commits] [diffoscope] 15/15: XXX nomeata
Jérémy Bobbio
lunar at moszumanska.debian.org
Thu Dec 3 11:08:00 UTC 2015
This is an automated email from the git hooks/post-receive script.
lunar pushed a commit to branch pu/parallel2
in repository diffoscope.
commit d5658106ad8de67414097b6d2d6b0085ab83d6b0
Author: Jérémy Bobbio <lunar at debian.org>
Date: Thu Dec 3 11:06:44 2015 +0000
XXX nomeata
---
diffoscope/__main__.py | 1 +
diffoscope/comparators/binary.py | 6 +++
diffoscope/comparators/deb.py | 4 +-
diffoscope/comparators/tar.py | 96 +---------------------------------------
diffoscope/comparators/utils.py | 3 +-
diffoscope/difference.py | 20 +++++++--
tests/comparators/test_bzip2.py | 8 ++--
tests/comparators/test_cbfs.py | 6 +--
tests/comparators/test_cpio.py | 6 +--
tests/comparators/test_deb.py | 14 +++---
tests/comparators/test_debian.py | 12 ++---
11 files changed, 52 insertions(+), 124 deletions(-)
diff --git a/diffoscope/__main__.py b/diffoscope/__main__.py
index 40f1ef7..5548788 100644
--- a/diffoscope/__main__.py
+++ b/diffoscope/__main__.py
@@ -125,6 +125,7 @@ def run_diffoscope(parsed_args):
set_locale()
difference = diffoscope.comparators.compare_root_paths(
parsed_args.file1, parsed_args.file2)
+ difference.finish_threads()
Config.general.executor.shutdown(wait=True)
if difference:
if parsed_args.html_output:
diff --git a/diffoscope/comparators/binary.py b/diffoscope/comparators/binary.py
index 693addc..c565ec0 100644
--- a/diffoscope/comparators/binary.py
+++ b/diffoscope/comparators/binary.py
@@ -210,6 +210,12 @@ class File(object, metaclass=ABCMeta):
return difference
return self.compare_bytes(other, source)
+ def synchronized_compare(self, other):
+ difference = self.compare(other)
+ if difference is not None:
+ difference.finish_threads()
+ return difference
+
class FilesystemFile(File):
def __init__(self, path):
diff --git a/diffoscope/comparators/deb.py b/diffoscope/comparators/deb.py
index 87fcd08..c71391d 100644
--- a/diffoscope/comparators/deb.py
+++ b/diffoscope/comparators/deb.py
@@ -25,7 +25,7 @@ from diffoscope.comparators.binary import File
from diffoscope.comparators.libarchive import LibarchiveContainer
from diffoscope.comparators.utils import \
Archive, ArchiveMember, get_ar_content
-from diffoscope.comparators.tar import TarContainer, TarListing
+from diffoscope.comparators.tar import TarListing
class DebContainer(LibarchiveContainer):
@@ -99,7 +99,7 @@ class Md5sumsFile(File):
return difference
-class DebTarContainer(TarContainer):
+class DebTarContainer(LibarchiveContainer):
def __init__(self, archive):
super().__init__(archive)
ignore_files = archive.container.source.container.source.files_with_same_content_in_data
diff --git a/diffoscope/comparators/tar.py b/diffoscope/comparators/tar.py
index 33e05b2..619da55 100644
--- a/diffoscope/comparators/tar.py
+++ b/diffoscope/comparators/tar.py
@@ -28,102 +28,10 @@ from diffoscope.difference import Difference
from diffoscope.comparators.binary import File
from diffoscope.comparators.device import Device
from diffoscope.comparators.directory import Directory
+from diffoscope.comparators.libarchive import LibarchiveContainer
from diffoscope.comparators.symlink import Symlink
from diffoscope.comparators.utils import Archive, ArchiveMember, Command, tool_required
-class TarMember(ArchiveMember):
- def is_directory(self):
- return False
-
- def is_symlink(self):
- return False
-
- def is_device(self):
- return False
-
-
-class TarDirectory(Directory, TarMember):
- def __init__(self, archive, member_name):
- ArchiveMember.__init__(self, archive, member_name)
-
- def compare(self, other, source=None):
- return None
-
- def has_same_content_as(self, other):
- return False
-
- @property
- def path(self):
- raise NotImplementedError('TarDirectory is not meant to be extracted.')
-
- def is_directory(self):
- return True
-
- def get_member_names(self):
- raise ValueError("Tar archives are compared as a whole.")
-
- def get_member(self, member_name):
- raise ValueError("Tar archives are compared as a whole.")
-
-class TarSymlink(Symlink, TarMember):
- def __init__(self, archive, member_name, destination):
- TarMember.__init__(self, archive, member_name)
- self._destination = destination
-
- def is_symlink(self):
- return True
-
- @property
- def symlink_destination(self):
- return self._destination
-
-
-class TarDevice(Device, TarMember):
- def __init__(self, archive, member_name, mode, major, minor):
- TarMember.__init__(self, archive, member_name)
- self._mode = mode
- self._major = major
- self._minor = minor
-
- def get_device(self):
- return (self._mode, self._major, self._minor)
-
- def is_device(self):
- return True
-
-
-class TarContainer(Archive):
- def open_archive(self):
- return tarfile.open(self.source.path, 'r')
-
- def close_archive(self):
- self.archive.close()
-
- def get_member_names(self):
- return self.archive.getnames()
-
- def extract(self, member_name, dest_dir):
- logger.debug('tar extracting %s to %s', member_name, dest_dir)
- self.archive.extract(member_name, dest_dir)
- return os.path.join(dest_dir, member_name)
-
- def get_member(self, member_name):
- tarinfo = self.archive.getmember(member_name)
- if tarinfo.isdir():
- return TarDirectory(self, member_name)
- elif tarinfo.issym():
- return TarSymlink(self, member_name, tarinfo.linkname)
- elif tarinfo.ischr() or tarinfo.isblk():
- mode = tarinfo.mode
- if tarinfo.isblk():
- mode |= stat.S_IFBLK
- else:
- mode |= stat.S_IFCHR
- return TarDevice(self, member_name, mode, tarinfo.devmajor, tarinfo.devminor)
- else:
- return TarMember(self, member_name)
-
-
class TarListing(Command):
@tool_required('tar')
def cmdline(self):
@@ -131,7 +39,7 @@ class TarListing(Command):
class TarFile(File):
- CONTAINER_CLASS = TarContainer
+ CONTAINER_CLASS = LibarchiveContainer
RE_FILE_TYPE = re.compile(r'\btar archive\b')
@staticmethod
diff --git a/diffoscope/comparators/utils.py b/diffoscope/comparators/utils.py
index 063d283..9c06f7a 100644
--- a/diffoscope/comparators/utils.py
+++ b/diffoscope/comparators/utils.py
@@ -194,7 +194,7 @@ class Container(object, metaclass=ABCMeta):
yield NonExistingFile('/dev/null', other_file), other_file, NO_NOTIFICATION
def compare(self, other, source=None):
- return starmap(diffoscope.comparators.compare_files_with_notification, self.comparisons(other))
+ return [Config.general.executor.submit(diffoscope.comparators.compare_files_with_notification, *args) for args in self.comparisons(other)]
class ArchiveMember(File):
@@ -225,6 +225,7 @@ class ArchiveMember(File):
if self._path is not None:
self._path = None
if self._temp_dir is not None:
+ logger.debug('cleanup %s', self._temp_dir)
self._temp_dir.cleanup()
self._temp_dir = None
super().cleanup()
diff --git a/diffoscope/difference.py b/diffoscope/difference.py
index 32dd62e..6728718 100644
--- a/diffoscope/difference.py
+++ b/diffoscope/difference.py
@@ -188,6 +188,7 @@ class ExThread(Thread):
else:
raise ex
+import time
def feed(feeder, f, end_nl_q):
# work-around unified diff limitation: if there's no newlines in both
@@ -249,7 +250,6 @@ def make_feeder_from_text_reader(in_file, filter=lambda text_buf: text_buf):
return filter(text_buf).encode('utf-8')
return make_feeder_from_raw_reader(in_file, encoding_filter)
-
def make_feeder_from_command(command, suppress_errrors):
def feeder(out_file):
end_nl = True
@@ -312,7 +312,8 @@ class Difference(object):
if comment:
difference.add_comment(comment)
#try: XXXXXXXXXX
- difference.unified_diff = Config.general.executor.submit(diff, feeder1, feeder2)
+ #difference.unified_diff = Config.general.executor.submit(diff, feeder1, feeder2)
+ difference.unified_diff = diff(feeder1, feeder2)
#except RequiredToolNotFound:
# difference.add_comment('diff is not available!')
return difference
@@ -382,6 +383,17 @@ class Difference(object):
def add_notification(self, notification):
self._notifications.append(notification)
+ def finish_threads(self):
+ finished_details = []
+ for detail in self._details:
+ if isinstance(detail, Future):
+ detail = detail.result()
+ if not detail:
+ continue
+ detail.finish_threads()
+ finished_details.append(detail)
+ self._details = finished_details
+
@property
def comment(self):
return '\n'.join(self._comments)
@@ -404,8 +416,8 @@ class Difference(object):
@property
def unified_diff(self):
- if isinstance(self._unified_diff, Future):
- self._unified_diff = self._unified_diff.result()
+ #if isinstance(self._unified_diff, Future):
+ # self._unified_diff = self._unified_diff.result()
return self._unified_diff
@unified_diff.setter
diff --git a/tests/comparators/test_bzip2.py b/tests/comparators/test_bzip2.py
index 946b444..06c0b2c 100644
--- a/tests/comparators/test_bzip2.py
+++ b/tests/comparators/test_bzip2.py
@@ -43,12 +43,12 @@ def test_identification(bzip1):
assert isinstance(bzip1, Bzip2File)
def test_no_differences(bzip1):
- difference = bzip1.compare(bzip1)
+ difference = bzip1.synchronized_compare(bzip1)
assert not difference
@pytest.fixture
def differences(bzip1, bzip2):
- return bzip1.compare(bzip2).details
+ return bzip1.synchronized_compare(bzip2).details
@pytest.mark.skipif(tool_missing('bzip2'), reason='missing bzip2')
def test_content_source(differences):
@@ -63,7 +63,7 @@ def test_content_source_without_extension(tmpdir):
shutil.copy(TEST_FILE2_PATH, path2)
bzip1 = specialize(FilesystemFile(path1))
bzip2 = specialize(FilesystemFile(path2))
- differences = bzip1.compare(bzip2).details
+ differences = bzip1.synchronized_compare(bzip2).details
assert differences[0].source1 == 'test1-content'
assert differences[0].source2 == 'test2-content'
@@ -75,5 +75,5 @@ def test_content_diff(differences):
@pytest.mark.skipif(tool_missing('bzip2'), reason='missing bzip2')
def test_compare_non_existing(monkeypatch, bzip1):
monkeypatch.setattr(Config, 'new_file', True)
- difference = bzip1.compare(NonExistingFile('/nonexisting', bzip1))
+ difference = bzip1.synchronized_compare(NonExistingFile('/nonexisting', bzip1))
assert difference.source2 == '/nonexisting'
diff --git a/tests/comparators/test_cbfs.py b/tests/comparators/test_cbfs.py
index f9b8646..b87fee4 100644
--- a/tests/comparators/test_cbfs.py
+++ b/tests/comparators/test_cbfs.py
@@ -64,12 +64,12 @@ def test_identification_without_offset(rom2):
@pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
def test_no_differences(rom1):
- difference = rom1.compare(rom1)
+ difference = rom1.synchronized_compare(rom1)
assert not difference
@pytest.fixture
def differences(rom1, rom2):
- difference = rom1.compare(rom2)
+ difference = rom1.synchronized_compare(rom2)
output_text(difference, print_func=print)
return difference.details
@@ -88,6 +88,6 @@ def test_content(differences):
@pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
def test_compare_non_existing(monkeypatch, rom1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = rom1.compare(NonExistingFile('/nonexisting', rom1))
+ difference = rom1.synchronized_compare(NonExistingFile('/nonexisting', rom1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_cpio.py b/tests/comparators/test_cpio.py
index 5421da0..3ed0432 100644
--- a/tests/comparators/test_cpio.py
+++ b/tests/comparators/test_cpio.py
@@ -41,12 +41,12 @@ def test_identification(cpio1):
assert isinstance(cpio1, CpioFile)
def test_no_differences(cpio1):
- difference = cpio1.compare(cpio1)
+ difference = cpio1.synchronized_compare(cpio1)
assert not difference
@pytest.fixture
def differences(cpio1, cpio2):
- return cpio1.compare(cpio2).details
+ return cpio1.synchronized_compare(cpio2).details
@pytest.mark.skipif(tool_missing('cpio'), reason='missing cpio')
def test_listing(differences):
@@ -70,7 +70,7 @@ def test_compressed_files(differences):
@pytest.mark.skipif(tool_missing('cpio'), reason='missing cpio')
def test_compare_non_existing(monkeypatch, cpio1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = cpio1.compare(NonExistingFile('/nonexisting', cpio1))
+ difference = cpio1.synchronized_compare(NonExistingFile('/nonexisting', cpio1))
output_text(difference, print_func=print)
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_deb.py b/tests/comparators/test_deb.py
index fbd12cf..7e429dc 100644
--- a/tests/comparators/test_deb.py
+++ b/tests/comparators/test_deb.py
@@ -41,12 +41,12 @@ def test_identification(deb1):
assert isinstance(deb1, DebFile)
def test_no_differences(deb1):
- difference = deb1.compare(deb1)
+ difference = deb1.synchronized_compare(deb1)
assert not difference
@pytest.fixture
def differences(deb1, deb2):
- return deb1.compare(deb2).details
+ return deb1.synchronized_compare(deb2).details
@pytest.mark.skipif(tool_missing('ar'), reason='missing ar')
def test_metadata(differences):
@@ -75,7 +75,7 @@ def test_identification_of_md5sums_in_deb(deb1, deb2, monkeypatch):
return ret
test_identification_of_md5sums_in_deb.found = False
monkeypatch.setattr(Md5sumsFile, 'recognizes', probe)
- deb1.compare(deb2)
+ deb1.synchronized_compare(deb2)
assert test_identification_of_md5sums_in_deb.found
@pytest.mark.skipif(tool_missing('ar'), reason='missing ar')
@@ -84,7 +84,7 @@ def test_md5sums(differences):
@pytest.mark.skipif(tool_missing('ar'), reason='missing ar')
def test_identical_files_in_md5sums(deb1, deb2):
- deb1.compare(deb2)
+ deb1.synchronized_compare(deb2)
assert deb1.files_with_same_content_in_data == set(['./usr/share/doc/test/README.Debian',
'./usr/share/doc/test/copyright'])
@@ -99,7 +99,7 @@ def test_identification_of_data_tar(deb1, deb2, monkeypatch):
return ret
test_identification_of_data_tar.found = False
monkeypatch.setattr(DebDataTarFile, 'recognizes', probe)
- deb1.compare(deb2)
+ deb1.synchronized_compare(deb2)
assert test_identification_of_data_tar.found
@pytest.mark.skipif(tool_missing('ar'), reason='missing ar')
@@ -110,12 +110,12 @@ def test_skip_comparison_of_known_identical_files(deb1, deb2, monkeypatch):
compared.add(file1.name)
return orig_func(file1, file2, source=None)
monkeypatch.setattr(diffoscope.comparators, 'compare_files', probe)
- deb1.compare(deb2)
+ deb1.synchronized_compare(deb2)
assert './usr/share/doc/test/README.Debian' not in compared
@pytest.mark.skipif(tool_missing('ar'), reason='missing ar')
def test_compare_non_existing(monkeypatch, deb1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = deb1.compare(NonExistingFile('/nonexisting', deb1))
+ difference = deb1.synchronized_compare(NonExistingFile('/nonexisting', deb1))
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/comparators/test_debian.py b/tests/comparators/test_debian.py
index 07d89d0..13ce9dd 100644
--- a/tests/comparators/test_debian.py
+++ b/tests/comparators/test_debian.py
@@ -65,12 +65,12 @@ def test_dot_changes_invalid(tmpdir):
assert not isinstance(identified, DotChangesFile)
def test_dot_changes_no_differences(dot_changes1):
- difference = dot_changes1.compare(dot_changes1)
+ difference = dot_changes1.synchronized_compare(dot_changes1)
assert not difference
@pytest.fixture
def dot_changes_differences(dot_changes1, dot_changes2):
- difference = dot_changes1.compare(dot_changes2)
+ difference = dot_changes1.synchronized_compare(dot_changes2)
output_text(difference, print_func=print)
return difference.details
@@ -87,7 +87,7 @@ def test_dot_changes_internal_diff(dot_changes_differences):
@pytest.mark.skipif(miss_debian_module, reason='debian module is not installed')
def test_dot_changes_compare_non_existing(monkeypatch, dot_changes1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = dot_changes1.compare(NonExistingFile('/nonexisting', dot_changes1))
+ difference = dot_changes1.synchronized_compare(NonExistingFile('/nonexisting', dot_changes1))
output_text(difference, print_func=print)
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
@@ -126,12 +126,12 @@ def test_dot_dsc_invalid(tmpdir, dot_dsc2):
assert not isinstance(identified, DotDscFile)
def test_dot_dsc_no_differences(dot_dsc1):
- difference = dot_dsc1.compare(dot_dsc1)
+ difference = dot_dsc1.synchronized_compare(dot_dsc1)
assert not difference
@pytest.fixture
def dot_dsc_differences(dot_dsc1, dot_dsc2):
- difference = dot_dsc1.compare(dot_dsc2)
+ difference = dot_dsc1.synchronized_compare(dot_dsc2)
output_text(difference, print_func=print)
return difference.details
@@ -142,7 +142,7 @@ def test_dot_dsc_internal_diff(dot_dsc_differences):
@pytest.mark.skipif(miss_debian_module, reason='debian module is not installed')
def test_dot_dsc_compare_non_existing(monkeypatch, dot_dsc1):
monkeypatch.setattr(Config.general, 'new_file', True)
- difference = dot_dsc1.compare(NonExistingFile('/nonexisting', dot_dsc1))
+ difference = dot_dsc1.synchronized_compare(NonExistingFile('/nonexisting', dot_dsc1))
output_text(difference, print_func=print)
assert difference.source2 == '/nonexisting'
assert difference.details[-1].source2 == '/dev/null'
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the Reproducible-commits
mailing list