[Reproducible-commits] [diffoscope] 01/15: xxx remove needs_content
Jérémy Bobbio
lunar at moszumanska.debian.org
Thu Dec 3 11:07:58 UTC 2015
This is an automated email from the git hooks/post-receive script.
lunar pushed a commit to branch pu/parallel2
in repository diffoscope.
commit 40e8f702e19e1d34c36a7fec0067f29ee0f3cf27
Author: Jérémy Bobbio <lunar at debian.org>
Date: Fri Sep 25 14:19:32 2015 +0000
xxx remove needs_content
---
diffoscope/comparators/__init__.py | 25 ++++++------
diffoscope/comparators/binary.py | 79 ++++++++++++++----------------------
diffoscope/comparators/bzip2.py | 3 +-
diffoscope/comparators/cbfs.py | 68 +++++++++++++++----------------
diffoscope/comparators/cpio.py | 3 +-
diffoscope/comparators/deb.py | 5 +--
diffoscope/comparators/debian.py | 61 ++++++++++++----------------
diffoscope/comparators/device.py | 23 +++++++----
diffoscope/comparators/directory.py | 28 +++++--------
diffoscope/comparators/elf.py | 4 +-
diffoscope/comparators/fonts.py | 3 +-
diffoscope/comparators/gettext.py | 3 +-
diffoscope/comparators/gzip.py | 2 -
diffoscope/comparators/haskell.py | 44 ++++++++++----------
diffoscope/comparators/iso9660.py | 3 +-
diffoscope/comparators/java.py | 3 +-
diffoscope/comparators/libarchive.py | 6 +--
diffoscope/comparators/mono.py | 3 +-
diffoscope/comparators/pdf.py | 3 +-
diffoscope/comparators/png.py | 3 +-
diffoscope/comparators/ppu.py | 10 ++---
diffoscope/comparators/rpm.py | 2 -
diffoscope/comparators/sqlite.py | 3 +-
diffoscope/comparators/squashfs.py | 9 ++--
diffoscope/comparators/symlink.py | 23 +++++++----
diffoscope/comparators/tar.py | 9 ++--
diffoscope/comparators/text.py | 6 +--
diffoscope/comparators/utils.py | 40 ++++++++++--------
diffoscope/comparators/xz.py | 3 +-
diffoscope/comparators/zip.py | 3 +-
30 files changed, 218 insertions(+), 262 deletions(-)
diff --git a/diffoscope/comparators/__init__.py b/diffoscope/comparators/__init__.py
index 4c111b9..c6777e2 100644
--- a/diffoscope/comparators/__init__.py
+++ b/diffoscope/comparators/__init__.py
@@ -91,19 +91,18 @@ def compare_root_paths(path1, path2):
def compare_files(file1, file2, source=None):
logger.debug('compare files %s and %s', file1, file2)
- with file1.get_content(), file2.get_content():
- if file1.has_same_content_as(file2):
- logger.debug('same content, skipping')
- return None
- specialize(file1)
- specialize(file2)
- if isinstance(file1, NonExistingFile):
- file1.other_file = file2
- elif isinstance(file2, NonExistingFile):
- file2.other_file = file1
- elif file1.__class__.__name__ != file2.__class__.__name__:
- return file1.compare_bytes(file2, source)
- return file1.compare(file2, source)
+ if file1.has_same_content_as(file2):
+ logger.debug('same content, skipping')
+ return None
+ specialize(file1)
+ specialize(file2)
+ if isinstance(file1, NonExistingFile):
+ file1.other_file = file2
+ elif isinstance(file2, NonExistingFile):
+ file2.other_file = file1
+ elif file1.__class__.__name__ != file2.__class__.__name__:
+ return file1.compare_bytes(file2, source)
+ return file1.compare(file2, source)
def compare_commented_files(file1, file2, comment=None, source=None):
difference = compare_files(file1, file2, source=source)
diff --git a/diffoscope/comparators/binary.py b/diffoscope/comparators/binary.py
index cc2d5d9..60b6101 100644
--- a/diffoscope/comparators/binary.py
+++ b/diffoscope/comparators/binary.py
@@ -71,14 +71,6 @@ def compare_binary_files(file1, file2, source=None):
SMALL_FILE_THRESHOLD = 65536 # 64 kiB
-# decorator for functions which needs to access the file content
-# (and so requires a path to be set)
-def needs_content(original_method):
- @wraps(original_method)
- def wrapper(self, other, *args, **kwargs):
- with self.get_content(), other.get_content():
- return original_method(self, other, *args, **kwargs)
- return wrapper
class File(object, metaclass=ABCMeta):
if hasattr(magic, 'open'): # use Magic-file-extensions from file
@@ -109,12 +101,22 @@ class File(object, metaclass=ABCMeta):
return self._mimedb_encoding.from_file(path).decode('utf-8')
def __repr__(self):
- return '<%s %s %s>' % (self.__class__, self.name, self.path)
+ return '<%s %s>' % (self.__class__, self.name)
- # Path should only be used when accessing the file content (through get_content())
+ # This should return a path that allows to access the file content
@property
+ @abstractmethod
def path(self):
- return self._path
+ raise NotImplemented
+
+ # Remove any temporary data associated with the file. The function
+ # should be idempotent and work during the destructor. It does nothing by
+ # default.
+ def cleanup(self):
+ pass
+
+ def __del__(self):
+ self.cleanup()
# This might be different from path and is used to do file extension matching
@property
@@ -124,33 +126,26 @@ class File(object, metaclass=ABCMeta):
@property
def magic_file_type(self):
if not hasattr(self, '_magic_file_type'):
- with self.get_content():
- self._magic_file_type = File.guess_file_type(self.path)
+ self._magic_file_type = File.guess_file_type(self.path)
return self._magic_file_type
if tlsh:
@property
def fuzzy_hash(self):
if not hasattr(self, '_fuzzy_hash'):
- with self.get_content():
- # tlsh is not meaningful with files smaller than 512 bytes
- if os.stat(self.path).st_size >= 512:
- h = tlsh.Tlsh()
- with open(self.path, 'rb') as f:
- for buf in iter(lambda: f.read(32768), b''):
- h.update(buf)
- h.final()
- self._fuzzy_hash = h.hexdigest()
- else:
- self._fuzzy_hash = None
+ # tlsh is not meaningful with files smaller than 512 bytes
+ if os.stat(self.path).st_size >= 512:
+ h = tlsh.Tlsh()
+ with open(self.path, 'rb') as f:
+ for buf in iter(lambda: f.read(32768), b''):
+ h.update(buf)
+ h.final()
+ self._fuzzy_hash = h.hexdigest()
+ else:
+ self._fuzzy_hash = None
return self._fuzzy_hash
@abstractmethod
- @contextmanager
- def get_content(self):
- raise NotImplemented
-
- @abstractmethod
def is_directory():
raise NotImplemented
@@ -162,7 +157,6 @@ class File(object, metaclass=ABCMeta):
def is_device():
raise NotImplemented
- @needs_content
def compare_bytes(self, other, source=None):
return compare_binary_files(self, other, source)
@@ -175,7 +169,6 @@ class File(object, metaclass=ABCMeta):
return difference
@tool_required('cmp')
- @needs_content
def has_same_content_as(self, other):
logger.debug('%s has_same_content %s', self, other)
# try comparing small files directly first
@@ -190,7 +183,6 @@ class File(object, metaclass=ABCMeta):
# To be specialized directly, or by implementing compare_details
- @needs_content
def compare(self, other, source=None):
if hasattr(self, 'compare_details'):
try:
@@ -227,17 +219,11 @@ class File(object, metaclass=ABCMeta):
class FilesystemFile(File):
def __init__(self, path):
- self._path = None
self._name = path
- @contextmanager
- def get_content(self):
- if self._path is not None:
- yield
- else:
- self._path = self._name
- yield
- self._path = None
+ @property
+ def path(self):
+ return self._name
def is_directory(self):
return not os.path.islink(self._name) and os.path.isdir(self._name)
@@ -261,11 +247,14 @@ class NonExistingFile(File):
return False
def __init__(self, path, other_file=None):
- self._path = None
self._name = path
self._other_file = other_file
@property
+ def path(self):
+ return '/dev/null'
+
+ @property
def other_file(self):
return self._other_file
@@ -276,12 +265,6 @@ class NonExistingFile(File):
def has_same_content_as(self, other):
return False
- @contextmanager
- def get_content(self):
- self._path = '/dev/null'
- yield
- self._path = None
-
def is_directory(self):
return False
diff --git a/diffoscope/comparators/bzip2.py b/diffoscope/comparators/bzip2.py
index 806e329..aef9c46 100644
--- a/diffoscope/comparators/bzip2.py
+++ b/diffoscope/comparators/bzip2.py
@@ -21,7 +21,7 @@ import os.path
import re
import subprocess
import diffoscope.comparators
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Archive, get_compressed_content_name, NO_COMMENT
from diffoscope import logger, tool_required
@@ -62,7 +62,6 @@ class Bzip2File(File):
def recognizes(file):
return Bzip2File.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
with Bzip2Container(self).open() as my_container, \
Bzip2Container(other).open() as other_container:
diff --git a/diffoscope/comparators/cbfs.py b/diffoscope/comparators/cbfs.py
index bf708dd..00efa5a 100644
--- a/diffoscope/comparators/cbfs.py
+++ b/diffoscope/comparators/cbfs.py
@@ -25,7 +25,7 @@ import subprocess
import stat
import struct
from diffoscope import logger, tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Archive, Command
from diffoscope.difference import Difference
@@ -94,40 +94,38 @@ def is_header_valid(buf, size, offset=0):
class CbfsFile(File):
@staticmethod
def recognizes(file):
- with file.get_content():
- size = os.stat(file.path).st_size
- if size < CBFS_HEADER_SIZE:
- return False
- with open(file.path, 'rb') as f:
- # pick at the latest byte as it should contain the relative offset of the header
- f.seek(-4, io.SEEK_END)
- # <pgeorgi> given the hardware we support so far, it looks like
- # that field is now bound to be little endian
- # -- #coreboot, 2015-10-14
- rel_offset = struct.unpack('<i', f.read(4))[0]
- if rel_offset < 0 and -rel_offset > CBFS_HEADER_SIZE and -rel_offset < size:
- f.seek(rel_offset, io.SEEK_END)
- logger.debug('looking for header at offset: %x', f.tell())
- if is_header_valid(f.read(CBFS_HEADER_SIZE), size):
- return True
- elif not file.name.endswith('.rom'):
- return False
- else:
- logger.debug('CBFS relative offset seems wrong, scanning whole image')
- f.seek(0, io.SEEK_SET)
- offset = 0
- buf = f.read(CBFS_HEADER_SIZE)
- while len(buf) >= CBFS_HEADER_SIZE:
- if is_header_valid(buf, size, offset):
- return True
- if len(buf) - offset <= CBFS_HEADER_SIZE:
- buf = f.read(32768)
- offset = 0
- else:
- offset += 1
- return False
-
- @needs_content
+ size = os.stat(file.path).st_size
+ if size < CBFS_HEADER_SIZE:
+ return False
+ with open(file.path, 'rb') as f:
+ # pick at the latest byte as it should contain the relative offset of the header
+ f.seek(-4, io.SEEK_END)
+ # <pgeorgi> given the hardware we support so far, it looks like
+ # that field is now bound to be little endian
+ # -- #coreboot, 2015-10-14
+ rel_offset = struct.unpack('<i', f.read(4))[0]
+ if rel_offset < 0 and -rel_offset > CBFS_HEADER_SIZE and -rel_offset < size:
+ f.seek(rel_offset, io.SEEK_END)
+ logger.debug('looking for header at offset: %x', f.tell())
+ if is_header_valid(f.read(CBFS_HEADER_SIZE), size):
+ return True
+ elif not file.name.endswith('.rom'):
+ return False
+ else:
+ logger.debug('CBFS relative offset seems wrong, scanning whole image')
+ f.seek(0, io.SEEK_SET)
+ offset = 0
+ buf = f.read(CBFS_HEADER_SIZE)
+ while len(buf) >= CBFS_HEADER_SIZE:
+ if is_header_valid(buf, size, offset):
+ return True
+ if len(buf) - offset <= CBFS_HEADER_SIZE:
+ buf = f.read(32768)
+ offset = 0
+ else:
+ offset += 1
+ return False
+
def compare_details(self, other, source=None):
differences = []
differences.append(Difference.from_command(CbfsListing, self.path, other.path))
diff --git a/diffoscope/comparators/cpio.py b/diffoscope/comparators/cpio.py
index d11ba78..4f2f76b 100644
--- a/diffoscope/comparators/cpio.py
+++ b/diffoscope/comparators/cpio.py
@@ -20,7 +20,7 @@
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.libarchive import LibarchiveContainer
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -39,7 +39,6 @@ class CpioFile(File):
def recognizes(file):
return CpioFile.RE_FILE_TYPE.search(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
differences.append(Difference.from_command(
diff --git a/diffoscope/comparators/deb.py b/diffoscope/comparators/deb.py
index b7e92cc..d27ac6d 100644
--- a/diffoscope/comparators/deb.py
+++ b/diffoscope/comparators/deb.py
@@ -21,7 +21,7 @@ import re
import os.path
from diffoscope import logger
from diffoscope.difference import Difference
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.libarchive import LibarchiveContainer
from diffoscope.comparators.utils import \
Archive, ArchiveMember, get_ar_content
@@ -49,7 +49,6 @@ class DebFile(File):
def set_files_with_same_content_in_data(self, files):
self._files_with_same_content_in_data = files
- @needs_content
def compare_details(self, other, source=None):
differences = []
my_content = get_ar_content(self.path)
@@ -81,7 +80,6 @@ class Md5sumsFile(File):
d[path] = md5sum
return d
- @needs_content
def compare(self, other, source=None):
if other.path is None:
return None
@@ -122,7 +120,6 @@ class DebDataTarFile(File):
file.container.source.name.startswith('data.tar.') and \
isinstance(file.container.source.container.source, DebFile)
- @needs_content
def compare_details(self, other, source=None):
differences = []
ignore_files = self.container.source.container.source.files_with_same_content_in_data
diff --git a/diffoscope/comparators/debian.py b/diffoscope/comparators/debian.py
index a361e5d..d6a54ba 100644
--- a/diffoscope/comparators/debian.py
+++ b/diffoscope/comparators/debian.py
@@ -25,7 +25,7 @@ import re
from debian.deb822 import Dsc
from diffoscope.changes import Changes
import diffoscope.comparators
-from diffoscope.comparators.binary import File, NonExistingFile, needs_content
+from diffoscope.comparators.binary import File, NonExistingFile
from diffoscope.comparators.utils import Container, NO_COMMENT
from diffoscope.config import Config
from diffoscope.difference import Difference
@@ -53,15 +53,9 @@ class DebControlMember(File):
def name(self):
return self._name
- @contextmanager
- def get_content(self):
- if self._path is not None:
- yield
- else:
- with self.container.source.get_content():
- self._path = os.path.join(os.path.dirname(self.container.source.path), self.name)
- yield
- self._path = None
+ @property
+ def path(self):
+ return os.path.join(os.path.dirname(self.container.source.path), self.name)
def is_directory(self):
return False
@@ -109,7 +103,6 @@ class DebControlFile(File):
def deb822(self):
return self._deb822
- @needs_content
def compare_details(self, other, source=None):
differences = []
@@ -142,14 +135,13 @@ class DotChangesFile(DebControlFile):
def recognizes(file):
if not DotChangesFile.RE_FILE_EXTENSION.search(file.name):
return False
- with file.get_content():
- changes = Changes(filename=file.path)
- try:
- changes.validate(check_signature=False)
- except FileNotFoundError:
- return False
- file._deb822 = changes
- return True
+ changes = Changes(filename=file.path)
+ try:
+ changes.validate(check_signature=False)
+ except FileNotFoundError:
+ return False
+ file._deb822 = changes
+ return True
class DotDscFile(DebControlFile):
RE_FILE_EXTENSION = re.compile(r'\.dsc$')
@@ -158,19 +150,18 @@ class DotDscFile(DebControlFile):
def recognizes(file):
if not DotDscFile.RE_FILE_EXTENSION.search(file.name):
return False
- with file.get_content():
- with open(file.path, 'rb') as f:
- dsc = Dsc(f)
- for d in dsc.get('Files'):
- md5 = hashlib.md5()
- # XXX: this will not work for containers
- in_dsc_path = os.path.join(os.path.dirname(file.path), d['Name'])
- if not os.path.exists(in_dsc_path):
- return False
- with open(in_dsc_path, 'rb') as f:
- for buf in iter(partial(f.read, 32768), b''):
- md5.update(buf)
- if md5.hexdigest() != d['md5sum']:
- return False
- file._deb822 = dsc
- return True
+ with open(file.path, 'rb') as f:
+ dsc = Dsc(f)
+ for d in dsc.get('Files'):
+ md5 = hashlib.md5()
+ # XXX: this will not work for containers
+ in_dsc_path = os.path.join(os.path.dirname(file.path), d['Name'])
+ if not os.path.exists(in_dsc_path):
+ return False
+ with open(in_dsc_path, 'rb') as f:
+ for buf in iter(partial(f.read, 32768), b''):
+ md5.update(buf)
+ if md5.hexdigest() != d['md5sum']:
+ return False
+ file._deb822 = dsc
+ return True
diff --git a/diffoscope/comparators/device.py b/diffoscope/comparators/device.py
index cd9aaf0..14d0216 100644
--- a/diffoscope/comparators/device.py
+++ b/diffoscope/comparators/device.py
@@ -20,7 +20,7 @@
from contextlib import contextmanager
import os
import tempfile
-from diffoscope.comparators.binary import File, FilesystemFile, needs_content
+from diffoscope.comparators.binary import File, FilesystemFile
from diffoscope.comparators.utils import format_device
from diffoscope.difference import Difference
from diffoscope import logger
@@ -39,16 +39,23 @@ class Device(File):
def has_same_content_as(self, other):
return self.get_device() == other.get_device()
- @contextmanager
- def get_content(self):
- with tempfile.NamedTemporaryFile(mode='w+', suffix='diffoscope') as f:
+ def create_placeholder(self):
+ with tempfile.NamedTemporaryFile(mode='w+', suffix='diffoscope', delete=False) as f:
f.write(format_device(*self.get_device()))
f.flush()
- self._path = f.name
- yield
- self._path = None
+ return f.name
+
+ @property
+ def path(self):
+ if not hasattr(self, '_placeholder'):
+ self._placeholder = self.create_placeholder()
+ return self._placeholder
+
+ def cleanup(self):
+ if hasattr(self, '_placeholder'):
+ os.remove(self._placeholder)
+ del self._placeholder
- @needs_content
def compare(self, other, source=None):
with open(self.path) as my_content, \
open(other.path) as other_content:
diff --git a/diffoscope/comparators/directory.py b/diffoscope/comparators/directory.py
index 2141572..07d7d9a 100644
--- a/diffoscope/comparators/directory.py
+++ b/diffoscope/comparators/directory.py
@@ -113,10 +113,6 @@ class FilesystemDirectory(object):
def name(self):
return self._path
- @contextmanager
- def get_content(self):
- yield
-
def is_directory(self):
return True
@@ -140,15 +136,14 @@ class FilesystemDirectory(object):
for name in sorted(set(my_names).intersection(other_names)):
my_file = my_container.get_member(name)
other_file = other_container.get_member(name)
- with my_file.get_content(), other_file.get_content():
- inner_difference = diffoscope.comparators.compare_files(
- my_file, other_file, source=name)
- meta_differences = compare_meta(my_file.name, other_file.name)
- if meta_differences and not inner_difference:
- inner_difference = Difference(None, my_file.path, other_file.path)
- if inner_difference:
- inner_difference.add_details(meta_differences)
- differences.append(inner_difference)
+ inner_difference = diffoscope.comparators.compare_files(
+ my_file, other_file, source=name)
+ meta_differences = compare_meta(my_file.name, other_file.name)
+ if meta_differences and not inner_difference:
+ inner_difference = Difference(None, my_file.path, other_file.path)
+ if inner_difference:
+ inner_difference.add_details(meta_differences)
+ differences.append(inner_difference)
if not differences:
return None
difference = Difference(None, self.path, other.path, source)
@@ -159,10 +154,9 @@ class FilesystemDirectory(object):
class DirectoryContainer(Container):
@contextmanager
def open(self):
- with self.source.get_content():
- self._path = self.source.path.rstrip('/') or '/'
- yield self
- self._path = None
+ self._path = self.source.path.rstrip('/') or '/'
+ yield self
+ self._path = None
def get_member_names(self):
names = []
diff --git a/diffoscope/comparators/elf.py b/diffoscope/comparators/elf.py
index 38cf665..30f6953 100644
--- a/diffoscope/comparators/elf.py
+++ b/diffoscope/comparators/elf.py
@@ -20,7 +20,7 @@
import os.path
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import get_ar_content, Command
from diffoscope.difference import Difference
@@ -90,7 +90,6 @@ class ElfFile(File):
def recognizes(file):
return ElfFile.RE_FILE_TYE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
return _compare_elf_data(self.path, other.path)
@@ -102,7 +101,6 @@ class StaticLibFile(File):
def recognizes(file):
return StaticLibFile.RE_FILE_TYPE.search(file.magic_file_type) and StaticLibFile.RE_FILE_EXTENSION.search(file.name)
- @needs_content
def compare_details(self, other, source=None):
differences = []
# look up differences in metadata
diff --git a/diffoscope/comparators/fonts.py b/diffoscope/comparators/fonts.py
index 859c924..187655a 100644
--- a/diffoscope/comparators/fonts.py
+++ b/diffoscope/comparators/fonts.py
@@ -19,7 +19,7 @@
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -40,6 +40,5 @@ class TtfFile(File):
def recognizes(file):
return TtfFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Showttf, self.path, other.path)]
diff --git a/diffoscope/comparators/gettext.py b/diffoscope/comparators/gettext.py
index 6ded32d..b739900 100644
--- a/diffoscope/comparators/gettext.py
+++ b/diffoscope/comparators/gettext.py
@@ -20,7 +20,7 @@
from io import BytesIO
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
from diffoscope import logger
@@ -63,6 +63,5 @@ class MoFile(File):
def recognizes(file):
return MoFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Msgunfmt, self.path, other.path)]
diff --git a/diffoscope/comparators/gzip.py b/diffoscope/comparators/gzip.py
index f151d48..481b028 100644
--- a/diffoscope/comparators/gzip.py
+++ b/diffoscope/comparators/gzip.py
@@ -22,7 +22,6 @@ import subprocess
import os.path
import diffoscope.comparators
from diffoscope import logger, tool_required
-from diffoscope.comparators.binary import needs_content
from diffoscope.comparators.utils import Archive, get_compressed_content_name, NO_COMMENT
from diffoscope.difference import Difference
@@ -63,7 +62,6 @@ class GzipFile(object):
def recognizes(file):
return GzipFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
differences.append(Difference.from_text(
diff --git a/diffoscope/comparators/haskell.py b/diffoscope/comparators/haskell.py
index 6f696fa..7bd4025 100644
--- a/diffoscope/comparators/haskell.py
+++ b/diffoscope/comparators/haskell.py
@@ -22,7 +22,7 @@ import platform
import struct
import subprocess
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
from diffoscope import logger
@@ -60,29 +60,27 @@ class HiFile(File):
if HiFile.hi_version is None:
return False
- with file.get_content():
- with open(file.path, 'rb') as fp:
- # read magic
- buf = fp.read(4)
- if buf != HI_MAGIC:
- logger.debug('Haskell interface magic mismatch. Found %r instead of %r or %r', buf, HI_MAGIC_32, HI_MAGIC_64)
- return False
- # skip some old descriptor thingy that has varying size
- if buf == HI_MAGIC_32:
- fp.read(4)
- elif buf == HI_MAGIC_64:
- fp.read(8)
- # skip way_descr
+ with open(file.path, 'rb') as fp:
+ # read magic
+ buf = fp.read(4)
+ if buf != HI_MAGIC:
+ logger.debug('Haskell interface magic mismatch. Found %r instead of %r or %r', buf, HI_MAGIC_32, HI_MAGIC_64)
+ return False
+ # skip some old descriptor thingy that has varying size
+ if buf == HI_MAGIC_32:
fp.read(4)
- # now read version
- buf = fp.read(16)
- version_found = ''.join(map(chr, struct.unpack_from('=3IB', buf)))
- if version_found != HiFile.hi_version:
- logger.debug('Haskell version mismatch. Found %s instead of %s.',
- version_found, HiFile.hi_version)
- return False
- return True
+ elif buf == HI_MAGIC_64:
+ fp.read(8)
+ # skip way_descr
+ fp.read(4)
+ # now read version
+ buf = fp.read(16)
+ version_found = ''.join(map(chr, struct.unpack_from('=3IB', buf)))
+ if version_found != HiFile.hi_version:
+ logger.debug('Haskell version mismatch. Found %s instead of %s.',
+ version_found, HiFile.hi_version)
+ return False
+ return True
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(ShowIface, self.path, other.path)]
diff --git a/diffoscope/comparators/iso9660.py b/diffoscope/comparators/iso9660.py
index cf21ccf..3eca892 100644
--- a/diffoscope/comparators/iso9660.py
+++ b/diffoscope/comparators/iso9660.py
@@ -20,7 +20,7 @@
import re
import subprocess
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.libarchive import LibarchiveContainer
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -68,7 +68,6 @@ class Iso9660File(File):
def recognizes(file):
return Iso9660File.RE_FILE_TYPE.search(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
differences.append(Difference.from_command(ISO9660PVD, self.path, other.path))
diff --git a/diffoscope/comparators/java.py b/diffoscope/comparators/java.py
index abf9972..f8b6b5e 100644
--- a/diffoscope/comparators/java.py
+++ b/diffoscope/comparators/java.py
@@ -21,7 +21,7 @@
import os.path
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -48,6 +48,5 @@ class ClassFile(File):
def recognizes(file):
return ClassFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Javap, self.path, other.path)]
diff --git a/diffoscope/comparators/libarchive.py b/diffoscope/comparators/libarchive.py
index f8d5b4f..f9a2e9d 100644
--- a/diffoscope/comparators/libarchive.py
+++ b/diffoscope/comparators/libarchive.py
@@ -63,9 +63,9 @@ class LibarchiveDirectory(Directory, LibarchiveMember):
def has_same_content_as(self, other):
return False
- @contextmanager
- def get_content(self):
- yield
+ @property
+ def path(self):
+ raise NotImplementedError('LibarchiveDirectory is not meant to be extracted.')
def is_directory(self):
return True
diff --git a/diffoscope/comparators/mono.py b/diffoscope/comparators/mono.py
index c563390..cfbf87d 100644
--- a/diffoscope/comparators/mono.py
+++ b/diffoscope/comparators/mono.py
@@ -20,7 +20,7 @@
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -38,6 +38,5 @@ class MonoExeFile(File):
def recognizes(file):
return MonoExeFile.RE_FILE_TYPE.search(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Pedump, self.path, other.path)]
diff --git a/diffoscope/comparators/pdf.py b/diffoscope/comparators/pdf.py
index 2f15733..d105c07 100644
--- a/diffoscope/comparators/pdf.py
+++ b/diffoscope/comparators/pdf.py
@@ -19,7 +19,7 @@
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -46,7 +46,6 @@ class PdfFile(File):
def recognizes(file):
return PdfFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
differences.append(Difference.from_command(Pdftotext, self.path, other.path))
diff --git a/diffoscope/comparators/png.py b/diffoscope/comparators/png.py
index 845441e..0328e33 100644
--- a/diffoscope/comparators/png.py
+++ b/diffoscope/comparators/png.py
@@ -20,7 +20,7 @@
from functools import partial
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -43,6 +43,5 @@ class PngFile(File):
def recognizes(file):
return PngFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Sng, self.path, other.path, source='sng')]
diff --git a/diffoscope/comparators/ppu.py b/diffoscope/comparators/ppu.py
index 6ea7b44..1b252a3 100644
--- a/diffoscope/comparators/ppu.py
+++ b/diffoscope/comparators/ppu.py
@@ -22,7 +22,7 @@
import os
import re
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -55,12 +55,10 @@ class PpuFile(File):
def recognizes(file):
if not PpuFile.RE_FILE_EXTENSION.search(file.name):
return False
- with file.get_content():
- with open(file.path, 'rb') as f:
- if re.match(rb'^PPU[0-9]+', f.read(32)):
- return True
+ with open(file.path, 'rb') as f:
+ if re.match(rb'^PPU[0-9]+', f.read(32)):
+ return True
return False
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Ppudump, self.path, other.path)]
diff --git a/diffoscope/comparators/rpm.py b/diffoscope/comparators/rpm.py
index 641dac1..afa3eff 100644
--- a/diffoscope/comparators/rpm.py
+++ b/diffoscope/comparators/rpm.py
@@ -25,7 +25,6 @@ import subprocess
import rpm
from diffoscope import logger, tool_required
from diffoscope.comparators.rpm_fallback import AbstractRpmFile
-from diffoscope.comparators.binary import needs_content
from diffoscope.comparators.utils import Archive, make_temp_directory
from diffoscope.difference import Difference
@@ -103,7 +102,6 @@ class RpmContainer(Archive):
class RpmFile(AbstractRpmFile):
- @needs_content
def compare_details(self, other, source=None):
differences = []
differences.append(compare_rpm_headers(self.path, other.path))
diff --git a/diffoscope/comparators/sqlite.py b/diffoscope/comparators/sqlite.py
index b3320c8..e361dea 100644
--- a/diffoscope/comparators/sqlite.py
+++ b/diffoscope/comparators/sqlite.py
@@ -18,7 +18,7 @@
# along with diffoscope. If not, see <http://www.gnu.org/licenses/>.
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Command
from diffoscope.difference import Difference
@@ -34,7 +34,6 @@ class Sqlite3Database(File):
def recognizes(file):
return file.magic_file_type == 'SQLite 3.x database'
- @needs_content
def compare_details(self, other, source=None):
return [Difference.from_command(Sqlite3Dump, self.path, other.path)]
diff --git a/diffoscope/comparators/squashfs.py b/diffoscope/comparators/squashfs.py
index a489742..fda4377 100644
--- a/diffoscope/comparators/squashfs.py
+++ b/diffoscope/comparators/squashfs.py
@@ -23,7 +23,7 @@ import re
import subprocess
import stat
from diffoscope import logger, tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.device import Device
from diffoscope.comparators.directory import Directory
from diffoscope.comparators.symlink import Symlink
@@ -83,9 +83,9 @@ class SquashfsDirectory(Directory, SquashfsMember):
def has_same_content_as(self, other):
return False
- @contextmanager
- def get_content(self):
- yield
+ @property
+ def path(self):
+ raise NotImplementedError('SquashfsDirectory is not meant to be extracted.')
def is_directory(self):
return True
@@ -194,7 +194,6 @@ class SquashfsFile(File):
def recognizes(file):
return SquashfsFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
differences.append(Difference.from_command(SquashfsSuperblock, self.path, other.path))
diff --git a/diffoscope/comparators/symlink.py b/diffoscope/comparators/symlink.py
index 93dad0f..defd89f 100644
--- a/diffoscope/comparators/symlink.py
+++ b/diffoscope/comparators/symlink.py
@@ -20,7 +20,7 @@
from contextlib import contextmanager
import os
import tempfile
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import format_symlink
from diffoscope.difference import Difference
from diffoscope import logger
@@ -35,16 +35,23 @@ class Symlink(File):
def symlink_destination(self):
return os.readlink(self.name)
- @contextmanager
- def get_content(self):
- with tempfile.NamedTemporaryFile('w+', suffix='diffoscope') as f:
+ def create_placeholder(self):
+ with tempfile.NamedTemporaryFile('w+', suffix='diffoscope', delete=False) as f:
f.write(format_symlink(self.symlink_destination))
f.flush()
- self._path = f.name
- yield
- self._path = None
+ return f.name
+
+ @property
+ def path(self):
+ if not hasattr(self, '_placeholder'):
+ self._placeholder = self.create_placeholder()
+ return self._placeholder
+
+ def cleanup(self):
+ if hasattr(self, '_placeholder'):
+ os.remove(self._placeholder)
+ del self._placeholder
- @needs_content
def compare(self, other, source=None):
logger.debug('my_content %s', self.path)
with open(self.path) as my_content, \
diff --git a/diffoscope/comparators/tar.py b/diffoscope/comparators/tar.py
index 27d4782..ed5f59b 100644
--- a/diffoscope/comparators/tar.py
+++ b/diffoscope/comparators/tar.py
@@ -25,7 +25,7 @@ import sys
import tarfile
from diffoscope import logger
from diffoscope.difference import Difference
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.device import Device
from diffoscope.comparators.directory import Directory
from diffoscope.comparators.symlink import Symlink
@@ -52,9 +52,9 @@ class TarDirectory(Directory, TarMember):
def has_same_content_as(self, other):
return False
- @contextmanager
- def get_content(self):
- yield
+ @property
+ def path(self):
+ raise NotImplementedError('TarDirectory is not meant to be extracted.')
def is_directory(self):
return True
@@ -137,7 +137,6 @@ class TarFile(File):
def recognizes(file):
return TarFile.RE_FILE_TYPE.search(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
with TarContainer(self).open() as my_container, \
diff --git a/diffoscope/comparators/text.py b/diffoscope/comparators/text.py
index 417238a..a48a50e 100644
--- a/diffoscope/comparators/text.py
+++ b/diffoscope/comparators/text.py
@@ -19,7 +19,7 @@
import codecs
import re
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.difference import Difference
@@ -33,11 +33,9 @@ class TextFile(File):
@property
def encoding(self):
if not hasattr(self, '_encoding'):
- with self.get_content():
- self._encoding = File.guess_encoding(self.path)
+ self._encoding = File.guess_encoding(self.path)
return self._encoding
- @needs_content
def compare(self, other, source=None):
my_encoding = self.encoding or 'utf-8'
other_encoding = other.encoding or 'utf-8'
diff --git a/diffoscope/comparators/utils.py b/diffoscope/comparators/utils.py
index 767f01b..7bcb838 100644
--- a/diffoscope/comparators/utils.py
+++ b/diffoscope/comparators/utils.py
@@ -26,6 +26,7 @@ import os
import shutil
from stat import S_ISCHR, S_ISBLK
import subprocess
+from tempfile import TemporaryDirectory
import tempfile
from threading import Thread
import diffoscope.comparators
@@ -213,6 +214,7 @@ class ArchiveMember(File):
def __init__(self, container, member_name):
self._container = container
self._name = member_name
+ self._temp_dir = None
self._path = None
@property
@@ -223,17 +225,22 @@ class ArchiveMember(File):
def name(self):
return self._name
- @contextmanager
- def get_content(self):
- logger.debug('%s get_content; path %s', self, self._path)
+ @property
+ def path(self):
+ if self._path is None:
+ logger.debug('unpacking %s', self._name)
+ assert self._temp_dir is None
+ self._temp_dir = TemporaryDirectory(suffix='diffoscope')
+ with self._container.open() as container:
+ self._path = container.extract(self._name, self._temp_dir.name)
+ return self._path
+
+ def cleanup(self):
if self._path is not None:
- yield
- else:
- with make_temp_directory() as temp_dir, \
- self._container.open() as container:
- self._path = container.extract(self._name, temp_dir)
- yield
- self._path = None
+ self._path = None
+ if self._temp_dir is not None:
+ self._temp_dir.cleanup()
+ self._temp_dir = None
def is_directory(self):
return False
@@ -257,13 +264,12 @@ class Archive(Container, metaclass=ABCMeta):
elif self._archive is not None:
yield self
else:
- with self.source.get_content():
- self._archive = self.open_archive(self.source.path)
- try:
- yield self
- finally:
- self.close_archive()
- self._archive = None
+ self._archive = self.open_archive(self.source.path)
+ try:
+ yield self
+ finally:
+ self.close_archive()
+ self._archive = None
@property
def archive(self):
diff --git a/diffoscope/comparators/xz.py b/diffoscope/comparators/xz.py
index 4848e65..9ff9713 100644
--- a/diffoscope/comparators/xz.py
+++ b/diffoscope/comparators/xz.py
@@ -21,7 +21,7 @@ import os.path
import re
import subprocess
import diffoscope.comparators
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.utils import Archive, get_compressed_content_name, NO_COMMENT
from diffoscope import logger, tool_required
@@ -62,7 +62,6 @@ class XzFile(File):
def recognizes(file):
return XzFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
with XzContainer(self).open() as my_container, \
XzContainer(other).open() as other_container:
diff --git a/diffoscope/comparators/zip.py b/diffoscope/comparators/zip.py
index dd7b915..9d7b0a9 100644
--- a/diffoscope/comparators/zip.py
+++ b/diffoscope/comparators/zip.py
@@ -25,7 +25,7 @@ import sys
import zipfile
from diffoscope.difference import Difference
from diffoscope import tool_required
-from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.binary import File
from diffoscope.comparators.directory import Directory
from diffoscope.comparators.utils import Archive, ArchiveMember, Command
@@ -106,7 +106,6 @@ class ZipFile(File):
def recognizes(file):
return ZipFile.RE_FILE_TYPE.match(file.magic_file_type)
- @needs_content
def compare_details(self, other, source=None):
differences = []
# look up differences in metadata
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the Reproducible-commits
mailing list