[Reproducible-commits] [diffoscope] 07/13: Add support for CBFS images
Jérémy Bobbio
lunar at moszumanska.debian.org
Thu Oct 15 16:04:35 UTC 2015
This is an automated email from the git hooks/post-receive script.
lunar pushed a commit to branch master
in repository diffoscope.
commit 5ebe5f822dcaaad6218ae1e79efad9aa283e7f0d
Author: Jérémy Bobbio <lunar at debian.org>
Date: Thu Oct 15 10:05:36 2015 +0000
Add support for CBFS images
This requires cbfstool which is not available in Debian (yet?). We need to
change `--list-tools` to cope with a tool that has no matching package.
We restrict looking for a CBFS header in the whole image (which takes some
time) to cases where the file name ends with `.rom`.
Closes: #788364
---
diffoscope/__init__.py | 1 +
diffoscope/__main__.py | 2 +-
diffoscope/comparators/__init__.py | 2 +
diffoscope/comparators/cbfs.py | 137 ++++++++++++++++++++++++++++++++++
tests/comparators/test_cbfs.py | 93 +++++++++++++++++++++++
tests/data/cbfs_listing_expected_diff | 9 +++
6 files changed, 243 insertions(+), 1 deletion(-)
diff --git a/diffoscope/__init__.py b/diffoscope/__init__.py
index 711a72c..52e3e59 100644
--- a/diffoscope/__init__.py
+++ b/diffoscope/__init__.py
@@ -36,6 +36,7 @@ ch.setFormatter(formatter)
class RequiredToolNotFound(Exception):
PROVIDERS = { 'ar': { 'debian': 'binutils-multiarch' }
, 'bzip2': { 'debian': 'bzip2' }
+ , 'cbfstool': {}
, 'cmp': { 'debian': 'diffutils' }
, 'cpio': { 'debian': 'cpio' }
, 'diff': { 'debian': 'diffutils' }
diff --git a/diffoscope/__main__.py b/diffoscope/__main__.py
index 04d84bc..5048c36 100644
--- a/diffoscope/__main__.py
+++ b/diffoscope/__main__.py
@@ -97,7 +97,7 @@ class ListToolsAction(argparse.Action):
print(', '.join(tool_required.all))
print()
print("Available in packages:")
- print(', '.join(sorted(set([RequiredToolNotFound.PROVIDERS[k]["debian"] for k in tool_required.all]))))
+ print(', '.join(sorted(filter(None, { RequiredToolNotFound.PROVIDERS[k].get('debian', None) for k in tool_required.all }))))
sys.exit(0)
diff --git a/diffoscope/comparators/__init__.py b/diffoscope/comparators/__init__.py
index 3af5669..dcde9dc 100644
--- a/diffoscope/comparators/__init__.py
+++ b/diffoscope/comparators/__init__.py
@@ -31,6 +31,7 @@ from diffoscope.comparators.binary import \
File, FilesystemFile, NonExistingFile, compare_binary_files
from diffoscope.comparators.bzip2 import Bzip2File
from diffoscope.comparators.java import ClassFile
+from diffoscope.comparators.cbfs import CbfsFile
from diffoscope.comparators.cpio import CpioFile
from diffoscope.comparators.deb import DebFile, Md5sumsFile, DebDataTarFile
from diffoscope.comparators.debian import DotChangesFile
@@ -117,6 +118,7 @@ FILE_CLASSES = (
DebDataTarFile,
TextFile,
Bzip2File,
+ CbfsFile,
CpioFile,
DebFile,
ElfFile,
diff --git a/diffoscope/comparators/cbfs.py b/diffoscope/comparators/cbfs.py
new file mode 100644
index 0000000..523a1d5
--- /dev/null
+++ b/diffoscope/comparators/cbfs.py
@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2015 Jérémy Bobbio <lunar at debian.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <http://www.gnu.org/licenses/>.
+
+import io
+import os
+import os.path
+import re
+import subprocess
+import stat
+import struct
+from diffoscope import logger, tool_required
+from diffoscope.comparators.binary import File, needs_content
+from diffoscope.comparators.utils import Archive, Command
+from diffoscope.difference import Difference
+
+
+class CbfsListing(Command):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._header_re = re.compile(r'^.*: ([^,]+, bootblocksize [0-9]+, romsize [0-9]+, offset 0x[0-9A-Fa-f]+)$')
+
+ @tool_required('cbfstool')
+ def cmdline(self):
+ return ['cbfstool', self.path, 'print']
+
+ def filter(self, line):
+ return self._header_re.sub('\\1', line.decode('utf-8')).encode('utf-8')
+
+
+class CbfsContainer(Archive):
+ @tool_required('cbfstool')
+ def entries(self, path):
+ cmd = ['cbfstool', path, 'print']
+ output = subprocess.check_output(cmd, shell=False).decode('utf-8')
+ header = True
+ for line in output.rstrip('\n').split('\n'):
+ if header:
+ if line.startswith('Name'):
+ header = False
+ continue
+ name = line.split()[0]
+ if name == '(empty)':
+ continue
+ yield name
+
+ def open_archive(self, path):
+ return self
+
+ def close_archive(self):
+ pass
+
+ def get_member_names(self):
+ return list(self.entries(self.source.path))
+
+ @tool_required('cbfstool')
+ def extract(self, member_name, dest_dir):
+ dest_path = os.path.join(dest_dir, os.path.basename(member_name))
+ cmd = ['cbfstool', self.source.path, 'extract', '-n', member_name, '-f', dest_path]
+ logger.debug("cbfstool extract %s to %s", member_name, dest_path)
+ subprocess.check_call(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+ return dest_path
+
+
+CBFS_HEADER_MAGIC = 0x4F524243
+CBFS_HEADER_VERSION1 = 0x31313131
+CBFS_HEADER_VERSION2 = 0x31313132
+CBFS_HEADER_SIZE = 8 * 4 # 8 * uint32_t
+
+
+def is_header_valid(buf, size, offset=0):
+ magic, version, romsize, bootblocksize, align, cbfs_offset, architecture, pad = struct.unpack_from('!IIIIIIII', buf, offset)
+ return magic == CBFS_HEADER_MAGIC and \
+ (version == CBFS_HEADER_VERSION1 or version == CBFS_HEADER_VERSION2) and \
+ (romsize <= size) and \
+ (cbfs_offset < romsize)
+
+
+class CbfsFile(File):
+ @staticmethod
+ def recognizes(file):
+ with file.get_content():
+ size = os.stat(file.path).st_size
+ if size < CBFS_HEADER_SIZE:
+ return False
+ with open(file.path, 'rb') as f:
+ # pick at the latest byte as it should contain the relative offset of the header
+ f.seek(-4, io.SEEK_END)
+ # <pgeorgi> given the hardware we support so far, it looks like
+ # that field is now bound to be little endian
+ # -- #coreboot, 2015-10-14
+ rel_offset = struct.unpack('<i', f.read(4))[0]
+ if rel_offset < 0 and -rel_offset > CBFS_HEADER_SIZE:
+ f.seek(rel_offset, io.SEEK_END)
+ logger.debug('looking for header at offset: %x', f.tell())
+ if is_header_valid(f.read(CBFS_HEADER_SIZE), size):
+ return True
+ elif not file.name.endswith('.rom'):
+ return False
+ else:
+ logger.debug('CBFS relative offset seems wrong, scanning whole image')
+ f.seek(0, io.SEEK_SET)
+ offset = 0
+ buf = f.read(CBFS_HEADER_SIZE)
+ while len(buf) >= CBFS_HEADER_SIZE:
+ if is_header_valid(buf, size, offset):
+ return True
+ if len(buf) - offset <= CBFS_HEADER_SIZE:
+ buf = f.read(32768)
+ offset = 0
+ else:
+ offset += 1
+ return False
+
+ @needs_content
+ def compare_details(self, other, source=None):
+ differences = []
+ differences.append(Difference.from_command(CbfsListing, self.path, other.path))
+ with CbfsContainer(self).open() as my_container, \
+ CbfsContainer(other).open() as other_container:
+ differences.extend(my_container.compare(other_container))
+ return differences
diff --git a/tests/comparators/test_cbfs.py b/tests/comparators/test_cbfs.py
new file mode 100644
index 0000000..1850f76
--- /dev/null
+++ b/tests/comparators/test_cbfs.py
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+#
+# diffoscope: in-depth comparison of files, archives, and directories
+#
+# Copyright © 2015 Jérémy Bobbio <lunar at debian.org>
+#
+# diffoscope is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# diffoscope is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with diffoscope. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+from subprocess import check_call
+import struct
+import pytest
+from diffoscope.comparators import specialize
+from diffoscope.comparators.binary import FilesystemFile, NonExistingFile
+from diffoscope.comparators.cbfs import CbfsFile
+from diffoscope.config import Config
+from diffoscope.presenters.text import output_text
+from conftest import tool_missing
+
+TEST_FILE1_PATH = os.path.join(os.path.dirname(__file__), '../data/text_ascii1')
+TEST_FILE2_PATH = os.path.join(os.path.dirname(__file__), '../data/text_ascii2')
+
+ at pytest.fixture
+def rom1(tmpdir):
+ path = str(tmpdir.join('coreboot1'))
+ check_call(['cbfstool', path, 'create', '-m', 'x86', '-s', '32768'], shell=False)
+ check_call(['cbfstool', path, 'add', '-f', TEST_FILE1_PATH, '-n', 'text', '-t', 'raw'], shell=False)
+ return specialize(FilesystemFile(path))
+
+ at pytest.fixture
+def rom2(tmpdir):
+ path = str(tmpdir.join('coreboot2.rom'))
+ size = 32768
+ check_call(['cbfstool', path, 'create', '-m', 'x86', '-s', '%s' % size], shell=False)
+ check_call(['cbfstool', path, 'add', '-f', TEST_FILE2_PATH, '-n', 'text', '-t', 'raw'], shell=False)
+ # Remove the last 4 bytes to exercice the full header search
+ buf = bytearray(size)
+ with open(path, 'rb') as f:
+ f.readinto(buf)
+ with open(path, 'wb') as f:
+ size = struct.unpack_from('!I', buf, offset=len(buf) - 4 - 32 + 8)[0]
+ struct.pack_into('!I', buf, len(buf) - 4 - 32 + 8, size - 4)
+ f.write(buf[:-4])
+ return specialize(FilesystemFile(path))
+
+ at pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
+def test_identification_using_offset(rom1):
+ assert isinstance(rom1, CbfsFile)
+
+ at pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
+def test_identification_without_offset(rom2):
+ assert isinstance(rom2, CbfsFile)
+
+ at pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
+def test_no_differences(rom1):
+ difference = rom1.compare(rom1)
+ assert difference is None
+
+ at pytest.fixture
+def differences(rom1, rom2):
+ difference = rom1.compare(rom2)
+ output_text(difference, print_func=print)
+ return difference.details
+
+ at pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
+def test_listing(differences):
+ expected_diff = open(os.path.join(os.path.dirname(__file__), '../data/cbfs_listing_expected_diff')).read()
+ assert differences[0].unified_diff == expected_diff
+
+ at pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
+def test_content(differences):
+ assert differences[1].source1 == 'text'
+ assert differences[1].source2 == 'text'
+ expected_diff = open(os.path.join(os.path.dirname(__file__), '../data/text_ascii_expected_diff')).read()
+ assert differences[1].unified_diff == expected_diff
+
+ at pytest.mark.skipif(tool_missing('cbfstool'), reason='missing cbfstool')
+def test_compare_non_existing(monkeypatch, rom1):
+ monkeypatch.setattr(Config.general, 'new_file', True)
+ difference = rom1.compare(NonExistingFile('/nonexisting', rom1))
+ assert difference.source2 == '/nonexisting'
+ assert difference.details[-1].source2 == '/dev/null'
diff --git a/tests/data/cbfs_listing_expected_diff b/tests/data/cbfs_listing_expected_diff
new file mode 100644
index 0000000..e40db68
--- /dev/null
+++ b/tests/data/cbfs_listing_expected_diff
@@ -0,0 +1,9 @@
+@@ -1,6 +1,3 @@
+-32 kB, bootblocksize 0, romsize 32768, offset 0x0
+-alignment: 64 bytes, architecture: x86
+-
+ Name Offset Type Size
+-text 0x0 raw 446
+-(empty) 0x200 null 32152
++text 0x0 raw 671
++(empty) 0x300 null 31896
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the Reproducible-commits
mailing list