[Reproducible-commits] [diffoscope] 10/15: XXX tentative parallel
Jérémy Bobbio
lunar at moszumanska.debian.org
Thu Dec 3 11:08:00 UTC 2015
This is an automated email from the git hooks/post-receive script.
lunar pushed a commit to branch pu/parallel2
in repository diffoscope.
commit 75c6e568a2822472a7a91ea51c2bca4456bec31d
Author: Jérémy Bobbio <lunar at debian.org>
Date: Wed Nov 18 16:57:15 2015 +0000
XXX tentative parallel
---
diffoscope/__main__.py | 6 ++++++
diffoscope/comparators/text.py | 16 ++++++++--------
diffoscope/config.py | 20 ++++++++++++++++++++
diffoscope/difference.py | 19 ++++++++++---------
4 files changed, 44 insertions(+), 17 deletions(-)
diff --git a/diffoscope/__main__.py b/diffoscope/__main__.py
index c027b54..40f1ef7 100644
--- a/diffoscope/__main__.py
+++ b/diffoscope/__main__.py
@@ -66,6 +66,10 @@ def create_parser():
help='maximum number of lines fed to diff (default: %d)' %
Config.general.max_diff_input_lines,
default=Config.general.max_diff_input_lines)
+ parser.add_argument('--max-workers', dest='max_workers', type=int,
+ help='maximum number of concurrent workers (default: %d)' %
+ Config.general.max_workers,
+ default=Config.general.max_workers)
parser.add_argument('--fuzzy-threshold', dest='fuzzy_threshold', type=int,
help='threshold for fuzzy-matching '
'(0 to disable, %d is default, 400 is high fuzziness)' %
@@ -115,11 +119,13 @@ def run_diffoscope(parsed_args):
Config.general.max_report_size = parsed_args.max_report_size
Config.general.fuzzy_threshold = parsed_args.fuzzy_threshold
Config.general.new_file = parsed_args.new_file
+ Config.general.max_workers = parsed_args.max_workers
if parsed_args.debug:
logger.setLevel(logging.DEBUG)
set_locale()
difference = diffoscope.comparators.compare_root_paths(
parsed_args.file1, parsed_args.file2)
+ Config.general.executor.shutdown(wait=True)
if difference:
if parsed_args.html_output:
with make_printer(parsed_args.html_output) as print_func:
diff --git a/diffoscope/comparators/text.py b/diffoscope/comparators/text.py
index e4e3fd5..daf790e 100644
--- a/diffoscope/comparators/text.py
+++ b/diffoscope/comparators/text.py
@@ -40,14 +40,14 @@ class TextFile(File):
my_encoding = self.encoding or 'utf-8'
other_encoding = other.encoding or 'utf-8'
try:
- with codecs.open(self.path, 'r', encoding=my_encoding) as my_content, \
- codecs.open(other.path, 'r', encoding=other_encoding) as other_content:
- difference = Difference.from_text_readers(my_content, other_content, self.name, other.name, source)
- if my_encoding != other_encoding:
- if not difference:
- difference = Difference(self.path, other.path, source)
- difference.add_details([Difference.from_text(my_encoding, other_encoding, None, None, source='encoding')])
- return difference
+ my_content = codecs.open(self.path, 'r', encoding=my_encoding)
+ other_content = codecs.open(other.path, 'r', encoding=other_encoding)
+ difference = Difference.from_text_readers(my_content, other_content, self.name, other.name, source)
+ if my_encoding != other_encoding:
+ if not difference:
+ difference = Difference(self.path, other.path, source)
+ difference.add_details([Difference.from_text(my_encoding, other_encoding, None, None, source='encoding')])
+ return difference
except (LookupError, UnicodeDecodeError):
# unknown or misdetected encoding
return self.compare_bytes(other, source)
diff --git a/diffoscope/config.py b/diffoscope/config.py
index 5087306..dae5dff 100644
--- a/diffoscope/config.py
+++ b/diffoscope/config.py
@@ -18,6 +18,10 @@
# along with diffoscope. If not, see <http://www.gnu.org/licenses/>.
+from concurrent.futures import ThreadPoolExecutor
+import os
+
+
# From http://stackoverflow.com/a/7864317
# Credits to kylealanhale
class classproperty(property):
@@ -32,6 +36,8 @@ class Config(object):
self._max_report_size = 2000 * 2 ** 10 # 2000 kB
self._fuzzy_threshold = 60
self._new_file = False
+ self._max_workers = os.cpu_count()
+ self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
@classproperty
def general(cls):
@@ -78,3 +84,17 @@ class Config(object):
@new_file.setter
def new_file(self, value):
self._new_file = value
+
+ @property
+ def max_workers(self):
+ return self._max_workers
+
+ @max_workers.setter
+ def max_workers(self, value):
+ self._max_workers = value
+ self._executor.shutdown(wait=False)
+ self._executor = ThreadPoolExecutor(max_workers=value)
+
+ @property
+ def executor(self):
+ return self._executor
diff --git a/diffoscope/difference.py b/diffoscope/difference.py
index a4756ee..4a37428 100644
--- a/diffoscope/difference.py
+++ b/diffoscope/difference.py
@@ -17,6 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with diffoscope. If not, see <http://www.gnu.org/licenses/>.
+from concurrent.futures import Future
from contextlib import contextmanager, suppress
from io import StringIO
import os
@@ -301,17 +302,17 @@ class Difference(object):
return '<Difference %s -- %s %s>' % (self._source1, self._source2, self._details)
def __bool__(self):
- return self._unified_diff is not None or len(self._notifications) > 0 or len(self._details) > 0
+ return self.unified_diff is not None or len(self._notifications) > 0 or len(self._details) > 0
@staticmethod
def from_feeder(feeder1, feeder2, path1, path2, source=None, comment=None):
difference = Difference(path1, path2, source)
- try:
- difference.unified_diff = diff(feeder1, feeder2)
- except RequiredToolNotFound:
- difference.add_comment('diff is not available!')
- if difference.unified_diff and comment:
+ if comment:
difference.add_comment(comment)
+ #try: XXXXXXXXXX
+ difference.unified_diff = Config.general.executor.submit(diff, feeder1, feeder2)
+ #except RequiredToolNotFound:
+ # difference.add_comment('diff is not available!')
return difference
@staticmethod
@@ -356,8 +357,6 @@ class Difference(object):
difference = None
with suppress(*suppress_errors):
difference = Difference.from_feeder(feeder1, feeder2, path1, path2, *args, **kwargs)
- if not difference:
- return None
if command1 and command1.stderr_content:
difference.add_comment('stderr from `%s`:' % ' '.join(command1.cmdline()))
difference.add_comment(command1.stderr_content)
@@ -405,6 +404,8 @@ class Difference(object):
@property
def unified_diff(self):
+ if isinstance(self._unified_diff, Future):
+ self._unified_diff = self._unified_diff.result()
return self._unified_diff
@unified_diff.setter
@@ -423,7 +424,7 @@ class Difference(object):
if self._unified_diff is None:
difference.unified_diff = None
else:
- difference.unified_diff = reverse_unified_diff(self._unified_diff)
+ difference.unified_diff = reverse_unified_diff(self.unified_diff)
logger.debug('reverse orig %s %s', self._source1, self._source2)
difference.add_details([d.get_reverse() for d in self._details])
return difference
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/diffoscope.git
More information about the Reproducible-commits
mailing list