[Pkg-bazaar-commits] r133 /bzr/pkg-bazaar/bzr-builddeb/people/jdw/merge_upstream: Refactor import_dsc to be a bit cleaner. Still needs some work.
James Westby
jw+debian at jameswestby.net
Sat Jun 30 14:17:45 UTC 2007
------------------------------------------------------------
revno: 133
committer: James Westby <jw+debian at jameswestby.net>
branch nick: merge_upstream
timestamp: Sat 2007-06-30 15:17:45 +0100
message:
Refactor import_dsc to be a bit cleaner. Still needs some work.
modified:
__init__.py
import_dsc.py
tests/test_import_dsc.py
-------------- next part --------------
=== modified file '__init__.py'
--- a/__init__.py 2007-06-26 20:55:43 +0000
+++ b/__init__.py 2007-06-30 14:17:45 +0000
@@ -339,7 +339,7 @@
if directory is None:
directory = package
importer = SnapshotImporter(package)
- importer.do_import(directory)
+ importer.import_dsc(directory)
register_command(cmd_import_snapshot)
=== modified file 'import_dsc.py'
--- a/import_dsc.py 2007-06-26 20:55:43 +0000
+++ b/import_dsc.py 2007-06-30 14:17:45 +0000
@@ -20,28 +20,23 @@
import gzip
import os
-from StringIO import StringIO
from subprocess import Popen, PIPE
import deb822
from debian_bundle.changelog import Version
from bzrlib import (bzrdir,
- generate_ids,
urlutils,
)
from bzrlib.errors import FileExists, BzrError
from bzrlib.trace import warning, info
-from bzrlib.transform import TreeTransform
from bzrlib.transport import get_transport
from bzrlib.plugins.bzrtools.upstream_import import (import_tar,
- common_directory,
)
from errors import ImportError
from merge_upstream import make_upstream_tag
-import patches
# TODO: support native packages (should be easy).
# TODO: Use a transport to retrieve the files, so that they can be got remotely
@@ -53,31 +48,37 @@
base_dir are not None, then path will be interpreted relative to base_dir.
"""
if transport is None:
- return open(path, 'rb')
+ base_dir, path = urlutils.split(path)
+ transport = get_transport(base_dir)
else:
if base_dir is not None:
path = urlutils.join(base_dir, path)
- return transport.get(path)
+ return (transport.get(path), transport)
class DscCache(object):
def __init__(self, transport=None):
self.cache = {}
+ self.transport_cache = {}
self.transport = transport
def get_dsc(self, name):
if name in self.cache:
dsc1 = self.cache[name]
else:
- f1 = open_file(name, self.transport)
+ (f1, transport) = open_file(name, self.transport)
try:
dsc1 = deb822.Dsc(f1)
finally:
f1.close()
self.cache[name] = dsc1
+ self.transport_cache[name] = transport
return dsc1
+ def get_transport(self, name):
+ return self.transport_cache[name]
+
class DscComp(object):
def __init__(self, cache):
@@ -95,122 +96,151 @@
return -1
-def import_orig(tree, origname, version, last_upstream=None, transport=None,
- base_dir=None):
- f = open_file(origname, transport, base_dir=base_dir)
- try:
- dangling_revid = None
- if last_upstream is not None:
- dangling_revid = tree.branch.last_revision()
- old_upstream_revid = tree.branch.tags.lookup_tag(
- make_upstream_tag(last_upstream))
- tree.revert([], tree.branch.repository.revision_tree(old_upstream_revid))
- import_tar(tree, f)
- if last_upstream is not None:
- tree.set_parent_ids([old_upstream_revid])
- revno = tree.branch.revision_id_to_revno(old_upstream_revid)
- tree.branch.set_last_revision_info(revno, old_upstream_revid)
- tree.commit('import upstream from %s' % (os.path.basename(origname)))
- upstream_version = version.upstream_version
- tree.branch.tags.set_tag(make_upstream_tag(upstream_version),
- tree.branch.last_revision())
- finally:
- f.close()
- return dangling_revid
-
-
-def import_diff(tree, diffname, version, dangling_revid=None,
- transport=None, base_dir=None):
- upstream_version = version.upstream_version
- up_revid = tree.branch.tags.lookup_tag(make_upstream_tag(upstream_version))
- up_tree = tree.branch.repository.revision_tree(up_revid)
- if dangling_revid is None:
- current_revid = tree.branch.last_revision()
- else:
- current_revid = dangling_revid
- current_tree = tree.branch.repository.revision_tree(current_revid)
- tree.revert([], tree.branch.repository.revision_tree(up_revid))
- f = open_file(diffname, transport, base_dir=base_dir)
- f = gzip.GzipFile(fileobj=f)
- try:
- cmd = ['patch', '--strip', '1', '--quiet', '--directory', tree.basedir]
+class DscImporter(object):
+
+ transport = None
+
+ def __init__(self, dsc_files):
+ self.dsc_files = dsc_files
+
+ def import_orig(self, tree, origname, version, last_upstream=None,
+ transport=None, base_dir=None):
+ f = open_file(origname, transport, base_dir=base_dir)[0]
+ try:
+ dangling_revid = None
+ if last_upstream is not None:
+ dangling_revid = tree.branch.last_revision()
+ old_upstream_revid = tree.branch.tags.lookup_tag(
+ make_upstream_tag(last_upstream))
+ tree.revert([],
+ tree.branch.repository.revision_tree(old_upstream_revid))
+ import_tar(tree, f)
+ if last_upstream is not None:
+ tree.set_parent_ids([old_upstream_revid])
+ revno = tree.branch.revision_id_to_revno(old_upstream_revid)
+ tree.branch.set_last_revision_info(revno, old_upstream_revid)
+ tree.commit('import upstream from %s' % (os.path.basename(origname)))
+ upstream_version = version.upstream_version
+ tree.branch.tags.set_tag(make_upstream_tag(upstream_version),
+ tree.branch.last_revision())
+ finally:
+ f.close()
+ return dangling_revid
+
+ def _patch_tree(self, patch, basedir):
+ cmd = ['patch', '--strip', '1', '--quiet', '--directory', basedir]
child_proc = Popen(cmd, stdin=PIPE)
- for line in f:
+ for line in patch:
child_proc.stdin.write(line)
child_proc.stdin.close()
r = child_proc.wait()
if r != 0:
raise BzrError('patch failed')
- f.seek(0)
+
+ def _get_touched_paths(self, patch):
cmd = ['lsdiff', '--strip', '1']
child_proc = Popen(cmd, stdin=PIPE, stdout=PIPE)
- for line in f:
+ for line in patch:
child_proc.stdin.write(line)
child_proc.stdin.close()
r = child_proc.wait()
if r != 0:
- raise BzrError('patch failed')
+ raise BzrError('lsdiff failed')
touched_paths = []
- for file in child_proc.stdout.readlines():
- if file.endswith('\n'):
- file = file[:-1]
- touched_paths.append(file)
- implied_parents = set()
- def add_implied_parents(path, file_ids_from=None):
- parent = os.path.dirname(path)
- if parent == '':
- return
- if parent in implied_parents:
- return
- implied_parents.add(parent)
- add_implied_parents(parent)
- if file_ids_from is None:
- tree.add([parent])
- else:
- file_id = file_ids_from.path2id(parent)
- if file_id is None:
+ for filename in child_proc.stdout.readlines():
+ if filename.endswith('\n'):
+ filename = filename[:-1]
+ touched_paths.append(filename)
+ return touched_paths
+
+ def import_diff(self, tree, diffname, version, dangling_revid=None,
+ transport=None, base_dir=None):
+ upstream_version = version.upstream_version
+ up_revid = tree.branch.tags.lookup_tag(make_upstream_tag(upstream_version))
+ up_tree = tree.branch.repository.revision_tree(up_revid)
+ if dangling_revid is None:
+ current_revid = tree.branch.last_revision()
+ else:
+ current_revid = dangling_revid
+ current_tree = tree.branch.repository.revision_tree(current_revid)
+ tree.revert([], tree.branch.repository.revision_tree(up_revid))
+ f = open_file(diffname, transport, base_dir=base_dir)[0]
+ f = gzip.GzipFile(fileobj=f)
+ try:
+ self._patch_tree(f, tree.basedir)
+ f.seek(0)
+ touched_paths = self._get_touched_paths(f)
+ implied_parents = set()
+
+ def add_implied_parents(path, file_ids_from=None):
+ parent = os.path.dirname(path)
+ if parent == '':
+ return
+ if parent in implied_parents:
+ return
+ implied_parents.add(parent)
+ add_implied_parents(parent)
+ if file_ids_from is None:
tree.add([parent])
else:
- tree.add([parent], [file_id])
- for path in touched_paths:
- if not tree.has_filename(path):
- tree.remove([path], verbose=False)
- if not current_tree.has_filename(path):
- add_implied_parents(path)
- tree.add([path])
- if not up_tree.has_filename(path) and current_tree.has_filename(path):
- add_implied_parents(path, file_ids_from=current_tree)
- file_id = current_tree.path2id(path)
- if file_id is None:
+ file_id = file_ids_from.path2id(parent)
+ if file_id is None:
+ tree.add([parent])
+ else:
+ tree.add([parent], [file_id])
+
+ for path in touched_paths:
+ if not tree.has_filename(path):
+ tree.remove([path], verbose=False)
+ if not current_tree.has_filename(path):
+ add_implied_parents(path)
tree.add([path])
- else:
- tree.add([path], [file_id])
- if dangling_revid is not None:
- tree.add_parent_tree_id(dangling_revid)
- tree.commit('merge packaging changes from %s' % \
- (os.path.basename(diffname)))
- finally:
- f.close()
-
-
-def import_dsc(target_dir, dsc_files, transport=None):
- if os.path.exists(target_dir):
- raise FileExists(target_dir)
- cache = DscCache(transport=transport)
- dsc_files.sort(cmp=DscComp(cache).cmp)
- safe_files = []
- package_name = None
- for dscname in dsc_files:
- dsc = cache.get_dsc(dscname)
- orig_file = None
- diff_file = None
- if package_name is not None and dsc['Source'] != package_name:
+ if not up_tree.has_filename(path) and current_tree.has_filename(path):
+ add_implied_parents(path, file_ids_from=current_tree)
+ file_id = current_tree.path2id(path)
+ if file_id is None:
+ tree.add([path])
+ else:
+ tree.add([path], [file_id])
+ if dangling_revid is not None:
+ tree.add_parent_tree_id(dangling_revid)
+ tree.commit('merge packaging changes from %s' % \
+ (os.path.basename(diffname)))
+ finally:
+ f.close()
+
+ def _add_to_safe(self, file, version, type, base, transport):
+ found = False
+ for safe_file in self.safe_files:
+ if file == safe_file[0]:
+ found = True
+ break
+ if not found:
+ self.safe_files.append((file, version, type, base, transport))
+
+ def _check_orig_exists(self, version):
+ found = False
+ for safe_file in self.safe_files:
+ if safe_file[0].endswith("_%s.orig.tar.gz" % version.upstream_version):
+ found = True
+ break
+ if found == False:
+ raise ImportError("There is no upstream tarball corresponding to %s" % \
+ version)
+
+ def _check_package_name(self, name):
+ if self.package_name is not None and name != self.package_name:
raise ImportError("The reported package name has changed from %s to "
"%s. I don't know what to do in this case. If this "
"case should be handled, please contact the author "
"with details of your case, and the expected outcome."
- % (package_name, dsc['Source']))
- package_name = dsc['Source']
+ % (self.package_name, name))
+ self.package_name = name
+
+ def _decode_dsc(self, dsc, dscname):
+ orig_file = None
+ diff_file = None
+ self._check_package_name(dsc['Source'])
for file_details in dsc['files']:
name = file_details['name']
if name.endswith('.orig.tar.gz'):
@@ -225,57 +255,54 @@
raise ImportError("%s contains only a .orig.tar.gz, it must contain a "
".diff.gz as well" % dscname)
version = Version(dsc['Version'])
- base_dir = urlutils.split(dscname)[0]
+ if self.transport is not None:
+ base_dir = urlutils.split(dscname)[0]
+ else:
+ base_dir = None
+ dsc_transport = self.cache.get_transport(dscname)
if orig_file is not None:
- found = False
- for safe_file in safe_files:
- if orig_file == safe_file[0]:
- found = True
- break
- if not found:
- safe_files.append((orig_file, version, 'orig', base_dir))
- found = False
- for safe_file in safe_files:
- if safe_file[0].endswith("_%s.orig.tar.gz" % version.upstream_version):
- found = True
- break
- if found == False:
- raise ImportError("There is no upstream version corresponding to %s" % \
- diff_file)
- found = False
- for safe_file in safe_files:
- if diff_file == safe_file[0]:
- found = True
- break
- if not found:
- safe_files.append((diff_file, version, 'diff', base_dir))
- os.mkdir(target_dir)
- format = bzrdir.format_registry.make_bzrdir('dirstate-tags')
- branch = bzrdir.BzrDir.create_branch_convenience(target_dir,
- format=format)
- tree = branch.bzrdir.open_workingtree()
- tree.lock_write()
- try:
- last_upstream = None
- dangling_revid = None
- for (filename, version, type, base_dir) in safe_files:
- if type == 'orig':
- dangling_revid = import_orig(tree, filename, version,
- last_upstream=last_upstream,
- transport=transport,
- base_dir=base_dir)
- info("imported %s" % filename)
- last_upstream = version.upstream_version
- elif type == 'diff':
- import_diff(tree, filename, version, dangling_revid=dangling_revid,
- transport=transport, base_dir=base_dir)
- info("imported %s" % filename)
- dangling_revid = None
- finally:
- tree.unlock()
-
-
-class SourcesImporter(object):
+ self._add_to_safe(orig_file, version, 'orig', base_dir, dsc_transport)
+ self._check_orig_exists(version)
+ self._add_to_safe(diff_file, version, 'diff', base_dir, dsc_transport)
+
+ def import_dsc(self, target_dir):
+ if os.path.exists(target_dir):
+ raise FileExists(target_dir)
+ self.cache = DscCache(transport=self.transport)
+ self.dsc_files.sort(cmp=DscComp(self.cache).cmp)
+ self.safe_files = []
+ self.package_name = None
+ for dscname in self.dsc_files:
+ dsc = self.cache.get_dsc(dscname)
+ self._decode_dsc(dsc, dscname)
+ os.mkdir(target_dir)
+ format = bzrdir.format_registry.make_bzrdir('dirstate-tags')
+ branch = bzrdir.BzrDir.create_branch_convenience(target_dir,
+ format=format)
+ tree = branch.bzrdir.open_workingtree()
+ tree.lock_write()
+ try:
+ last_upstream = None
+ dangling_revid = None
+ for (filename, version, type, base_dir, transport) in self.safe_files:
+ if type == 'orig':
+ dangling_revid = self.import_orig(tree, filename, version,
+ last_upstream=last_upstream,
+ transport=transport,
+ base_dir=base_dir)
+ info("imported %s" % filename)
+ last_upstream = version.upstream_version
+ elif type == 'diff':
+ self.import_diff(tree, filename, version,
+ dangling_revid=dangling_revid,
+ transport=transport, base_dir=base_dir)
+ info("imported %s" % filename)
+ dangling_revid = None
+ finally:
+ tree.unlock()
+
+
+class SourcesImporter(DscImporter):
"""For importing all the .dsc files from a Sources file."""
def __init__(self, base, sources_path):
@@ -291,16 +318,8 @@
if isinstance(sources_path, unicode):
sources_path = sources_path.encode('utf-8')
self.sources_path = sources_path
-
- def do_import(self, target):
- """Perform the import, with the resulting branch in ``target``.
-
- :param target: the path to the branch that should be created for the
- import. The path cannot already exist.
- :type target: string.
- """
- transport = get_transport(self.base)
- sources_file = transport.get(self.sources_path)
+ self.transport = get_transport(self.base)
+ sources_file = self.transport.get(self.sources_path)
if self.sources_path.endswith(".gz"):
sources_file = gzip.GzipFile(fileobj=sources_file)
dsc_files = []
@@ -315,7 +334,7 @@
name = file_info['name']
if name.endswith('.dsc'):
dsc_files.append(urlutils.join(base_dir, name))
- import_dsc(target, dsc_files, transport=transport)
+ super(SourcesImporter, self).__init__(dsc_files)
def _check_basedir(self, base_dir):
return True
=== modified file 'tests/test_import_dsc.py'
--- a/tests/test_import_dsc.py 2007-06-26 20:55:43 +0000
+++ b/tests/test_import_dsc.py 2007-06-30 14:17:45 +0000
@@ -27,7 +27,7 @@
from bzrlib.workingtree import WorkingTree
from errors import ImportError
-from import_dsc import import_dsc
+from import_dsc import DscImporter
def write_to_file(filename, contents):
f = open(filename, 'wb')
@@ -43,7 +43,7 @@
finally:
f.close()
-class TestImportDsc(TestCaseWithTransport):
+class TestDscImporter(TestCaseWithTransport):
basedir = 'package'
target = 'target'
@@ -134,12 +134,12 @@
Build-Depends: debhelper (>= 5.0.0)
Files:
8636a3e8ae81664bac70158503aaf53a 1328218 %s
-""" % (package, version, file1))
+""" % (package, version, os.path.basename(file1)))
i = 1
for extra_file in extra_files:
append_to_file(filename,
" 1acd97ad70445afd5f2a64858296f21%d 20709 %s\n" % \
- (i, extra_file))
+ (i, os.path.basename(extra_file)))
i += 1
def make_dsc_1(self):
@@ -166,44 +166,45 @@
def import_dsc_1(self):
self.make_dsc_1()
- import_dsc(self.target, [self.dsc_1])
+ DscImporter([self.dsc_1]).import_dsc(self.target)
def import_dsc_1b(self):
self.make_dsc_1()
self.make_dsc_1b()
- import_dsc(self.target, [self.dsc_1, self.dsc_1b])
+ DscImporter([self.dsc_1, self.dsc_1b]).import_dsc(self.target)
def import_dsc_1b_repeated_diff(self):
self.make_dsc_1()
self.make_dsc_1b()
- import_dsc(self.target, [self.dsc_1, self.dsc_1b, self.dsc_1b])
+ DscImporter([self.dsc_1, self.dsc_1b, self.dsc_1b]).import_dsc(self.target)
def import_dsc_1c(self):
self.make_dsc_1()
self.make_dsc_1b()
self.make_dsc_1c()
- import_dsc(self.target, [self.dsc_1, self.dsc_1c, self.dsc_1b])
+ DscImporter([self.dsc_1, self.dsc_1c, self.dsc_1b]).import_dsc(self.target)
def import_dsc_2(self):
self.make_dsc_1()
self.make_dsc_1b()
self.make_dsc_1c()
self.make_dsc_2()
- import_dsc(self.target,
- [self.dsc_1, self.dsc_1b, self.dsc_1c, self.dsc_2])
+ importer = DscImporter([self.dsc_1, self.dsc_1b, self.dsc_1c, self.dsc_2])
+ importer.import_dsc(self.target)
def import_dsc_2_repeated_orig(self):
self.make_dsc_1()
self.make_dsc_1b_repeated_orig()
self.make_dsc_1c()
self.make_dsc_2()
- import_dsc(self.target,
- [self.dsc_1, self.dsc_1b, self.dsc_1c, self.dsc_2])
+ importer = DscImporter([self.dsc_1, self.dsc_1b, self.dsc_1c, self.dsc_2])
+ importer.import_dsc(self.target)
def test_import_dsc_target_extant(self):
os.mkdir(self.target)
write_to_file('package_0.1.dsc', '')
- self.assertRaises(FileExists, import_dsc, self.target, ['package_0.1.dsc'])
+ importer = DscImporter([self.dsc_1])
+ self.assertRaises(FileExists, importer.import_dsc, self.target)
def test_import_one_dsc_tree(self):
self.import_dsc_1()
@@ -474,17 +475,21 @@
def test_import_dsc_restrictions_on_dscs(self):
"""Test that errors are raised for confusing sets of .dsc files."""
self.make_dsc(self.dsc_1, '0.1-1', self.diff_1)
- self.assertRaises(ImportError, import_dsc, self.target, [self.dsc_1])
+ importer = DscImporter([self.dsc_1])
+ self.assertRaises(ImportError, importer.import_dsc, self.target)
self.make_dsc(self.dsc_1, '0.1-1', self.orig_1)
- self.assertRaises(ImportError, import_dsc, self.target, [self.dsc_1])
+ importer = DscImporter([self.dsc_1])
+ self.assertRaises(ImportError, importer.import_dsc, self.target)
self.make_dsc(self.dsc_1, '0.1-1', self.orig_1, [self.diff_1, self.diff_1])
- self.assertRaises(ImportError, import_dsc, self.target, [self.dsc_1])
+ importer = DscImporter([self.dsc_1])
+ self.assertRaises(ImportError, importer.import_dsc, self.target)
self.make_dsc(self.dsc_1, '0.1-1', self.orig_1, [self.orig_1, self.diff_1])
- self.assertRaises(ImportError, import_dsc, self.target, [self.dsc_1])
+ importer = DscImporter([self.dsc_1])
+ self.assertRaises(ImportError, importer.import_dsc, self.target)
self.make_dsc(self.dsc_1, '0.1-1', self.orig_1, [self.diff_1])
self.make_dsc(self.dsc_1b, '0.1-2', self.diff_1b, package='otherpackage')
- self.assertRaises(ImportError, import_dsc, self.target,
- [self.dsc_1, self.dsc_1b])
+ importer = DscImporter([self.dsc_1, self.dsc_1b])
+ self.assertRaises(ImportError, importer.import_dsc, self.target)
def test_import_four_dsc_two_upstream_history_repeated_orig(self):
self.import_dsc_2_repeated_orig()
@@ -529,3 +534,22 @@
self.assertEqual(changes.modified[0][0], 'debian/changelog')
self.assertEqual(changes.modified[0][2], 'file')
+ def test_import_dsc_different_dir(self):
+ source = 'source'
+ os.mkdir(source)
+ self.diff_1 = os.path.join(source, self.diff_1)
+ self.orig_1 = os.path.join(source, self.orig_1)
+ self.dsc_1 = os.path.join(source, self.dsc_1)
+ self.import_dsc_1()
+ self.failUnlessExists(self.target)
+ tree = WorkingTree.open(self.target)
+ tree.lock_read()
+ expected_inv = ['README', 'CHANGELOG', 'Makefile', 'debian/',
+ 'debian/changelog', 'debian/install']
+ try:
+ self.check_inventory_shape(tree.inventory, expected_inv)
+ finally:
+ tree.unlock()
+ for path in expected_inv:
+ self.failUnlessExists(os.path.join(self.target, path))
+
More information about the Pkg-bazaar-commits
mailing list