[ros-rosdistro] 01/03: Imported Upstream version 0.4.7
Jochen Sprickerhof
jspricke-guest at moszumanska.debian.org
Sat Jun 18 08:26:24 UTC 2016
This is an automated email from the git hooks/post-receive script.
jspricke-guest pushed a commit to branch master
in repository ros-rosdistro.
commit 50dcb882784f8f7215b8426918ab73946e602791
Author: Jochen Sprickerhof <git at jochen.sprickerhof.de>
Date: Sat Jun 18 10:21:25 2016 +0200
Imported Upstream version 0.4.7
---
scripts/rosdistro_freeze_source | 87 +++++++++++++++++++++
setup.py | 1 +
src/rosdistro/__init__.py | 6 +-
src/rosdistro/_version.py | 2 +-
src/rosdistro/distribution_cache.py | 15 ++++
src/rosdistro/distribution_cache_generator.py | 18 ++++-
src/rosdistro/freeze_source.py | 108 ++++++++++++++++++++++++++
src/rosdistro/manifest_provider/git.py | 55 ++++++++-----
src/rosdistro/release_cache_generator.py | 12 ++-
9 files changed, 274 insertions(+), 30 deletions(-)
diff --git a/scripts/rosdistro_freeze_source b/scripts/rosdistro_freeze_source
new file mode 100755
index 0000000..0a4a129
--- /dev/null
+++ b/scripts/rosdistro_freeze_source
@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+# Software License Agreement (BSD License)
+#
+# Copyright (c) 2016, Clearpath Robotics
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Open Source Robotics Foundation, Inc. nor
+# the names of its contributors may be used to endorse or promote
+# products derived from this software without specific prior
+# written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import os.path
+import sys
+import yaml
+
+from rosdistro import get_distribution
+from rosdistro.index import Index
+from rosdistro.freeze_source import freeze_distribution_sources, CONCURRENT_DEFAULT
+from rosdistro.writer import yaml_from_distribution_file
+
+
+def parse_args(args=sys.argv[1:]):
+ parser = argparse.ArgumentParser(
+ description='''
+ Freeze a rosdistro\'s source branch versions to hashes or tags. If neither --release-version
+ nor --release-tag are specified, the hashes of the current devel branches are used.
+ ''')
+ parser.add_argument('index', type=argparse.FileType(),
+ help='Path to a local index.yaml file.')
+ parser.add_argument('dist_names', nargs='*', help='The names of the distributions (default: all)')
+ parser.add_argument('-j', '--jobs', type=int, default=CONCURRENT_DEFAULT,
+ help='How many worker threads to use.')
+ parser.add_argument('-q', '--quiet', action="store_true",
+ help='Suppress updating status bar (for script/CI usage).')
+
+ mode = parser.add_mutually_exclusive_group()
+ mode.add_argument('--release-version', action="store_true",
+ help='Freeze to the hash of current release tag.')
+ mode.add_argument('--release-tag', action="store_true",
+ help='Freeze to name of current release tag.')
+ return parser.parse_args(args)
+
+
+def main():
+ args = parse_args()
+ index = Index(yaml.safe_load(args.index),
+ 'file://%s' % os.path.dirname(os.path.abspath(args.index.name)))
+
+ if not args.dist_names:
+ args.dist_names = sorted(index.distributions.keys())
+
+ for dist_name in args.dist_names:
+ dist = get_distribution(index, dist_name)
+ freeze_distribution_sources(dist, release_version=args.release_version, release_tag=args.release_tag,
+ concurrent_ops=args.jobs, quiet=args.quiet)
+ dist_file_local = index.distributions[dist_name]['distribution'].split('://')[1]
+ with open(dist_file_local, 'w') as f:
+ f.write(yaml_from_distribution_file(dist))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/setup.py b/setup.py
index 2b64aeb..bf7143b 100755
--- a/setup.py
+++ b/setup.py
@@ -19,6 +19,7 @@ setup(
scripts=[
# 'scripts/rosdistro',
'scripts/rosdistro_build_cache',
+ 'scripts/rosdistro_freeze_source',
# 'scripts/rosdistro_convert',
# 'scripts/rosdistro_generate_cache',
'scripts/rosdistro_migrate_to_rep_141',
diff --git a/src/rosdistro/__init__.py b/src/rosdistro/__init__.py
index 5a39c01..8577c64 100644
--- a/src/rosdistro/__init__.py
+++ b/src/rosdistro/__init__.py
@@ -164,7 +164,7 @@ def get_cached_distribution(index, dist_name, cache=None, allow_lazy_load=False)
def get_distribution_cache(index, dist_name):
if dist_name not in index.distributions.keys():
- raise RuntimeError("Unknown distribution: '{0}'. Valid distribution names are: {1}".format(dist_name, ', '.join(["'%s'" % d for d in index.distributions.keys()])))
+ raise RuntimeError("Unknown distribution: '{0}'. Valid distribution names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys()))))
dist = index.distributions[dist_name]
if 'distribution_cache' not in dist.keys():
raise RuntimeError("Distribution has no cache: '{0}'".format(dist_name))
@@ -223,7 +223,7 @@ def get_release_file(index, dist_name):
def get_release_cache(index, dist_name):
print('# rosdistro.get_release_cache() has been deprecated in favor of the new function rosdistro.get_distribution_cache() - please check that you have the latest versions of the Python tools (e.g. on Ubuntu/Debian use: sudo apt-get update && sudo apt-get install --only-upgrade python-bloom python-rosdep python-rosinstall python-rosinstall-generator)', file=sys.stderr)
if dist_name not in index.distributions.keys():
- raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(["'%s'" % d for d in index.distributions.keys()])))
+ raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys()))))
dist = index.distributions[dist_name]
if 'distribution_cache' not in dist.keys():
raise RuntimeError("Release has no cache: '{0}'".format(dist_name))
@@ -305,7 +305,7 @@ def get_doc_build_files(index, dist_name):
def _get_dist_file_data(index, dist_name, type_):
if dist_name not in index.distributions.keys():
- raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(["'%s'" % d for d in index.distributions.keys()])))
+ raise RuntimeError("Unknown release: '{0}'. Valid release names are: {1}".format(dist_name, ', '.join(sorted(index.distributions.keys()))))
dist = index.distributions[dist_name]
if type_ not in dist.keys():
raise RuntimeError('unknown release type "%s"' % type_)
diff --git a/src/rosdistro/_version.py b/src/rosdistro/_version.py
index 9a8e054..1e4826d 100644
--- a/src/rosdistro/_version.py
+++ b/src/rosdistro/_version.py
@@ -1 +1 @@
-__version__ = '0.4.4'
+__version__ = '0.4.7'
diff --git a/src/rosdistro/distribution_cache.py b/src/rosdistro/distribution_cache.py
index 07330ab..38b10c1 100644
--- a/src/rosdistro/distribution_cache.py
+++ b/src/rosdistro/distribution_cache.py
@@ -78,6 +78,21 @@ class DistributionCache(object):
# remove packages which are not in the old distribution file
self._remove_obsolete_entries()
+ # determine differences in doc and source entries
+ if len(distribution_file_data) == len(self._distribution_file_data):
+ for old_data, new_data in zip(self._distribution_file_data, distribution_file_data):
+ for repo_name in sorted(new_data['repositories'].keys()):
+ repo = new_data['repositories'][repo_name]
+ for section in ['doc', 'source']:
+ if section not in repo:
+ continue
+ if repo_name in old_data['repositories'] and \
+ section in old_data['repositories'][repo_name] and \
+ old_data['repositories'][repo_name][section] == repo[section]:
+ continue
+ # section is either different or does't exist before
+ print(" - updated '%s' entry for repository '%s'" % (section, repo_name))
+
self._distribution_file_data = distribution_file_data
dist_file = create_distribution_file(self.distribution_file.name, self._distribution_file_data)
diff --git a/src/rosdistro/distribution_cache_generator.py b/src/rosdistro/distribution_cache_generator.py
index 314e2f7..60faba9 100644
--- a/src/rosdistro/distribution_cache_generator.py
+++ b/src/rosdistro/distribution_cache_generator.py
@@ -84,6 +84,9 @@ def generate_distribution_cache(index, dist_name, preclean=False, ignore_local=F
sys.stdout.write('.')
sys.stdout.flush()
# check that package.xml is fetchable
+ old_package_xml = None
+ if cache and pkg_name in cache.release_package_xmls:
+ old_package_xml = cache.release_package_xmls[pkg_name]
package_xml = dist.get_release_package_xml(pkg_name)
if not package_xml:
errors.append('%s: missing package.xml file for package "%s"' % (dist_name, pkg_name))
@@ -98,6 +101,9 @@ def generate_distribution_cache(index, dist_name, preclean=False, ignore_local=F
if not re.match('^%s-[\dA-z~\+\.]+$' % re.escape(pkg.version), repo.version):
errors.append('%s: different version in package.xml (%s) for package "%s" than for the repository (%s) (after removing the debian increment)' % (dist_name, pkg.version, pkg_name, repo.version))
+ if package_xml != old_package_xml:
+ print(" - updated manifest of package '%s' to version '%s'" % (pkg_name, pkg.version))
+
if not debug:
print('')
@@ -130,14 +136,18 @@ def _get_cached_distribution(index, dist_name, preclean=False, ignore_local=Fals
print('- trying to fetch cache')
# get distribution cache
cache = get_distribution_cache(index, dist_name)
- # get current distribution file
- rel_file_data = _get_dist_file_data(index, dist_name, 'distribution')
- # update cache with current distribution file
- cache.update_distribution(rel_file_data)
except Exception as e:
print('- failed to fetch old cache: %s' % e)
+
if cache:
print('- update cache')
+ # get current distribution file
+ rel_file_data = _get_dist_file_data(index, dist_name, 'distribution')
+ # since format 2 of the index file might contain a single value rather then a list
+ if not isinstance(rel_file_data, list):
+ rel_file_data = [rel_file_data]
+ # update cache with current distribution file
+ cache.update_distribution(rel_file_data)
else:
print('- build cache from scratch')
# get empty cache with distribution file
diff --git a/src/rosdistro/freeze_source.py b/src/rosdistro/freeze_source.py
new file mode 100644
index 0000000..facdc91
--- /dev/null
+++ b/src/rosdistro/freeze_source.py
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+# Software License Agreement (BSD License)
+#
+# Copyright (c) 2016, Clearpath Robotics
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Open Source Robotics Foundation, Inc. nor
+# the names of its contributors may be used to endorse or promote
+# products derived from this software without specific prior
+# written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import print_function
+
+import sys
+import subprocess
+import threading
+import time
+import yaml
+
+try:
+ import queue
+except ImportError:
+ import Queue as queue
+
+CONCURRENT_DEFAULT = 16
+
+
+def freeze_distribution_sources(dist, release_version=False, release_tag=False,
+ concurrent_ops=CONCURRENT_DEFAULT, quiet=False):
+ # Populate this queue with tuples of repositories instances to be updated,
+ # so that this work can be spread across multiple threads.
+ work_queue = queue.Queue()
+ for repo_name, repo in dist.repositories.iteritems():
+ # Only manipulate distribution entries with a source repo listed.
+ if repo.source_repository:
+ # Decide which git ref string we'll be using as the replacement match.
+ if repo.release_repository and (release_version or release_tag):
+ version = repo.release_repository.version.split('-')[0]
+ else:
+ version = repo.source_repository.version
+ work_queue.put((repo.source_repository, version, release_tag))
+
+ total_items = work_queue.qsize()
+
+ for i in range(concurrent_ops):
+ threading.Thread(target=_worker, args=[work_queue]).start()
+
+ # Wait until the threads have done all the work and exited.
+ while not work_queue.empty():
+ time.sleep(0.1)
+ if not quiet:
+ sys.stdout.write("Updating source repo versions (%d/%d) \r" %
+ (total_items - work_queue.qsize(), total_items))
+ sys.stdout.flush()
+ work_queue.join()
+
+ # Clear past the updating line.
+ if not quiet:
+ print("")
+
+
+def _worker(work_queue):
+ while True:
+ try:
+ source_repo, freeze_version, freeze_to_tag = work_queue.get(block=False)
+ cmd = ['git', 'ls-remote', source_repo.url]
+ ls_remote_lines = subprocess.check_output(cmd).splitlines()
+ for line in ls_remote_lines:
+ hash, ref = line.split('\t', 1)
+ if ref.endswith(freeze_version):
+ if freeze_to_tag and ref.startswith('refs/tags/'):
+ source_repo.version = ref.split('refs/tags/')[1]
+ else:
+ source_repo.version = hash
+ break
+
+ work_queue.task_done()
+
+ except subprocess.CalledProcessError:
+ print("Non-zero return code for: %s" % ' '.join(cmd), file=sys.stderr)
+ work_queue.task_done()
+
+ except queue.Empty:
+ break
diff --git a/src/rosdistro/manifest_provider/git.py b/src/rosdistro/manifest_provider/git.py
index 5288310..a53c5da 100644
--- a/src/rosdistro/manifest_provider/git.py
+++ b/src/rosdistro/manifest_provider/git.py
@@ -31,6 +31,7 @@
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
+from distutils.version import LooseVersion
import os
import shutil
import subprocess
@@ -53,35 +54,52 @@ def git_manifest_provider(_dist_name, repo, pkg_name):
raise RuntimeError('Unable to fetch package.xml: %s' % e)
+def _git_version_gte(version):
+ global _git_client_version
+ if not _git_client_version:
+ cmd = [_git_client_executable, '--version']
+ result = _run_command(cmd)
+ _git_client_version = result['output'].split()[-1]
+ return LooseVersion(_git_client_version) >= LooseVersion(version)
+
+
def _get_package_xml(url, tag):
base = tempfile.mkdtemp('rosdistro')
try:
- # git 1.7.9 does not support cloning a tag directly, so doing it in two steps
assert _git_client_executable is not None, "'git' not found"
- cmd = [_git_client_executable, 'clone', url, base]
- result = _run_command(cmd, base)
- if result['returncode'] != 0:
- raise RuntimeError('Could not clone repository "%s"' % url)
+ if _git_version_gte('1.8.0'):
+ # Directly clone the required tag with least amount of additional history. This behaviour
+ # has been available since git 1.8.0:
+ # https://git.kernel.org/cgit/git/git.git/tree/Documentation/git-clone.txt?h=v1.8.0#n158
+ cmd = [_git_client_executable, 'clone', url, base, '--depth', '1', '--branch', tag]
+ result = _run_command(cmd, base)
+ if result['returncode'] != 0:
+ raise RuntimeError('Could not clone repository "%s" at tag "%s"' % (url, tag))
+ else:
+ # Old git doesn't support cloning a tag directly, so check it out after a full clone.
+ cmd = [_git_client_executable, 'clone', url, base]
+ result = _run_command(cmd, base)
+ if result['returncode'] != 0:
+ raise RuntimeError('Could not clone repository "%s"' % url)
- cmd = [_git_client_executable, 'tag', '-l']
- result = _run_command(cmd, base)
- if result['returncode'] != 0:
- raise RuntimeError('Could not get tags of repository "%s"' % url)
+ cmd = [_git_client_executable, 'tag', '-l']
+ result = _run_command(cmd, base)
+ if result['returncode'] != 0:
+ raise RuntimeError('Could not get tags of repository "%s"' % url)
- if tag not in result['output'].splitlines():
- raise RuntimeError('Specified tag "%s" is not a git tag of repository "%s"' % (tag, url))
+ if tag not in result['output'].splitlines():
+ raise RuntimeError('Specified tag "%s" is not a git tag of repository "%s"' % (tag, url))
- cmd = [_git_client_executable, 'checkout', tag]
- result = _run_command(cmd, base)
- if result['returncode'] != 0:
- raise RuntimeError('Could not checkout tag "%s" of repository "%s"' % (tag, url))
+ cmd = [_git_client_executable, 'checkout', tag]
+ result = _run_command(cmd, base)
+ if result['returncode'] != 0:
+ raise RuntimeError('Could not checkout tag "%s" of repository "%s"' % (tag, url))
filename = os.path.join(base, 'package.xml')
if not os.path.exists(filename):
raise RuntimeError('Could not find package.xml in repository "%s"' % url)
with open(filename, 'r') as f:
- package_xml = f.read()
- return package_xml
+ return f.read()
finally:
shutil.rmtree(base)
@@ -104,7 +122,7 @@ def check_remote_tag_exists(url, tag):
return False
-def _run_command(cmd, cwd, env=None):
+def _run_command(cmd, cwd=None, env=None):
result = {'cmd': ' '.join(cmd), 'cwd': cwd}
try:
proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
@@ -127,3 +145,4 @@ def _find_executable(file_name):
return None
_git_client_executable = _find_executable('git')
+_git_client_version = None
diff --git a/src/rosdistro/release_cache_generator.py b/src/rosdistro/release_cache_generator.py
index 95f8fc7..d44eae1 100644
--- a/src/rosdistro/release_cache_generator.py
+++ b/src/rosdistro/release_cache_generator.py
@@ -129,14 +129,18 @@ def _get_cached_release(index, dist_name, preclean=False):
print('- trying to fetch cache')
# get release cache
cache = get_release_cache(index, dist_name)
- # get current release file
- rel_file_data = _get_dist_file_data(index, dist_name, 'release')
- # update cache with current release file
- cache.update_distribution(rel_file_data)
except:
print('- failed to fetch old cache')
+
if cache:
print('- update cache')
+ # get current release file
+ rel_file_data = _get_dist_file_data(index, dist_name, 'release')
+ # since format 2 of the index file might contain a single value rather then a list
+ if not isinstance(rel_file_data, list):
+ rel_file_data = [rel_file_data]
+ # update cache with current release file
+ cache.update_distribution(rel_file_data)
else:
print('- build cache from scratch')
# get empty cache with distribution file
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/ros/ros-rosdistro.git
More information about the debian-science-commits
mailing list