[Debian-NP-Commits] r300 - people/vagrant/simple-cdd
Vagrant Cascadian
debian-np-devel@lists.alioth.debian.org
Fri, 03 Dec 2004 12:06:33 -0700
Author: vagrant-guest
Date: Fri Dec 3 12:05:58 2004
New Revision: 300
Added:
people/vagrant/simple-cdd/patched-debpartial-mirror (contents, props changed)
Modified:
people/vagrant/simple-cdd/build-simple-cdd
Log:
use patched debpartial-mirror for better output and include debian-cd tasks dir.
Modified: people/vagrant/simple-cdd/build-simple-cdd
==============================================================================
--- people/vagrant/simple-cdd/build-simple-cdd (original)
+++ people/vagrant/simple-cdd/build-simple-cdd Fri Dec 3 12:05:58 2004
@@ -96,7 +96,8 @@
echo include-task=$TASK >> $debpartial_conf
# generate the mirror
- debpartial-mirror -c $debpartial_conf
+ # FIXME use regular debpartial-mirror
+ ./patched-debpartial-mirror -c $debpartial_conf
fi
if [ ! -d "$MIRROR" ]; then
Added: people/vagrant/simple-cdd/patched-debpartial-mirror
==============================================================================
--- (empty file)
+++ people/vagrant/simple-cdd/patched-debpartial-mirror Fri Dec 3 12:05:58 2004
@@ -0,0 +1,352 @@
+#!/usr/bin/env python
+
+# debpartial-mirror - partial debian mirror package tool
+# (c) 2004 Otavio Salvador <otavio@debian.org>, Henrique Vilela <jacare@ucpel.tche.br>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+# $Id$
+
+import getopt
+import gzip
+import os
+import re
+import sys
+import time
+import urllib
+
+sys.path.append("/usr/share/debpartial-mirror")
+
+import Config
+import LocalPackages
+import Package
+import RemotePackages
+
+class Mirror:
+ def __init__(self, cfg_file, getFiles, getPackages, Simulate = False):
+ try:
+ self.__cfg = Config.Config(cfg_file)
+
+ if not Simulate:
+ self.__getPackagesIndex()
+ except Exception, msg:
+ print msg
+ sys.exit(1)
+
+ if getFiles:
+ for filename in self.__cfg.getFiles():
+ self.__getFile(self.__cfg.getServer() + filename, self.__cfg.getLocalDirectory() + filename, True, Simulate)
+
+ if getPackages:
+ rules = {}
+ excludes = {}
+ for d in self.__cfg.getDists():
+ for s in self.__cfg.getSections(d):
+ for f in self.__getPackagesURI(d, s, self.__cfg.getLocalDirectory()):
+ rules[f] = self.__cfg.getFilters(d, s)
+ excludes[f] = self.__cfg.getExcludes(d)
+
+ if self.__cfg.getIncludeTask(d):
+ for a in self.__cfg.getArchs():
+ try:
+ for i in os.popen('cpp -I /usr/share/debian-cd/tasks -I ' + os.path.dirname(self.__cfg.getIncludeTask(d)) + ' -DARCH_' + a + ' ' + self.__cfg.getIncludeTask(d) + '| grep -v \'^$\|^#.*\'', 'r').readlines():
+ rules[f].append({'section':'*', 'priority':'*', 'package':i[:-1]})
+ except Exception, msg:
+ print msg
+ sys.exit(1)
+
+ if self.__cfg.getExcludeTask(d):
+ for a in self.__cfg.getArchs():
+ try:
+ for i in os.popen('cpp -I ' + os.path.dirname(self.__cfg.getExcludeTask(d)) + ' -DARCH_' + a + ' ' + self.__cfg.getExcludeTask(d) + '| grep -v \'^$\|^#.*\'', 'r').readlines():
+ excludes[f].append(i[:-1])
+ except Exception, msg:
+ print msg
+ sys.exit(1)
+
+ self.__selected = RemotePackages.RemotePackages(rules,
+ excludes,
+ self.__cfg.getSuggests(),
+ self.__cfg.getRecomends(),
+ self.__cfg.getProvides()).getSelectedPackages()
+ self.__local = LocalPackages.LocalPackages(self.__cfg.getLocalDirectory() + 'pool/').getLocalPackages()
+
+
+ remove = []
+ for local_pkg in self.__local.itervalues():
+ for local_version in local_pkg:
+ if not self.__checkRemotePackage(local_version):
+ remove.append(local_version)
+
+ c = 0
+ for i in remove:
+ c=c+1
+ print "\n", c, " of ", len(remove)
+ self.__removePath(i.getFilename(), Simulate);
+
+ get = []
+ for filename in self.__selected:
+ for selected_version in self.__selected[filename].itervalues():
+ if not self.__checkLocalPackage(selected_version):
+ get.append(selected_version)
+ c = 0
+ for i in get:
+ c=c+1
+ print c, " of ", len(get)
+ self.__getFile(self.__cfg.getServer() + i.getFilename(), self.__cfg.getLocalDirectory() + i.getFilename(), Simulate)
+
+ def __checkRemotePackage(self, local):
+ for filename in self.__selected:
+ if self.__selected[filename].has_key(local.getName()):
+ if self.__selected[filename][local.getName()].getMD5Sum() == local.getMD5Sum():
+ return True
+ return False
+
+ def __checkLocalPackage(self, remote):
+ for local_pkg in self.__local.itervalues():
+ if self.__local.has_key(remote.getName()):
+ for local_version in local_pkg:
+ if local_version.getMD5Sum() == remote.getMD5Sum():
+ return True
+ return False
+
+ def __getPackagesURI(self, dist, section, where, append=""):
+ files = []
+
+ for arch in self.__cfg.getArchs():
+ files.append(where + 'dists/' + dist + '/' + section + '/binary-' + arch + '/Packages' + append)
+
+ return files
+
+ def __getPackagesIndex(self):
+ for dist in self.__cfg.getDists().keys():
+ self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Release",
+ self.__cfg.getLocalDirectory() + "dists/" + dist + "/Release",
+ True);
+
+ self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Release.gpg",
+ self.__cfg.getLocalDirectory() + "dists/" + dist + "/Release.gpg",
+ True);
+
+ for section in self.__cfg.getSections(dist).keys():
+ for arch in self.__cfg.getArchs():
+ self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/" + section + "/binary-" + arch + "/Release",
+ self.__cfg.getLocalDirectory() + "dists/" + dist + "/" + section + "/binary-" + arch + "/Release",
+ True);
+
+ self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Contents-" + arch + ".gz",
+ self.__cfg.getLocalDirectory() + "dists/" + dist + "/Contents-" + arch + ".gz",
+ True);
+
+ for package in self.__getPackagesURI(dist, section, '', '.gz'):
+ origin = self.__cfg.getServer() + package
+ destiny = self.__cfg.getLocalDirectory() + package
+ if self.__getFile(origin, destiny, True):
+ self.__uncompressFile(destiny)
+
+ def __removePath(self, filename, Simulate=False):
+ if os.path.isdir(filename):
+ files = os.listdir(filename)
+ if not len(files):
+ print "Removing " + filename
+ if not Simulate:
+ os.rmdir(filename)
+ matches = re.compile('(.*)/.*').findall(filename);
+ if matches:
+ self.__removePath(matches[0])
+ else:
+ print "Removing " + filename
+ if not Simulate:
+ os.unlink(filename)
+ self.__removePath(os.path.dirname(filename))
+
+ def __getFile(self, origin, destiny, checkNew = False, Simulate = False):
+ matches = re.compile('(.*)\((.*)\)', re.IGNORECASE | re.MULTILINE ).findall(origin)
+ if matches:
+ origin=matches[0][0]
+
+ expression=""
+ matches = re.compile('(.*)\((.*)\)', re.IGNORECASE | re.MULTILINE ).findall(destiny)
+ if matches:
+ destiny=matches[0][0]
+ expression=matches[0][1]
+
+ self.__createPath(destiny)
+
+ existSize = -1
+ if os.path.isfile(destiny):
+ existSize = os.path.getsize(destiny)
+
+ webPage = urllib.urlopen(origin)
+
+ #If the file exists, but we already have the whole thing, don't download again
+ if checkNew and webPage.headers.has_key('content-length') and int(webPage.headers['content-length']) == existSize:
+ if webPage.headers.has_key('Last-Modified') and int(time.mktime(time.strptime(webPage.headers['Last-Modified'], "%a, %d %b %Y %H:%M:%S %Z"))) < (os.path.getmtime(destiny) + time.altzone):
+ print 'Match:', destiny
+ webPage.close()
+ return False
+
+ if not Simulate:
+ # Get the file
+ content = ""
+ if origin[-1] == '/':
+ directory = True
+ else:
+ directory = False
+
+ if not directory and webPage.headers.has_key('Content-Length'):
+ content_length = int(webPage.headers['content-length'])
+
+ numBytes = 0
+ line = "Getting " + origin + "\n"
+ sys.stdout.write(line)
+ while numBytes < content_length:
+ percent_line = "... %d%%" % (numBytes * 100 / content_length)
+ sys.stdout.write(percent_line + '\r')
+ sys.stdout.flush()
+
+ data = webPage.read(8 * 1024)
+ if not data:
+ break
+ content += data
+ numBytes = numBytes + len(data)
+
+ sys.stdout.write("\rDone " + line)
+ else:
+ if directory:
+ content = webPage.read()
+ else:
+ print "Error: " + origin + " does not exist."
+ webPage.close()
+ return False;
+
+ webPage.close()
+
+ if directory:
+ matches = re.compile('<a href="([a-zA-Z0-9_][a-zA-Z0-9\-\./_]+)">', re.IGNORECASE | re.MULTILINE ).findall(content)
+ if matches:
+ for filename in matches:
+ if re.match(expression, filename):
+ self.__getFile(origin + filename, destiny + filename, True, Simulate)
+ else:
+ print "Filed ", filename
+ else:
+ try:
+ outputFile = open(destiny, "wb")
+ outputFile.write(content)
+ outputFile.close()
+ except IOError, msg:
+ print msg
+ sys.exit(1)
+
+ return True
+
+ def __uncompressFile(self, filename):
+ print 'Uncompressing', filename
+ # uncompress the file
+ compressedFile = gzip.GzipFile(filename, 'rb')
+ try:
+ outputFile = open(filename.split('.gz')[0], "wb")
+ except IOError, msg:
+ print msg
+ sys.exit(1)
+
+ numBytes = 0
+ while True:
+ data = compressedFile.read(64 * 1024)
+ if not data:
+ break
+ outputFile.write(data)
+ numBytes = numBytes + len(data)
+
+ compressedFile.close()
+ outputFile.close()
+ return True
+
+ def __createPath(self, path):
+ dir_name = os.path.dirname(path)
+ if not os.path.exists(dir_name):
+ print "Making directory " + dir_name + "..."
+ os.makedirs(dir_name)
+
+# Main function, no further comment needed. :)
+def main():
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], 'hvs:c:', ["help",
+ "version",
+ "skip-packages",
+ "skip-files",
+ "simulate",
+ "configfile="])
+ except getopt.GetoptError:
+ print "ERROR"
+ usage()
+
+ configurefile = '/etc/debpartial-mirror.conf'
+
+ getFiles = True
+ getPackages = True
+ Simulate = False
+
+ for o, a in opts:
+ if o in ("-h", "--help"):
+ usage()
+ sys.exit()
+
+ if o in ("-v", "--version"):
+ version()
+ sys.exit()
+
+ if o in ("-c", "--configfile"):
+ if a == '':
+ usage()
+ configurefile = a
+
+ if o == "-s" or o in ("--skip-files",
+ "--skip-packages",
+ "--simulate"):
+ if o == "--skip-files" or a == "f":
+ getFiles = False
+
+ if o == "--skip-packages" or a == "p":
+ getPackages = False
+
+ if o == "--simulate" or a == "":
+ Simulate = True
+
+ version()
+ mirror = Mirror(configurefile, getFiles, getPackages, Simulate)
+
+def version():
+ print "debpartial-mirror 0.2.9 - Partial mirroring tool for Debian - Thu, 25 Nov 2004 18:33:20 -0200"
+ print "(c) 2004 Otavio Salvador <otavio@debian.org>, Henrique Vilela <jacare@ucpel.tche.br>"
+ print "This program is free software and was released under the terms of the GNU General Public License"
+ print
+
+def usage():
+ print "Usage: debpartial-mirros [OPTION]"
+ print
+ print "-h --help\t\t\tDisplay this help end exit"
+ print "-c<file> --configfile=<file>\tSelect a config file"
+ print "-v --version\t\t\tShow the version"
+ print
+ sys.exit(2)
+
+
+# Main
+if __name__ == '__main__':
+ try:
+ main()
+ except KeyboardInterrupt:
+ print "exiting due to user interrupt."