r217 - branches/rewrite/src
Otavio Salvador
partial-mirror-devel@lists.alioth.debian.org
Mon, 02 Aug 2004 08:18:58 -0600
Author: otavio
Date: Mon Aug 2 08:18:57 2004
New Revision: 217
Modified:
branches/rewrite/src/debpartial-mirror.in
Log:
Initial work on how this will operate ... Really a draft!
Modified: branches/rewrite/src/debpartial-mirror.in
==============================================================================
--- branches/rewrite/src/debpartial-mirror.in (original)
+++ branches/rewrite/src/debpartial-mirror.in Mon Aug 2 08:18:57 2004
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# debpartial-mirror - partial debian mirror package tool
-# (c) 2004 Otavio Salvador <otavio@debian.org>, Henrique Vilela <jacare@ucpel.tche.br>
+# (c) 2004 Otavio Salvador <otavio@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -18,316 +18,10 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# $Id$
-import getopt
-import gzip
-import os
-import re
-import sys
-import time
-import urllib
-
-sys.path.append("/usr/share/debpartial-mirror")
-
-import Config
-import LocalPackages
-import Package
-import RemotePackages
-
-class Mirror:
- def __init__(self, cfg_file, getFiles, getPackages, Simulate = False):
- try:
- self.__cfg = Config.Config(cfg_file)
-
- if not Simulate:
- self.__getPackagesIndex()
- except Exception, msg:
- print msg
- sys.exit(1)
-
- if getFiles:
- for filename in self.__cfg.getFiles():
- self.__getFile(self.__cfg.getServer() + filename, self.__cfg.getLocalDirectory() + filename, True, Simulate)
-
- if getPackages:
- rules = {}
- excludes = {}
- for d in self.__cfg.getDists():
- for s in self.__cfg.getSections(d):
- for f in self.__getPackagesURI(d, s, self.__cfg.getLocalDirectory()):
- rules[f] = self.__cfg.getFilters(d, s)
- excludes[f] = self.__cfg.getExcludes(d)
-
- if self.__cfg.getIncludeTask(d):
- for a in self.__cfg.getArchs():
- try:
- for i in os.popen('cpp -I ' + os.path.dirname(self.__cfg.getIncludeTask(d)) + ' -DARCH_' + a + ' ' + self.__cfg.getIncludeTask(d) + '| grep -v \'^$\|^#.*\'', 'r').readlines():
- rules[f].append({'section':'*', 'priority':'*', 'package':i[:-1]})
- except Exception, msg:
- print msg
- sys.exit(1)
-
- if self.__cfg.getExcludeTask(d):
- for a in self.__cfg.getArchs():
- try:
- for i in os.popen('cpp -I ' + os.path.dirname(self.__cfg.getExcludeTask(d)) + ' -DARCH_' + a + ' ' + self.__cfg.getExcludeTask(d) + '| grep -v \'^$\|^#.*\'', 'r').readlines():
- excludes[f].append(i[:-1])
- except Exception, msg:
- print msg
- sys.exit(1)
-
- self.__selected = RemotePackages.RemotePackages(rules,
- excludes,
- self.__cfg.getSuggests(),
- self.__cfg.getRecomends(),
- self.__cfg.getProvides()).getSelectedPackages()
- self.__local = LocalPackages.LocalPackages(self.__cfg.getLocalDirectory() + 'pool/').getLocalPackages()
-
-
- remove = []
- for local_pkg in self.__local.itervalues():
- for local_version in local_pkg:
- if not self.__checkRemotePackage(local_version):
- remove.append(local_version)
-
- c = 0
- for i in remove:
- c=c+1
- print "\n", c, " of ", len(remove)
- self.__removePath(i.getFilename(), Simulate);
-
- get = []
- for filename in self.__selected:
- for selected_version in self.__selected[filename].itervalues():
- if not self.__checkLocalPackage(selected_version):
- get.append(selected_version)
- c = 0
- for i in get:
- c=c+1
- print c, " of ", len(get)
- self.__getFile(self.__cfg.getServer() + i.getFilename(), self.__cfg.getLocalDirectory() + i.getFilename(), Simulate)
-
- def __checkRemotePackage(self, local):
- for filename in self.__selected:
- if self.__selected[filename].has_key(local.getName()):
- if self.__selected[filename][local.getName()].getMD5Sum() == local.getMD5Sum():
- return True
- return False
-
- def __checkLocalPackage(self, remote):
- for local_pkg in self.__local.itervalues():
- if self.__local.has_key(remote.getName()):
- for local_version in local_pkg:
- if local_version.getMD5Sum() == remote.getMD5Sum():
- return True
- return False
-
- def __getPackagesURI(self, dist, section, where, append=""):
- files = []
-
- for arch in self.__cfg.getArchs():
- files.append(where + 'dists/' + dist + '/' + section + '/binary-' + arch + '/Packages' + append)
-
- return files
-
- def __getPackagesIndex(self):
- for dist in self.__cfg.getDists().keys():
- self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Release",
- self.__cfg.getLocalDirectory() + "dists/" + dist + "/Release",
- True);
-
- self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Release.gpg",
- self.__cfg.getLocalDirectory() + "dists/" + dist + "/Release.gpg",
- True);
-
- for section in self.__cfg.getSections(dist).keys():
- for arch in self.__cfg.getArchs():
- self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/" + section + "/binary-" + arch + "/Release",
- self.__cfg.getLocalDirectory() + "dists/" + dist + "/" + section + "/binary-" + arch + "/Release",
- True);
-
- self.__getFile(self.__cfg.getServer() + "dists/" + dist + "/Contents-" + arch + ".gz",
- self.__cfg.getLocalDirectory() + "dists/" + dist + "/Contents-" + arch + ".gz",
- True);
-
- for package in self.__getPackagesURI(dist, section, '', '.gz'):
- origin = self.__cfg.getServer() + package
- destiny = self.__cfg.getLocalDirectory() + package
- if self.__getFile(origin, destiny, True):
- self.__uncompressFile(destiny)
-
- def __removePath(self, filename, Simulate=False):
- if os.path.isdir(filename):
- files = os.listdir(filename)
- if not len(files):
- print "Removing " + filename
- if not Simulate:
- os.rmdir(filename)
- matches = re.compile('(.*)/.*').findall(filename);
- if matches:
- self.__removePath(matches[0])
- else:
- print "Removing " + filename
- if not Simulate:
- os.unlink(filename)
- self.__removePath(os.path.dirname(filename))
-
- def __getFile(self, origin, destiny, checkNew = False, Simulate = False):
- matches = re.compile('(.*)\((.*)\)', re.IGNORECASE | re.MULTILINE ).findall(origin)
- if matches:
- origin=matches[0][0]
-
- expression=""
- matches = re.compile('(.*)\((.*)\)', re.IGNORECASE | re.MULTILINE ).findall(destiny)
- if matches:
- destiny=matches[0][0]
- expression=matches[0][1]
-
- self.__createPath(destiny)
-
- existSize = -1
- if os.path.isfile(destiny):
- existSize = os.path.getsize(destiny)
-
- webPage = urllib.urlopen(origin)
-
- #If the file exists, but we already have the whole thing, don't download again
- if checkNew and webPage.headers.has_key('content-length') and int(webPage.headers['content-length']) == existSize:
- if webPage.headers.has_key('Last-Modified') and int(time.mktime(time.strptime(webPage.headers['Last-Modified'], "%a, %d %b %Y %H:%M:%S %Z"))) < (os.path.getmtime(destiny) + time.altzone):
- print 'Match:', destiny
- webPage.close()
- return False
-
- if not Simulate:
- # Get the file
- content = ""
- if origin[-1] == '/':
- directory = True
- else:
- directory = False
-
- if not directory and webPage.headers.has_key('Content-Length'):
- content_length = int(webPage.headers['content-length'])
-
- numBytes = 0
- while numBytes < content_length:
- line = "Getting " + origin + "... %d%%" % (numBytes * 100 / content_length)
- sys.stdout.write(line + '\r')
-
- data = webPage.read(8 * 1024)
- if not data:
- break
- content += data
- numBytes = numBytes + len(data)
-
- sys.stdout.write("Getting " + origin + " done. \n")
- else:
- if directory:
- content = webPage.read()
- else:
- print "Error: " + origin + " not exists."
- webPage.close()
- return False;
-
- webPage.close()
-
- if directory:
- matches = re.compile('<a href="([a-zA-Z0-9_][a-zA-Z0-9\-\./_]+)">', re.IGNORECASE | re.MULTILINE ).findall(content)
- if matches:
- for filename in matches:
- if re.match(expression, filename):
- self.__getFile(origin + filename, destiny + filename, True, Simulate)
- else:
- print "Filed ", filename
- else:
- try:
- outputFile = open(destiny, "wb")
- outputFile.write(content)
- outputFile.close()
- except IOError, msg:
- print msg
- sys.exit(1)
-
- return True
-
- def __uncompressFile(self, filename):
- print 'Uncompressing', filename
- # uncompress the file
- compressedFile = gzip.GzipFile(filename, 'rb')
- try:
- outputFile = open(filename.split('.gz')[0], "wb")
- except IOError, msg:
- print msg
- sys.exit(1)
-
- numBytes = 0
- while True:
- data = compressedFile.read(64 * 1024)
- if not data:
- break
- outputFile.write(data)
- numBytes = numBytes + len(data)
-
- compressedFile.close()
- outputFile.close()
- return True
-
- def __createPath(self, path):
- dir_name = os.path.dirname(path)
- if not os.path.exists(dir_name):
- print "Making directory " + dir_name + "..."
- os.makedirs(dir_name)
-
-# Main function, no further comment needed. :)
-def main():
- try:
- opts, args = getopt.getopt(sys.argv[1:], 'hvs:c:', ["help",
- "version",
- "skip-packages",
- "skip-files",
- "simulate",
- "configfile="])
- except getopt.GetoptError:
- print "ERROR"
- usage()
-
- configurefile = '/etc/debpartial-mirror.conf'
-
- getFiles = True
- getPackages = True
- Simulate = False
-
- for o, a in opts:
- if o in ("-h", "--help"):
- usage()
- sys.exit()
-
- if o in ("-v", "--version"):
- version()
- sys.exit()
-
- if o in ("-c", "--configfile"):
- if a == '':
- usage()
- configurefile = a
-
- if o == "-s" or o in ("--skip-files",
- "--skip-packages",
- "--simulate"):
- if o == "--skip-files" or a == "f":
- getFiles = False
-
- if o == "--skip-packages" or a == "p":
- getPackages = False
-
- if o == "--simulate" or a == "":
- Simulate = True
-
- version()
- mirror = Mirror(configurefile, getFiles, getPackages, Simulate)
+from Config import *
def version():
print "debpartial-mirror @VERSION@ - Partial mirroring tool for Debian - @DATE@"
- print "(c) 2004 Otavio Salvador <otavio@debian.org>, Henrique Vilela <jacare@ucpel.tche.br>"
print "This program is free software and was released under the terms of the GNU General Public License"
print
@@ -340,10 +34,23 @@
print
sys.exit(2)
-
-# Main
-if __name__ == '__main__':
- try:
- main()
- except KeyboardInterrupt:
- print "exiting due to user interrupt."
+version()
+try:
+ cnf = Config("../etc/debpartial-mirror.conf")
+except InvalidOption, msg:
+ print("Wrong option [%s] found on [%s] section."
+ % (msg.option, msg.section))
+ exit(1)
+except InvalidSection, msg:
+ print("Wrong section [%s] found." % msg.section)
+ exit(1)
+except RequiredOptionMissing, msg:
+ print("Required option [%s] was missing on [%s] section."
+ % (msg.option, msg.section))
+ exit(1)
+
+for backend in cnf.getBackends(ConfigBackendMirror):
+ print cnf.getOption("server", backend.section)
+
+for backend in cnf.getBackends(ConfigBackendMerge):
+ print cnf.getOption("backends", backend.section)