[apt-proxy-devel] r537 - in people/halls/rework: . apt_proxy apt_proxy/test debian
apt-proxy-devel@lists.alioth.debian.org
apt-proxy-devel@lists.alioth.debian.org
Sat, 09 Jul 2005 19:38:14 +0000
Author: halls
Date: Fri Jul 8 22:21:34 2005
New Revision: 537
Modified:
people/halls/rework/apt_proxy/apt_proxy.py
people/halls/rework/apt_proxy/apt_proxy_conf.py
people/halls/rework/apt_proxy/packages.py
people/halls/rework/apt_proxy/test/test_apt_proxy.py
people/halls/rework/debian/changelog
people/halls/rework/runtests
Log:
More work, mostly on packages
Modified: people/halls/rework/apt_proxy/apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy.py (original)
+++ people/halls/rework/apt_proxy/apt_proxy.py Fri Jul 8 22:21:34 2005
@@ -1253,7 +1253,7 @@
def get_packages_db(self):
"Return packages parser object for the backend, creating one if necessary"
if self.packages == None:
- self.packages = packages.AptPackages(self, self.factory.config.cache_dir)
+ self.packages = packages.AptPackages(self.base, self.factory.config.cache_dir)
return self.packages
def get_path(self, path):
@@ -1697,6 +1697,7 @@
self.packages: all versions of a certain package name.
"""
+ databases=('update_times', 'access_times', 'packages')
def periodic(self):
"Called periodically as configured mainly to do mirror maintanace."
@@ -1706,7 +1707,13 @@
log.debug("Periodic cleaning done")
if (self.config.cleanup_freq != None):
reactor.callLater(self.config.cleanup_freq, self.periodic)
-
+ def __del__(self):
+ for f in self.databases:
+ try:
+ if hasattr(self, f):
+ getattr(self, f).close()
+ except Exception:
+ pass
def __init__ (self, config):
self.runningFetchers = {}
self.backends = []
@@ -1923,7 +1930,6 @@
keys = db.keys()
for key in keys:
func(key,db[key])
-
if log.isEnabled('db'):
log.msg("=========================",'db')
log.msg("Dumping update times",'db')
Modified: people/halls/rework/apt_proxy/apt_proxy_conf.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy_conf.py (original)
+++ people/halls/rework/apt_proxy/apt_proxy_conf.py Fri Jul 8 22:21:34 2005
@@ -160,7 +160,7 @@
filehandle.close()
return conf
- def setDebug(self, debug):
+ def setDebug(self):
"Set logger debug level"
for domain in self.debug.split():
#print "domain:",domain
@@ -179,7 +179,7 @@
self.debug=config.get(DEFAULTSECT, 'debug')
else:
self.debug='all:3'
- self.setDebug(self.debug)
+ self.setDebug()
# read default values
for name,default,getmethod in self.CONFIG_ITEMS:
Modified: people/halls/rework/apt_proxy/packages.py
==============================================================================
--- people/halls/rework/apt_proxy/packages.py (original)
+++ people/halls/rework/apt_proxy/packages.py Fri Jul 8 22:21:34 2005
@@ -49,21 +49,58 @@
key, value = line.split(': ', 1)
self.data[key] = value
+class PackageFileList:
+ """
+ Manages a list of package files belonging to a backend
+ """
+ def __init__(self, backendName, cache_dir):
+ self.cache_dir = cache_dir
+ packagedb_dir = cache_dir+'/'+ apt_proxy.status_dir + \
+ '/backends/' + backendName
+ if not os.path.exists(packagedb_dir):
+ os.makedirs(packagedb_dir)
+ self.packages = shelve.open(packagedb_dir+'/packages.db')
+ def __del__(self):
+ try:
+ self.packages.close()
+ except:
+ pass
+
+ def update_file(self, uri):
+ """
+ Called from apt_proxy.py when files get updated so we can update our
+ fake lists/ directory and sources.list.
+
+ @param uri Filename of cached file (without cache_dir prefix)
+ """
+ if basename(uri)=="Packages" or basename(uri)=="Release":
+ log.msg("REGISTERING PACKAGE:"+uri,'apt_pkg',4)
+ stat_result = os.stat(self.cache_dir+'/'+uri)
+ self.packages[uri] = stat_result
+ def get_files(self):
+ """
+ Get list of files in database. Each file will be checked that it exists
+ """
+ files = self.packages.keys()
+ for f in files:
+ if not os.path.exists(self.cache_dir + '/' + f):
+ log.debug("File in packages database has been deleted: "+f, 'apt_pkg')
+ del files[files.index(f)]
+ del self.packages[f]
+ return files
+
class AptPackages:
"""
Uses AptPackagesServer to answer queries about packages.
Makes a fake configuration for python-apt for each backend.
-
- self.packages: a list of files which should go in the fake source.list
- along with their mtime.
"""
apt_config = {
#'APT' : '',
'APT::Architecture' : apt_pkg.CPU,
#'APT::Default-Release' : 'unstable',
- 'Dir':'/home/ranty/work/apt-proxy/twisted/cache/', # /
+ 'Dir':'.', # /
'Dir::State' : 'apt/', # var/lib/apt/
'Dir::State::Lists': 'lists/', # lists/
#'Dir::State::cdroms' : 'cdroms.list',
@@ -95,18 +132,18 @@
'apt/lists/partial')
essential_files = ('apt/dpkg/status', 'apt/etc/sources.list',)
- def __init__(self, backend, cache_dir):
+ def __init__(self, backendName, cache_dir):
"""
Construct new packages manager
- backend: Backend associated with this database
+ backend: Name of backend associated with this packages file
cache_dir: cache directory from config file
"""
- self.backend = backend
+ self.backendName = backendName
self.cache_dir = cache_dir
self.apt_config = copy.copy(self.apt_config)
self.status_dir = (cache_dir+'/'+ aptpkg_dir
- +'/backends/'+backend.base)
+ +'/backends/'+backendName)
for dir in self.essential_dirs:
path = self.status_dir+'/'+dir
if not os.path.exists(path):
@@ -119,23 +156,18 @@
del f
self.apt_config['Dir'] = self.status_dir
-
- self.packages = shelve.open(cache_dir+'/'+ apt_proxy.status_dir
- +'/backends/'+backend.base+'/packages.db')
-
+ self.packages = PackageFileList(backendName, cache_dir)
self.loaded = 0
- def packages_file(self, uri):
+ def __del__(self):
+ self.cleanup()
+ def file_updated(self, uri):
"""
- Called from apt_proxy.py when files get updated so we can update our
- fake lists/ directory and sources.list.
+ A file in the backend has changed. If this affects us, unload our apt database
"""
- if basename(uri)=="Packages" or basename(uri)=="Release":
- log.msg("REGISTERING PACKAGE:"+uri,'apt_pkg')
- mtime = os.stat(self.cache_dir+'/'+uri)
- self.packages[uri] = mtime
+ if self.packages.update_file(uri):
self.unload()
-
+
def __save_stdout(self):
self.real_stdout_fd = os.dup(1)
os.close(1)
@@ -155,9 +187,10 @@
os.makedirs(self.status_dir+'/apt/lists/partial')
sources = open(self.status_dir+'/'+'apt/etc/sources.list', 'w')
sources_count = 0
- for file in self.packages.keys():
+ for file in self.packages.get_files():
# we should probably clear old entries from self.packages and
# take into account the recorded mtime as optimization
+ filepath = self.cache_dir + file
fake_uri='http://apt-proxy:'+file
source_line='deb '+dirname(fake_uri)+'/ /'
listpath=(self.status_dir+'/apt/lists/'
@@ -175,11 +208,10 @@
sources.close()
if sources_count == 0:
- log.msg("No Packages files available for %s backend"%(self.backend.base), 'apt_pkg')
+ log.msg("No Packages files available for %s backend"%(self.backendName), 'apt_pkg')
return False
log.msg("Loading Packages database for "+self.status_dir,'apt_pkg')
- raise 1
for key, value in self.apt_config.items():
apt_pkg.Config[key] = value
@@ -190,8 +222,10 @@
else:
# apt_pkg prints progress messages to stdout, disable
self.__save_stdout()
- self.cache = apt_pkg.GetCache()
- self.__restore_stdout()
+ try:
+ self.cache = apt_pkg.GetCache()
+ finally:
+ self.__restore_stdout()
self.records = apt_pkg.GetPkgRecords(self.cache)
#for p in self.cache.Packages:
@@ -209,7 +243,6 @@
def cleanup(self):
self.unload()
- self.packages.close()
def get_mirror_path(self, name, version):
"Find the path for version 'version' of package 'name'"
@@ -226,7 +259,13 @@
def get_mirror_versions(self, package_name):
- "Find the available versions of the package name given"
+ """
+ Find the available versions of the package name given
+ @type package_name: string
+ @param package_name: package name to search for e.g. ;apt'
+ @return: A list of mirror versions available, None if not available
+
+ """
vers = []
if not self.load(): return vers
try:
@@ -234,6 +273,8 @@
vers.append(pack_vers.VerStr)
except KeyError:
pass
+ if vers == []:
+ return None
return vers
Modified: people/halls/rework/apt_proxy/test/test_apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/test/test_apt_proxy.py (original)
+++ people/halls/rework/apt_proxy/test/test_apt_proxy.py Fri Jul 8 22:21:34 2005
@@ -16,7 +16,7 @@
"""Unit test for apt_proxy.py"""
-import copy, tempfile, os
+import copy, tempfile, os, shutil
from twisted.trial import unittest
from StringIO import StringIO
@@ -30,7 +30,6 @@
port=9999
address=
cleanup_freq=off
-cache_dir = test_data/cache
[backend1]
backends = http://a.b.c/d
@@ -48,7 +47,6 @@
port=8888
address=1.2.3.4
cleanup_freq=off
-cache_dir = test_data/cache
# Backend 1 deleted
@@ -69,37 +67,55 @@
backends = http://t.u.v/w
"""
-class FactoryInitTest(unittest.TestCase):
+class apTestHelper(unittest.TestCase):
+ default_config = "[DEFAULT]\ndebug=all:9\n" # Config string to use
def setUp(self):
- self.c = apConfig(StringIO(config1))
+ self.cache_dir = tempfile.mkdtemp('.aptproxy')
+ self.config = self.default_config.replace('[DEFAULT]','[DEFAULT]\ncache_dir=' + self.cache_dir)
+ def tearDown(self):
+ shutil.rmtree(self.cache_dir)
+
+class FactoryInitTest(apTestHelper):
+ def setUp(self):
+ self.default_config = config1
+ apTestHelper.setUp(self)
+ self.c = apConfig(StringIO(self.config))
def testFactoryInit(self):
factory = Factory(self.c)
self.assertEquals(factory.config, self.c)
+ del factory
def testFactoryBackendInit(self):
factory = Factory(self.c)
factory.configurationChanged()
self.assertEquals(len(factory.backends),3)
self.assertEquals(factory.backends.keys(), ['backend1', 'backend2', 'backend3'])
self.assertEquals(factory.backends['backend1'].uris[0].host, 'a.b.c')
+ del factory
class StartFactoryTest(unittest.TestCase):
def setUp(self):
self.cache_dir = tempfile.mkdtemp('.aptproxy')
- config = config1 + '\ncache_dir=' + self.cache_dir
+ config = config1.replace('[DEFAULT]','[DEFAULT]\ncache_dir=' + self.cache_dir)
self.c = apConfig(StringIO(config))
def tearDown(self):
- os.removedirs(self.cache_dir)
+ shutil.rmtree(self.cache_dir)
def testFactoryStart(self):
factory = Factory(self.c)
factory.startFactory
class ConfigChangeTest(unittest.TestCase):
def setUp(self):
- self.cOld = apConfig(StringIO(config1))
+ self.tempdir = tempfile.mkdtemp('.aptproxy')
+ configOld = config1.replace('[DEFAULT]','[DEFAULT]\ncache_dir=%s/old'%(self.tempdir))
+ self.cOld = apConfig(StringIO(configOld))
self.factory = Factory(self.cOld)
self.factory.configurationChanged()
+ def tearDown(self):
+ del(self.factory)
+ shutil.rmtree(self.tempdir)
def loadNewConfig(self):
- self.cNew = apConfig(StringIO(config2))
+ configNew = config2.replace('[DEFAULT]','[DEFAULT]\ncache_dir=%s/new'%(self.tempdir))
+ self.cNew = apConfig(StringIO(configNew))
self.factory.config = copy.copy(self.cNew)
self.factory.configurationChanged(self.cOld)
def testNotAllowedChanges(self):
Modified: people/halls/rework/debian/changelog
==============================================================================
--- people/halls/rework/debian/changelog (original)
+++ people/halls/rework/debian/changelog Fri Jul 8 22:21:34 2005
@@ -3,8 +3,12 @@
* Add support for file:// backends. Thanks to Michael Vogt for
the patch. FetcherFile is renamed to FetcherCachedFile and
FetcherFile is now the file: backend.
+ * Seperate out configuration file loading code into standalone classes
+ that can be used (and tested independently)
+ * Seperate management of the list of Packages files associated with a
+ backend,
- -- Chris Halls <halls@debian.org> Wed, 22 Jun 2005 10:12:23 +0100
+ -- Chris Halls <chris.halls@credativ.co.uk> Tue, 5 Jul 2005 20:23:06 +0100
apt-proxy (1.9.31) unstable; urgency=low
Modified: people/halls/rework/runtests
==============================================================================
--- people/halls/rework/runtests (original)
+++ people/halls/rework/runtests Fri Jul 8 22:21:34 2005
@@ -16,4 +16,7 @@
[ -f test_data/packages/Packages ] || make_packages_dir
pwd
+#if [ $# -eq 0 ]; then
+# 1=apt_proxy.test
+#fi
PYTHONPATH="`pwd`" trial --verbose --logfile `pwd`/unitttests.log apt_proxy.test