[apt-proxy-devel] r635 - in trunk: apt_proxy apt_proxy/test bin
debian doc
Chris Halls
halls at costa.debian.org
Sun Sep 10 22:52:40 UTC 2006
Author: halls
Date: Sun Sep 10 22:52:38 2006
New Revision: 635
Modified:
trunk/apt_proxy/apt_proxy.py
trunk/apt_proxy/apt_proxy_conf.py
trunk/apt_proxy/cache.py
trunk/apt_proxy/clients.py
trunk/apt_proxy/fetchers.py
trunk/apt_proxy/test/test_cache.py
trunk/apt_proxy/test/test_config.py
trunk/apt_proxy/test/test_requests.py
trunk/bin/apt-proxy
trunk/debian/changelog
trunk/doc/apt-proxy.conf
trunk/doc/apt-proxy.conf.5
Log:
* Set process name to apt-proxy
* Start changing refresh_delay parameter meaning
* Add gz / bzip2 uncompression classes
Modified: trunk/apt_proxy/apt_proxy.py
==============================================================================
--- trunk/apt_proxy/apt_proxy.py (original)
+++ trunk/apt_proxy/apt_proxy.py Sun Sep 10 22:52:38 2006
@@ -112,7 +112,10 @@
"A cache entry is finished and clients are disconnected"
#if self.entries.has_key(entry.path):
log.debug("entry_done: %s" %(entry.path), 'Backend')
- del self.entries[entry.path]
+ try:
+ del self.entries[entry.path]
+ except KeyError:
+ pass # In case this is called twice
def get_packages_db(self):
"Return packages parser object for the backend, creating one if necessary"
Modified: trunk/apt_proxy/apt_proxy_conf.py
==============================================================================
--- trunk/apt_proxy/apt_proxy_conf.py (original)
+++ trunk/apt_proxy/apt_proxy_conf.py Sun Sep 10 22:52:38 2006
@@ -115,7 +115,8 @@
['passive_ftp', None, 'boolean'],
['backends', '', 'stringlist'],
['http_proxy', None , 'proxyspec'],
- ['bandwidth_limit', None, '*int']
+ ['bandwidth_limit', None, '*int'],
+ ['min_refresh_delay', 30, 'time'],
]
DEFAULT_CONFIG_FILE = ['/etc/apt-proxy/apt-proxy-v2.conf',
Modified: trunk/apt_proxy/cache.py
==============================================================================
--- trunk/apt_proxy/cache.py (original)
+++ trunk/apt_proxy/cache.py Sun Sep 10 22:52:38 2006
@@ -123,8 +123,10 @@
self.get()
else:
# Subsequent request - client must be brought up to date
- if self.state in (self.STATE_DOWNLOAD, self.STATE_SENT, self.STATE_SENDFILE):
+ if self.state in (self.STATE_DOWNLOAD, self.STATE_SENT):
self.send_cached_file(request=request)
+ elif self.state == self.STATE_SENDFILE:
+ self.transfer_file(request=request)
def remove_request(self,request):
"""
@@ -138,7 +140,7 @@
if len(self.requests) != 0:
return
- log.debug("Last request removed",'cacheEntry')
+ log.debug("Last request removed for %s" % (self.file_path),'cacheEntry')
self.requests_done()
# TODO - fixme
@@ -243,7 +245,7 @@
"""
if self.file_mtime is not None:
log.msg("sending file from cache:" + self.file_path, "CacheEntry")
- self.transfer_file(self.file_path, request=request)
+ self.transfer_file(request=request)
else:
log.msg("sending hits to all clients (%s)" % (self.file_path), "CacheEntry")
for req in self.requests:
@@ -255,7 +257,7 @@
"""
self.file_sent()
- def transfer_file(self, filename,request=None):
+ def transfer_file(self, filename=None, request=None):
"""
Send given file to clients
"""
@@ -265,25 +267,30 @@
else:
# Start one request
requests = [request]
-
- log.msg("transfer_file:" + filename, "CacheEntry")
+
+ if filename is not None:
+ self.file_path = filename
+
+ log.msg("transfer_file:" + self.file_path, "CacheEntry")
+
try:
- stat_tuple = os.stat(filename)
+ stat_tuple = os.stat(self.file_path)
mtime = stat_tuple[stat.ST_MTIME]
size = stat_tuple[stat.ST_SIZE]
self.state = self.STATE_SENDFILE
if size > 0:
- log.debug("Sending file to clients:%s size:%s" % (filename, size), 'CacheEntry')
- self.streamfile = open(filename,'rb')
+ log.debug("Sending file to clients:%s size:%s" % (self.file_path, size), 'CacheEntry')
+ if not self.streamfile: # Only open file if not already open
+ self.streamfile = open(self.file_path,'rb')
#fcntl.lockf(file.fileno(), fcntl.LOCK_SH)
for request in self.requests:
if request.start_streaming(size, mtime):
basic.FileSender().beginFileTransfer(self.streamfile, request) \
- .addBoth(self.file_transfer_complete, request, filename)
+ .addBoth(self.file_transfer_complete, request, self.file_path)
else:
- log.debug("Sending empty file to clients:%s" % (filename), 'CacheEntry')
+ log.debug("Sending empty file to clients:%s" % (self.file_path), 'CacheEntry')
for request in self.requests:
if request.start_streaming(size, mtime):
request.finish()
@@ -345,28 +352,6 @@
self.fetcher = fetcher
self.file_mtime = mtime
- """
- Use post_convert and gzip_convert regular expresions of the Fetcher
- to gzip/gunzip file before and after download.
- """
-
- if self.filename == 'Packages.gz':
- log.msg('TODO postconvert Packages.gz',CacheEntry)
-# if (fetcher and fetcher.post_convert.search(req.uri)
-# and not running.has_key(req.uri[:-3])):
-# log.debug("post converting: "+req.uri,'convert')
-# loop = LoopbackRequest(req)
-# loop.uri = req.uri[:-3]
-# loop.local_file = req.local_file[:-3]
-# loop.process()
-# loop.serve_if_cached=0
-# #FetcherGzip will attach as a request of the
-# #original Fetcher, efectively waiting for the
-# #original file if needed
-# gzip = FetcherGzip()
-# gzip.activate(loop, postconverting=1)
-
-
for req in self.requests:
req.start_streaming(size, mtime)
@@ -407,8 +392,8 @@
log.debug('Finish ' + str(req), 'CacheEntry')
try:
req.finish()
- except e:
- log.err('Error finishing request %s - %s' % (req, e), 'CacheEntry')
+ except Exception, e:
+ log.err('ERROR finishing request %s - %s' % (req, e), 'CacheEntry')
self.file_sent()
def download_failure(self, http_code, reason):
@@ -442,7 +427,7 @@
self.factory.file_served(self.cache_path)
self.factory.update_times[self.cache_path] = time.time()
self.state = self.STATE_NEW
-
+ self.remove_request(None) # In case there are no requests
def init_tempfile(self):
#log.msg("init_tempfile:" + self.file_path, "CacheEntry")
self.create_directory()
Modified: trunk/apt_proxy/clients.py
==============================================================================
--- trunk/apt_proxy/clients.py (original)
+++ trunk/apt_proxy/clients.py Sun Sep 10 22:52:38 2006
@@ -184,6 +184,7 @@
logname = 'UncompressClient' # Name for log messages
if_modified_since = None
+ can_uncompress_stream = True # We can deal with chunks
class FilenameError(Exception):
def __init__(self, filename, msg):
@@ -212,12 +213,17 @@
return -1
def finishCode(self, responseCode, message=None):
- "Request aborted"
- self.finish()
+ "Request aborted"
+ self.dest.download_failure(responseCode, message)
+ self.remove_from_cache_entry()
def finish(self):
self.dest.download_data_end()
- if self.source:
+ self.remove_from_cache_entry()
+
+ def remove_from_cache_entry(self):
+ if self.source:
+ reactor.callLater(0, self.source.remove_request, self)
self.source = None
def start_streaming(self, size, mtime):
@@ -228,32 +234,33 @@
log.debug("Skipping decompression of file (%s mtime=%s), destination file (%s, mtime=%s) is newer"
% (self.source.path, mtime, self.dest.filename, self.dest.file_mtime), self.logname)
self.finish()
+ return False
+
+ log.debug("Decompressing %s -> %s" % (self.source.path, self.dest.filename), self.logname)
+ self.dest.init_tempfile() # Open file for streaming
+ self.dest.download_started(None, size, mtime)
+ if self.source.state == self.source.STATE_SENDFILE:
+ self.finish()
+ return False # We don't want file to be streamed directly
+
+ if self.can_uncompress_stream:
+ return True # Send us chunks of data please
else:
- log.debug("Decompressing %s -> %s" % (self.source.path, self.dest.filename), self.logname)
- self.dest.init_tempfile() # Open file for streaming
- self.dest.download_started(None, size, mtime)
+ return False
+
def write(self, data):
log.debug("Decompressing %s bytes" % (len(data)), self.logname)
uncompressed = self.uncompress(data)
self.dest.download_data_received(uncompressed)
-class GzUncompressClient(UncompressClient):
- """
- Uncompress file using gzip (e.g. Packages.gz)
- """
- logname = 'GzUncompressClient'
- ext = '.gz'
-
+class UncompressClientFile(UncompressClient):
+ can_uncompress_stream = False
def __init__(self, compressedCacheEntry):
- self.string = StringIO()
- self.unzipper = gzip.GzipFile(compresslevel=0, fileobj = self.string)
UncompressClient.__init__(self, compressedCacheEntry)
- def uncompress(self, data):
- buflen = len(data)
- self.string.write(data)
- self.string.seek(-buflen, 1)
- buf = self.unzipper.read()
- return buf
+ def finish(self):
+ self.uncompressStart()
+ def write(self,data):
+ pass # Nothing to do
class Bz2UncompressClient(UncompressClient):
"""
@@ -267,4 +274,58 @@
UncompressClient.__init__(self, compressedCacheEntry)
def uncompress(self, data):
return self.decompressor.decompress(data)
-
\ No newline at end of file
+
+# Note: this class does not work because gzip.GzipFile wants to seek
+# in the file
+#class GzUncompressClient(UncompressClient):
+ #"""
+ #Uncompress file using gzip (e.g. Packages.gz)
+ #"""
+ #logname = 'GzUncompressClient'
+ #ext = '.gz'
+
+ #def __init__(self, compressedCacheEntry):
+ #self.string = StringIO()
+ #self.unzipper = gzip.GzipFile(compresslevel=0, fileobj = self.string, mode='r')
+ #UncompressClient.__init__(self, compressedCacheEntry)
+ #def uncompress(self, data):
+ #buflen = len(data)
+ #self.string.write(data)
+ #self.string.seek(-buflen, 1)
+ #buf = self.unzipper.read()
+ #return buf
+
+class GzUncompressClient(UncompressClientFile):
+ """
+ Uncompress file using gzip (e.g. Packages.gz)
+ """
+ logname = 'GzUncompressClient'
+ ext = '.gz'
+
+ def __init__(self, compressedCacheEntry):
+ UncompressClientFile.__init__(self, compressedCacheEntry)
+
+
+ def uncompressStart(self):
+ # Stream all data
+ log.debug("Uncompressing file %s" % (self.source.file_path), self.logname)
+ self.inputFile = open(self.source.file_path)
+ self.unzipper = gzip.GzipFile(compresslevel=0, fileobj = self.inputFile, mode='r')
+ reactor.callLater(0, self.uncompressFileChunk)
+ self.uncompressedLen = 0
+
+ def uncompressFileChunk(self):
+ uncompressed = self.unzipper.read()
+ chunkLen = len(uncompressed)
+ if chunkLen == 0:
+ self.uncompressDone()
+ return
+
+ self.uncompressedLen += chunkLen
+ self.dest.download_data_received(uncompressed)
+ reactor.callLater(0, self.uncompressFileChunk)
+
+ def uncompressDone(self):
+ log.debug("Uncompressed file %s, len=%s" % (self.dest.filename, self.uncompressedLen), self.logname)
+ self.unzipper.close()
+ UncompressClient.finish(self)
Modified: trunk/apt_proxy/fetchers.py
==============================================================================
--- trunk/apt_proxy/fetchers.py (original)
+++ trunk/apt_proxy/fetchers.py Sun Sep 10 22:52:38 2006
@@ -231,7 +231,7 @@
"""
Send a complete file (used by FileFetcher)
"""
- self.cacheEntry.transfer_file(filename)
+ self.cacheEntry.transfer_file(filename=filename)
def up_to_date(self):
"""
Modified: trunk/apt_proxy/test/test_cache.py
==============================================================================
--- trunk/apt_proxy/test/test_cache.py (original)
+++ trunk/apt_proxy/test/test_cache.py Sun Sep 10 22:52:38 2006
@@ -177,7 +177,7 @@
self.entry.file_mtime = time.time()+1000
self.failUnless(self.entry.check_age())
- def testCheckAgeMmutable(self):
+ def testCheckAgeMutable(self):
# pretend that testfile.deb is immutable, i.e.
# it will be updated like Packages, Release
self.entry.filetype = copy.deepcopy(self.entry.filetype) # Take a copy of the filetype object
Modified: trunk/apt_proxy/test/test_config.py
==============================================================================
--- trunk/apt_proxy/test/test_config.py (original)
+++ trunk/apt_proxy/test/test_config.py Sun Sep 10 22:52:38 2006
@@ -37,12 +37,14 @@
timeout = 888
bandwidth_limit = 2323
http_proxy = somehost:9876
+min_refresh_delay = 1h
[backend1]
backends = ftp://a.b.c
timeout = 999
bandwidth_limit = 3434
http_proxy = user:secret at otherhost:webcache
+min_refresh_delay = 2h
[backend2]
backends =
@@ -97,6 +99,10 @@
self.assertEquals(self.c.backends['backend1'].http_proxy.port, 'webcache')
self.assertEquals(self.c.backends['backend1'].http_proxy.user, 'user')
self.assertEquals(self.c.backends['backend1'].http_proxy.password, 'secret')
+ def testMinRefreshDelay(self):
+ self.assertEquals(self.c.min_refresh_delay,60*60)
+ self.assertEquals(self.c.backends['backend1'].min_refresh_delay,2*60*60)
+ self.assertEquals(self.c.backends['backend2'].min_refresh_delay,60*60)
class BrokenTimeoutTest(unittest.TestCase):
def testBrokenTimeout(self):
@@ -109,3 +115,5 @@
self.assertEquals(self.c.bandwidth_limit, None)
def testDefaultHttpProxy(self):
self.assertEquals(self.c.http_proxy, None)
+ def testDefaultMinRefreshDelay(self):
+ self.assertEquals(self.c.min_refresh_delay, 30)
Modified: trunk/apt_proxy/test/test_requests.py
==============================================================================
--- trunk/apt_proxy/test/test_requests.py (original)
+++ trunk/apt_proxy/test/test_requests.py Sun Sep 10 22:52:38 2006
@@ -206,6 +206,10 @@
del(self.factory)
apTestHelper.tearDown(self)
self.assertRaises(OSError, os.stat, self.cache_dir)
+
+ # The ftp classes use callLater(0, ...) several times, so allow
+ # those calls to complete
+ reactor.runUntilCurrent()
def doRequest(self, *data):
portno = self.port.getHost().port
@@ -245,6 +249,7 @@
def PackagesFile2(self, x):
backend = self.factory.getBackend('files')
# Check that request was deleted from backend
+ reactor.runUntilCurrent() # Gzip decompression runs using reactor.callLater(..)
self.assertEquals(len(backend.entries), 0)
def testForbidden(self):
@@ -359,6 +364,7 @@
testPackagesGzFile.timeout = 2
def PackagesUncompressed(self, x):
# Check that Packages file was registered
+ reactor.runUntilCurrent() # Gzip compression runs using reactor.callLater(..)
filename = self.calcFilePaths(self.packagesTestFile)[0][1:] # Remove leading '/' from path
backend = self.factory.getBackend(self.backendName)
self.assertEquals(backend.get_packages_db().packages.get_files(), [filename])
@@ -559,11 +565,6 @@
self.ftpserver.stop()
BackendTestBase.tearDown(self)
- # The ftp classes use callLater(0, ...) several times, so allow
- # those calls to complete
- reactor.iterate()
- reactor.iterate()
-
class RsyncBackendTest(TestRequestHelper, BackendTestBase):
def setUp(self):
"""
Modified: trunk/bin/apt-proxy
==============================================================================
--- trunk/bin/apt-proxy (original)
+++ trunk/bin/apt-proxy Sun Sep 10 22:52:38 2006
@@ -55,7 +55,9 @@
shell.port = config.telnet_port
# application to be started by twistd
-application = service.Application("AptProxy", uid, gid)
+application = service.Application("apt-proxy", uid, gid)
+print service.IProcess(application).processName
+service.IProcess(application).processName = 'apt-proxy'
for address in config.address:
if config.telnet_port:
Modified: trunk/debian/changelog
==============================================================================
--- trunk/debian/changelog (original)
+++ trunk/debian/changelog Sun Sep 10 22:52:38 2006
@@ -1,9 +1,13 @@
apt-proxy (1.9.36~svn) unstable; urgency=low
+ * Change the meaning of min_refresh_delay parameter, so the
+ delay is measured from the modification time of the file on the
+ backend instead of the time a client last requested this file.
* Uncompress Packages.gz and Packages.bz2 on the fly, and
update databases from these files (Closes: #TODO)
+ * Set process name to apt-proxy
- -- Chris Halls <halls at debian.org> Tue, 22 Aug 2006 11:07:26 +0100
+ -- Chris Halls <halls at debian.org> Fri, 1 Sep 2006 12:51:05 +0100
apt-proxy (1.9.35) unstable; urgency=low
Modified: trunk/doc/apt-proxy.conf
==============================================================================
--- trunk/doc/apt-proxy.conf (original)
+++ trunk/doc/apt-proxy.conf Sun Sep 10 22:52:38 2006
@@ -10,7 +10,7 @@
;; Control files (Packages/Sources/Contents) refresh rate
;;
-;; Minimum time between attempts to refresh a file
+;; Minimum age of a file before it is refreshed
min_refresh_delay = 1h
;; Minimum age of a file before attempting an update (NOT YET IMPLEMENTED)
@@ -42,14 +42,11 @@
;passive_ftp = on
;; Use HTTP proxy?
-;http_proxy = host:port
+;http_proxy = [username:password@]host:port
;; Limit download rate from backend servers (http and rsync only), in bytes/sec
;bandwidth_limit = 100000
-;; Enable HTTP pipelining within apt-proxy (for test purposes)
-;disable_pipelining=0
-
;;--------------------------------------------------------------
;; Cache housekeeping
@@ -78,9 +75,6 @@
;; You can override the default timeout like this:
;timeout = 30
-;; Rsync server used to rsync the Packages file (NOT YET IMPLEMENTED)
-;;rsyncpackages = rsync://ftp.de.debian.org/debian
-
;; Backend servers, in order of preference
backends =
http://ftp.us.debian.org/debian
@@ -88,39 +82,35 @@
http://ftp2.de.debian.org/debian
ftp://ftp.uk.debian.org/debian
+min_refresh_delay = 1d
+
[security]
;; Debian security archive
backends =
http://security.debian.org/debian-security
http://ftp2.de.debian.org/debian-security
+min_refresh_delay = 1m
+
[ubuntu]
;; Ubuntu archive
backends = http://archive.ubuntu.com/ubuntu
+min_refresh_delay = 15m
[ubuntu-security]
;; Ubuntu security updates
backends = http://security.ubuntu.com/ubuntu
-
-;[openoffice]
-;; OpenOffice.org packages
-;backends =
-; http://ftp.freenet.de/pub/debian-openoffice
-; http://ftp.sh.cvut.cz/MIRRORS/OpenOffice.deb
-; http://borft.student.utwente.nl/debian
-
-;[apt-proxy]
-;; Apt-proxy new versions
-;backends = http://apt-proxy.sourceforge.net/apt-proxy
+min_refresh_delay = 1m
;[backports.org]
;; backports.org
;backends = http://backports.org/debian
+min_refresh_delay = 1d
;[blackdown]
;; Blackdown Java
;backends = http://ftp.gwdg.de/pub/languages/java/linux/debian
-
+min_refresh_delay = 1d
;[debian-people]
;; people.debian.org
@@ -134,6 +124,5 @@
;; An example using an rsync server. This is not recommended
;; unless http is not available, becuause rsync is only more
;; efficient for transferring uncompressed files and puts much
-;; more overhead on the server. See the rsyncpacakges parameter
-;; for a way of rsyncing just the Packages files.
+;; more overhead on the server.
;backends = rsync://ftp.uk.debian.org/debian
Modified: trunk/doc/apt-proxy.conf.5
==============================================================================
--- trunk/doc/apt-proxy.conf.5 (original)
+++ trunk/doc/apt-proxy.conf.5 Sun Sep 10 22:52:38 2006
@@ -33,8 +33,7 @@
.TP
.B min_refresh_delay
If different from \fBoff\fP, means that Packages and other control
-files will not be refreshed more frequently than this number of
-seconds\&.
+files will not be refreshed until they are at least this age.
.TP
.B timeout
@@ -63,7 +62,8 @@
\&.deb to keep. This is the number of versions per distribution, for example
setting max_versions to 2 will ensure that a maximum of 6 packages would be kept:
the last 2 stable versions, the last 2 testing versions and the last 2 unstable
-versions.
+versions. NOTE: If your system clock is very inaccurate this will not work
+properly and should be disabled.
.TP
.B passive_ftp
@@ -129,6 +129,11 @@
Specify \fB[username:password@]hostname:port\fP to use an upstream proxy,
overriding the global http_proxy.
+.TP
+.B min_refresh_delay
+Override the global min_refresh_delay for this backend. Set this to the interval at
+which this archive is usually refreshed.
+
.SH CONFIGURATION EXAMPLES
To access a resource that's listed under a specific section name, simply append
More information about the apt-proxy-devel
mailing list