[apt-proxy-devel] r615 - in trunk: apt_proxy apt_proxy/test
Chris Halls
halls at costa.debian.org
Fri Aug 4 21:06:42 UTC 2006
Author: halls
Date: Fri Aug 4 21:06:41 2006
New Revision: 615
Modified:
trunk/apt_proxy/apt_proxy.py
trunk/apt_proxy/cache.py
trunk/apt_proxy/fetchers.py
trunk/apt_proxy/test/test_apt_proxy.py
trunk/apt_proxy/test/test_requests.py
trunk/debian/changelog
trunk/debian/control
Log:
* Reorganise changelog
* Fix exception if HIT is returned for a file which has not already been cached
* Build-depend on twisted-web instead of twisted
* testAbort modified, but not yet working
Modified: trunk/apt_proxy/apt_proxy.py
==============================================================================
--- trunk/apt_proxy/apt_proxy.py (original)
+++ trunk/apt_proxy/apt_proxy.py Fri Aug 4 21:06:41 2006
@@ -278,6 +278,12 @@
scheme, netloc, path, params, query, fragment = urlparse.urlparse(uri)
return os.path.normpath(path)
+ def not_modified(self):
+ """
+ File is not modified - send http hit
+ """
+ self.setHeader("content-length", 0)
+ self.finishCode(http.NOT_MODIFIED, 'File is up to date')
def start_streaming(self, size, mtime):
"""
@@ -294,8 +300,7 @@
return True
else:
log.debug("file not modified: mtime=%s if_modified_since=%s" % (mtime, self.if_modified_since) , 'Request')
- self.setHeader("content-length", 0)
- self.finishCode(http.NOT_MODIFIED, 'File is up to date')
+ self.not_modified()
return False
def finishCode(self, responseCode, message=None):
Modified: trunk/apt_proxy/cache.py
==============================================================================
--- trunk/apt_proxy/cache.py (original)
+++ trunk/apt_proxy/cache.py Fri Aug 4 21:06:41 2006
@@ -80,8 +80,8 @@
(self.filebase, self.fileext) = os.path.splitext(self.filename)
# self.create_directory()
- self.file_mtime = None
- self.file_size = None
+ self.file_mtime = None # Modification time of cache file
+ self.file_size = None # Size of cache file
self.fetcher = None
@@ -125,10 +125,12 @@
#log.debug("cancelling download (set complete_clientless_downloads to continue)",'cacheEntry')
#self.fetcher.cancel_download()
- if self.streamfile is not None:
- # File was streamed to clients
- self.streamfile.close()
- self.streamfile = None
+ #if self.streamfile is not None and self.state == self.STATE_DOWNLOAD:
+ #name = self.streamfile.name
+ #log.debug("Deleting temporary file (%s)" % name,'cacheEntry')
+ #self.streamfile.close()
+ #self.streamfile = None
+ #os.remove(name)
def start_request_stream(self, request):
"""
@@ -217,8 +219,13 @@
"""
File is up to date - send complete file from cache to clients
"""
- log.msg("sending file from cache:" + self.file_path, "CacheEntry")
- self.transfer_file(self.file_path)
+ if self.file_mtime is not None:
+ log.msg("sending file from cache:" + self.file_path, "CacheEntry")
+ self.transfer_file(self.file_path)
+ else:
+ log.msg("sending hits to all clients (%s)" % (self.file_path), "CacheEntry")
+ for req in self.requests:
+ req.not_modified()
def end_send_cached(self):
"""
@@ -274,8 +281,30 @@
Start file transfer from backend server
"""
log.msg("start download:" + self.path, "CacheEntry")
+
self.backend.start_download(self)
+ def get_request_mtime(self):
+ """
+ Return the latest modification time for which a HIT can be given instead
+ of downloading the file
+ """
+ backend_mtime = None # modification time to request from backend
+
+ # Find the earliest if-modified-since of the requests
+ for req in self.requests:
+ if req.if_modified_since is not None:
+ if backend_mtime is None:
+ backend_mtime = req.if_modified_since
+ elif req.if_modified_since < backend_mtime:
+ backend_mtime = req.if_modified_since
+
+ # If our cached file is newer, use that time instead
+ if self.file_mtime is not None and backend_mtime < self.file_mtime:
+ backend_mtime = self.file_mtime
+
+ return backend_mtime
+
def download_started(self, fetcher, size, mtime):
"""
Callback from Fetcher
Modified: trunk/apt_proxy/fetchers.py
==============================================================================
--- trunk/apt_proxy/fetchers.py (original)
+++ trunk/apt_proxy/fetchers.py Fri Aug 4 21:06:41 2006
@@ -196,8 +196,9 @@
self.download()
def download(self):
- log.debug('downloading:%s mtime:%s' % (self.cacheEntry.path, self.cacheEntry.file_mtime), 'Fetcher')
- self.fetcher.download(self, self.cacheEntry.path, self.cacheEntry.file_mtime)
+ mtime = self.cacheEntry.get_request_mtime()
+ log.debug('downloading:%s mtime:%s' % (self.cacheEntry.path, mtime), 'Fetcher')
+ self.fetcher.download(self, self.cacheEntry.path, mtime)
def disconnect(self):
if self.fetcher is not None:
Modified: trunk/apt_proxy/test/test_apt_proxy.py
==============================================================================
--- trunk/apt_proxy/test/test_apt_proxy.py (original)
+++ trunk/apt_proxy/test/test_apt_proxy.py Fri Aug 4 21:06:41 2006
@@ -19,6 +19,7 @@
import copy, tempfile, os, shutil
from twisted.trial import unittest
+from twisted.internet import reactor
from StringIO import StringIO
from apt_proxy.apt_proxy_conf import apConfig
@@ -75,6 +76,10 @@
self.cache_dir = tempfile.mkdtemp('.aptproxy')
self.config = self.default_config.replace('[DEFAULT]','[DEFAULT]\ncache_dir=' + self.cache_dir)
def tearDown(self):
+ # Allow connections to close down etc.
+ reactor.iterate(0.1)
+ reactor.iterate(0.1)
+ reactor.iterate(0.1)
log.debug('Removing temporary directory: ' + self.cache_dir)
shutil.rmtree(self.cache_dir)
self.assertRaises(OSError, os.stat, self.cache_dir)
Modified: trunk/apt_proxy/test/test_requests.py
==============================================================================
--- trunk/apt_proxy/test/test_requests.py (original)
+++ trunk/apt_proxy/test/test_requests.py Fri Aug 4 21:06:41 2006
@@ -71,7 +71,7 @@
log.debug('Lost connection. Reason:'+ str(reason))
def clientConnectionFailed(self, connector, reason):
log.err('Connection failed. Reason:', reason, 'requestFactory')
- self.request.failed()
+ self.request.failed(reason)
def __init__(self, host, *testData):
"""
@@ -120,6 +120,7 @@
log.debug("waiting for test results for: " + self.nextTest.filename, 'uriRequester')
else:
log.debug('test passed', 'uriRequester')
+ self.connection.disconnect()
self.deferred.callback(None)
#def handleStatus(self, version, code, message):
@@ -134,8 +135,11 @@
if self.nextTest.abortTransfer == False:
http.HTTPClient.dataReceived(self, data)
else:
- self.passed() # Trigger disconnection of connection
-
+ log.debug("aborting transfer", 'uriRequester')
+ self.disconnect()
+ self.deferred.callback(None)
+ #self.passed() # Trigger disconnection of connection
+
class ResponseError:
pass
class SizeError:
@@ -159,8 +163,10 @@
self.getNextTest()
def failed(self, data):
log.debug('test failed', 'uriRequester')
+ self.disconnect()
self.deferred.errback(data)
-
+ def disconnect(self):
+ reactor.callLater(0, self.connection.disconnect)
class TestRequestHelper(apTestHelper):
def setUp(self, config):
@@ -187,8 +193,8 @@
client = uriRequester(host, *data)
connection = reactor.connectTCP("127.0.0.1", portno, client.factory)
self.connection = connection
+ client.connection = connection
- client.deferred.addBoth(lambda x: connection.disconnect())
self.lastRequestFactory = client
return client.deferred
@@ -251,6 +257,7 @@
# Name of test backend
backendName = 'test_data'
+ packagesTestFile = '/packages/Packages.gz'
def setUp(self, backend_uri):
"""
@@ -270,7 +277,7 @@
return self.doRequest(uriData('/test_data/NotHere.gz', http.NOT_FOUND))
testNotFound.timeout = 2
- def downloadFile(self, file='/packages/Packages.gz'):
+ def downloadFile(self, file=packagesTestFile):
"""
Download a file to cache
self.backend is set to backend name
@@ -320,11 +327,11 @@
d = self.downloadFile()
self.testResult = defer.Deferred()
d.addCallback(self.NotModifiedGreater2)
- d.addErrback(lambda x: self.testResult.errback(failure.Failure()))
+ d.addErrback(lambda x: self.testResult.errback(failure.Failure(self.UnknownFailure)))
return self.testResult
def NotModifiedGreater2(self, x):
log.debug("testNotModifiedGreater: starting second client", self.debugname)
- d = self.doRequest(uriData(self.filename, http.NOT_MODIFIED, time.time()))
+ d = self.doRequest(uriData(self.filename, http.NOT_MODIFIED, if_modified_since=time.time()))
d.chainDeferred(self.testResult)
testNotModifiedGreater.timeout = 3
@@ -332,13 +339,31 @@
d= self.downloadFile()
self.testResult = defer.Deferred()
d.addCallback(self.NotModifiedGreater2)
- d.addErrback(lambda x: self.testResult.errback(failure.Failure()))
+ d.addErrback(lambda x: self.testResult.errback(failure.Failure(self.UnknownFailure)))
return self.testResult
def NotModifiedExact2(self, x):
- d = self.doRequest(uriData(self.filename, http.NOT_MODIFIED, os.path.getmtime(self.filepath)))
+ d = self.doRequest(uriData(self.filename, http.NOT_MODIFIED, if_modified_since=os.path.getmtime(self.filepath)))
d.chainDeferred(self.testResult)
testNotModifiedExact.timeout = 2
+ class UnknownFailure:
+ pass
+
+ def testNotModifiedUncached(self):
+ "Check http not modified sent when file not in cache"
+ # Using /packages/Packages as a file that is registered in the package database
+ filename, sourcepath, self.destpath = self.getFilePaths('/packages/Packages')
+ mtime = os.path.getmtime(sourcepath)
+ self.testResult = defer.Deferred()
+ d = self.doRequest(uriData(filename, http.NOT_MODIFIED, if_modified_since=mtime))
+ d.addCallback(self.NotModifiedUncached2)
+ d.addErrback(lambda x: self.testResult.errback(failure.Failure(self.UnknownFailure)))
+ return self.testResult
+ def NotModifiedUncached2(self, x):
+ self.assertRaises(OSError, os.stat, self.destpath)
+ self.testResult.callback(None)
+ testNotModifiedUncached.timeout = 2
+
def testCloseFetcherImmediately(self):
DownloadQueue.closeTimeout = 0 # Close fetcher immediately
return self.downloadFile().addCallback(self.CloseFetcherImmediately2)
@@ -360,7 +385,7 @@
d = self.downloadFile()
self.autoclosedeferred = defer.Deferred()
d.addCallback(self.AutoCloseFetcher2)
- d.addErrback(lambda x: self.autoclosedeferred.errback(failure.Failure()))
+ d.addErrback(lambda x: self.autoclosedeferred.errback(failure.Failure(self.UnknownFailure)))
return self.autoclosedeferred
def AutoCloseFetcher2(self, x):
# File is downloaded, now check fetcher state
@@ -405,19 +430,24 @@
def testAbort(self):
"Abort with complete_clientless_downloads=off"
- import twisted
- twisted.internet.base.DelayedCall.debug = True
+ self.testResult = defer.Deferred()
b = self.factory.getBackend(self.backendName)
b.config.bandwidth_limit = 10
# We're not testing here that limiting is applied, just that the code runs
filename, sourcepath, destpath = self.getFilePaths('/packages/apt_0.0.1_test.deb')
d = self.doRequest(uriData(filename, http.OK, filePath=sourcepath, abortTransfer=True))
d.addCallback(self.Abort2)
- return d
- testBwLimit.timeout = 2
+ d.addErrback(lambda x: self.testResult.errback(failure.Failure(self.UnknownFailure)))
+ return self.testResult
def Abort2(self, x):
"Connection was aborted, check that fetchers were closed"
-
+ log.debug("testAbort2", self.debugname)
+ #backendQueues = self.factory.getBackend(self.backendName).queue.queues
+ #self.assertEquals(len(backendQueues), 0)
+ log.debug("testAbort2a", self.debugname)
+ self.testResult.callback(None)
+ testAbort.timeout = 2
+ testAbort.todo = "Test not yet working"
# This test does not work with current twisted http client :(
#def testPipeline(self):
Modified: trunk/debian/changelog
==============================================================================
--- trunk/debian/changelog (original)
+++ trunk/debian/changelog Fri Aug 4 21:06:41 2006
@@ -1,46 +1,57 @@
apt-proxy (1.9.33+svn) unstable; urgency=low
* Acknowledge NMU by Luk Claes, thanks! (Closes: #359798)
+
* Change maintainer to myself and add Otavio to Uploaders, at
Otavio's request. Thanks Otavio for all your work.
- * Fix breakage caused by new twisted (Closes: #375677)
- * http_proxy can now be set in each [backend] section
- * Add support for username and password in http_proxy parameter.
- Thanks to Thomas Champagne for the patch
- (Closes: #323147, #327239)
- * Move fetchers and cache management into separate files
- * Add bandwidth_limit configuration parameter to limit download
- rates (Closes: #306095, #259011)
- * Add support for rsync port specification
- * Always check cache directory and logfile permissions when package
- is installed, thanks Ben Hutchings for the patch (Closes: #312969)
- * Add more unit tests
- * Remove obsolete debian/TODO from source package
- * Update doc/TODO, removing fixed items
- * Recognise apt package diff files (*.diff/Index). Thanks
- Florian Weimer for the patch (Closes: #336433)
- * Add debhelper to Build-Depends, needed for dh_clean in clean target
- * Remove http scheme, host and port from requested URL (Closes: #374405)
- * Add download queueing mechanism. Clients can now use HTTP pipelining to
- request files, and each file will be queued at the corresponding backend.
- Each separate apt client connection generates a connection to the
- backend. (Closes: #261802)
- * HTTP pipelining now works and is enabled by default
- (Closes: #272206, #141312)
- * Fix shutdown code (Closes: #359805)
- * Remove reference to v1 in description (Closes: #337966)
- * Give a meaningful error message if an empty time is given in the
- configuration file (Closes: #304611)
- * Reorganise download process to be more logical, fixing several problems
- (Closes: #329764)
- * Remove references to -i parameter in apt-proxy.conf manpage.
- (Closes: #328983)
- * In example apt-proxy.conf, remove references to non-US archive
- (Closes: #329935)
- * Support updated Python policy, thanks Matthias Klose. (Closes: #377322)
- * Standards-Version 3.7.2 (no changes necessary)
- -- Chris Halls <halls at debian.org> Fri, 4 Aug 2006 12:34:38 +0100
+ * Caching behaviour changes:
+ - Reorganise download process to be more logical, fixing several problems
+ (Closes: #329764)
+ - Recognise apt package diff files (*.diff/Index). Thanks
+ Florian Weimer for the patch (Closes: #336433)
+ - Remove http scheme, host and port from URL requested by clients
+ (Closes: #374405)
+ - Add download queueing mechanism. Clients can now use HTTP pipelining to
+ request files, and each file will be queued at the corresponding backend.
+ Each separate apt client connection generates a connection to the
+ backend. (Closes: #261802)
+ - HTTP pipelining now works and is enabled by default
+ (Closes: #272206, #141312)
+
+ * Code reorganisation:
+ - Fix breakage caused by new twisted (Closes: #375677)
+ - Move fetchers and cache management into separate files
+ - Add more unit tests
+ - Fix shutdown code (Closes: #359805)
+
+ * apt-proxy.conf:
+ - http_proxy can now be set in each [backend] section
+ - Add support for username and password in http_proxy parameter.
+ Thanks to Thomas Champagne for the patch (Closes: #323147, #327239)
+ - Add bandwidth_limit configuration parameter to limit download
+ rates (Closes: #306095, #259011)
+ - Add support for rsync port specification
+ - Give a meaningful error message if an empty time is given in the
+ configuration file (Closes: #304611)
+
+ * Packaging changes:
+ - Always check cache directory and logfile permissions when package
+ is installed, thanks Ben Hutchings for the patch (Closes: #312969)
+ - Add debhelper to Build-Depends, needed for dh_clean in clean target
+ - Remove reference to v1 in description (Closes: #337966)
+ - Support updated Python policy, thanks Matthias Klose. (Closes: #377322)
+ - Standards-Version 3.7.2 (no changes necessary)
+
+ * Documentation updates:
+ - Remove obsolete debian/TODO from source package
+ - Update doc/TODO, removing fixed items
+ - Remove references to -i parameter in apt-proxy.conf manpage.
+ (Closes: #328983)
+ - In example apt-proxy.conf, remove references to non-US archive
+ (Closes: #329935)
+
+ -- Chris Halls <halls at debian.org> Fri, 4 Aug 2006 21:50:56 +0100
apt-proxy (1.9.33-0.1) unstable; urgency=high
Modified: trunk/debian/control
==============================================================================
--- trunk/debian/control (original)
+++ trunk/debian/control Fri Aug 4 21:06:41 2006
@@ -6,7 +6,7 @@
Standards-Version: 3.7.2
XS-Python-Version: current
Build-Depends: debhelper
-Build-Depends-Indep: debhelper (>= 4.1.13), po-debconf, help2man, python-twisted (>= 2.4), python (>= 2.3.5-1), python-apt, po4a (>= 0.18.1), python-central (>= 0.5)
+Build-Depends-Indep: debhelper (>= 4.1.13), po-debconf, help2man, python-twisted-web (>= 0.6), python (>= 2.3.5-1), python-apt, po4a (>= 0.18.1), python-central (>= 0.5)
Package: apt-proxy
Architecture: all
More information about the apt-proxy-devel
mailing list