[apt-proxy-devel] r630 - in trunk/apt_proxy: test
Chris Halls
halls at costa.debian.org
Mon Aug 14 22:00:51 UTC 2006
Author: halls
Date: Mon Aug 14 22:00:45 2006
New Revision: 630
Modified:
trunk/apt_proxy/apt_proxy.py
trunk/apt_proxy/cache.py
trunk/apt_proxy/packages.py
trunk/apt_proxy/test/test_cache.py
trunk/apt_proxy/test/test_fetchers.py
trunk/apt_proxy/test/test_packages.py
trunk/apt_proxy/test/test_requests.py
Log:
Fix test suite so almost all tests pass
Modified: trunk/apt_proxy/apt_proxy.py
==============================================================================
--- trunk/apt_proxy/apt_proxy.py (original)
+++ trunk/apt_proxy/apt_proxy.py Mon Aug 14 22:00:45 2006
@@ -138,6 +138,14 @@
"""
self.queue.addFile(entry)
+ def close(self):
+ "Clean up backend and associated structures"
+ if self.queue is not None:
+ self.queue.stop()
+ self.queue = None
+ if self.packages is not None:
+ del(self.packages)
+
class BackendServer:
"""
A repository server. A BackendServer is created for each URI defined in 'backends'
@@ -292,11 +300,11 @@
"""
if self.if_modified_since is None or self.if_modified_since < mtime:
log.debug("start_streaming size=%s mtime=%s if_modified_since=%s" % (size, mtime, self.if_modified_since) , 'Request')
- self.setResponseCode(http.OK, 'Streaming file')
if mtime is not None:
self.setHeader('last-modified', http.datetimeToString(mtime))
if size is not None:
self.setHeader('content-length', size)
+ self.setResponseCode(http.OK, 'Streaming file')
return True
else:
log.debug("file not modified: mtime=%s if_modified_since=%s" % (mtime, self.if_modified_since) , 'Request')
@@ -629,8 +637,7 @@
# Stop all DownloadQueues and their fetchers
for b in self.backends.values():
- b.queue.stop()
- b.queue = None
+ b.close()
self.backends = {}
packages.cleanup(self)
if self.recycler is not None:
Modified: trunk/apt_proxy/cache.py
==============================================================================
--- trunk/apt_proxy/cache.py (original)
+++ trunk/apt_proxy/cache.py Mon Aug 14 22:00:45 2006
@@ -431,7 +431,8 @@
"""
def __init__ (self, regex, contype, mutable):
- self.regex = regex
+ self.name = regex
+ self.regex = re.compile(regex)
self.contype = contype
self.mutable = mutable
@@ -441,35 +442,38 @@
return 1
else:
return 0
+ def __str__(self):
+ return "FileType regex=%s mimetype=%s mutable=%s" % (self.name,self.contype,self.mutable)
+
# Set up the list of filetypes that we are prepared to deal with.
# If it is not in this list, then we will ignore the file and
# return an error.
filetypes = (
- FileType(re.compile(r"\.u?deb$"), "application/dpkg", 0),
- FileType(re.compile(r"\.tar\.gz$"), "application/x-gtar", 0),
- FileType(re.compile(r"\.dsc$"),"text/plain", 0),
- FileType(re.compile(r"\.diff\.gz$"), "x-gzip", 0),
- FileType(re.compile(r"\.bin$"), "application/octet-stream", 0),
- FileType(re.compile(r"\.tgz$"), "application/x-gtar", 0),
- FileType(re.compile(r"\.txt$"), "text/plain", 1),
- FileType(re.compile(r"\.html$"), "text/html", 1),
+ FileType(r"\.u?deb$", "application/dpkg", 0),
+ FileType(r"\.tar\.gz$", "application/x-gtar", 0),
+ FileType(r"\.dsc$","text/plain", 0),
+ FileType(r"\.diff\.gz$", "x-gzip", 0),
+ FileType(r"\.bin$", "application/octet-stream", 0),
+ FileType(r"\.tgz$", "application/x-gtar", 0),
+ FileType(r"\.txt$", "text/plain", 1),
+ FileType(r"\.html$", "text/html", 1),
- FileType(re.compile(r"(?:^|/)(?:Packages|Release(?:\.gpg)?|Sources|(?:Contents|Translation)-[a-z0-9]+)"
- r"(?:\.(?:gz|bz2))?$"),
+ FileType(r"(?:^|/)(?:Packages|Release(?:\.gpg)?|Sources|(?:Contents|Translation)-[a-z0-9]+)"
+ r"(?:\.(?:gz|bz2))?$",
"text/plain", 1),
- FileType(re.compile(r"(?:^|/)(?:Packages|Sources|Contents-[a-z0-9]+)\.diff/Index$"),
+ FileType(r"(?:^|/)(?:Packages|Sources|Contents-[a-z0-9]+)\.diff/Index$",
"text/plain", 1),
- FileType(re.compile(r"(?:^|/)(?:Packages|Sources|Contents-[a-z0-9]+)\.diff/[a-z0-9.-]+"
- r"(?:\.(?:gz|bz2))?$"),
+ FileType(r"(?:^|/)(?:Packages|Sources|Contents-[a-z0-9]+)\.diff/[a-z0-9.-]+"
+ r"(?:\.(?:gz|bz2))?$",
"text/plain", 0),
- FileType(re.compile(r"\.rpm$"), "application/rpm", 0),
+ FileType(r"\.rpm$", "application/rpm", 0),
- FileType(re.compile(r"(?:^|/)(?:pkglist|release|srclist)(?:\.(?:\w|-)+)?"
- r"(?:\.(?:gz|bz2))?$"),
+ FileType(r"(?:^|/)(?:pkglist|release|srclist)(?:\.(?:\w|-)+)?"
+ r"(?:\.(?:gz|bz2))?$",
"text/plain", 1),
- FileType(re.compile(r"\.gz$"), "x-gzip", 1)
+ FileType(r"\.gz$", "x-gzip", 1)
)
Modified: trunk/apt_proxy/packages.py
==============================================================================
--- trunk/apt_proxy/packages.py (original)
+++ trunk/apt_proxy/packages.py Mon Aug 14 22:00:45 2006
@@ -56,16 +56,19 @@
"""
def __init__(self, backendName, cache_dir):
self.cache_dir = cache_dir
- packagedb_dir = cache_dir+'/'+ apt_proxy.status_dir + \
+ self.packagedb_dir = cache_dir+'/'+ apt_proxy.status_dir + \
'/backends/' + backendName
- if not os.path.exists(packagedb_dir):
- os.makedirs(packagedb_dir)
- self.packages = shelve.open(packagedb_dir+'/packages.db')
- def __del__(self):
- try:
+ if not os.path.exists(self.packagedb_dir):
+ os.makedirs(self.packagedb_dir)
+ self.packages = None
+ self.open()
+
+ def open(self):
+ if self.packages is None:
+ self.packages = shelve.open(self.packagedb_dir+'/packages.db')
+ def close(self):
+ if self.packages is not None:
self.packages.close()
- except:
- pass
def update_file(self, entry):
"""
@@ -169,7 +172,7 @@
def __del__(self):
self.cleanup()
#print "start aptPackages [%s] %s " % (self.backendName, self.cache_dir)
- del self.packages
+ self.packages.close()
#print "Deleted aptPackages [%s] %s " % (self.backendName, self.cache_dir)
def file_updated(self, entry):
"""
Modified: trunk/apt_proxy/test/test_cache.py
==============================================================================
--- trunk/apt_proxy/test/test_cache.py (original)
+++ trunk/apt_proxy/test/test_cache.py Mon Aug 14 22:00:45 2006
@@ -16,14 +16,14 @@
"""Unit test for cache.py"""
-import os, time, shutil
+import os, time, shutil, copy
from twisted.trial import unittest
from twisted.internet import reactor, defer
from StringIO import StringIO
from apt_proxy.apt_proxy_conf import apConfig
from apt_proxy.test.test_apt_proxy import apTestHelper
-from apt_proxy.cache import CacheEntry, findFileType
+from apt_proxy.cache import CacheEntry, findFileType, filetypes
from apt_proxy.apt_proxy import Factory
from apt_proxy.misc import log
from apt_proxy.fetchers import DownloadQueue
@@ -32,6 +32,7 @@
def __init__(self):
self.finished = False
self.streamed = 0
+ self.if_modified_since = None
def finishCode(self, code, reason):
self.finished = True
def start_streaming(self, file_size, file_mtime):
@@ -138,6 +139,8 @@
pass
class StreamedError:
pass
+ class UnknownError:
+ pass
def start_download(entry):
# This test function replaces the normal
# Backend.start_download so we can see that
@@ -148,7 +151,7 @@
self.testResult.errback(failure.Failure(VerifySizeError()))
if self.request.streamed:
self.testResult.errback(failure.Failure(StreamedError()))
- self.testResult.callback(None)
+ self.VerifyFailCleanup.callback(None)
self.backend.start_download = start_download
entry = CacheEntry(self.backend, "testdir/test.deb")
entry.test_download = False
@@ -157,8 +160,14 @@
f.write('this is not a real .deb')
f.close()
entry.add_request(self.request)
+ self.VerifyFailCleanup = defer.Deferred()
+ self.VerifyFailCleanup.addCallback(self.VerifyFail2)
+ self.VerifyFailCleanup.addErrback(lambda x: self.testResult.errback(self.UnknownError))
return self.testResult
testVerifyFail.timeout = 2
+ def VerifyFail2(self, x):
+ reactor.iterate(0.1) # Allow process to be reaped
+ self.testResult.callback(None)
def testCheckAgeImmutable(self):
# testfile.deb is immutable
@@ -171,6 +180,7 @@
def testCheckAgeMmutable(self):
# pretend that testfile.deb is immutable, i.e.
# it will be updated like Packages, Release
+ self.entry.filetype = copy.deepcopy(self.entry.filetype) # Take a copy of the filetype object
self.entry.filetype.mutable = True
self.entry.file_mtime = 0
self.failIf(self.entry.check_age())
@@ -227,5 +237,6 @@
log.debug('Testing filetype, name=%s mimetype=%s mutable=%s' % (name, mimetype, mutable))
result = findFileType(name)
self.assertNotEquals(result, None)
+ log.debug(' -> result contype=%s mutable=%s' % (result.contype, result.mutable))
self.assertEquals(mimetype, result.contype)
self.assertEquals(mutable, result.mutable)
Modified: trunk/apt_proxy/test/test_fetchers.py
==============================================================================
--- trunk/apt_proxy/test/test_fetchers.py (original)
+++ trunk/apt_proxy/test/test_fetchers.py Mon Aug 14 22:00:45 2006
@@ -108,11 +108,16 @@
#delayeds = reactor.getDelayedCalls()
#for d in delayeds:
# d.cancel()
+ reactor.iterate(0.1) # Process FTP callbacks before disconnecting
self.ftpFetcher.disconnect()
self.ftpFetcher = None
self.ftpserver.stop()
self.ftpserver = None
+ # Allow FTP code to shutdown
FactoryTestHelper.tearDown(self)
+ reactor.iterate(0.1)
+ reactor.iterate(0.1)
+ reactor.iterate(0.1)
class FetcherFtpTest(FetcherFtpTestHelper):
def setUp(self):
@@ -136,7 +141,7 @@
return d
testConnectFail.timeout = 2
-class DummyFetcher:
+class FetchersDummyFetcher:
def __init__(self, deferred):
self.deferred = deferred
self.error_code = None # Anticipated error
@@ -159,7 +164,7 @@
def setUp(self):
FetcherFtpTestHelper.setUp(self)
self.resultCallback = defer.Deferred()
- self.fetcher = DummyFetcher(self.resultCallback)
+ self.fetcher = FetchersDummyFetcher(self.resultCallback)
self.fetcher.backendServer = self.backendServer
def tearDown(self):
@@ -238,13 +243,15 @@
backends = rsync://127.0.0.1:0/test
"""
- class DummyFetcher:
+ class RsyncDummyFetcher:
def __init__(self, backend, backendServer):
self.backend = backend
self.backendServer = backendServer
self.cacheEntry = backend.get_cache_entry("testdir/testfile.deb")
def fetcher_internal_error(self, message):
log.debug('fetcher_internal_error: %s' % (message))
+ def server_mtime(self, time):
+ pass
def setUp(self):
"""
@@ -262,7 +269,7 @@
return self.f.connect() # connect returns a deferred that fires
def testDownload(self):
self.f.connect()
- dummyFetcher = self.DummyFetcher(self.backend, self.backendServer)
+ dummyFetcher = self.RsyncDummyFetcher(self.backend, self.backendServer)
self.f.download(dummyFetcher, 'test', time.time())
class RsyncServer(protocol.ProcessProtocol):
@@ -378,7 +385,7 @@
def setUp(self):
FetcherRsyncTestHelper.setUp(self)
self.resultCallback = defer.Deferred()
- self.fetcher = DummyFetcher(self.resultCallback)
+ self.fetcher = FetchersDummyFetcher(self.resultCallback)
self.fetcher.backendServer = self.backendServer
def tearDown(self):
@@ -413,6 +420,8 @@
def download(self, fetcher, uri, mtime):
fetcher.cacheEntry.state = CacheEntry.STATE_DOWNLOAD
pass
+ def server_mtime(self, time):
+ pass
class DummyServer:
fetcher = DummyFetcher
@@ -437,6 +446,8 @@
self.file_mtime = None
self.requests = []
self.state = self.STATE_NEW
+ def get_request_mtime(self):
+ return None
class DownloadQueueTest(FactoryTestHelper):
Modified: trunk/apt_proxy/test/test_packages.py
==============================================================================
--- trunk/apt_proxy/test/test_packages.py (original)
+++ trunk/apt_proxy/test/test_packages.py Mon Aug 14 22:00:45 2006
@@ -34,6 +34,8 @@
self.path = file
self.cache_path = backend + os.sep + file
self.file_path = cache_dir + os.sep + self.cache_path
+ def get_request_mtime(self):
+ return None
class PackageFileListTest(apTestHelper):
def setUp(self):
Modified: trunk/apt_proxy/test/test_requests.py
==============================================================================
--- trunk/apt_proxy/test/test_requests.py (original)
+++ trunk/apt_proxy/test/test_requests.py Mon Aug 14 22:00:45 2006
@@ -83,6 +83,7 @@
self.http_status = None
self.received_len = 0
self.tests=testData
+ self.chunked = False
def connectionMade(self):
"""
@@ -123,12 +124,18 @@
self.connection.disconnect()
self.deferred.callback(None)
- #def handleStatus(self, version, code, message):
def handleStatus(self, version, code, message):
log.debug('handleStatus: (%s) %s - %s, expected:%s' %
(version, code, message, self.nextTest.expectedResponse), 'uriRequester')
self.http_status = int(code)
+ def handleHeader(self, key, value):
+ if key.lower() == 'transfer-encoding' and value.lower() == 'chunked':
+ # There doesn't seem to be any support for chunked mode in twisted-web's http
+ # client, so use fromChunk ourselves
+ log.debug("Chunked mode")
+ self.chunked = True
+
def dataReceived(self, data):
self.received_len = self.received_len + len(data)
log.debug("data received, len: %s" % (self.received_len), 'uriRequester')
@@ -140,22 +147,36 @@
self.deferred.callback(None)
#self.passed() # Trigger disconnection of connection
- class ResponseError:
- pass
- class SizeError:
- pass
+ class ResponseError(Exception):
+ def __init__(self, received, expected):
+ self.received = received
+ self.expected = expected
+ def __str__(self):
+ return "ResponseError - received %s but expected %s" % (self.received,self.expected)
+ class SizeError(Exception):
+ def __init__(self, received, expected):
+ self.received = received
+ self.expected = expected
+ def __str__(self):
+ return "SizeError - received %s bytes but expected %s" % (self.received,self.expected)
def handleResponse(self, buffer):
+ if self.chunked:
+ buffer, rest = http.fromChunk(buffer)
+ while len(rest)>0:
+ log.debug("buf size=%s rest size=%s" % (len(buffer), len(rest)))
+ data, rest = http.fromChunk(rest)
+ buffer += data
received_len = len(buffer)
log.debug('data received: %s bytes, expected:%s' % (received_len, self.nextTest.expectedSize), 'uriRequester')
if self.http_status != self.nextTest.expectedResponse:
log.debug('test FAILED: response code (%s) is not %s' %
(self.http_status, self.nextTest.expectedResponse), 'uriRequester')
- self.failed(self.ResponseError())
+ raise self.ResponseError(self.http_status, self.nextTest.expectedResponse)
elif self.nextTest.expectedSize is not None and received_len != self.nextTest.expectedSize:
log.debug('test FAILED: received %s bytes, but expected %s' %
(received_len, self.nextTest.expectedSize), 'uriRequester')
- self.failed(self.SizeError())
+ raise self.SizeError(received_len, self.nextTest.expectedSize)
else:
self.passed()
@@ -257,7 +278,7 @@
# Name of test backend
backendName = 'test_data'
- packagesTestFile = '/packages/Packages.gz'
+ packagesTestFile = '/packages/Packages'
def setUp(self, backend_uri):
"""
@@ -346,7 +367,7 @@
d.chainDeferred(self.testResult)
testNotModifiedExact.timeout = 2
- class UnknownFailure:
+ class UnknownFailure(Exception):
pass
def testNotModifiedUncached(self):
@@ -385,7 +406,7 @@
d = self.downloadFile()
self.autoclosedeferred = defer.Deferred()
d.addCallback(self.AutoCloseFetcher2)
- d.addErrback(lambda x: self.autoclosedeferred.errback(failure.Failure(self.UnknownFailure)))
+ d.addErrback(lambda x: self.autoclosedeferred.errback(failure.Failure()))
return self.autoclosedeferred
def AutoCloseFetcher2(self, x):
# File is downloaded, now check fetcher state
@@ -416,8 +437,10 @@
def CachedPass(self, x):
self.testResult.callback(None)
def CachedError(self, x):
+ class CachedErrorException(Exception):
+ pass
log.debug("testCached ERROR", self.debugname)
- self.testResult.errback(failure.Failure())
+ raise CachedErrorException()
testCached.timeout = 2
def testBwLimit(self):
@@ -550,5 +573,5 @@
if file is not None:
self.testResult.callback("Tempfile is %s" %(file))
else:
- self.testResult.errback(failure.Failure())
+ raise UnknownFailure()
testTempFile.timeout=2
\ No newline at end of file
More information about the apt-proxy-devel
mailing list