[apt-proxy-devel] r599 - in people/halls/rework/apt_proxy: . test
Chris Halls
halls at costa.debian.org
Wed Mar 29 09:20:22 UTC 2006
Author: halls
Date: Wed Mar 29 09:20:13 2006
New Revision: 599
Modified:
people/halls/rework/apt_proxy/apt_proxy.py
people/halls/rework/apt_proxy/fetchers.py
people/halls/rework/apt_proxy/test/test_cache.py
people/halls/rework/apt_proxy/test/test_requests.py
Log:
More fixes
Modified: people/halls/rework/apt_proxy/apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy.py (original)
+++ people/halls/rework/apt_proxy/apt_proxy.py Wed Mar 29 09:20:13 2006
@@ -596,9 +596,10 @@
log.debug('Main factory stop', 'factory')
import packages
# self.dumpdbs()
+
+ # Stop all DownloadQueues and their fetchers
for b in self.backends.values():
- log.debug('backend: %s queue: %s' %(b, b.queue))
- del b.queue
+ b.queue.stop()
b.queue = None
self.backends = {}
packages.cleanup(self)
Modified: people/halls/rework/apt_proxy/fetchers.py
==============================================================================
--- people/halls/rework/apt_proxy/fetchers.py (original)
+++ people/halls/rework/apt_proxy/fetchers.py Wed Mar 29 09:20:13 2006
@@ -73,7 +73,7 @@
self.connectToBackend()
def connectToBackend(self):
- log.debug('Connecting to backend server', 'fetcher')
+ log.debug('Connecting to backend server %s' % (self.backendServer), 'fetcher')
self.fetcher = self.backendServer.fetcher(self.backendServer)
d = self.fetcher.connect()
d.addCallback(self.connected)
@@ -82,6 +82,9 @@
return True
+ def __str__(self):
+ return 'Fetcher server=%s file=%s' % (str(self.backendServer), self.cacheEntry.path)
+
def start_download(self):
"""
Begin streaming file
@@ -189,7 +192,10 @@
self.fetcher.download(self, self.cacheEntry.path, self.cacheEntry.file_mtime)
def disconnect(self):
- self.fetcher = None # TODO loseConnection?
+ if self.fetcher is not None:
+ log.debug('disconnect %s' % (self.cacheEntry.path), 'Fetcher')
+ self.fetcher.disconnect()
+ self.fetcher = None
def connection_closed(self):
"""
@@ -249,6 +255,9 @@
# start the transfer
self.parent.send_complete_file(self.local_file)
+ def disconnect(self):
+ pass
+
class FetcherHttpClient(http.HTTPClient):
"""
This class represents an Http conncetion to a backend
@@ -352,7 +361,8 @@
log.debug("File transfer complete",'http_client')
self.fetcher.download_complete()
if self.close_on_completion:
- self.transport.loseConnection()
+ self.parent.disconnect()
+ #self.transport.loseConnection()
else:
log.err("File transfer overrun! Expected size:%s Received size:%s" %
(self.server_size, self.fetcher.len_received), 'http_client')
@@ -448,6 +458,10 @@
"""
self.connection.download(fetcher, uri, mtime)
+ def disconnect(self):
+ if self.isConnected:
+ self.connection.transport.loseConnection()
+ self.isConnected = False
class FtpFetcher(Fetcher, protocol.Protocol):
"""
@@ -605,7 +619,7 @@
d.addErrback(self.ftpFetchFailed)
def ftpFetchResult(self, msg):
self.parent.download_complete()
- def ftpFetchFailed(self, msg):
+ def ftpFetchFailed(self, msgs):
log.debug("ftp fetch failed: %s" % (msgs), 'ftp_client')
self.parent.file_not_found()
@@ -613,8 +627,8 @@
self.parent.data_received(data)
def disconnect(self):
- log.debug('disconnecting', 'ftp_client')
if self.ftpclient is not None:
+ log.debug('disconnecting', 'ftp_client')
self.ftpclient.quit()
self.ftpclient.transport.loseConnection()
self.ftpclient = None
@@ -979,7 +993,6 @@
"""
closeTimeout = 5 # Time to close fetcher connections after lsat download (seconds)
def __init__(self):
- log.debug("-------new queue: %s" % (self), 'DownloadQueue')
#import traceback
#traceback.print_stack()
self.queue = [] # List of cacheEntry classes waiting
@@ -994,8 +1007,8 @@
@return Deferred that is triggered when file has been downloaded
"""
if len(self.queue) == 0 and self.timeoutCB is not None:
- log.debug("cancel timeout: %s, %s" % (self, self.timeoutCB), 'DownloadQueue')
self.timeoutCB.cancel()
+ self.timeoutCB = None
self.queue.append(cacheEntry)
if self.activeFile is None:
self.startNextDownload()
@@ -1036,24 +1049,19 @@
log.debug("download queue is empty", 'DownloadQueue')
if self.closeTimeout:
self.timeoutCB = reactor.callLater(self.closeTimeout, self.closeFetcher)
- log.debug("create timeout: %s, %s" % (self, self.timeoutCB), 'DownloadQueue')
- print "ACTIVE" + str(self.timeoutCB)
- #timeoutCB.cancel()
+ else:
+ self.closeFetcher()
def closeFetcher(self):
"Close active fetcher - called after queue has been empty for closeTimeout seconds"
-
- log.debug("closing fetcher", 'DownloadQueue')
self.timeoutCB = None
if self.fetcher is not None:
- log.debug("queue is idle - closing fetcher", 'DownloadQueue')
+ log.debug("closing fetcher [%s]" % (self.fetcher.backendServer), 'DownloadQueue')
self.fetcher.disconnect()
+ self.fetcher = None
def stop(self):
- log.debug("stop queue: %s, %s" % (self, self.timeoutCB), 'DownloadQueue')
if self.timeoutCB is not None:
- log.debug("cancel timeout %s" (dir(self.timeoutCB)), 'DownloadQueue')
- print dir(self.timeoutCB)
- #self.timeoutCB.cancel()
+ self.timeoutCB.cancel()
self.closeFetcher()
Modified: people/halls/rework/apt_proxy/test/test_cache.py
==============================================================================
--- people/halls/rework/apt_proxy/test/test_cache.py (original)
+++ people/halls/rework/apt_proxy/test/test_cache.py Wed Mar 29 09:20:13 2006
@@ -26,6 +26,7 @@
from apt_proxy.cache import CacheEntry
from apt_proxy.apt_proxy import Factory
from apt_proxy.misc import log
+from apt_proxy.fetchers import DownloadQueue
class DummyRequest:
def __init__(self):
@@ -43,6 +44,7 @@
[files]
backends=file:///<path to test packages directory>
"""
+ DownloadQueue.closeTimeout = 0 # Close fetcher immediately
apTestHelper.setUp(self)
packagedir = self.cache_dir+"/packages"
filedir = os.path.normpath(os.getcwd()+"/../test_data/packages")
Modified: people/halls/rework/apt_proxy/test/test_requests.py
==============================================================================
--- people/halls/rework/apt_proxy/test/test_requests.py (original)
+++ people/halls/rework/apt_proxy/test/test_requests.py Wed Mar 29 09:20:13 2006
@@ -18,7 +18,7 @@
import os, time
from twisted.trial import unittest
-from twisted.internet import protocol, reactor
+from twisted.internet import protocol, reactor, defer
from twisted import web
from twisted.web import http
from StringIO import StringIO
@@ -28,6 +28,7 @@
from apt_proxy.cache import CacheEntry
from apt_proxy.apt_proxy import Factory
from apt_proxy.misc import log
+from apt_proxy.fetchers import DownloadQueue
class uriRequester(http.HTTPClient):
"""
@@ -247,6 +248,30 @@
self.downloadFile()
self.doRequest(self.filename, http.NOT_MODIFIED, os.path.getmtime(self.filepath))
+ def testCloseFetcherImmediately(self):
+ DownloadQueue.closeTimeout = 0 # Close fetcher immediately
+ self.downloadFile()
+ f = self.factory.getBackend(self.backend).queue.fetcher
+ self.assertEquals(f, None)
+
+ def testLeaveFetcherOpen(self):
+ DownloadQueue.closeTimeout = 2 # 2 second delay to close
+ self.downloadFile()
+ f = self.factory.getBackend(self.backend).queue.fetcher
+ self.assertNotEquals(f, None)
+
+ def testAutoCloseFetcher(self):
+ DownloadQueue.closeTimeout = 0.1
+ self.downloadFile()
+ self.f = self.factory.getBackend(self.backend).queue.fetcher
+ d = defer.Deferred()
+ reactor.callLater(0.2, self.AutoCloseFetcherResult, d)
+ return d
+ def AutoCloseFetcherResult(self, deferred):
+ f = self.factory.getBackend(self.backend).queue.fetcher
+ self.assertEquals(f, None)
+ deferred.callback(None)
+
def testCached(self):
self.downloadFile()
More information about the apt-proxy-devel
mailing list