[apt-proxy-devel] r599 - in people/halls/rework/apt_proxy: . test

Chris Halls halls at costa.debian.org
Wed Mar 29 09:20:22 UTC 2006


Author: halls
Date: Wed Mar 29 09:20:13 2006
New Revision: 599

Modified:
   people/halls/rework/apt_proxy/apt_proxy.py
   people/halls/rework/apt_proxy/fetchers.py
   people/halls/rework/apt_proxy/test/test_cache.py
   people/halls/rework/apt_proxy/test/test_requests.py
Log:
More fixes


Modified: people/halls/rework/apt_proxy/apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy.py	(original)
+++ people/halls/rework/apt_proxy/apt_proxy.py	Wed Mar 29 09:20:13 2006
@@ -596,9 +596,10 @@
         log.debug('Main factory stop', 'factory')
         import packages
         # self.dumpdbs()
+        
+        # Stop all DownloadQueues and their fetchers
         for b in self.backends.values():
-            log.debug('backend: %s queue: %s' %(b, b.queue))
-            del b.queue
+            b.queue.stop()
             b.queue = None
         self.backends = {}
         packages.cleanup(self)

Modified: people/halls/rework/apt_proxy/fetchers.py
==============================================================================
--- people/halls/rework/apt_proxy/fetchers.py	(original)
+++ people/halls/rework/apt_proxy/fetchers.py	Wed Mar 29 09:20:13 2006
@@ -73,7 +73,7 @@
         self.connectToBackend()
 
     def connectToBackend(self):
-        log.debug('Connecting to backend server', 'fetcher')
+        log.debug('Connecting to backend server %s' % (self.backendServer), 'fetcher')
         self.fetcher = self.backendServer.fetcher(self.backendServer)
         d = self.fetcher.connect()
         d.addCallback(self.connected)
@@ -82,6 +82,9 @@
 
         return True
 
+    def __str__(self):
+        return 'Fetcher server=%s file=%s' % (str(self.backendServer), self.cacheEntry.path)
+
     def start_download(self):
         """
         Begin streaming file
@@ -189,7 +192,10 @@
         self.fetcher.download(self, self.cacheEntry.path, self.cacheEntry.file_mtime)
 
     def disconnect(self):
-        self.fetcher = None # TODO loseConnection?
+        if self.fetcher is not None:
+            log.debug('disconnect %s' % (self.cacheEntry.path), 'Fetcher')
+            self.fetcher.disconnect()
+            self.fetcher = None
 
     def connection_closed(self):
         """
@@ -249,6 +255,9 @@
         # start the transfer
         self.parent.send_complete_file(self.local_file)
 
+    def disconnect(self):
+        pass
+
 class FetcherHttpClient(http.HTTPClient):
     """
     This class represents an Http conncetion to a backend
@@ -352,7 +361,8 @@
                         log.debug("File transfer complete",'http_client')
                         self.fetcher.download_complete()
                         if self.close_on_completion:
-                            self.transport.loseConnection()
+                            self.parent.disconnect()
+                            #self.transport.loseConnection()
                     else:
                         log.err("File transfer overrun! Expected size:%s Received size:%s" % 
                                 (self.server_size, self.fetcher.len_received), 'http_client')
@@ -448,6 +458,10 @@
         """
         self.connection.download(fetcher, uri, mtime)
 
+    def disconnect(self):
+        if self.isConnected:
+            self.connection.transport.loseConnection()
+            self.isConnected = False
 
 class FtpFetcher(Fetcher, protocol.Protocol):
     """
@@ -605,7 +619,7 @@
         d.addErrback(self.ftpFetchFailed)
     def ftpFetchResult(self, msg):
         self.parent.download_complete()
-    def ftpFetchFailed(self, msg):
+    def ftpFetchFailed(self, msgs):
         log.debug("ftp fetch failed: %s" % (msgs), 'ftp_client')
         self.parent.file_not_found()
 
@@ -613,8 +627,8 @@
         self.parent.data_received(data)
 
     def disconnect(self):
-        log.debug('disconnecting', 'ftp_client')
         if self.ftpclient is not None:
+            log.debug('disconnecting', 'ftp_client')
             self.ftpclient.quit()
             self.ftpclient.transport.loseConnection()
             self.ftpclient = None
@@ -979,7 +993,6 @@
     """
     closeTimeout = 5 # Time to close fetcher connections after lsat download (seconds)
     def __init__(self):
-        log.debug("-------new queue:  %s" % (self), 'DownloadQueue')
         #import traceback
         #traceback.print_stack()
         self.queue = [] # List of cacheEntry classes waiting
@@ -994,8 +1007,8 @@
         @return Deferred that is triggered when file has been downloaded
         """
         if len(self.queue) == 0 and self.timeoutCB is not None:
-            log.debug("cancel timeout: %s, %s" % (self, self.timeoutCB), 'DownloadQueue')
             self.timeoutCB.cancel()
+            self.timeoutCB = None
         self.queue.append(cacheEntry)
         if self.activeFile is None:
             self.startNextDownload()
@@ -1036,24 +1049,19 @@
             log.debug("download queue is empty", 'DownloadQueue')
             if self.closeTimeout:
                 self.timeoutCB = reactor.callLater(self.closeTimeout, self.closeFetcher)
-                log.debug("create timeout: %s, %s" % (self, self.timeoutCB), 'DownloadQueue')
-            print "ACTIVE" + str(self.timeoutCB)
-            #timeoutCB.cancel()
+            else:
+                self.closeFetcher()
 
     def closeFetcher(self):
         "Close active fetcher - called after queue has been empty for closeTimeout seconds"
-        
-        log.debug("closing fetcher", 'DownloadQueue')
 
         self.timeoutCB = None
         if self.fetcher is not None:
-            log.debug("queue is idle - closing fetcher", 'DownloadQueue')
+            log.debug("closing fetcher [%s]" % (self.fetcher.backendServer), 'DownloadQueue')
             self.fetcher.disconnect()
+            self.fetcher = None
 
     def stop(self):
-        log.debug("stop queue:  %s, %s" % (self, self.timeoutCB), 'DownloadQueue')
         if self.timeoutCB is not None:
-            log.debug("cancel timeout %s" (dir(self.timeoutCB)), 'DownloadQueue')
-            print dir(self.timeoutCB)
-            #self.timeoutCB.cancel()
+            self.timeoutCB.cancel()
         self.closeFetcher()

Modified: people/halls/rework/apt_proxy/test/test_cache.py
==============================================================================
--- people/halls/rework/apt_proxy/test/test_cache.py	(original)
+++ people/halls/rework/apt_proxy/test/test_cache.py	Wed Mar 29 09:20:13 2006
@@ -26,6 +26,7 @@
 from apt_proxy.cache import CacheEntry
 from apt_proxy.apt_proxy import Factory
 from apt_proxy.misc import log
+from apt_proxy.fetchers import DownloadQueue
 
 class DummyRequest:
     def __init__(self):
@@ -43,6 +44,7 @@
         [files]
         backends=file:///<path to test packages directory>
         """
+        DownloadQueue.closeTimeout = 0 # Close fetcher immediately
         apTestHelper.setUp(self)
         packagedir = self.cache_dir+"/packages"
         filedir = os.path.normpath(os.getcwd()+"/../test_data/packages")

Modified: people/halls/rework/apt_proxy/test/test_requests.py
==============================================================================
--- people/halls/rework/apt_proxy/test/test_requests.py	(original)
+++ people/halls/rework/apt_proxy/test/test_requests.py	Wed Mar 29 09:20:13 2006
@@ -18,7 +18,7 @@
 
 import os, time
 from twisted.trial import unittest
-from twisted.internet import protocol, reactor
+from twisted.internet import protocol, reactor, defer
 from twisted import web
 from twisted.web import http
 from StringIO import StringIO
@@ -28,6 +28,7 @@
 from apt_proxy.cache import CacheEntry
 from apt_proxy.apt_proxy import Factory
 from apt_proxy.misc import log
+from apt_proxy.fetchers import DownloadQueue
 
 class uriRequester(http.HTTPClient):
     """
@@ -247,6 +248,30 @@
         self.downloadFile()
         self.doRequest(self.filename, http.NOT_MODIFIED, os.path.getmtime(self.filepath))
 
+    def testCloseFetcherImmediately(self):
+        DownloadQueue.closeTimeout = 0 # Close fetcher immediately
+        self.downloadFile()
+        f = self.factory.getBackend(self.backend).queue.fetcher
+        self.assertEquals(f, None)
+
+    def testLeaveFetcherOpen(self):
+        DownloadQueue.closeTimeout = 2 # 2 second delay to close
+        self.downloadFile()
+        f = self.factory.getBackend(self.backend).queue.fetcher
+        self.assertNotEquals(f, None)
+
+    def testAutoCloseFetcher(self):
+        DownloadQueue.closeTimeout = 0.1
+        self.downloadFile()
+        self.f = self.factory.getBackend(self.backend).queue.fetcher
+        d = defer.Deferred()
+        reactor.callLater(0.2, self.AutoCloseFetcherResult, d)
+        return d
+    def AutoCloseFetcherResult(self, deferred):
+        f = self.factory.getBackend(self.backend).queue.fetcher
+        self.assertEquals(f, None)
+        deferred.callback(None)
+
     def testCached(self):
         self.downloadFile()
 



More information about the apt-proxy-devel mailing list