[apt-proxy-devel] r588 - people/halls/rework/apt_proxy

Chris Halls halls at costa.debian.org
Fri Jan 13 11:47:29 UTC 2006


Author: halls
Date: Fri Jan 13 11:47:27 2006
New Revision: 588

Modified:
   people/halls/rework/apt_proxy/apt_proxy.py
   people/halls/rework/apt_proxy/apt_proxy_conf.py
   people/halls/rework/apt_proxy/cache.py
   people/halls/rework/apt_proxy/fetchers.py
Log:
* More work on implementation of FtpFetcher
* Fixed various syntax errors


Modified: people/halls/rework/apt_proxy/apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy.py	(original)
+++ people/halls/rework/apt_proxy/apt_proxy.py	Fri Jan 13 11:47:27 2006
@@ -25,7 +25,7 @@
 
 import memleak
 import fetchers, cache, packages
-from misc import log
+from misc import log, MirrorRecycler
 import twisted_compat
 
 #from posixfile import SEEK_SET, SEEK_CUR, SEEK_END
@@ -91,7 +91,7 @@
         """
         if self.entries.has_key(path):
             return entries[path]
-        else
+        else:
             e = CacheEntry(self, path)
             entries[path] = e
             return e
@@ -245,7 +245,7 @@
 
         self.cacheEntry.add_request(self)
 
-`   def start_streaming(self, size, mtime):
+    def start_streaming(self, size, mtime):
         """
         Prepare client to stream file
         Return false if streaming is not necessary (i.e. cache hit)
@@ -255,7 +255,7 @@
             self.setHeader('last-modified', http.datetimeToString(mtime))
             self.setHeader('content-length', size)
             return True
-        else
+        else:
             self.setHeader("content-length", 0)
             self.finishCode(http.NOT_MODIFIED, 'File is up to date')
             return False
@@ -414,7 +414,7 @@
     def startFactory(self):
         #start periodic updates
         self.configurationChanged()
-        self.recycler = misc.MirrorRecycler(self, 1)
+        self.recycler = MirrorRecycler(self, 1)
         self.recycler.start()
 
     def configurationChanged(self, oldconfig = None):

Modified: people/halls/rework/apt_proxy/apt_proxy_conf.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy_conf.py	(original)
+++ people/halls/rework/apt_proxy/apt_proxy_conf.py	Fri Jan 13 11:47:27 2006
@@ -63,7 +63,7 @@
         p = ProxyConfig(self.get(section,option))
         if p.host is not None:
             return p
-        else
+        else:
             return None
 
 class apConfig:
@@ -111,8 +111,8 @@
     BACKEND_CONFIG_ITEMS = [
         ['timeout', None, 'time'],
         ['passive_ftp', None, 'boolean'],
-        ['backends', '', 'stringlist']
-        ['http_proxy', None , 'proxyspec'],
+        ['backends', '', 'stringlist'],
+        ['http_proxy', None , 'proxyspec']
         ]
 
     DEFAULT_CONFIG_FILE = ['/etc/apt-proxy/apt-proxy-v2.conf',

Modified: people/halls/rework/apt_proxy/cache.py
==============================================================================
--- people/halls/rework/apt_proxy/cache.py	(original)
+++ people/halls/rework/apt_proxy/cache.py	Fri Jan 13 11:47:27 2006
@@ -83,12 +83,14 @@
             raise RuntimeError, \
                   'this request is already assigned to this CacheEntry'
         self.requests.append(request)
-        if(len(self.requests)==1)
+        if(len(self.requests)==1):
             # First request
             self.get()
-        else
+        else:
             # Subsequent request - client must be brought up to date
-            if self.state == STATE_DOWNLOADING
+            if self.state == STATE_DOWNLOADING:
+                raise RuntimeError, \
+                      'TODO: multiple clients not implemented yet'
 
     def remove_request(self,request):
         """
@@ -250,7 +252,7 @@
         to gzip/gunzip file before and after download.
         """
 
-        if self.filename == 'Packages.gz'
+        if self.filename == 'Packages.gz':
             log.msg('TODO postconvert Packages.gz',CacheEntry)
 #             if (fetcher and fetcher.post_convert.search(req.uri)
 #                 and not running.has_key(req.uri[:-3])):
@@ -295,7 +297,8 @@
         """
         Download is not possible
         """
-        self.state = 
+        for request in self.requests:
+            request.finishCode(http.NOT_FOUND, reason)
 
     def file_sent(self):
         """

Modified: people/halls/rework/apt_proxy/fetchers.py
==============================================================================
--- people/halls/rework/apt_proxy/fetchers.py	(original)
+++ people/halls/rework/apt_proxy/fetchers.py	Fri Jan 13 11:47:27 2006
@@ -25,6 +25,7 @@
 from twisted.web import static, http
 from twisted.internet import protocol
 
+
 class Fetcher:
     """
     This class manages the selection of a BackendServer and downloading from
@@ -160,12 +161,12 @@
         log.err(msg)
         self.fail_over(http.SERVICE_UNAVAILABLE, reason)
 
-    def connected()
+    def connected():
         log.debug("Connected to  "+ self.request.backend_uri, 'Fetcher')
         log.debug('downloading:%s mtime:%s' % (uri, mtime), 'Fetcher')
         self.fetcher.download(self.cacheEntry.path, self.cacheEntry.file_mtime)
 
-    def file_not_found()
+    def file_not_found():
         log.msg("[%s] file not found: %s" % (self.backendServer.path, self.request.backend_uri))
         # TODO - failover?
         self.download_failed(reason)
@@ -177,6 +178,10 @@
         self.cacheEntry.transfer_file(filename)
 
     def up_to_date():
+        """
+        Fetcher has determined that our cached file is up to date
+        so the file is sent from our cache
+        """
         self.cacheEntry.send_cached_file()
 
 class FileFetcher:
@@ -215,12 +220,17 @@
 class FetcherHttpClient(http.HTTPClient):
     """
     This class represents an Http conncetion to a backend
-    server
+    server. It is generated by the HttpFetcher class when
+    a connection is made to an http server
     """
     def __init__(self, parent):
         self.parent = parent # HttpFetcher
         self.proxy = parent.backendServer.config.http_proxy
     def connectionMade(self):
+        """
+        Http connection made - inform parent, which will
+        trigger callbacks
+        """
         self.parent.connected(self)
 
     def download(self, fetcher, uri, mtime):
@@ -236,7 +246,7 @@
             serverpath = serverpath + "/" + backendServer.path 
 
         #self.sendCommand(self.request.method, 
-        self.sendCommand("GET" serverpath + "/" + uri)
+        self.sendCommand("GET", serverpath + "/" + uri)
 
         self.sendHeader('host', backendServer.host)
         if self.proxy.user:
@@ -252,13 +262,10 @@
     def handleStatus(self, version, code, message):
         __pychecker__ = 'unusednames=version,message'
         log.debug('handleStatus %s - %s' % (code, message), 'http_client')
-        self.status_code = int(code)
-        
-        # Keep a record of server response even if overriden later by setReponseCode
-        self.http_status = self.status_code  
+        self.http_status = int(code)
+
+        self.setResponseCode(self.http_status)
 
-        self.setResponseCode(self.status_code)
-        
     def handleHeader(self, key, value):
 
         log.debug("Received: " + key + " " + str(value), 'http_client')
@@ -266,7 +273,7 @@
 
         if key == 'last-modified':
             self.server_mtime = http.stringToDatetime(value)
-        elsif key == 'content-length':
+        elif key == 'content-length':
             self.server_size = http.stringToDatetime(value)
 
     def handleEndHeaders(self):
@@ -275,16 +282,16 @@
             self.parent.up_to_date()
 
     def rawDataReceived(self, data):
-        self.apDataReceived(data)
+        self.parent.data_received(data)
 
-    def handleResponse(self, buffer):
-        if self.length == 0:
-            self.setResponseCode(http.NOT_FOUND)
-        # print "length: " + str(self.length), "response:", self.status_code
-        if self.http_status == http.NOT_MODIFIED:
-            self.apDataEnd(self.transfered, False)
-        else:
-            self.apDataEnd(self.transfered, True)
+#     def handleResponse(self, buffer):
+#         if self.length == 0:
+#             self.setResponseCode(http.NOT_FOUND)
+#         # print "length: " + str(self.length), "response:", self.status_code
+#         if self.http_status == http.NOT_MODIFIED:
+#             self.apDataEnd(self.transfered, False)
+#         else:
+#             self.apDataEnd(self.transfered, True)
 
     def lineReceived(self, line):
         """
@@ -310,12 +317,12 @@
 
     def sendHeader(self, name, value):
         "log and handle to the base class."
-        log.debug(name + ":" + value,'http_client')
+        log.debug(name + " sendHeader:" + value,'http_client')
         http.HTTPClient.sendHeader(self, name, value)
 
 class HttpFetcher(protocol.ClientFactory):
     """
-    A Fetcher that retrieves files via HTTP
+    A Fetcher factory that retrieves files via HTTP
     """
     def __init__(self, backendServer):
         self.backendServer = backendServer
@@ -326,10 +333,11 @@
         if not self.proxy.host:
             host = request.backendServer.host
             port = request.backendServer.port
-        else
+        else:
             host = self.proxy.host
             port = self.proxy.port
         reactor.connectTCP(host, port, self, request.backend.config.timeout)
+        return self.connectCallback
 
     def buildProtocol(self, addr):
         return FetcherHttpClient(self)
@@ -369,22 +377,45 @@
     NOTE: Twisted's FTPClient code uses it's own timeouts here and there,
     so the timeout specified for the backend may not always be used
     """
-    def activate (self, request):
-        Fetcher.activate(self, request)
-        if not request.apFetcher:
-            return
+    
+    def __init__(self, backendServer):
+        self.backendServer = backendServer
+        self.isConnected = False
 
-        self.passive_ftp = self.request.backend.config.passive_ftp
-        
-        self.remote_file = (self.request.backendServer.path + "/" 
-                            + self.request.backend_uri)
+    def connect(self):
+        """
+        Establish connection to ftp server specified by backendServer
+        """
+        self.connectCallback = deferred()
+        if not self.proxy.host:
+            host = self.backendServer.host
+            port = self.backendServer.port
+        else:
+            host = self.proxy.host
+            port = self.proxy.port
+        reactor.connectTCP(host, port, self, self.backendServer.config.timeout)
+        return self.connectCallback
 
-        from twisted.internet.protocol import ClientCreator
+    def download(self, fetcher, uri, mtime):
+        """
+        Request download
+        %param fetcher: Fetcher class to receive callbacks
+        %param uri: URI of file to be downloaded within backend
+        %param mtime: Modification time of current file in cache
+        """
+        self.parent = fetcher
+        self.cache_mtime = mtime
+        self.request_uri = uri
+
+        self.passive_ftp = self.backendServer.config.passive_ftp
+        
+        self.remote_file = (self.backendServer.path + "/" 
+                            + uri)
 
         if not request.backendServer.username:
-            creator = ClientCreator(reactor, ftp.FTPClient, passive=0)
+            creator = protocol.ClientCreator(reactor, ftp.FTPClient, passive=0)
         else:
-            creator = ClientCreator(reactor, ftp.FTPClient, request.backendServer.username,
+            creator = protocol.ClientCreator(reactor, ftp.FTPClient, request.backendServer.username,
                                     request.backendServer.password, passive=0)
         d = creator.connectTCP(request.backendServer.host, request.backendServer.port,
                                request.backend.config.timeout)
@@ -880,7 +911,7 @@
             if self.fetcher.Backend != self.activeFile.Backend:
                 self.fetcher.closeConnection()
                 self.fetcher = Fetcher()
-        else
+        else:
             self.fetcher = Fetcher()
         deferred = self.fetcher.start(activeFile)
         deferred.addCallback(self.downloadComplete)



More information about the apt-proxy-devel mailing list