[apt-proxy-devel] r582 - people/halls/rework/apt_proxy
Chris Halls
halls at costa.debian.org
Wed Sep 7 18:06:33 UTC 2005
Author: halls
Date: Wed Sep 7 18:06:32 2005
New Revision: 582
Modified:
people/halls/rework/apt_proxy/apt_proxy.py
people/halls/rework/apt_proxy/cache.py
people/halls/rework/apt_proxy/fetchers.py
Log:
* Remove FetcherClient skeleton. I don't think it will ever make sense to call anything but the CacheEntry, and for unit testing we can create a fake CacheEntry for testing
* Rename Backend.base to Backend.name
* Start a new Fetcher class. This will no longer be the parent of <Protocol>Fetcher. Instead the protocol fetcher will be a class member. Therefore this class can now take the code to handle fallback to different BackendServers. Much cleaner than being in the Request :)
* Fetcher renamed to FetcherOld. This will probably die once all the code has moved elsewhere
Modified: people/halls/rework/apt_proxy/apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy.py (original)
+++ people/halls/rework/apt_proxy/apt_proxy.py Wed Sep 7 18:06:32 2005
@@ -53,7 +53,7 @@
"Packages database for this backend"
packages = None
- base = None
+ name = None
def __init__(self, factory, config):
self.factory = factory
@@ -269,40 +269,6 @@
self.cacheEntry.remove_request(self)
#self.finish()
-class LoopbackRequest(Request):
- """
- This is just a fake Request so a Fetcher can attach to another
- Fetcher and be notified when then transaction is completed.
-
- Look at FetcherGzip for a sample.
- """
- __pychecker__ = 'no-callinit'
- import cStringIO
- local_mtime = None
- headers = {}
- content = cStringIO.StringIO()
-
- def __init__(self, other_req, finish=None):
-
- self.finish_cb = finish
- http.Request.__init__(self, None, 1)
- self.backend = other_req.backend
- self.factory = other_req.factory
- self.filetype = other_req.filetype
- self.method = other_req.method
- self.clientproto = other_req.clientproto
- def process(self):
- self.backend_uri = self.backend.get_path(self.uri)
- def write(self, data):
- "We don't care for the data, just want to know then it is served."
- pass
- def finish(self):
- "If he wanted to know, tell daddy that we are served."
- if self.finish_cb:
- self.finish_cb()
- self.transport = None
- pass
-
class Channel(http.HTTPChannel):
"""
This class encapsulates a channel (an HTTP socket connection with a single
Modified: people/halls/rework/apt_proxy/cache.py
==============================================================================
--- people/halls/rework/apt_proxy/cache.py (original)
+++ people/halls/rework/apt_proxy/cache.py Wed Sep 7 18:06:32 2005
@@ -189,20 +189,19 @@
"""
pass
- def start_download(self):
+
+ def download_started(self, fetcher):
+ """
+ Callback from Fetcher
+ A fetcher has begun streaming this file
"""
- Begin streaming file
- Serve from cache or through the appropriate Fetcher
- depending on the asociated backend.
+ self.state = STATE_DOWNLOAD
+ self.fetcher = fetcher
+ """
Use post_convert and gzip_convert regular expresions of the Fetcher
to gzip/gunzip file before and after download.
"""
- log.debug("Downloading: " + self.path, 'CacheEntry')
- self.transfered = TempFile()
-
- self.state = STATE_DOWNLOADING
- self.backendServer = self.backend.get_first_server()
if self.filename == 'Packages.gz'
log.msg('TODO postconvert Packages.gz',CacheEntry)
@@ -220,23 +219,19 @@
# gzip = FetcherGzip()
# gzip.activate(loop, postconverting=1)
-
- def activateNextBackendServer(self, fetcher):
+ def download_data_rcvd(self, data):
"""
- The attempt to retrieve a file from the BackendServer failed.
- Look for the next possible BackendServer and transfer requests to that
- Returns true if another BackendServer was found
- """
- self.backendServer = self.backend.get_next_server(self.backendServer)
- if(self.backendServer == None):
- log.debug("no more Backends", "fetcher")
- return False
-
- fetcher_class = self.backendServer.fetcher
- log.debug('Trying next backendServer', 'fetcher')
- fetcher.apEndTransfer(fetcher_class)
+ Callback from Fetcher
+ A block of data has been received from the streaming backend server
+ """
+ pass
- return True
+ def download_data_end(self):
+ """
+ Callback from Fetcher
+ File streaming is complete
+ """
+ pass
Modified: people/halls/rework/apt_proxy/fetchers.py
==============================================================================
--- people/halls/rework/apt_proxy/fetchers.py (original)
+++ people/halls/rework/apt_proxy/fetchers.py Wed Sep 7 18:06:32 2005
@@ -25,32 +25,73 @@
from twisted.web import static, http
from twisted.internet import protocol
-class FetcherClient:
+class Fetcher:
"""
- This class receives streamed data from backend servers
- and notification of file downloads
+ This class manages the selection of a BackendServer and downloading from
+ that backend
+
"""
- def download_started(self, fetcher):
+ cacheEntry = None
+ fetcher = None # connection-specific fetcher
+
+ def init_tempfile(self):
+ self.transfered = TempFile()
+
+ def start(self, cacheEntry):
+ self.init_tempfile()
+ self.cacheEntry = cacheEntry
+ self.backend = cacheEntry.backend
+
+ def getFetcher(self):
+ """
+ get next fetcher for backend
"""
- A fetcher has begun streaming this file
+ def activateNextBackendServer(self, fetcher):
"""
- self.state = STATE_DOWNLOAD
- self.fetcher = fetcher
+ Returns true if another BackendServer was found
+ """
+ if self.backendServer is None:
+ self.backendServer = self.backend.get_first_server()
+ if(self.backendServer == None):
+ log.err("No backend server found for backend " + self.backend.name, "fetcher")
+ return False
+ else:
+ # Look for the next possible BackendServer and transfer requests to that
+ # The attempt to retrieve a file from the BackendServer failed.
+ self.backendServer = self.backend.get_next_server(self.backendServer)
+
+ if(self.backendServer == None):
+ log.debug("no more Backends", "fetcher")
+ return False
+
+ fetcher_class = self.backendServer.fetcher
+ log.debug('Trying next backendServer', 'fetcher')
+ fetcher.apEndTransfer(fetcher_class)
+
+ return True
- def download_data_rcvd(self, data):
+ def start_download(self):
"""
- A block of data has been received from the streaming backend server
+ Begin streaming file
+ Serve from cache or through the appropriate Fetcher
+ depending on the asociated backend.
+
+ Use post_convert and gzip_convert regular expresions of the Fetcher
+ to gzip/gunzip file before and after download.
"""
- pass
+ log.debug("Downloading: " + self.path, 'CacheEntry')
+ self.transfered = TempFile()
+ self.activateNextBackendServer(self.fetcher)
- def download_data_end(self):
+ def download_complete(self):
"""
- File streaming is complete
+ Download was successful
"""
pass
+ def download_failed(self):
+ pass
-
-class Fetcher:
+class FetcherOld:
"""
This is the base class for all Fetcher*, it tries to hold as much
common code as posible.
@@ -64,14 +105,7 @@
status_message = None
length = None
transport = None
- cacheEntry = None
-
- def __init__(self, cacheEntry):
- self.init_temfile()
- self.cacheEntry = cacheEntry
- def init_tempfile(self):
- self.transfered = TempFile()
def setResponseCode(self, code, message=None):
"Set response code for all requests"
@@ -231,7 +265,7 @@
Tell our requests that the connection with the server failed.
"""
msg = '[%s] Connection Failed: %s/%s'%(
- self.request.backend.base,
+ self.request.backend.name,
self.request.backendServer.path, self.request.backend_uri)
if reason:
@@ -934,6 +968,7 @@
"""
queuedFiles = []
activeFile = None
+ fetcher = None
def addFile(self, cacheEntry):
"""
@@ -946,3 +981,19 @@
def startNextDownload(self):
activeFile = queuedFiles[0]
queuedFiles = queuedFiles[1:]
+
+ if self.fetcher is not None:
+ if self.fetcher.Backend != self.activeFile.Backend:
+ self.fetcher.closeConnection()
+ self.fetcher = Fetcher()
+ else
+ self.fetcher = Fetcher()
+ deferred = self.fetcher.start(activeFile)
+ deferred.addCallback(self.downloadComplete)
+ deferred.addErrback(self.downloadFailed)
+
+ def downloadCopmlete(self):
+ startNextDownload(self)
+ def downloadFailed(self):
+ pass # TODO
+
More information about the apt-proxy-devel
mailing list