[apt-proxy-devel] r588 - people/halls/rework/apt_proxy
Chris Halls
halls at costa.debian.org
Fri Jan 13 11:47:29 UTC 2006
Author: halls
Date: Fri Jan 13 11:47:27 2006
New Revision: 588
Modified:
people/halls/rework/apt_proxy/apt_proxy.py
people/halls/rework/apt_proxy/apt_proxy_conf.py
people/halls/rework/apt_proxy/cache.py
people/halls/rework/apt_proxy/fetchers.py
Log:
* More work on implementation of FtpFetcher
* Fixed various syntax errors
Modified: people/halls/rework/apt_proxy/apt_proxy.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy.py (original)
+++ people/halls/rework/apt_proxy/apt_proxy.py Fri Jan 13 11:47:27 2006
@@ -25,7 +25,7 @@
import memleak
import fetchers, cache, packages
-from misc import log
+from misc import log, MirrorRecycler
import twisted_compat
#from posixfile import SEEK_SET, SEEK_CUR, SEEK_END
@@ -91,7 +91,7 @@
"""
if self.entries.has_key(path):
return entries[path]
- else
+ else:
e = CacheEntry(self, path)
entries[path] = e
return e
@@ -245,7 +245,7 @@
self.cacheEntry.add_request(self)
-` def start_streaming(self, size, mtime):
+ def start_streaming(self, size, mtime):
"""
Prepare client to stream file
Return false if streaming is not necessary (i.e. cache hit)
@@ -255,7 +255,7 @@
self.setHeader('last-modified', http.datetimeToString(mtime))
self.setHeader('content-length', size)
return True
- else
+ else:
self.setHeader("content-length", 0)
self.finishCode(http.NOT_MODIFIED, 'File is up to date')
return False
@@ -414,7 +414,7 @@
def startFactory(self):
#start periodic updates
self.configurationChanged()
- self.recycler = misc.MirrorRecycler(self, 1)
+ self.recycler = MirrorRecycler(self, 1)
self.recycler.start()
def configurationChanged(self, oldconfig = None):
Modified: people/halls/rework/apt_proxy/apt_proxy_conf.py
==============================================================================
--- people/halls/rework/apt_proxy/apt_proxy_conf.py (original)
+++ people/halls/rework/apt_proxy/apt_proxy_conf.py Fri Jan 13 11:47:27 2006
@@ -63,7 +63,7 @@
p = ProxyConfig(self.get(section,option))
if p.host is not None:
return p
- else
+ else:
return None
class apConfig:
@@ -111,8 +111,8 @@
BACKEND_CONFIG_ITEMS = [
['timeout', None, 'time'],
['passive_ftp', None, 'boolean'],
- ['backends', '', 'stringlist']
- ['http_proxy', None , 'proxyspec'],
+ ['backends', '', 'stringlist'],
+ ['http_proxy', None , 'proxyspec']
]
DEFAULT_CONFIG_FILE = ['/etc/apt-proxy/apt-proxy-v2.conf',
Modified: people/halls/rework/apt_proxy/cache.py
==============================================================================
--- people/halls/rework/apt_proxy/cache.py (original)
+++ people/halls/rework/apt_proxy/cache.py Fri Jan 13 11:47:27 2006
@@ -83,12 +83,14 @@
raise RuntimeError, \
'this request is already assigned to this CacheEntry'
self.requests.append(request)
- if(len(self.requests)==1)
+ if(len(self.requests)==1):
# First request
self.get()
- else
+ else:
# Subsequent request - client must be brought up to date
- if self.state == STATE_DOWNLOADING
+ if self.state == STATE_DOWNLOADING:
+ raise RuntimeError, \
+ 'TODO: multiple clients not implemented yet'
def remove_request(self,request):
"""
@@ -250,7 +252,7 @@
to gzip/gunzip file before and after download.
"""
- if self.filename == 'Packages.gz'
+ if self.filename == 'Packages.gz':
log.msg('TODO postconvert Packages.gz',CacheEntry)
# if (fetcher and fetcher.post_convert.search(req.uri)
# and not running.has_key(req.uri[:-3])):
@@ -295,7 +297,8 @@
"""
Download is not possible
"""
- self.state =
+ for request in self.requests:
+ request.finishCode(http.NOT_FOUND, reason)
def file_sent(self):
"""
Modified: people/halls/rework/apt_proxy/fetchers.py
==============================================================================
--- people/halls/rework/apt_proxy/fetchers.py (original)
+++ people/halls/rework/apt_proxy/fetchers.py Fri Jan 13 11:47:27 2006
@@ -25,6 +25,7 @@
from twisted.web import static, http
from twisted.internet import protocol
+
class Fetcher:
"""
This class manages the selection of a BackendServer and downloading from
@@ -160,12 +161,12 @@
log.err(msg)
self.fail_over(http.SERVICE_UNAVAILABLE, reason)
- def connected()
+ def connected():
log.debug("Connected to "+ self.request.backend_uri, 'Fetcher')
log.debug('downloading:%s mtime:%s' % (uri, mtime), 'Fetcher')
self.fetcher.download(self.cacheEntry.path, self.cacheEntry.file_mtime)
- def file_not_found()
+ def file_not_found():
log.msg("[%s] file not found: %s" % (self.backendServer.path, self.request.backend_uri))
# TODO - failover?
self.download_failed(reason)
@@ -177,6 +178,10 @@
self.cacheEntry.transfer_file(filename)
def up_to_date():
+ """
+ Fetcher has determined that our cached file is up to date
+ so the file is sent from our cache
+ """
self.cacheEntry.send_cached_file()
class FileFetcher:
@@ -215,12 +220,17 @@
class FetcherHttpClient(http.HTTPClient):
"""
This class represents an Http conncetion to a backend
- server
+ server. It is generated by the HttpFetcher class when
+ a connection is made to an http server
"""
def __init__(self, parent):
self.parent = parent # HttpFetcher
self.proxy = parent.backendServer.config.http_proxy
def connectionMade(self):
+ """
+ Http connection made - inform parent, which will
+ trigger callbacks
+ """
self.parent.connected(self)
def download(self, fetcher, uri, mtime):
@@ -236,7 +246,7 @@
serverpath = serverpath + "/" + backendServer.path
#self.sendCommand(self.request.method,
- self.sendCommand("GET" serverpath + "/" + uri)
+ self.sendCommand("GET", serverpath + "/" + uri)
self.sendHeader('host', backendServer.host)
if self.proxy.user:
@@ -252,13 +262,10 @@
def handleStatus(self, version, code, message):
__pychecker__ = 'unusednames=version,message'
log.debug('handleStatus %s - %s' % (code, message), 'http_client')
- self.status_code = int(code)
-
- # Keep a record of server response even if overriden later by setReponseCode
- self.http_status = self.status_code
+ self.http_status = int(code)
+
+ self.setResponseCode(self.http_status)
- self.setResponseCode(self.status_code)
-
def handleHeader(self, key, value):
log.debug("Received: " + key + " " + str(value), 'http_client')
@@ -266,7 +273,7 @@
if key == 'last-modified':
self.server_mtime = http.stringToDatetime(value)
- elsif key == 'content-length':
+ elif key == 'content-length':
self.server_size = http.stringToDatetime(value)
def handleEndHeaders(self):
@@ -275,16 +282,16 @@
self.parent.up_to_date()
def rawDataReceived(self, data):
- self.apDataReceived(data)
+ self.parent.data_received(data)
- def handleResponse(self, buffer):
- if self.length == 0:
- self.setResponseCode(http.NOT_FOUND)
- # print "length: " + str(self.length), "response:", self.status_code
- if self.http_status == http.NOT_MODIFIED:
- self.apDataEnd(self.transfered, False)
- else:
- self.apDataEnd(self.transfered, True)
+# def handleResponse(self, buffer):
+# if self.length == 0:
+# self.setResponseCode(http.NOT_FOUND)
+# # print "length: " + str(self.length), "response:", self.status_code
+# if self.http_status == http.NOT_MODIFIED:
+# self.apDataEnd(self.transfered, False)
+# else:
+# self.apDataEnd(self.transfered, True)
def lineReceived(self, line):
"""
@@ -310,12 +317,12 @@
def sendHeader(self, name, value):
"log and handle to the base class."
- log.debug(name + ":" + value,'http_client')
+ log.debug(name + " sendHeader:" + value,'http_client')
http.HTTPClient.sendHeader(self, name, value)
class HttpFetcher(protocol.ClientFactory):
"""
- A Fetcher that retrieves files via HTTP
+ A Fetcher factory that retrieves files via HTTP
"""
def __init__(self, backendServer):
self.backendServer = backendServer
@@ -326,10 +333,11 @@
if not self.proxy.host:
host = request.backendServer.host
port = request.backendServer.port
- else
+ else:
host = self.proxy.host
port = self.proxy.port
reactor.connectTCP(host, port, self, request.backend.config.timeout)
+ return self.connectCallback
def buildProtocol(self, addr):
return FetcherHttpClient(self)
@@ -369,22 +377,45 @@
NOTE: Twisted's FTPClient code uses it's own timeouts here and there,
so the timeout specified for the backend may not always be used
"""
- def activate (self, request):
- Fetcher.activate(self, request)
- if not request.apFetcher:
- return
+
+ def __init__(self, backendServer):
+ self.backendServer = backendServer
+ self.isConnected = False
- self.passive_ftp = self.request.backend.config.passive_ftp
-
- self.remote_file = (self.request.backendServer.path + "/"
- + self.request.backend_uri)
+ def connect(self):
+ """
+ Establish connection to ftp server specified by backendServer
+ """
+ self.connectCallback = deferred()
+ if not self.proxy.host:
+ host = self.backendServer.host
+ port = self.backendServer.port
+ else:
+ host = self.proxy.host
+ port = self.proxy.port
+ reactor.connectTCP(host, port, self, self.backendServer.config.timeout)
+ return self.connectCallback
- from twisted.internet.protocol import ClientCreator
+ def download(self, fetcher, uri, mtime):
+ """
+ Request download
+ %param fetcher: Fetcher class to receive callbacks
+ %param uri: URI of file to be downloaded within backend
+ %param mtime: Modification time of current file in cache
+ """
+ self.parent = fetcher
+ self.cache_mtime = mtime
+ self.request_uri = uri
+
+ self.passive_ftp = self.backendServer.config.passive_ftp
+
+ self.remote_file = (self.backendServer.path + "/"
+ + uri)
if not request.backendServer.username:
- creator = ClientCreator(reactor, ftp.FTPClient, passive=0)
+ creator = protocol.ClientCreator(reactor, ftp.FTPClient, passive=0)
else:
- creator = ClientCreator(reactor, ftp.FTPClient, request.backendServer.username,
+ creator = protocol.ClientCreator(reactor, ftp.FTPClient, request.backendServer.username,
request.backendServer.password, passive=0)
d = creator.connectTCP(request.backendServer.host, request.backendServer.port,
request.backend.config.timeout)
@@ -880,7 +911,7 @@
if self.fetcher.Backend != self.activeFile.Backend:
self.fetcher.closeConnection()
self.fetcher = Fetcher()
- else
+ else:
self.fetcher = Fetcher()
deferred = self.fetcher.start(activeFile)
deferred.addCallback(self.downloadComplete)
More information about the apt-proxy-devel
mailing list