r99 - in /debtorrent/branches/http-listen/DebTorrent: BT1/AptListener.py launchmanycore.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Wed Jun 13 03:05:22 UTC 2007
Author: camrdale-guest
Date: Wed Jun 13 03:05:21 2007
New Revision: 99
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=99
Log:
Getting package (.deb) files from torrents works.
Fixed info_page dispplay (though its boring).
Modified:
debtorrent/branches/http-listen/DebTorrent/BT1/AptListener.py
debtorrent/branches/http-listen/DebTorrent/launchmanycore.py
Modified: debtorrent/branches/http-listen/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/http-listen/DebTorrent/BT1/AptListener.py?rev=99&op=diff
==============================================================================
--- debtorrent/branches/http-listen/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/branches/http-listen/DebTorrent/BT1/AptListener.py Wed Jun 13 03:05:21 2007
@@ -7,6 +7,8 @@
@type alas: C{string}
@var alas: the message to send when the data is not found
+ at type VERSION: C{string}
+ at var VERSION: the Server identifier sent to all sites
"""
@@ -38,7 +40,8 @@
import signal
import re
import DebTorrent.__init__
-from DebTorrent.__init__ import version, createPeerID
+from DebTorrent.__init__ import version, createPeerID, product_name,version_short
+
try:
True
except:
@@ -47,6 +50,8 @@
bool = lambda x: not not x
DEBUG = True
+
+VERSION = product_name+'/'+version_short
def statefiletemplate(x):
"""Check the saved state file for corruption.
@@ -130,6 +135,8 @@
class AptListener:
"""Listen for Apt requests to download files.
+ @type handler: unknown
+ @ivar handler: the download handler to use
@type config: C{dictionary}
@ivar config: the configuration parameters
@type dfile: C{string}
@@ -179,19 +186,28 @@
@ivar uq_broken: unknown
@type Filter: unknown
@ivar Filter: unknown
+ @type request_queue: C{dictionary}
+ @ivar request_queue: the pending HTTP get requests that are waiting for download.
+ Keys are L{DebTorrent.HTTPHandler.HTTPConnection} objects, values are
+ (L{DebTorrent.download_bt1.BT1Download}, C{int}, C{list} of C{int}, C{float})
+ which are the torrent downloader, file index, list of pieces needed, and
+ the time of the original request.
"""
- def __init__(self, config, rawserver):
+ def __init__(self, handler, config, rawserver):
"""Initialize the instance.
+ @type handler: unknown
+ @param handler: the download handler to use
@type config: C{dictionary}
@param config: the configuration parameters
@type rawserver: L{DebTorrent.RawServer}
@param rawserver: the server to use for scheduling
"""
-
+
+ self.handler = handler
self.config = config
self.dfile = config['dfile']
favicon = config['favicon']
@@ -283,6 +299,53 @@
self.uq_broken = unquote('+') != ' '
self.Filter = Filter(rawserver.add_task)
+
+ self.request_queue = {}
+ rawserver.add_task(self.process_queue, 1)
+
+ def enqueue_request(self, connection, downloader, file_num, pieces_needed):
+ """Add a new download request to the queue of those waiting for pieces.
+
+ @type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
+ @param connection: the conection the request came in on
+ @type downloader: L{DebTorrent.download_bt1.BT1Download}
+ @param downloader: the torrent download that has the file
+ @type file_num: C{int}
+ @param file_num: the index of the file in the torrent
+ @type pieces_needed: C{list} of C{int}
+ @param pieces_needed: the list of pieces in the torrent that still
+ need to download
+
+ """
+
+ assert not self.request_queue.has_key(connection)
+
+ if DEBUG:
+ print 'queueing request as file', file_num, 'needs pieces:', pieces_needed
+
+ self.request_queue[connection] = (downloader, file_num, pieces_needed, clock())
+
+ def process_queue(self):
+ """Process the queue of waiting requests."""
+
+ # Schedule it again
+ self.rawserver.add_task(self.process_queue, 1)
+
+ for c, v in self.request_queue.items():
+ # Remove the downloaded pieces from the list of needed ones
+ for piece in list(v[2]):
+ if v[0].storagewrapper.do_I_have(piece):
+ if DEBUG:
+ print 'queued request for file', v[1], 'got piece', piece
+ v[2].remove(piece)
+
+ # If no more pieces are needed, return the answer and remove the request
+ if not v[2]:
+ if DEBUG:
+ print 'queued request for file', v[1], 'is complete'
+ del self.request_queue[c]
+ self.answer_package(c, v[0], v[1])
+
def get_infopage(self):
"""Format the info page to display for normal browsers.
@@ -297,10 +360,10 @@
try:
if not self.config['show_infopage']:
- return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+ return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
red = self.config['infopage_redirect']
if red:
- return (302, 'Found', {'Content-Type': 'text/html', 'Location': red},
+ return (302, 'Found', {'Server': VERSION, 'Content-Type': 'text/html', 'Location': red},
'<A HREF="'+red+'">Click Here</A>')
s = StringIO()
@@ -314,76 +377,80 @@
'<li><strong>client version:</strong> %s</li>\n' \
'<li><strong>client time:</strong> %s</li>\n' \
'</ul>\n' % (version, isotime()))
- if self.config['allowed_dir']:
- if self.show_names:
- names = [ (self.allowed[hash]['name'],hash)
- for hash in self.allowed.keys() ]
- else:
- names = [ (None,hash)
- for hash in self.allowed.keys() ]
- else:
- names = [ (None,hash) for hash in self.downloads.keys() ]
- if not names:
- s.write('<p>not downloading any files yet...</p>\n')
- else:
- names.sort()
- tn = 0
- tc = 0
- td = 0
- tt = 0 # Total transferred
- ts = 0 # Total size
- nf = 0 # Number of files displayed
- if self.config['allowed_dir'] and self.show_names:
- s.write('<table summary="files" border="1">\n' \
- '<tr><th>info hash</th><th>torrent name</th><th align="right">size</th><th align="right">complete</th><th align="right">downloading</th><th align="right">downloaded</th><th align="right">transferred</th></tr>\n')
- else:
- s.write('<table summary="files">\n' \
- '<tr><th>info hash</th><th align="right">complete</th><th align="right">downloading</th><th align="right">downloaded</th></tr>\n')
- for name,hash in names:
- l = self.downloads[hash]
- n = self.completed.get(hash, 0)
- tn = tn + n
- c = self.seedcount[hash]
- tc = tc + c
- d = len(l) - c
- td = td + d
- if self.config['allowed_dir'] and self.show_names:
- if self.allowed.has_key(hash):
- nf = nf + 1
- sz = self.allowed[hash]['length'] # size
- ts = ts + sz
- szt = sz * n # Transferred for this torrent
- tt = tt + szt
- if self.allow_get == 1:
- linkname = '<a href="/file?info_hash=' + quote(hash) + '">' + name + '</a>'
- else:
- linkname = name
- s.write('<tr><td><code>%s</code></td><td>%s</td><td align="right">%s</td><td align="right">%i</td><td align="right">%i</td><td align="right">%i</td><td align="right">%s</td></tr>\n' \
- % (b2a_hex(hash), linkname, size_format(sz), c, d, n, size_format(szt)))
- else:
- s.write('<tr><td><code>%s</code></td><td align="right"><code>%i</code></td><td align="right"><code>%i</code></td><td align="right"><code>%i</code></td></tr>\n' \
- % (b2a_hex(hash), c, d, n))
- if self.config['allowed_dir'] and self.show_names:
- s.write('<tr><td align="right" colspan="2">%i files</td><td align="right">%s</td><td align="right">%i</td><td align="right">%i</td><td align="right">%i</td><td align="right">%s</td></tr>\n'
- % (nf, size_format(ts), tc, td, tn, size_format(tt)))
- else:
- s.write('<tr><td align="right">%i files</td><td align="right">%i</td><td align="right">%i</td><td align="right">%i</td></tr>\n'
- % (nf, tc, td, tn))
- s.write('</table>\n' \
- '<ul>\n' \
- '<li><em>info hash:</em> SHA1 hash of the "info" section of the metainfo (*.dtorrent)</li>\n' \
- '<li><em>complete:</em> number of connected clients with the complete file</li>\n' \
- '<li><em>downloading:</em> number of connected clients still downloading</li>\n' \
- '<li><em>downloaded:</em> reported complete downloads</li>\n' \
- '<li><em>transferred:</em> torrent size * total downloaded (does not include partial transfers)</li>\n' \
- '</ul>\n')
+# if self.config['allowed_dir']:
+# if self.show_names:
+# names = [ (self.allowed[hash]['name'],hash)
+# for hash in self.allowed.keys() ]
+# else:
+# names = [ (None,hash)
+# for hash in self.allowed.keys() ]
+# else:
+# names = [ (None,hash) for hash in self.downloads.keys() ]
+# if not names:
+# s.write('<p>not downloading any files yet...</p>\n')
+# else:
+# names.sort()
+# tn = 0
+# tc = 0
+# td = 0
+# tt = 0 # Total transferred
+# ts = 0 # Total size
+# nf = 0 # Number of files displayed
+# if self.config['allowed_dir'] and self.show_names:
+# s.write('<table summary="files" border="1">\n' \
+# '<tr><th>info hash</th><th>torrent name</th><th align="right">size</th><th align="right">complete</th><th align="right">downloading</th><th align="right">downloaded</th><th align="right">transferred</th></tr>\n')
+# else:
+# s.write('<table summary="files">\n' \
+# '<tr><th>info hash</th><th align="right">complete</th><th align="right">downloading</th><th align="right">downloaded</th></tr>\n')
+# for name,hash in names:
+# l = self.downloads[hash]
+# n = self.completed.get(hash, 0)
+# tn = tn + n
+# c = self.seedcount[hash]
+# tc = tc + c
+# d = len(l) - c
+# td = td + d
+# if self.config['allowed_dir'] and self.show_names:
+# if self.allowed.has_key(hash):
+# nf = nf + 1
+# sz = self.allowed[hash]['length'] # size
+# ts = ts + sz
+# szt = sz * n # Transferred for this torrent
+# tt = tt + szt
+# if self.allow_get == 1:
+# linkname = '<a href="/file?info_hash=' + quote(hash) + '">' + name + '</a>'
+# else:
+# linkname = name
+# s.write('<tr><td><code>%s</code></td><td>%s</td><td align="right">%s</td><td align="right">%i</td><td align="right">%i</td><td align="right">%i</td><td align="right">%s</td></tr>\n' \
+# % (b2a_hex(hash), linkname, size_format(sz), c, d, n, size_format(szt)))
+# else:
+# s.write('<tr><td><code>%s</code></td><td align="right"><code>%i</code></td><td align="right"><code>%i</code></td><td align="right"><code>%i</code></td></tr>\n' \
+# % (b2a_hex(hash), c, d, n))
+# if self.config['allowed_dir'] and self.show_names:
+# s.write('<tr><td align="right" colspan="2">%i files</td><td align="right">%s</td><td align="right">%i</td><td align="right">%i</td><td align="right">%i</td><td align="right">%s</td></tr>\n'
+# % (nf, size_format(ts), tc, td, tn, size_format(tt)))
+# else:
+# s.write('<tr><td align="right">%i files</td><td align="right">%i</td><td align="right">%i</td><td align="right">%i</td></tr>\n'
+# % (nf, tc, td, tn))
+# s.write('</table>\n' \
+# '<ul>\n' \
+# '<li><em>info hash:</em> SHA1 hash of the "info" section of the metainfo (*.dtorrent)</li>\n' \
+# '<li><em>complete:</em> number of connected clients with the complete file</li>\n' \
+# '<li><em>downloading:</em> number of connected clients still downloading</li>\n' \
+# '<li><em>downloaded:</em> reported complete downloads</li>\n' \
+# '<li><em>transferred:</em> torrent size * total downloaded (does not include partial transfers)</li>\n' \
+# '</ul>\n')
s.write('</body>\n' \
'</html>\n')
- return (200, 'OK', {'Content-Type': 'text/html; charset=iso-8859-1'}, s.getvalue())
+ return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, s.getvalue())
except:
print_exc()
- return (500, 'Internal Server Error', {'Content-Type': 'text/html; charset=iso-8859-1'}, 'Server Error')
+ return (500, 'Internal Server Error', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, 'Server Error')
+
+
+ def get_meow(self):
+ return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/html; charset=iso-8859-1'}, """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">\n<html><head><title>Meow</title>\n</head>\n<body style="color: rgb(255, 255, 255); background-color: rgb(0, 0, 0);">\n<div><big style="font-weight: bold;"><big><big><span style="font-family: arial,helvetica,sans-serif;">I IZ TAKIN BRAKE</span></big></big></big><br></div>\n<pre><b><tt> .-o=o-.<br> , /=o=o=o=\ .--.<br> _|\|=o=O=o=O=| \<br> __.' a`\=o=o=o=(`\ /<br> '. a 4/`|.-""'`\ \ ;'`) .---.<br> \ .' / .--' |_.' / .-._)<br> `) _.' / /`-.__.' /<br> `'-.____; /'-.___.-'<br> `\"""`</tt></b></pre>\n<div><big style="font-weight: bold;"><big><big><span style="font-family: arial,helvetica,sans-serif;">FRM GETIN UR PACKAGES</span></big></big></big><br></div>\n</body>\n</html>""")
def get_file(self, path):
@@ -417,38 +484,91 @@
except:
status = 404
msg = 'Unknown error occurred'
- return (status, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, msg)
-
- def get_package(self, path):
+ return (status, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, msg)
+
+ def get_package(self, connection, path):
"""Download a package file from a torrent.
+ @type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
+ @param connection: the conection the request came in on
@type path: C{list} of C{string}
@param path: the path of the file to download, starting with the mirror name
@rtype: (C{int}, C{string}, C{dictionary}, C{string})
- @return: the HTTP status code, status message, headers, and bencoded
- metainfo file
+ @return: the HTTP status code, status message, headers, and package data
+ (or None if the package is to be downloaded)
"""
-
- d, f = self.handler.find_file(path[1:])
+
+ # Find the file in one of the torrent downloads
+ d, f = self.handler.find_file(path[0], path[1:])
if d is None:
- return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
-
- # TODO: check if the torrent is running/not paused
- d.fileselector.storage.enable_file(f)
-
- # TODO: make this threaded using rawserver
+ return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+
+ # Check if the file has already been downloaded
data = ''
+ pieces_needed = []
start_piece, end_piece = d.fileselector.storage.file_pieces[f]
for piece in xrange(start_piece, end_piece+1):
- while not d.storagewrapper.do_I_have(piece):
- sleep(2)
- data.append(d.storagewrapper.get_piece(piece, 0, -1).getarray())
-
- # TODO: check for waiting too long
- # TODO: add headers here
- return (200, 'OK', {}, data)
+ if not d.storagewrapper.do_I_have(piece):
+ pieces_needed.append(piece)
+ elif not pieces_needed:
+ data = data + d.storagewrapper.get_piece(piece, 0, -1).getarray().tostring()
+
+ if not pieces_needed:
+ return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/plain'}, data)
+
+ # Check if the torrent is running/not paused
+ if d.doneflag.isSet():
+ return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+
+ if not d.unpauseflag.isSet():
+ d.Unpause()
+
+ # Enable the download of the piece
+ d.fileselector.set_priority(f, 1)
+
+ # Add the connection to the list of those needing responses
+ self.enqueue_request(connection, d, f, pieces_needed)
+
+ return None
+
+
+ def answer_package(self, connection, d, f):
+ """Send the newly downloaded package file to the requester.
+
+ @type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
+ @param connection: the conection the request came in on
+ @type d: L{DebTorrent.download_bt1.BT1Download}
+ @param d: the torrent download that has the file
+ @type f: C{int}
+ @param f: the index of the file in the torrent
+
+ """
+
+ # Check to make sure the requester is still waiting
+ if connection.closed:
+ return
+
+ # Check if the file has been downloaded
+ data = ''
+ pieces_needed = []
+ start_piece, end_piece = d.fileselector.storage.file_pieces[f]
+ for piece in xrange(start_piece, end_piece+1):
+ if not d.storagewrapper.do_I_have(piece):
+ pieces_needed.append(piece)
+ elif not pieces_needed:
+ data = data + d.storagewrapper.get_piece(piece, 0, -1).getarray().tostring()
+
+ if not pieces_needed:
+ connection.answer((200, 'OK', {'Server': VERSION, 'Content-Type': 'text/plain'}, data))
+ return
+
+ # Something strange has happened, requeue it
+ if DEBUG:
+ print 'request for', f, 'still needs pieces:', pieces_needed
+ self.enqueue_request(connection, d, f, pieces_needed)
+
def get(self, connection, path, headers):
"""Respond to a GET request.
@@ -457,7 +577,7 @@
calling the helper functions above if needed. Return the response to
be returned to the requester.
- @type connection: unknown
+ @type connection: L{DebTorrent.HTTPHandler.HTTPConnection}
@param connection: the conection the request came in on
@type path: C{string}
@param path: the URL being requested
@@ -482,7 +602,7 @@
if ( (self.allowed_IPs and not self.allowed_IPs.includes(ip))
or (self.banned_IPs and self.banned_IPs.includes(ip)) ):
- return (400, 'Not Authorized', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'},
+ return (400, 'Not Authorized', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'},
bencode({'failure reason':
'your IP is not allowed on this proxy'}))
@@ -521,28 +641,30 @@
if path == '' or path == 'index.html':
return self.get_infopage()
+ if path == 'meow':
+ return self.get_meow()
if path == 'favicon.ico':
if self.favicon is not None:
- return (200, 'OK', {'Content-Type' : 'image/x-icon'}, self.favicon)
+ return (200, 'OK', {'Server': VERSION, 'Content-Type' : 'image/x-icon'}, self.favicon)
else:
- return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+ return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
# Process the rest as a proxy
path = path.split('/')
if 'Packages.diff' in path:
- return (404, 'Not Found', {'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+ return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
# if path[-1] in ('Packages', 'Packages.gz', 'Packages.bz2'):
# return self.get_Packages(path)
if path[-1][-4:] == '.deb':
- return self.get_package(path)
+ return self.get_package(connection, path)
return self.get_file(path)
except ValueError, e:
- return (400, 'Bad Request', {'Content-Type': 'text/plain'},
+ return (400, 'Bad Request', {'Server': VERSION, 'Content-Type': 'text/plain'},
'you sent me garbage - ' + str(e))
Modified: debtorrent/branches/http-listen/DebTorrent/launchmanycore.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/http-listen/DebTorrent/launchmanycore.py?rev=99&op=diff
==============================================================================
--- debtorrent/branches/http-listen/DebTorrent/launchmanycore.py (original)
+++ debtorrent/branches/http-listen/DebTorrent/launchmanycore.py Wed Jun 13 03:05:21 2007
@@ -372,7 +372,7 @@
self.failed("Couldn't listen - " + str(e))
return
- self.aptlistener = AptListener(config, self.rawserver)
+ self.aptlistener = AptListener(self, config, self.rawserver)
self.rawserver.bind(config['port'], config['bind'],
reuse = True, ipv6_socket_style = config['ipv6_binds_v4'])
self.rawserver.set_handler(HTTPHandler(self.aptlistener.get,
@@ -570,12 +570,14 @@
return saveas
- def find_file(self, path):
+ def find_file(self, mirror, path):
"""Find which running torrent has the file.
Checks the metainfo of each torrent in the cache to find one that
has a file whose 'path' matches the given file's path.
+ @type mirror: C{string}
+ @param mirror: mirror name to find the download in
@type path: C{list} of C{string}
@param path: the path of the file to find
@rtype: L{download_bt1.BT1Download}, C{int}
@@ -587,14 +589,24 @@
file = '/'.join(path)
if DEBUG:
print 'Trying to find file:', file
+
+ # Check each torrent in the cache
for hash, data in self.torrent_cache.items():
+ # Make sure this torrent is from the mirror in question
+ # (TODO: later make this more certain by not prepending 'dt_' to the name)
+ if data['metainfo']['name'].find(mirror) == -1:
+ continue
+
file_num = -1
for f in data['metainfo']['info']['files']:
file_num += 1
- if '/'.join(f['path']) == file:
+
+ # Check that the file ends with the desired file name (TODO: security risk?)
+ if file.endswith('/'.join(f['path'])):
if DEBUG:
print 'Found file in:', binascii.b2a_hex(hash)
- return self.downloads[hash], file_num
+ return self.downloads[hash].d, file_num
+
if DEBUG:
print 'Failed to find file.'
return None, None
More information about the Debtorrent-commits
mailing list