r108 - in /debtorrent/trunk: DebTorrent/BT1/AptListener.py DebTorrent/BT1/makemetafile.py DebTorrent/download_bt1.py TODO
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Thu Jun 14 17:47:31 UTC 2007
Author: camrdale-guest
Date: Thu Jun 14 17:47:31 2007
New Revision: 108
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=108
Log:
AptListener uses the getpieces functions from makemetafile.
Modified:
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/TODO
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=108&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Thu Jun 14 17:47:31 2007
@@ -37,9 +37,7 @@
from types import StringType, IntType, LongType, ListType, DictType
from binascii import b2a_hex, a2b_hex, a2b_base64
from string import lower
-from time import sleep
-from makemetafile import uniconvertl, uniconvert
-from os.path import split
+from makemetafile import getpieces, getsubpieces, uniconvert
import sys, os
import signal
import re
@@ -618,35 +616,13 @@
print 'ERROR: Packages file could not be converted to a torrent'
return r
- # TODO: use routines from BT1.makemetafile instead
- pieces = []
- lengths = []
- fs = []
-
- p = [None, None, None]
- for line in h:
- line = line.rstrip()
- if line == "":
- if (p[0] and p[1] and p[2]):
- fpath = []
- while p[1]:
- p[1],d = split(p[1])
- fpath.insert(0,d)
- fs.append({'length': p[0], 'path': fpath})
- lengths.append(p[0])
- pieces.append(p[2])
- p = [None, None, None]
- if line[:9] == "Filename:":
- p[1] = line[10:]
- if line[:5] == "Size:":
- p[0] = long(line[6:])
- if line[:5] == "SHA1:":
- p[2] = a2b_hex(line[6:])
-
- response = {'info': {'pieces': ''.join(pieces),
- 'piece lengths': lengths, 'files': fs },
- 'announce': 'http://dttracker.debian.net:6969/announce',
- 'name': name }
+ sub_pieces = getsubpieces(name)
+
+ info = getpieces(h, sub_pieces = sub_pieces)
+
+ response = {'info': info,
+ 'announce': 'http://dttracker.debian.net:6969/announce',
+ 'name': uniconvert(name)}
if path.count('dists'):
mirror = 'http://' + '/'.join(path[:path.index('dists')]) + '/'
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=108&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Thu Jun 14 17:47:31 2007
@@ -75,7 +75,12 @@
raise UnicodeError('bad filename: '+join(l))
return r
-def uniconvert(s, e):
+def uniconvert(s, e = None):
+ if not e:
+ e = ENCODING
+ if not e:
+ e = 'ascii'
+
try:
s = unicode(s,e)
except UnicodeError:
@@ -117,10 +122,6 @@
encoding = None
if params.has_key('filesystem_encoding'):
encoding = params['filesystem_encoding']
- if not encoding:
- encoding = ENCODING
- if not encoding:
- encoding = 'ascii'
info = makeinfo(file, piece_length, encoding, progress, params['pieces_file'])
@@ -160,7 +161,7 @@
def getsubpieces(file, pieces_file = ''):
pieces = {}
packages = 0
-
+
if pieces_file:
try:
f = open(pieces_file)
@@ -183,6 +184,10 @@
pass
piece_url = 'http://merkel.debian.org/~ajt/extrapieces/dists_'
piece_url += '_'.join(parts[parts.index('dists')+1:])
+ if piece_url.endswith('.gz'):
+ piece_url = piece_url[:-3]
+ if piece_url.endswith('.bz2'):
+ piece_url = piece_url[:-4]
piece_url += '-extrapieces.gz'
piece_file = urlopen(piece_url)
piece_data = piece_file.read()
@@ -224,7 +229,13 @@
return pieces
-def getpieces(f, encoding, progress = lambda x: None, sub_pieces = {}):
+def getpieces(f, encoding = None, progress = lambda x: None, sub_pieces = {}):
+
+ if not encoding:
+ encoding = ENCODING
+ if not encoding:
+ encoding = 'ascii'
+
pieces = []
lengths = []
fs = []
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=108&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Thu Jun 14 17:47:31 2007
@@ -52,12 +52,6 @@
from gzip import GzipFile
from StringIO import StringIO
import binascii
-try:
- from sys import getfilesystemencoding
- ENCODING = getfilesystemencoding()
-except:
- from sys import getdefaultencoding
- ENCODING = getdefaultencoding()
try:
True
@@ -466,14 +460,6 @@
"""
- encoding = None
-# if params.has_key('filesystem_encoding'):
-# encoding = params['filesystem_encoding']
- if not encoding:
- encoding = ENCODING
- if not encoding:
- encoding = 'ascii'
-
try:
if file:
name = "dt_" + split(file)[1]
@@ -516,11 +502,11 @@
sub_pieces = getsubpieces(name)
- info = getpieces(h, encoding, sub_pieces = sub_pieces)
+ info = getpieces(h, sub_pieces = sub_pieces)
response = {'info': info,
'announce': 'http://dttracker.debian.net:6969/announce',
- 'name': uniconvert(name, encoding)}
+ 'name': uniconvert(name)}
except IOError, e:
# errorfunc('problem getting Packages info - ' + str(e))
Modified: debtorrent/trunk/TODO
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/TODO?rev=108&op=diff
==============================================================================
--- debtorrent/trunk/TODO (original)
+++ debtorrent/trunk/TODO Thu Jun 14 17:47:31 2007
@@ -1,4 +1,29 @@
Below are some things that still need to be done, in order of priority.
+
+
+AptListener queue waiting for downloads should be implemented using callbacks
+
+AptListener currently queues requests for packages that have not completed
+downloading, and then checks this queue every 1 second. This could be made
+more efficient by adding callbacks to PiecePicker or StorageWrapper, so that
+when a piece comes in and passes the hash check, then the AptListener will
+process any queued requests for that piece.
+
+
+HTTPHandler should support HTTP/1.1 and persistent connections/pipelining
+
+Currently HTTPHandler is HTTP/1.0, and so doesn't support persistent
+connections. These would be useful as APT could then pipeline multiple requests
+at a time to DebTorrent for processing. This would require something like the
+AptListener callbacks, as the connections would then have to support multiple
+queued package requests.
+
+
+Change the DEBUG printing to use debug levels
+
+Instead of a per-module DEBUG setting, implement a program-wide debug level,
+set by a configuration option. Each level would print differing amounts of
+debug information to the output/log.
Make the URL download better
More information about the Debtorrent-commits
mailing list