[Debtorrent-commits] r92 - in /debtorrent/branches/hippy/DebTorrent: BT1/makemetafile.py download_bt1.py zurllib.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Tue Jun 5 03:24:32 UTC 2007
Author: camrdale-guest
Date: Tue Jun 5 03:24:32 2007
New Revision: 92
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=92
Log:
Move all Packages file parsing to makemetafile.py.
Modify download_bt1 to call the makemetafile.py functions.
Modify download_bt1 to check merkel.d.o for extrapieces files.
Modified:
debtorrent/branches/hippy/DebTorrent/BT1/makemetafile.py
debtorrent/branches/hippy/DebTorrent/download_bt1.py
debtorrent/branches/hippy/DebTorrent/zurllib.py
Modified: debtorrent/branches/hippy/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/hippy/DebTorrent/BT1/makemetafile.py?rev=92&op=diff
==============================================================================
--- debtorrent/branches/hippy/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/branches/hippy/DebTorrent/BT1/makemetafile.py Tue Jun 5 03:24:32 2007
@@ -122,18 +122,50 @@
if not encoding:
encoding = 'ascii'
- sub_pieces = {}
- if params.has_key('pieces_file') and params['pieces_file']:
+ info = makeinfo(file, piece_length, encoding, progress, params['pieces_file'])
+
+ check_info(info)
+ h = open(f, 'wb')
+ data = {'info': info, 'announce': strip(url),
+ 'name': uniconvert("dt_" + split(file)[1], encoding),
+ 'creation date': long(time())}
+
+ if params.has_key('comment') and params['comment']:
+ data['comment'] = params['comment']
+
+ if params.has_key('real_announce_list'): # shortcut for progs calling in from outside
+ data['announce-list'] = params['real_announce_list']
+ elif params.has_key('announce_list') and params['announce_list']:
+ l = []
+ for tier in params['announce_list'].split('|'):
+ l.append(tier.split(','))
+ data['announce-list'] = l
+
+ if params.has_key('real_httpseeds'): # shortcut for progs calling in from outside
+ data['httpseeds'] = params['real_httpseeds']
+ elif params.has_key('httpseeds') and params['httpseeds']:
+ data['httpseeds'] = params['httpseeds'].split('|')
+
+ h.write(bencode(data))
+ h.close()
+
+def calcsize(file):
+ if not isdir(file):
+ return getsize(file)
+ total = 0L
+ for s in subfiles(abspath(file)):
+ total += getsize(s[1])
+ return total
+
+def getsubpieces(file, pieces_file = ''):
+ pieces = {}
+ packages = 0
+
+ if pieces_file:
try:
- h = open(params['pieces_file'])
- sub_pieces = getsubpieces(h)
+ f = open(pieces_file)
except:
- print 'The specified pieces file could not be read, it will be ignored'
- sub_pieces = {}
- try:
- h.close()
- except:
- pass
+ return {}
elif 'dists' in file.split('_'):
try:
parts = file.split('_')
@@ -158,57 +190,10 @@
piece_file.close()
except:
pass
- if url[-3:] == ".gz":
- compressed = StringIO(piece_data)
- piece_file = GzipFile(fileobj = compressed)
- piece_data = piece_file.read()
- h = piece_data.split('\n')
- sub_pieces = getsubpieces(h)
+ f = piece_data.split('\n')
except:
- sub_pieces = {}
-
- if not sub_pieces:
- print 'WARNING: Pieces file could not be found, not using sub-package pieces.'
-
- info = makeinfo(file, piece_length, encoding, progress, sub_pieces)
-
- check_info(info)
- h = open(f, 'wb')
- data = {'info': info, 'announce': strip(url),
- 'name': uniconvert("dt_" + split(file)[1], encoding),
- 'creation date': long(time())}
-
- if params.has_key('comment') and params['comment']:
- data['comment'] = params['comment']
-
- if params.has_key('real_announce_list'): # shortcut for progs calling in from outside
- data['announce-list'] = params['real_announce_list']
- elif params.has_key('announce_list') and params['announce_list']:
- l = []
- for tier in params['announce_list'].split('|'):
- l.append(tier.split(','))
- data['announce-list'] = l
-
- if params.has_key('real_httpseeds'): # shortcut for progs calling in from outside
- data['httpseeds'] = params['real_httpseeds']
- elif params.has_key('httpseeds') and params['httpseeds']:
- data['httpseeds'] = params['httpseeds'].split('|')
-
- h.write(bencode(data))
- h.close()
-
-def calcsize(file):
- if not isdir(file):
- return getsize(file)
- total = 0L
- for s in subfiles(abspath(file)):
- total += getsize(s[1])
- return total
-
-def getsubpieces(f):
- pieces = {}
- packages = 0
-
+ return {}
+
p = [None, [], []]
read_data = False
for line in f:
@@ -229,17 +214,27 @@
if read_data == True and line[:1] == " ":
p[1].append(binascii.a2b_hex(line[1:41]))
p[2].append(int(line[42:]))
+
+ try:
+ f.close()
+ except:
+ pass
+
return pieces
-def makeinfo(file, piece_length, encoding, progress, sub_pieces = {}):
- file = abspath(file)
+def getpieces(f, encoding, progress = lambda x: None, sub_pieces = {},
+ installed_versions = {}, status_to_download = 0):
pieces = []
lengths = []
fs = []
packages = 0
- p = [None, None, None]
- f = open(file)
+ if installed_versions:
+ priority = {}
+ else:
+ priority = None
+
+ p = [None, None, None, None, None]
for line in f:
line = line.rstrip()
if line == "":
@@ -251,6 +246,13 @@
lengths.append(p[0])
pieces.append(p[2])
+ if (p[3] and installed_versions.has_key(p[3])):
+ if status_to_download == 2:
+ priority[p[1]] = 1
+ elif (status_to_download == 1 and p[4]):
+ ret = system('dpkg --compare-versions "' + installed_versions[p[3]] + '" lt "' + p[4] + '"')
+ if ret == 0:
+ priority[p[1]] = 1
path = []
while p[1]:
p[1],d = split(p[1])
@@ -258,17 +260,34 @@
fs.append({'length': p[0], 'path': uniconvertl(path, encoding)})
packages += 1
progress(packages)
- p = [None, None, None]
+ p = [None, None, None, None, None]
if line[:9] == "Filename:":
p[1] = line[10:]
if line[:5] == "Size:":
p[0] = long(line[6:])
if line[:5] == "SHA1:":
p[2] = binascii.a2b_hex(line[6:])
+ if line[:8] == "Package:":
+ p[3] = line[9:]
+ if line[:8] == "Version:":
+ p[4] = line[9:]
+
+ return ({'pieces': ''.join(pieces), 'piece lengths': lengths, 'files': fs},
+ priority)
+
+def makeinfo(file, piece_length, encoding, progress, pieces_file = ''):
+
+ sub_pieces = getsubpieces(file, pieces_file)
+
+ if not sub_pieces:
+ print 'WARNING: Pieces file could not be found, not using sub-package pieces.'
+
+ file = abspath(file)
+ f = open(file)
+ info = getpieces(f, encoding, progress, sub_pieces = sub_pieces)
f.close()
- return {'pieces': ''.join(pieces),
- 'piece lengths': lengths, 'files': fs}
+ return info
def subfiles(d):
r = []
Modified: debtorrent/branches/hippy/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/hippy/DebTorrent/download_bt1.py?rev=92&op=diff
==============================================================================
--- debtorrent/branches/hippy/DebTorrent/download_bt1.py (original)
+++ debtorrent/branches/hippy/DebTorrent/download_bt1.py Tue Jun 5 03:24:32 2007
@@ -47,7 +47,7 @@
from clock import clock
from BTcrypto import CRYPTO_OK
from __init__ import createPeerID
-from BT1.makemetafile import uniconvertl, uniconvert
+from BT1.makemetafile import getpieces, getsubpieces, uniconvert
from os.path import split
from gzip import GzipFile
from StringIO import StringIO
@@ -463,9 +463,7 @@
if not encoding:
encoding = 'ascii'
- get_priority = False
installed_versions = {}
- priority = None
if status_to_download > 0:
try:
status = open("/var/lib/dpkg/status")
@@ -491,10 +489,6 @@
status.close()
except:
pass
-
- if installed_versions:
- get_priority = True
- priority = {}
try:
if file:
@@ -536,46 +530,14 @@
# errorfunc(url+' bad url')
return (None, None)
- pieces = []
- lengths = []
- fs = []
-
- p = [None, None, None, None, None]
- for line in h:
- line = line.rstrip()
- if line == "":
- if (p[0] and p[1] and p[2]):
- file_path = []
- temp = p[1]
- while p[1]:
- p[1],d = split(p[1])
- file_path.insert(0,d)
- fs.append({'length': p[0], 'path': uniconvertl(file_path, encoding)})
- lengths.append(p[0])
- pieces.append(p[2])
- if (get_priority and p[3] and installed_versions.has_key(p[3])):
- if status_to_download == 2:
- priority[temp] = 1
- elif (status_to_download == 1 and p[4]):
- ret = system('dpkg --compare-versions "' + installed_versions[p[3]] + '" lt "' + p[4] + '"')
- if ret == 0:
- priority[temp] = 1
- p = [None, None, None, None, None]
- if line[:9] == "Filename:":
- p[1] = line[10:]
- if line[:5] == "Size:":
- p[0] = long(line[6:])
- if line[:5] == "SHA1:":
- p[2] = binascii.a2b_hex(line[6:])
- if line[:8] == "Package:":
- p[3] = line[9:]
- if line[:8] == "Version:":
- p[4] = line[9:]
-
- response = {'info': {'pieces': ''.join(pieces),
- 'piece lengths': lengths, 'files': fs },
- 'announce': 'http://dttracker.debian.net:6969/announce',
- 'name': uniconvert(name, encoding) }
+ sub_pieces = getsubpieces(name)
+
+ (info, priority) = getpieces(h, encoding, sub_pieces = sub_pieces,
+ installed_versions = installed_versions,
+ status_to_download = status_to_download)
+ response = {'info': info,
+ 'announce': 'http://dttracker.debian.net:6969/announce',
+ 'name': uniconvert(name, encoding)}
except IOError, e:
# errorfunc('problem getting Packages info - ' + str(e))
Modified: debtorrent/branches/hippy/DebTorrent/zurllib.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/hippy/DebTorrent/zurllib.py?rev=92&op=diff
==============================================================================
--- debtorrent/branches/hippy/DebTorrent/zurllib.py (original)
+++ debtorrent/branches/hippy/DebTorrent/zurllib.py Tue Jun 5 03:24:32 2007
@@ -34,10 +34,11 @@
pass
class urlopen:
- def __init__(self, url):
+ def __init__(self, url, ungzip = True):
self.tries = 0
self._open(url.strip())
self.error_return = None
+ self.ungzip = ungzip
def _open(self, url):
self.tries += 1
@@ -90,7 +91,8 @@
def _read(self):
data = self.response.read()
- if self.response.getheader('Content-Encoding','').find('gzip') >= 0:
+ if self.ungzip and (self.response.getheader('Content-Type','').find('gzip') >= 0 or
+ self.response.getheader('Content-Encoding','').find('gzip') >= 0):
try:
compressed = StringIO(data)
f = GzipFile(fileobj = compressed)
More information about the Debtorrent-commits
mailing list