r104 - in /debtorrent/trunk: ./ DebTorrent/BT1/makemetafile.py DebTorrent/download_bt1.py DebTorrent/zurllib.py btmakemetafile.py hippy.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Wed Jun 13 21:51:09 UTC 2007
Author: camrdale-guest
Date: Wed Jun 13 21:51:09 2007
New Revision: 104
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=104
Log:
Merged revisions 88-103 via svnmerge from
svn+ssh://camrdale-guest@svn.debian.org/svn/debtorrent/debtorrent/branches/hippy
........
r89 | camrdale-guest | 2007-06-04 16:30:19 -0700 (Mon, 04 Jun 2007) | 1 line
Add option to btmakemetafile to use another file for sub-package pieces.
........
r90 | camrdale-guest | 2007-06-04 17:31:03 -0700 (Mon, 04 Jun 2007) | 2 lines
Make btmakemetafile check merkel.d.o for extrapieces files. (Warning, release names hardcoded.)
........
r91 | camrdale-guest | 2007-06-04 18:21:10 -0700 (Mon, 04 Jun 2007) | 1 line
Remove some no longer used parameters.
........
r92 | camrdale-guest | 2007-06-04 20:24:32 -0700 (Mon, 04 Jun 2007) | 3 lines
Move all Packages file parsing to makemetafile.py.
Modify download_bt1 to call the makemetafile.py functions.
Modify download_bt1 to check merkel.d.o for extrapieces files.
........
r93 | camrdale-guest | 2007-06-08 15:35:44 -0700 (Fri, 08 Jun 2007) | 1 line
Added hippy file to generate -extrapieces files.
........
r94 | camrdale-guest | 2007-06-08 17:58:08 -0700 (Fri, 08 Jun 2007) | 1 line
Minor fix as btmakemetafile wasn't working.
........
Added:
debtorrent/trunk/hippy.py
- copied unchanged from r94, debtorrent/branches/hippy/hippy.py
Modified:
debtorrent/trunk/ (props changed)
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/DebTorrent/zurllib.py
debtorrent/trunk/btmakemetafile.py
Propchange: debtorrent/trunk/
------------------------------------------------------------------------------
--- svnmerge-integrated (original)
+++ svnmerge-integrated Wed Jun 13 21:51:09 2007
@@ -1,1 +1,1 @@
-/debtorrent/branches/hippy:1-87 /debtorrent/branches/http-listen:1-102
+/debtorrent/branches/hippy:1-103 /debtorrent/branches/http-listen:1-102
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=104&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Wed Jun 13 21:51:09 2007
@@ -15,6 +15,9 @@
from threading import Event
from time import time
from traceback import print_exc
+from DebTorrent.zurllib import urlopen
+from gzip import GzipFile
+from StringIO import StringIO
import binascii
try:
from sys import getfilesystemencoding
@@ -36,7 +39,8 @@
"optional specification for filesystem encoding " +
"(set automatically in recent Python versions)"),
('target', '',
- "optional target file for the torrent")
+ "optional target file for the torrent"),
+ ('pieces_file', '', 'the file that contains the sub-package piece information'),
]
default_piece_len_exp = 18
@@ -78,8 +82,7 @@
raise UnicodeError('bad filename: '+s)
return s.encode('utf-8')
-def make_meta_file(file, url, params = {}, flag = Event(),
- progress = lambda x: None, progress_percent = 1):
+def make_meta_file(file, url, params = {}, progress = lambda x: None):
if params.has_key('piece_size_pow2'):
piece_len_exp = params['piece_size_pow2']
else:
@@ -118,10 +121,9 @@
encoding = ENCODING
if not encoding:
encoding = 'ascii'
-
- info = makeinfo(file, piece_length, encoding, flag, progress, progress_percent)
- if flag.isSet():
- return
+
+ info = makeinfo(file, piece_length, encoding, progress, params['pieces_file'])
+
check_info(info)
h = open(f, 'wb')
data = {'info': info, 'announce': strip(url),
@@ -155,39 +157,137 @@
total += getsize(s[1])
return total
-def makeinfo(file, piece_length, encoding, flag, progress, progress_percent=1):
- file = abspath(file)
+def getsubpieces(file, pieces_file = ''):
+ pieces = {}
+ packages = 0
+
+ if pieces_file:
+ try:
+ f = open(pieces_file)
+ except:
+ return {}
+ elif 'dists' in file.split('_'):
+ try:
+ parts = file.split('_')
+ try:
+ parts[parts.index('stable', parts.index('dists'))] = 'etch'
+ except:
+ pass
+ try:
+ parts[parts.index('testing', parts.index('dists'))] = 'lenny'
+ except:
+ pass
+ try:
+ parts[parts.index('unstable', parts.index('dists'))] = 'sid'
+ except:
+ pass
+ piece_url = 'http://merkel.debian.org/~ajt/extrapieces/dists_'
+ piece_url += '_'.join(parts[parts.index('dists')+1:])
+ piece_url += '-extrapieces.gz'
+ piece_file = urlopen(piece_url)
+ piece_data = piece_file.read()
+ try:
+ piece_file.close()
+ except:
+ pass
+ f = piece_data.split('\n')
+ except:
+ return {}
+
+ p = [None, [], []]
+ read_data = False
+ for line in f:
+ line = line.rstrip()
+ if line == "":
+ if (p[0] and p[1] and p[2]):
+ pieces[p[0]] = (p[1], p[2])
+ packages += 1
+ #progress(packages)
+ p = [None, [], []]
+ read_data = False
+ if read_data == True and line[:1] != " ":
+ read_data = False
+ if line[:9] == "Filename:":
+ p[0] = line[10:]
+ if line == "SHA1-Pieces:":
+ read_data = True
+ if read_data == True and line[:1] == " ":
+ p[1].append(binascii.a2b_hex(line[1:41]))
+ p[2].append(int(line[42:]))
+
+ try:
+ f.close()
+ except:
+ pass
+
+ return pieces
+
+def getpieces(f, encoding, progress = lambda x: None, sub_pieces = {},
+ installed_versions = {}, status_to_download = 0):
pieces = []
lengths = []
fs = []
packages = 0
- p = [None, None, None]
- f = open(file)
+ if installed_versions:
+ priority = {}
+ else:
+ priority = None
+
+ p = [None, None, None, None, None]
for line in f:
line = line.rstrip()
if line == "":
if (p[0] and p[1] and p[2]):
+ if sub_pieces.has_key(p[1]):
+ lengths.extend(sub_pieces[p[1]][1])
+ pieces.extend(sub_pieces[p[1]][0])
+ else:
+ lengths.append(p[0])
+ pieces.append(p[2])
+
+ if (p[3] and installed_versions.has_key(p[3])):
+ if status_to_download == 2:
+ priority[p[1]] = 1
+ elif (status_to_download == 1 and p[4]):
+ ret = system('dpkg --compare-versions "' + installed_versions[p[3]] + '" lt "' + p[4] + '"')
+ if ret == 0:
+ priority[p[1]] = 1
path = []
while p[1]:
p[1],d = split(p[1])
path.insert(0,d)
fs.append({'length': p[0], 'path': uniconvertl(path, encoding)})
- lengths.append(p[0])
- pieces.append(p[2])
packages += 1
progress(packages)
- p = [None, None, None]
+ p = [None, None, None, None, None]
if line[:9] == "Filename:":
p[1] = line[10:]
if line[:5] == "Size:":
p[0] = long(line[6:])
if line[:5] == "SHA1:":
p[2] = binascii.a2b_hex(line[6:])
+ if line[:8] == "Package:":
+ p[3] = line[9:]
+ if line[:8] == "Version:":
+ p[4] = line[9:]
+
+ return ({'pieces': ''.join(pieces), 'piece lengths': lengths, 'files': fs},
+ priority)
+
+def makeinfo(file, piece_length, encoding, progress, pieces_file = ''):
+
+ sub_pieces = getsubpieces(file, pieces_file)
+
+ if not sub_pieces:
+ print 'WARNING: Pieces file could not be found, not using sub-package pieces.'
+
+ file = abspath(file)
+ f = open(file)
+ (info, temp) = getpieces(f, encoding, progress, sub_pieces = sub_pieces)
f.close()
- return {'pieces': ''.join(pieces),
- 'piece lengths': lengths, 'files': fs}
+ return info
def subfiles(d):
r = []
@@ -203,8 +303,7 @@
return r
-def completedir(dir, url, params = {}, flag = Event(),
- vc = lambda x: None, fc = lambda x: None):
+def completedir(dir, url, params = {}, vc = lambda x: None, fc = lambda x: None):
files = listdir(dir)
files.sort()
ext = '.dtorrent'
@@ -225,6 +324,6 @@
if t not in ignore and t[0] != '.':
if target != '':
params['target'] = join(target,t+ext)
- make_meta_file(i, url, params, flag, progress = vc, progress_percent = 0)
+ make_meta_file(i, url, params, progress = vc)
except ValueError:
print_exc()
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=104&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Wed Jun 13 21:51:09 2007
@@ -47,7 +47,7 @@
from clock import clock
from BTcrypto import CRYPTO_OK
from __init__ import createPeerID
-from BT1.makemetafile import uniconvertl, uniconvert
+from BT1.makemetafile import getpieces, getsubpieces, uniconvert
from os.path import split
from gzip import GzipFile
from StringIO import StringIO
@@ -514,34 +514,14 @@
# errorfunc(url+' bad url')
return None
- pieces = []
- lengths = []
- fs = []
-
- p = [None, None, None]
- for line in h:
- line = line.rstrip()
- if line == "":
- if (p[0] and p[1] and p[2]):
- path = []
- while p[1]:
- p[1],d = split(p[1])
- path.insert(0,d)
- fs.append({'length': p[0], 'path': uniconvertl(path, encoding)})
- lengths.append(p[0])
- pieces.append(p[2])
- p = [None, None, None]
- if line[:9] == "Filename:":
- p[1] = line[10:]
- if line[:5] == "Size:":
- p[0] = long(line[6:])
- if line[:5] == "SHA1:":
- p[2] = binascii.a2b_hex(line[6:])
-
- response = {'info': {'pieces': ''.join(pieces),
- 'piece lengths': lengths, 'files': fs },
- 'announce': 'http://dttracker.debian.net:6969/announce',
- 'name': uniconvert(name, encoding) }
+ sub_pieces = getsubpieces(name)
+
+ (info, priority) = getpieces(h, encoding, sub_pieces = sub_pieces,
+ installed_versions = installed_versions,
+ status_to_download = status_to_download)
+ response = {'info': info,
+ 'announce': 'http://dttracker.debian.net:6969/announce',
+ 'name': uniconvert(name, encoding)}
except IOError, e:
# errorfunc('problem getting Packages info - ' + str(e))
Modified: debtorrent/trunk/DebTorrent/zurllib.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/zurllib.py?rev=104&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/zurllib.py (original)
+++ debtorrent/trunk/DebTorrent/zurllib.py Wed Jun 13 21:51:09 2007
@@ -62,7 +62,7 @@
"""
- def __init__(self, url):
+ def __init__(self, url, ungzip = True):
"""Initialize the instance and call the open method.
@type url: C{string}
@@ -73,6 +73,7 @@
self.tries = 0
self._open(url.strip())
self.error_return = None
+ self.ungzip = ungzip
def _open(self, url):
"""Open a connection and request the URL, saving the response.
@@ -147,7 +148,8 @@
"""
data = self.response.read()
- if self.response.getheader('Content-Encoding','').find('gzip') >= 0:
+ if self.ungzip and (self.response.getheader('Content-Type','').find('gzip') >= 0 or
+ self.response.getheader('Content-Encoding','').find('gzip') >= 0):
try:
compressed = StringIO(data)
f = GzipFile(fileobj = compressed)
Modified: debtorrent/trunk/btmakemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btmakemetafile.py?rev=104&op=diff
==============================================================================
--- debtorrent/trunk/btmakemetafile.py (original)
+++ debtorrent/trunk/btmakemetafile.py Wed Jun 13 21:51:09 2007
@@ -54,6 +54,7 @@
config, args = parseargs(argv[1:], defaults, 2, None)
for file in args[1:]:
make_meta_file(file, args[0], config, progress = prog)
+ print ''
except ValueError, e:
print 'error: ' + str(e)
print 'run with no args for parameter explanations'
More information about the Debtorrent-commits
mailing list