r217 - in /debtorrent/branches/unique: DebTorrent/BT1/btformats.py DebTorrent/BT1/makemetafile.py btmakemetafile.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Sat Aug 11 02:37:31 UTC 2007
Author: camrdale-guest
Date: Sat Aug 11 02:37:31 2007
New Revision: 217
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=217
Log:
Update the makemetafile routines to use the new piece ordering files.
Modified:
debtorrent/branches/unique/DebTorrent/BT1/btformats.py
debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py
debtorrent/branches/unique/btmakemetafile.py
Modified: debtorrent/branches/unique/DebTorrent/BT1/btformats.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/btformats.py?rev=217&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/btformats.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/btformats.py Sat Aug 11 02:37:31 2007
@@ -47,7 +47,7 @@
total_length = 0L
piece_bounds = [0L]
for length in piecelengths:
- if type(length) not in ints or length <= 0:
+ if type(length) not in ints or length < 0:
raise ValueError, 'bad metainfo - bad piece length'
total_length += length
piece_bounds.append(total_length)
@@ -67,7 +67,7 @@
if piece_bounds[bisect(piece_bounds,total_length)-1] != total_length:
raise ValueError, 'bad metainfo - file does not end on piece boundary'
path = f.get('path')
- if type(path) != ListType or path == []:
+ if type(path) != ListType:
raise ValueError, 'bad metainfo - bad path'
for p in path:
if type(p) != StringType:
Modified: debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py?rev=217&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py Sat Aug 11 02:37:31 2007
@@ -43,6 +43,7 @@
logger = logging.getLogger('DebTorrent.BT1.makemetafile')
defaults = [
+ ('announce', '', 'the announce URL to use for the torrent'),
('announce_list', '',
'a list of announce URLs - explained below'),
('deb_mirrors', '',
@@ -57,7 +58,10 @@
('target', '',
"optional target file for the torrent"),
('pieces_file', '', 'the file that contains the sub-package piece information'),
- ('separate_all', 0, 'create a separate torrent for the architecture:all packages'),
+ ('separate_all', 1, 'create a separate torrent for the architecture:all packages'),
+ ('ordering_file', '', 'the file that contains the piece ordering info'),
+ ('ordering_all_file', '', 'the file that contains the piece ordering info ' +
+ 'for the architecture:all torrent (only used if separate_all = 1)'),
]
default_piece_len_exp = 18
@@ -141,7 +145,7 @@
"""
- (f_all, n) = subn(r'binary-[a-zA-Z0-9]+([^a-zA-Z0-9]?)', r'binary-all\1', f)
+ (f_all, n) = subn(r'binary-[a-zA-Z0-9-]+([^a-zA-Z0-9-]?)', r'binary-all\1', f)
if n == 0:
# Otherwise add '-all' before the extension
(f_all, n) = subn(r'\.([^.]*)$', r'-all.\1', f)
@@ -150,13 +154,11 @@
f_all = f + '-all'
return f_all
-def make_meta_file(file, url, params = {}, progress = lambda x: None):
+def make_meta_file(file, params = {}, progress = lambda x: None):
"""Create the torrent files from a Packages file.
@type file: C{string}
@param file: the Packages file to parse to create the torrent
- @type url: C{string}
- @param url: the announce address to use
@type params: C{dictionary}
@param params: the command-line parameters to use
@type progress: C{method}
@@ -201,37 +203,51 @@
if params.has_key('filesystem_encoding'):
encoding = params['filesystem_encoding']
- (info, info_all) = makeinfo(file, piece_length, encoding, progress,
- params['separate_all'], params['pieces_file'])
+ (info, info_all, ordering_headers, ordering_all_headers) = \
+ makeinfo(file, piece_length, encoding, progress,
+ params['separate_all'], params['pieces_file'],
+ params['ordering_file'], params['ordering_all_file'])
if info:
- create_file(f, info, url, uniconvert(name, encoding), params)
+ create_file(f, info, uniconvert(name, encoding), params,
+ ordering_headers)
if info_all:
- create_file(convert_all(f), info_all, url, uniconvert(convert_all(name), encoding), params)
-
-def create_file(f, info, url, name, params):
+ create_file(convert_all(f), info_all,
+ uniconvert(convert_all(name), encoding), params,
+ ordering_all_headers)
+
+def create_file(f, info, name, params, ordering_headers):
"""Actually write the torrent data to a file.
@type f: C{string}
@param f: the file name to write
@type info: C{dictionary}
@param info: the torrent data to write
- @type url: C{string}
- @param url: the announce address for the torrent
@type name: C{string}
@param name: the internal name of the torrent
@type params: C{dictionary}
@param params: the command-line parameters
+ @type ordering_headers: C{dictionary}
+ @param ordering_headers: the headers from the ordering file for the torrent
+ @raise ValueError: if the data is not correct
"""
check_info(info)
- h = open(f, 'wb')
- data = {'info': info, 'announce': strip(url),
- 'name': name,
- 'creation date': long(time())}
-
+ data = {'info': info, 'name': name, 'creation date': long(time())}
+
+ if "Tracker" in ordering_headers:
+ data['announce'] = ordering_headers["Tracker"].strip()
+ del ordering_headers["Tracker"]
+ if "Torrent" in ordering_headers:
+ data['identifier'] = ordering_headers["Torrent"]
+ del ordering_headers["Torrent"]
+ for header, value in ordering_headers.items():
+ data[header] = value
+
+ if params.has_key('announce') and params['announce']:
+ data['announce'] = params['announce'].strip()
if params.has_key('comment') and params['comment']:
data['comment'] = params['comment']
@@ -248,6 +264,10 @@
elif params.has_key('deb_mirrors') and params['deb_mirrors']:
data['deb_mirrors'] = params['deb_mirrors'].split('|')
+ if "announce" not in data:
+ raise ValueError, "The announce URL must be specified, either on the command line or in the downloaded torrrent file"
+
+ h = open(f, 'wb')
h.write(bencode(data))
h.close()
@@ -269,7 +289,7 @@
return total
def getsubpieces(file, pieces_file = ''):
- """Retrieve the sub-package piece imformation for the Packages file.
+ """Retrieve the sub-package piece information for the Packages file.
@type file: C{string}
@param file: the Packages file name to retrieve piece information for
@@ -358,7 +378,167 @@
return pieces
-def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 0, sub_pieces = {}):
+def getordering(file, torrent_file = '', all = False):
+ """Retrieve unique piece piece ordering information for the Packages file.
+
+ @type file: C{string}
+ @param file: the Packages file name to retrieve piece ordering information for
+ @type torrent_file: C{string}
+ @param torrent_file: the file that contains the piece ordering information
+ (optional, defaults to retrieving the info from the web)
+ @type all: C{boolean}
+ @param all: whether to get it for the architecture, or for arch:all
+ (optional, defaults to the specific architecture)
+ @rtype: C{dictionary}
+ @return: the piece ordering info, keys are the starting piece numbers to
+ use for the files, values are the file names
+
+ """
+
+ pieces = {}
+ headers = {}
+ piece_url = ''
+
+ if torrent_file:
+ try:
+ f = open(torrent_file)
+ except:
+ logger.exception('torrent ordering file not found: '+torrent_file)
+ return (pieces, headers)
+ elif 'dists' in file.split('_'):
+ try:
+ parts = file.split('_')
+ try:
+ parts[parts.index('stable', parts.index('dists'))] = 'etch'
+ except:
+ pass
+ try:
+ parts[parts.index('testing', parts.index('dists'))] = 'lenny'
+ except:
+ pass
+ try:
+ parts[parts.index('unstable', parts.index('dists'))] = 'sid'
+ except:
+ pass
+ piece_url = 'http://debian.camrdale.org/debtorrent/dists_'
+ piece_url += '_'.join(parts[parts.index('dists')+1:])
+ if piece_url.endswith('.gz'):
+ piece_url = piece_url[:-3]
+ if piece_url.endswith('.bz2'):
+ piece_url = piece_url[:-4]
+ piece_url += '-torrent.gz'
+ if all:
+ piece_url = convert_all(piece_url)
+ piece_file = urlopen(piece_url)
+ piece_data = piece_file.read()
+ try:
+ piece_file.close()
+ except:
+ pass
+ f = piece_data.split('\n')
+ except:
+ logger.exception('torrent ordering URL not working: '+piece_url)
+ return (pieces, headers)
+ else:
+ logger.warning('unable to find torrent ordering data')
+ return (pieces, headers)
+
+ headers_done = False
+ for line in f:
+ line = line.rstrip()
+
+ if not headers_done:
+ # Read the headers from the file
+ h, v = line.split(":", 1)
+ if h == "PieceNumbers":
+ headers_done = True
+ continue
+
+ headers[h] = v[1:]
+ else:
+ # Read the piece ordering data from the file
+ if line[:1] != " ":
+ break
+
+ piece, file = line.split()
+ pieces[int(piece)] = file
+
+ try:
+ f.close()
+ except:
+ pass
+
+ logger.info('successfully retrieved torrent ordering data for '+str(len(pieces))+' pieces')
+
+ return (pieces, headers)
+
+def orderpieces(fs, pieces, lengths, separate_all = 1, piece_ordering = {}, num_pieces = 0):
+ """Order the pieces appropriately in the info dictionary.
+
+ @type fs: C{dictionary}
+ @param fs: the files data for the torrent, keys are the file names from the
+ Packages file, values are the dictionaries to use for them in the torrent
+ @type pieces: C{dictionary}
+ @param pieces: piece hashes for the torrent, keys are the file names from the
+ Packages file, values are lists of the piece hashes for the file
+ @type lengths: C{dictionary}
+ @param lengths: lengths of the pieces for the torrent, keys are the file
+ names from the Packages file, values are lists of the piece lengths
+ @type separate_all: C{boolean}
+ @param separate_all: whether to separate the architecture:all packages into
+ a separate torrent (optional, defaults to True)
+ @type piece_ordering: C{dictionary}
+ @param piece_ordering: the piece ordering info to use for the torrent
+ (optional, defaults to being ordered by file name)
+ @type num_pieces: C{int}
+ @param num_pieces: the number of pieces in the piece ordering
+ (optional, but must be specified if using ordering data)
+ @rtype: C{dictionary}
+ @return: the properly ordered info section of the torrent
+
+ """
+
+ if piece_ordering and separate_all:
+ pieces_list = ['\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']*num_pieces
+ lengths_list = [0]*num_pieces
+ fs_list = []
+
+ ordered_pieces = piece_ordering.keys()
+ ordered_pieces.sort()
+ cur_piece = 0
+ for next_piece in ordered_pieces:
+ if next_piece > cur_piece:
+ fs_list.append({'length': 0, 'path': []})
+ cur_piece = next_piece
+ elif next_piece < cur_piece:
+ raise ValueError, 'piece ordering is invalid'
+
+ len_pieces = len(pieces[piece_ordering[cur_piece]])
+ pieces_list[cur_piece:(cur_piece+len_pieces)] = \
+ pieces[piece_ordering[cur_piece]]
+ lengths_list[cur_piece:(cur_piece+len_pieces)] = \
+ lengths[piece_ordering[cur_piece]]
+ fs_list.append(fs[piece_ordering[cur_piece]])
+ cur_piece = cur_piece + len_pieces
+ else:
+ pieces_list = []
+ lengths_list = []
+ fs_list = []
+
+ files = fs.keys()
+ files.sort()
+ for file in files:
+ pieces_list.extend(pieces[file])
+ lengths_list.extend(lengths[file])
+ fs_list.append(fs[file])
+
+ return {'pieces': ''.join(pieces_list), 'piece lengths': lengths_list,
+ 'files': fs_list}
+
+
+def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 1,
+ sub_pieces = {}, piece_ordering = {}, piece_ordering_all = {},
+ num_pieces = 0, num_all_pieces = 0):
"""Extract the piece information from the Packages file.
@type f: C{iterable}
@@ -371,11 +551,23 @@
(optional, defaults to not printing progress updates)
@type separate_all: C{boolean}
@param separate_all: whether to separate the architecture:all packages into
- a separate torrent (optional, defaults to False)
+ a separate torrent (optional, defaults to True)
@type sub_pieces: C{dictionary}
@param sub_pieces: the sub-package piece info, keys are the file names,
values are tuples of a list of piece SHA1 hashes and a list of piece
sizes (optional, defaults to not using sub-package pieces)
+ @type piece_ordering: C{dictionary}
+ @param piece_ordering: the piece ordering info to use for the architecture
+ specific torrent (optional, defaults to being ordered by file name)
+ @type piece_ordering_all: C{dictionary}
+ @param piece_ordering_all: the piece ordering info to use for the architecture:all
+ torrent (optional, defaults to being ordered by file name)
+ @type num_pieces: C{int}
+ @param num_pieces: the number of pieces in the piece ordering for the architecture
+ specific torrent (optional, but must be specified if using ordering data)
+ @type num_all_pieces: C{int}
+ @param num_all_pieces: the number of pieces in the piece ordering for the architecture:all
+ torrent (optional, but must be specified if using ordering data)
@rtype: (C{dictionary}, C{dictionary})
@return: the two torrents, the second is the architecture:all one, if that
was requested, otherwise it is None
@@ -387,9 +579,9 @@
if not encoding:
encoding = 'ascii'
- pieces = ([], [])
- lengths = ([], [])
- fs = ([], [])
+ pieces = ({}, {})
+ lengths = ({}, {})
+ fs = ({}, {})
packages = [0, 0]
info = None
info_all = None
@@ -405,17 +597,18 @@
all = 1
if sub_pieces.has_key(p[1]):
- lengths[all].extend(sub_pieces[p[1]][1])
- pieces[all].extend(sub_pieces[p[1]][0])
+ lengths[all][p[1]] = sub_pieces[p[1]][1]
+ pieces[all][p[1]] = sub_pieces[p[1]][0]
else:
- lengths[all].append(p[0])
- pieces[all].append(p[2])
+ lengths[all][p[1]] = [p[0]]
+ pieces[all][p[1]] = [p[2]]
path = []
- while p[1]:
- p[1],d = split(p[1])
+ temp = p[1]
+ while temp:
+ temp,d = split(temp)
path.insert(0,d)
- fs[all].append({'length': p[0], 'path': uniconvertl(path, encoding)})
+ fs[all][p[1]] = {'length': p[0], 'path': uniconvertl(path, encoding)}
packages[all] += 1
progress(packages[0] + packages[1])
p = [None, None, None, None, None, None]
@@ -433,17 +626,22 @@
p[5] = line[14:]
if packages[0] > 0:
- info = {'pieces': ''.join(pieces[0]), 'piece lengths': lengths[0], 'files': fs[0]}
- logger.info('got metainfo for torrent of '+str(len(pieces[0]))+
- ' pieces for '+str(len(fs[0]))+' files')
+ info = orderpieces(fs[0], pieces[0], lengths[0], separate_all,
+ piece_ordering, num_pieces)
+ logger.info('got metainfo for torrent of '+str(len(info['piece lengths']))+
+ ' pieces for '+str(len(info['files']))+' files')
+
if packages[1] > 0:
- info_all = {'pieces': ''.join(pieces[1]), 'piece lengths': lengths[1], 'files': fs[1]}
- logger.info('got metainfo for torrent of '+str(len(pieces[0]))+
- ' pieces for '+str(len(fs[0]))+' files')
+ info_all = orderpieces(fs[1], pieces[1], lengths[1], separate_all,
+ piece_ordering_all, num_all_pieces)
+ logger.info('got metainfo for arch:all torrent of ' +
+ str(len(info_all['piece lengths'])) +
+ ' pieces for '+str(len(info_all['files']))+' files')
return (info, info_all)
-def makeinfo(file, piece_length, encoding, progress, separate_all = 0, pieces_file = ''):
+def makeinfo(file, piece_length, encoding, progress, separate_all = 1,
+ pieces_file = '', torrent_file = '', torrent_all_file = ''):
"""
@type file: C{string}
@@ -456,24 +654,41 @@
@param progress: the method to call with updates on the progress
@type separate_all: C{boolean}
@param separate_all: whether to separate the architecture:all packages into
- a separate torrent (optional, defaults to False)
+ a separate torrent (optional, defaults to True)
@type pieces_file: C{string}
@param pieces_file: the file that contains the piece information
(optional, defaults to retrieving the info from the web)
- @rtype: (C{dictionary}, C{dictionary})
+ @type torrent_file: C{string}
+ @param torrent_file: the file that contains the piece ordering information
+ (optional, defaults to retrieving the info from the web)
+ @type torrent_all_file: C{string}
+ @param torrent_all_file: the file that contains the piece ordering information
+ for arch:all (optional, defaults to retrieving the info from the web)
+ @rtype: (C{dictionary}, C{dictionary}, C{dictionary}, C{dictionary})
@return: the two torrents, the second is the architecture:all one, if that
- was requested, otherwise it is None
+ was requested, otherwise it is None, followed by the ordering headers
+ for the torrents
"""
sub_pieces = getsubpieces(file, pieces_file)
+
+ (piece_ordering, ordering_headers) = getordering(file, torrent_file)
+ if separate_all:
+ (piece_ordering_all, ordering_all_headers) = getordering(file, torrent_all_file, True)
+ else:
+ piece_ordering_all = {}
+ ordering_all_headers = {}
file = abspath(file)
f = open(file)
- (info, info_all) = getpieces(f, encoding, progress, separate_all = separate_all, sub_pieces = sub_pieces)
+ (info, info_all) = getpieces(f, encoding, progress, separate_all, sub_pieces,
+ piece_ordering, piece_ordering_all,
+ int(ordering_headers.get('NextPiece', 0)),
+ int(ordering_all_headers.get('NextPiece', 0)))
f.close()
- return (info, info_all)
+ return (info, info_all, ordering_headers, ordering_all_headers)
def subfiles(d):
"""Process a directory structure to find all the files in it.
Modified: debtorrent/branches/unique/btmakemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/btmakemetafile.py?rev=217&op=diff
==============================================================================
--- debtorrent/branches/unique/btmakemetafile.py (original)
+++ debtorrent/branches/unique/btmakemetafile.py Sat Aug 11 02:37:31 2007
@@ -41,9 +41,9 @@
print '%d packages found\r' % amount,
-if len(argv) < 3:
+if len(argv) < 2:
a,b = split(argv[0])
- print 'Usage: ' + b + ' <trackerurl> <Packages file> [Packages file...] [params...]'
+ print 'Usage: ' + b + ' <Packages file> [Packages file...] [params...]'
print
print formatDefinitions(defaults, 80)
print_announcelist_details()
@@ -51,9 +51,9 @@
exit(2)
try:
- config, args = parseargs(argv[1:], defaults, 2, None)
- for file in args[1:]:
- make_meta_file(file, args[0], config, progress = prog)
+ config, args = parseargs(argv[1:], defaults, 1, None)
+ for file in args[0:]:
+ make_meta_file(file, config, progress = prog)
print ''
except ValueError, e:
print 'error: ' + str(e)
More information about the Debtorrent-commits
mailing list