r217 - in /debtorrent/branches/unique: DebTorrent/BT1/btformats.py DebTorrent/BT1/makemetafile.py btmakemetafile.py

camrdale-guest at users.alioth.debian.org camrdale-guest at users.alioth.debian.org
Sat Aug 11 02:37:31 UTC 2007


Author: camrdale-guest
Date: Sat Aug 11 02:37:31 2007
New Revision: 217

URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=217
Log:
Update the makemetafile routines to use the new piece ordering files.

Modified:
    debtorrent/branches/unique/DebTorrent/BT1/btformats.py
    debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py
    debtorrent/branches/unique/btmakemetafile.py

Modified: debtorrent/branches/unique/DebTorrent/BT1/btformats.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/btformats.py?rev=217&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/btformats.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/btformats.py Sat Aug 11 02:37:31 2007
@@ -47,7 +47,7 @@
     total_length = 0L
     piece_bounds = [0L]
     for length in piecelengths:
-        if type(length) not in ints or length <= 0:
+        if type(length) not in ints or length < 0:
             raise ValueError, 'bad metainfo - bad piece length'
         total_length += length
         piece_bounds.append(total_length)
@@ -67,7 +67,7 @@
         if piece_bounds[bisect(piece_bounds,total_length)-1] != total_length:
             raise ValueError, 'bad metainfo - file does not end on piece boundary'
         path = f.get('path')
-        if type(path) != ListType or path == []:
+        if type(path) != ListType:
             raise ValueError, 'bad metainfo - bad path'
         for p in path:
             if type(p) != StringType:

Modified: debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py?rev=217&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/makemetafile.py Sat Aug 11 02:37:31 2007
@@ -43,6 +43,7 @@
 logger = logging.getLogger('DebTorrent.BT1.makemetafile')
 
 defaults = [
+    ('announce', '', 'the announce URL to use for the torrent'),
     ('announce_list', '',
         'a list of announce URLs - explained below'),
     ('deb_mirrors', '',
@@ -57,7 +58,10 @@
     ('target', '',
         "optional target file for the torrent"),
     ('pieces_file', '', 'the file that contains the sub-package piece information'),
-    ('separate_all', 0, 'create a separate torrent for the architecture:all packages'),
+    ('separate_all', 1, 'create a separate torrent for the architecture:all packages'),
+    ('ordering_file', '', 'the file that contains the piece ordering info'),
+    ('ordering_all_file', '', 'the file that contains the piece ordering info ' + 
+         'for the architecture:all torrent (only used if separate_all = 1)'),
     ]
 
 default_piece_len_exp = 18
@@ -141,7 +145,7 @@
     
     """
     
-    (f_all, n) = subn(r'binary-[a-zA-Z0-9]+([^a-zA-Z0-9]?)', r'binary-all\1', f)
+    (f_all, n) = subn(r'binary-[a-zA-Z0-9-]+([^a-zA-Z0-9-]?)', r'binary-all\1', f)
     if n == 0:
         # Otherwise add '-all' before the extension
         (f_all, n) = subn(r'\.([^.]*)$', r'-all.\1', f)
@@ -150,13 +154,11 @@
             f_all = f + '-all'
     return f_all
 
-def make_meta_file(file, url, params = {}, progress = lambda x: None):
+def make_meta_file(file, params = {}, progress = lambda x: None):
     """Create the torrent files from a Packages file.
     
     @type file: C{string}
     @param file: the Packages file to parse to create the torrent
-    @type url: C{string}
-    @param url: the announce address to use
     @type params: C{dictionary}
     @param params: the command-line parameters to use
     @type progress: C{method}
@@ -201,37 +203,51 @@
     if params.has_key('filesystem_encoding'):
         encoding = params['filesystem_encoding']
 
-    (info, info_all) = makeinfo(file, piece_length, encoding, progress, 
-                                params['separate_all'], params['pieces_file'])
+    (info, info_all, ordering_headers, ordering_all_headers) = \
+        makeinfo(file, piece_length, encoding, progress, 
+                 params['separate_all'], params['pieces_file'],
+                 params['ordering_file'], params['ordering_all_file'])
 
     if info:
-        create_file(f, info, url, uniconvert(name, encoding), params)
+        create_file(f, info, uniconvert(name, encoding), params,
+                    ordering_headers)
         
     if info_all:
-        create_file(convert_all(f), info_all, url, uniconvert(convert_all(name), encoding), params)
-        
-def create_file(f, info, url, name, params):
+        create_file(convert_all(f), info_all,
+                    uniconvert(convert_all(name), encoding), params,
+                    ordering_all_headers)
+        
+def create_file(f, info, name, params, ordering_headers):
     """Actually write the torrent data to a file.
     
     @type f: C{string}
     @param f: the file name to write
     @type info: C{dictionary}
     @param info: the torrent data to write
-    @type url: C{string}
-    @param url: the announce address for the torrent
     @type name: C{string}
     @param name: the internal name of the torrent
     @type params: C{dictionary}
     @param params: the command-line parameters
+    @type ordering_headers: C{dictionary}
+    @param ordering_headers: the headers from the ordering file for the torrent
+    @raise ValueError: if the data is not correct
    
     """
     
     check_info(info)
-    h = open(f, 'wb')
-    data = {'info': info, 'announce': strip(url), 
-        'name': name,
-        'creation date': long(time())}
-    
+    data = {'info': info, 'name': name, 'creation date': long(time())}
+    
+    if "Tracker" in ordering_headers:
+        data['announce'] = ordering_headers["Tracker"].strip()
+        del ordering_headers["Tracker"]
+    if "Torrent" in ordering_headers:
+        data['identifier'] = ordering_headers["Torrent"]
+        del ordering_headers["Torrent"]
+    for header, value in ordering_headers.items():
+        data[header] = value
+    
+    if params.has_key('announce') and params['announce']:
+        data['announce'] = params['announce'].strip()
     if params.has_key('comment') and params['comment']:
         data['comment'] = params['comment']
         
@@ -248,6 +264,10 @@
     elif params.has_key('deb_mirrors') and params['deb_mirrors']:
         data['deb_mirrors'] = params['deb_mirrors'].split('|')
         
+    if "announce" not in data:
+        raise ValueError, "The announce URL must be specified, either on the command line or in the downloaded torrrent file"
+        
+    h = open(f, 'wb')
     h.write(bencode(data))
     h.close()
 
@@ -269,7 +289,7 @@
     return total
 
 def getsubpieces(file, pieces_file = ''):
-    """Retrieve the sub-package piece imformation for the Packages file.
+    """Retrieve the sub-package piece information for the Packages file.
     
     @type file: C{string}
     @param file: the Packages file name to retrieve piece information for
@@ -358,7 +378,167 @@
 
     return pieces
 
-def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 0, sub_pieces = {}):
+def getordering(file, torrent_file = '', all = False):
+    """Retrieve unique piece piece ordering information for the Packages file.
+    
+    @type file: C{string}
+    @param file: the Packages file name to retrieve piece ordering information for
+    @type torrent_file: C{string}
+    @param torrent_file: the file that contains the piece ordering information
+        (optional, defaults to retrieving the info from the web)
+    @type all: C{boolean}
+    @param all: whether to get it for the architecture, or for arch:all
+        (optional, defaults to the specific architecture)
+    @rtype: C{dictionary}
+    @return: the piece ordering info, keys are the starting piece numbers to
+        use for the files, values are the file names
+
+    """
+    
+    pieces = {}
+    headers = {}
+    piece_url = ''
+    
+    if torrent_file:
+        try:
+            f = open(torrent_file)
+        except:
+            logger.exception('torrent ordering file not found: '+torrent_file)
+            return (pieces, headers)
+    elif 'dists' in file.split('_'):
+        try:
+            parts = file.split('_')
+            try:
+                parts[parts.index('stable', parts.index('dists'))] = 'etch'
+            except:
+                pass
+            try:
+                parts[parts.index('testing', parts.index('dists'))] = 'lenny'
+            except:
+                pass
+            try:
+                parts[parts.index('unstable', parts.index('dists'))] = 'sid'
+            except:
+                pass
+            piece_url = 'http://debian.camrdale.org/debtorrent/dists_'
+            piece_url += '_'.join(parts[parts.index('dists')+1:])
+            if piece_url.endswith('.gz'):
+                piece_url = piece_url[:-3]
+            if piece_url.endswith('.bz2'):
+                piece_url = piece_url[:-4]
+            piece_url += '-torrent.gz'
+            if all:
+                piece_url = convert_all(piece_url)
+            piece_file = urlopen(piece_url)
+            piece_data = piece_file.read()
+            try:
+                piece_file.close()
+            except:
+                pass
+            f = piece_data.split('\n')
+        except:
+            logger.exception('torrent ordering URL not working: '+piece_url)
+            return (pieces, headers)
+    else:
+        logger.warning('unable to find torrent ordering data')
+        return (pieces, headers)
+
+    headers_done = False
+    for line in f:
+        line = line.rstrip()
+
+        if not headers_done:
+            # Read the headers from the file
+            h, v = line.split(":", 1)
+            if h == "PieceNumbers":
+                headers_done = True
+                continue
+            
+            headers[h] = v[1:]
+        else:
+            # Read the piece ordering data from the file
+            if line[:1] != " ":
+                break
+
+            piece, file = line.split()
+            pieces[int(piece)] = file
+    
+    try:
+        f.close()
+    except:
+        pass
+
+    logger.info('successfully retrieved torrent ordering data for '+str(len(pieces))+' pieces')
+
+    return (pieces, headers)
+
+def orderpieces(fs, pieces, lengths, separate_all = 1, piece_ordering = {}, num_pieces = 0):
+    """Order the pieces appropriately in the info dictionary.
+    
+    @type fs: C{dictionary}
+    @param fs: the files data for the torrent, keys are the file names from the
+        Packages file, values are the dictionaries to use for them in the torrent
+    @type pieces: C{dictionary}
+    @param pieces: piece hashes for the torrent, keys are the file names from the
+        Packages file, values are lists of the piece hashes for the file
+    @type lengths: C{dictionary}
+    @param lengths: lengths of the pieces for the torrent, keys are the file
+        names from the Packages file, values are lists of the piece lengths
+    @type separate_all: C{boolean}
+    @param separate_all: whether to separate the architecture:all packages into
+        a separate torrent (optional, defaults to True)
+    @type piece_ordering: C{dictionary}
+    @param piece_ordering: the piece ordering info to use for the torrent
+        (optional, defaults to being ordered by file name)
+    @type num_pieces: C{int}
+    @param num_pieces: the number of pieces in the piece ordering 
+        (optional, but must be specified if using ordering data)
+    @rtype: C{dictionary}
+    @return: the properly ordered info section of the torrent
+    
+    """
+
+    if piece_ordering and separate_all:
+        pieces_list = ['\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']*num_pieces
+        lengths_list = [0]*num_pieces
+        fs_list = []
+        
+        ordered_pieces = piece_ordering.keys()
+        ordered_pieces.sort()
+        cur_piece = 0
+        for next_piece in ordered_pieces:
+            if next_piece > cur_piece:
+                fs_list.append({'length': 0, 'path': []})
+                cur_piece = next_piece
+            elif next_piece < cur_piece:
+                raise ValueError, 'piece ordering is invalid'
+
+            len_pieces = len(pieces[piece_ordering[cur_piece]])
+            pieces_list[cur_piece:(cur_piece+len_pieces)] = \
+                pieces[piece_ordering[cur_piece]]
+            lengths_list[cur_piece:(cur_piece+len_pieces)] = \
+                lengths[piece_ordering[cur_piece]]
+            fs_list.append(fs[piece_ordering[cur_piece]])
+            cur_piece = cur_piece + len_pieces
+    else:
+        pieces_list = []
+        lengths_list = []
+        fs_list = []
+
+        files = fs.keys()
+        files.sort()
+        for file in files:
+            pieces_list.extend(pieces[file])
+            lengths_list.extend(lengths[file])
+            fs_list.append(fs[file])
+        
+    return {'pieces': ''.join(pieces_list), 'piece lengths': lengths_list,
+            'files': fs_list}
+
+
+def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 1, 
+              sub_pieces = {}, piece_ordering = {}, piece_ordering_all = {},
+              num_pieces = 0, num_all_pieces = 0):
     """Extract the piece information from the Packages file.
     
     @type f: C{iterable}
@@ -371,11 +551,23 @@
         (optional, defaults to not printing progress updates)
     @type separate_all: C{boolean}
     @param separate_all: whether to separate the architecture:all packages into
-        a separate torrent (optional, defaults to False)
+        a separate torrent (optional, defaults to True)
     @type sub_pieces: C{dictionary}
     @param sub_pieces: the sub-package piece info, keys are the file names,
         values are tuples of a list of piece SHA1 hashes and a list of piece
         sizes (optional, defaults to not using sub-package pieces)
+    @type piece_ordering: C{dictionary}
+    @param piece_ordering: the piece ordering info to use for the architecture
+        specific torrent (optional, defaults to being ordered by file name)
+    @type piece_ordering_all: C{dictionary}
+    @param piece_ordering_all: the piece ordering info to use for the architecture:all
+        torrent (optional, defaults to being ordered by file name)
+    @type num_pieces: C{int}
+    @param num_pieces: the number of pieces in the piece ordering for the architecture
+        specific torrent (optional, but must be specified if using ordering data)
+    @type num_all_pieces: C{int}
+    @param num_all_pieces: the number of pieces in the piece ordering for the architecture:all
+        torrent (optional, but must be specified if using ordering data)
     @rtype: (C{dictionary}, C{dictionary})
     @return: the two torrents, the second is the architecture:all one, if that
         was requested, otherwise it is None
@@ -387,9 +579,9 @@
     if not encoding:
         encoding = 'ascii'
     
-    pieces = ([], [])
-    lengths = ([], [])
-    fs = ([], [])
+    pieces = ({}, {})
+    lengths = ({}, {})
+    fs = ({}, {})
     packages = [0, 0]
     info = None
     info_all = None
@@ -405,17 +597,18 @@
                     all = 1
 
                 if sub_pieces.has_key(p[1]):
-                    lengths[all].extend(sub_pieces[p[1]][1])
-                    pieces[all].extend(sub_pieces[p[1]][0])
+                    lengths[all][p[1]] = sub_pieces[p[1]][1]
+                    pieces[all][p[1]] = sub_pieces[p[1]][0]
                 else:
-                    lengths[all].append(p[0])
-                    pieces[all].append(p[2])
+                    lengths[all][p[1]] = [p[0]]
+                    pieces[all][p[1]] = [p[2]]
 
                 path = []
-                while p[1]:
-                    p[1],d = split(p[1])
+                temp = p[1]
+                while temp:
+                    temp,d = split(temp)
                     path.insert(0,d)
-                fs[all].append({'length': p[0], 'path': uniconvertl(path, encoding)})
+                fs[all][p[1]] = {'length': p[0], 'path': uniconvertl(path, encoding)}
                 packages[all] += 1
                 progress(packages[0] + packages[1])
             p = [None, None, None, None, None, None]
@@ -433,17 +626,22 @@
             p[5] = line[14:]
 
     if packages[0] > 0:
-        info = {'pieces': ''.join(pieces[0]), 'piece lengths': lengths[0], 'files': fs[0]}
-        logger.info('got metainfo for torrent of '+str(len(pieces[0]))+
-                    ' pieces for '+str(len(fs[0]))+' files')
+        info = orderpieces(fs[0], pieces[0], lengths[0], separate_all,
+                           piece_ordering, num_pieces)
+        logger.info('got metainfo for torrent of '+str(len(info['piece lengths']))+
+                    ' pieces for '+str(len(info['files']))+' files')
+
     if packages[1] > 0:
-        info_all = {'pieces': ''.join(pieces[1]), 'piece lengths': lengths[1], 'files': fs[1]}
-        logger.info('got metainfo for torrent of '+str(len(pieces[0]))+
-                    ' pieces for '+str(len(fs[0]))+' files')
+        info_all = orderpieces(fs[1], pieces[1], lengths[1], separate_all,
+                               piece_ordering_all, num_all_pieces)
+        logger.info('got metainfo for arch:all torrent of ' + 
+                    str(len(info_all['piece lengths'])) +
+                    ' pieces for '+str(len(info_all['files']))+' files')
 
     return (info, info_all)
 
-def makeinfo(file, piece_length, encoding, progress, separate_all = 0, pieces_file = ''):
+def makeinfo(file, piece_length, encoding, progress, separate_all = 1,
+             pieces_file = '', torrent_file = '', torrent_all_file = ''):
     """
     
     @type file: C{string}
@@ -456,24 +654,41 @@
     @param progress: the method to call with updates on the progress
     @type separate_all: C{boolean}
     @param separate_all: whether to separate the architecture:all packages into
-        a separate torrent (optional, defaults to False)
+        a separate torrent (optional, defaults to True)
     @type pieces_file: C{string}
     @param pieces_file: the file that contains the piece information
         (optional, defaults to retrieving the info from the web)
-    @rtype: (C{dictionary}, C{dictionary})
+    @type torrent_file: C{string}
+    @param torrent_file: the file that contains the piece ordering information
+        (optional, defaults to retrieving the info from the web)
+    @type torrent_all_file: C{string}
+    @param torrent_all_file: the file that contains the piece ordering information
+        for arch:all (optional, defaults to retrieving the info from the web)
+    @rtype: (C{dictionary}, C{dictionary}, C{dictionary}, C{dictionary})
     @return: the two torrents, the second is the architecture:all one, if that
-        was requested, otherwise it is None
+        was requested, otherwise it is None, followed by the ordering headers
+        for the torrents
     
     """
 
     sub_pieces = getsubpieces(file, pieces_file)
+
+    (piece_ordering, ordering_headers) = getordering(file, torrent_file)
+    if separate_all:
+        (piece_ordering_all, ordering_all_headers) = getordering(file, torrent_all_file, True)
+    else:
+        piece_ordering_all = {}
+        ordering_all_headers = {}
 
     file = abspath(file)
     f = open(file)
-    (info, info_all) = getpieces(f, encoding, progress, separate_all = separate_all, sub_pieces = sub_pieces)
+    (info, info_all) = getpieces(f, encoding, progress, separate_all, sub_pieces,
+                                 piece_ordering, piece_ordering_all,
+                                 int(ordering_headers.get('NextPiece', 0)),
+                                 int(ordering_all_headers.get('NextPiece', 0)))
     f.close()
     
-    return (info, info_all)
+    return (info, info_all, ordering_headers, ordering_all_headers)
 
 def subfiles(d):
     """Process a directory structure to find all the files in it.

Modified: debtorrent/branches/unique/btmakemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/btmakemetafile.py?rev=217&op=diff
==============================================================================
--- debtorrent/branches/unique/btmakemetafile.py (original)
+++ debtorrent/branches/unique/btmakemetafile.py Sat Aug 11 02:37:31 2007
@@ -41,9 +41,9 @@
     
     print '%d packages found\r' % amount,
 
-if len(argv) < 3:
+if len(argv) < 2:
     a,b = split(argv[0])
-    print 'Usage: ' + b + ' <trackerurl> <Packages file> [Packages file...] [params...]'
+    print 'Usage: ' + b + ' <Packages file> [Packages file...] [params...]'
     print
     print formatDefinitions(defaults, 80)
     print_announcelist_details()
@@ -51,9 +51,9 @@
     exit(2)
 
 try:
-    config, args = parseargs(argv[1:], defaults, 2, None)
-    for file in args[1:]:
-        make_meta_file(file, args[0], config, progress = prog)
+    config, args = parseargs(argv[1:], defaults, 1, None)
+    for file in args[0:]:
+        make_meta_file(file, config, progress = prog)
         print ''
 except ValueError, e:
     print 'error: ' + str(e)




More information about the Debtorrent-commits mailing list