r333 - in /debtorrent/trunk: ./ DebTorrent/ DebTorrent/BT1/
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Sat Jan 19 21:32:11 UTC 2008
Author: camrdale-guest
Date: Sat Jan 19 21:32:11 2008
New Revision: 333
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=333
Log:
Merged revisions 209-213,216-217,219,222-225 via svnmerge from
svn+ssh://camrdale-guest@svn.debian.org/svn/debtorrent/debtorrent/branches/unique
........
r209 | camrdale-guest | 2007-08-09 00:22:24 -0700 (Thu, 09 Aug 2007) | 1 line
Add initial uniquely script to manage unique piece numbers (not yet working).
........
r210 | camrdale-guest | 2007-08-09 11:21:55 -0700 (Thu, 09 Aug 2007) | 1 line
Mostly finished uniquely script.
........
r211 | camrdale-guest | 2007-08-09 13:55:05 -0700 (Thu, 09 Aug 2007) | 1 line
Add output flushing to the uniquely script.
........
r212 | camrdale-guest | 2007-08-09 15:28:09 -0700 (Thu, 09 Aug 2007) | 1 line
Change uniquely so a new torrent is created if some of the Release file fields change.
........
r213 | camrdale-guest | 2007-08-09 16:28:49 -0700 (Thu, 09 Aug 2007) | 1 line
Make uniquely remove old torrent files.
........
r216 | camrdale-guest | 2007-08-09 22:38:02 -0700 (Thu, 09 Aug 2007) | 1 line
Add the Tracker field to uniquely.
........
r217 | camrdale-guest | 2007-08-10 19:37:31 -0700 (Fri, 10 Aug 2007) | 1 line
Update the makemetafile routines to use the new piece ordering files.
........
r219 | camrdale-guest | 2007-08-11 13:48:51 -0700 (Sat, 11 Aug 2007) | 1 line
Update uniquely to order the pieces by full path name, and rewrite for readability.
........
r222 | camrdale-guest | 2007-08-11 17:47:24 -0700 (Sat, 11 Aug 2007) | 1 line
Use the new makemetafile piece ordering routines.
........
r223 | camrdale-guest | 2007-08-11 18:25:46 -0700 (Sat, 11 Aug 2007) | 1 line
Adjust bitfield messages to their proper length, and don't drop connections due to unknown Have messages.
........
r224 | camrdale-guest | 2007-08-11 19:03:14 -0700 (Sat, 11 Aug 2007) | 1 line
Add a force_tracker option for testing purposes.
........
r225 | camrdale-guest | 2007-08-12 10:58:57 -0700 (Sun, 12 Aug 2007) | 1 line
Switch to using the new torrent identifier instead of the info hash.
........
Added:
debtorrent/trunk/uniquely.py
- copied, changed from r213, debtorrent/branches/unique/uniquely.py
Modified:
debtorrent/trunk/ (props changed)
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/Connecter.py
debtorrent/trunk/DebTorrent/BT1/btformats.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/DebTorrent/bitfield.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/DebTorrent/launchmanycore.py
debtorrent/trunk/btcompletedir.py
debtorrent/trunk/btmakemetafile.py
debtorrent/trunk/btshowmetainfo.py
debtorrent/trunk/debtorrent-client.conf
debtorrent/trunk/test.py
Propchange: debtorrent/trunk/
------------------------------------------------------------------------------
--- svnmerge-integrated (original)
+++ svnmerge-integrated Sat Jan 19 21:32:11 2008
@@ -1,1 +1,1 @@
-/debtorrent/branches/http1.1:1-257 /debtorrent/branches/unique:1-204
+/debtorrent/branches/http1.1:1-257 /debtorrent/branches/unique:1-204,209-213,216-217,219,222-225
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Sat Jan 19 21:32:11 2008
@@ -373,10 +373,10 @@
logger.info('retrieving the cached Packages file to start the torrent')
r2 = self.Cache.cache_get(path)
TorrentCreator(path, r2[3], self.start_torrent,
- self.rawserver.add_task, self.config['separate_all'])
+ self.rawserver.add_task, self.config)
else:
TorrentCreator(path, r[3], self.start_torrent,
- self.rawserver.add_task, self.config['separate_all'])
+ self.rawserver.add_task, self.config)
return r
@@ -409,7 +409,7 @@
# If it's a torrent file, start it
if r[0] == 200 and path[-1] in ('Packages', 'Packages.gz', 'Packages.bz2'):
TorrentCreator(path, r[3], self.start_torrent,
- self.rawserver.add_task, self.config['separate_all'])
+ self.rawserver.add_task, self.config)
for (connection, httpreq) in connections:
# Check to make sure the requester is still waiting
@@ -605,9 +605,6 @@
def start_torrent(self, response, name, path):
-
- if 'announce' not in response or not response['announce']:
- response['announce'] = self.config['default_tracker']
infohash = sha(bencode(response['info'])).digest()
Modified: debtorrent/trunk/DebTorrent/BT1/Connecter.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Connecter.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Connecter.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Connecter.py Sat Jan 19 21:32:11 2008
@@ -541,8 +541,7 @@
return
i = struct.unpack('>i', message[1:])[0]
if i >= self.numpieces:
- logger.warning(c.get_ip()+': bad piece number, closing connection')
- connection.close()
+ logger.debug(c.get_ip()+': dropping an unknown piece number')
return
if c.download.got_have(i):
c.upload.got_not_interested()
Modified: debtorrent/trunk/DebTorrent/BT1/btformats.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/btformats.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/btformats.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/btformats.py Sat Jan 19 21:32:11 2008
@@ -47,7 +47,7 @@
total_length = 0L
piece_bounds = [0L]
for length in piecelengths:
- if type(length) not in ints or length <= 0:
+ if type(length) not in ints or length < 0:
raise ValueError, 'bad metainfo - bad piece length'
total_length += length
piece_bounds.append(total_length)
@@ -67,7 +67,7 @@
if piece_bounds[bisect(piece_bounds,total_length)-1] != total_length:
raise ValueError, 'bad metainfo - file does not end on piece boundary'
path = f.get('path')
- if type(path) != ListType or path == []:
+ if type(path) != ListType:
raise ValueError, 'bad metainfo - bad path'
for p in path:
if type(p) != StringType:
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Sat Jan 19 21:32:11 2008
@@ -44,6 +44,7 @@
logger = logging.getLogger('DebTorrent.BT1.makemetafile')
defaults = [
+ ('announce', '', 'the announce URL to use for the torrent'),
('announce_list', '',
'a list of announce URLs - explained below'),
('deb_mirrors', '',
@@ -58,7 +59,10 @@
('target', '',
"optional target file for the torrent"),
('pieces_file', '', 'the file that contains the sub-package piece information'),
- ('separate_all', 0, 'create a separate torrent for the architecture:all packages'),
+ ('separate_all', 1, 'create a separate torrent for the architecture:all packages'),
+ ('ordering_file', '', 'the file that contains the piece ordering info'),
+ ('ordering_all_file', '', 'the file that contains the piece ordering info ' +
+ 'for the architecture:all torrent (only used if separate_all = 1)'),
]
default_piece_len_exp = 18
@@ -142,7 +146,7 @@
"""
- (f_all, n) = subn(r'binary-[a-zA-Z0-9]+([^a-zA-Z0-9]?)', r'binary-all\1', f)
+ (f_all, n) = subn(r'binary-[a-zA-Z0-9-]+([^a-zA-Z0-9-]?)', r'binary-all\1', f)
if n == 0:
# Otherwise add '-all' before the extension
(f_all, n) = subn(r'\.([^.]*)$', r'-all.\1', f)
@@ -151,13 +155,11 @@
f_all = f + '-all'
return f_all
-def make_meta_file(file, url, params = {}, progress = lambda x: None):
+def make_meta_file(file, params = {}, progress = lambda x: None):
"""Create the torrent files from a Packages file.
@type file: C{string}
@param file: the Packages file to parse to create the torrent
- @type url: C{string}
- @param url: the announce address to use
@type params: C{dictionary}
@param params: the command-line parameters to use
@type progress: C{method}
@@ -202,37 +204,51 @@
if params.has_key('filesystem_encoding'):
encoding = params['filesystem_encoding']
- (info, info_all) = makeinfo(file, piece_length, encoding, progress,
- params['separate_all'], params['pieces_file'])
+ (info, info_all, ordering_headers, ordering_all_headers) = \
+ makeinfo(file, piece_length, encoding, progress,
+ params['separate_all'], params['pieces_file'],
+ params['ordering_file'], params['ordering_all_file'])
if info:
- create_file(f, info, url, uniconvert(name, encoding), params)
+ create_file(f, info, uniconvert(name, encoding), params,
+ ordering_headers)
if info_all:
- create_file(convert_all(f), info_all, url, uniconvert(convert_all(name), encoding), params)
-
-def create_file(f, info, url, name, params):
+ create_file(convert_all(f), info_all,
+ uniconvert(convert_all(name), encoding), params,
+ ordering_all_headers)
+
+def create_file(f, info, name, params, ordering_headers):
"""Actually write the torrent data to a file.
@type f: C{string}
@param f: the file name to write
@type info: C{dictionary}
@param info: the torrent data to write
- @type url: C{string}
- @param url: the announce address for the torrent
@type name: C{string}
@param name: the internal name of the torrent
@type params: C{dictionary}
@param params: the command-line parameters
+ @type ordering_headers: C{dictionary}
+ @param ordering_headers: the headers from the ordering file for the torrent
+ @raise ValueError: if the data is not correct
"""
check_info(info)
- h = open(f, 'wb')
- data = {'info': info, 'announce': strip(url),
- 'name': name,
- 'creation date': long(time())}
-
+ data = {'info': info, 'name': name, 'creation date': long(time())}
+
+ if "Tracker" in ordering_headers:
+ data['announce'] = ordering_headers["Tracker"].strip()
+ del ordering_headers["Tracker"]
+ if "Torrent" in ordering_headers:
+ data['identifier'] = binascii.a2b_hex(ordering_headers["Torrent"].strip())
+ del ordering_headers["Torrent"]
+ for header, value in ordering_headers.items():
+ data[header] = value.strip()
+
+ if params.has_key('announce') and params['announce']:
+ data['announce'] = params['announce'].strip()
if params.has_key('comment') and params['comment']:
data['comment'] = params['comment']
@@ -249,6 +265,10 @@
elif params.has_key('deb_mirrors') and params['deb_mirrors']:
data['deb_mirrors'] = params['deb_mirrors'].split('|')
+ if "announce" not in data:
+ raise ValueError, "The announce URL must be specified, either on the command line or in the downloaded torrrent file"
+
+ h = open(f, 'wb')
h.write(bencode(data))
h.close()
@@ -270,7 +290,7 @@
return total
def getsubpieces(file, pieces_file = ''):
- """Retrieve the sub-package piece imformation for the Packages file.
+ """Retrieve the sub-package piece information for the Packages file.
@type file: C{string}
@param file: the Packages file name to retrieve piece information for
@@ -359,7 +379,167 @@
return pieces
-def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 0, sub_pieces = {}):
+def getordering(file, torrent_file = '', all = False):
+ """Retrieve unique piece piece ordering information for the Packages file.
+
+ @type file: C{string}
+ @param file: the Packages file name to retrieve piece ordering information for
+ @type torrent_file: C{string}
+ @param torrent_file: the file that contains the piece ordering information
+ (optional, defaults to retrieving the info from the web)
+ @type all: C{boolean}
+ @param all: whether to get it for the architecture, or for arch:all
+ (optional, defaults to the specific architecture)
+ @rtype: C{dictionary}
+ @return: the piece ordering info, keys are the starting piece numbers to
+ use for the files, values are the file names
+
+ """
+
+ pieces = {}
+ headers = {}
+ piece_url = ''
+
+ if torrent_file:
+ try:
+ f = open(torrent_file)
+ except:
+ logger.exception('torrent ordering file not found: '+torrent_file)
+ return (pieces, headers)
+ elif 'dists' in file.split('_'):
+ try:
+ parts = file.split('_')
+ try:
+ parts[parts.index('stable', parts.index('dists'))] = 'etch'
+ except:
+ pass
+ try:
+ parts[parts.index('testing', parts.index('dists'))] = 'lenny'
+ except:
+ pass
+ try:
+ parts[parts.index('unstable', parts.index('dists'))] = 'sid'
+ except:
+ pass
+ piece_url = 'http://debian.camrdale.org/debtorrent/dists_'
+ piece_url += '_'.join(parts[parts.index('dists')+1:])
+ if piece_url.endswith('.gz'):
+ piece_url = piece_url[:-3]
+ if piece_url.endswith('.bz2'):
+ piece_url = piece_url[:-4]
+ piece_url += '-torrent.gz'
+ if all:
+ piece_url = convert_all(piece_url)
+ piece_file = urlopen(piece_url)
+ piece_data = piece_file.read()
+ try:
+ piece_file.close()
+ except:
+ pass
+ f = piece_data.split('\n')
+ except:
+ logger.exception('torrent ordering URL not working: '+piece_url)
+ return (pieces, headers)
+ else:
+ logger.warning('unable to find torrent ordering data')
+ return (pieces, headers)
+
+ headers_done = False
+ for line in f:
+ line = line.rstrip()
+
+ if not headers_done:
+ # Read the headers from the file
+ h, v = line.split(":", 1)
+ if h == "PieceNumbers":
+ headers_done = True
+ continue
+
+ headers[h] = v[1:]
+ else:
+ # Read the piece ordering data from the file
+ if line[:1] != " ":
+ break
+
+ piece, file = line.split()
+ pieces[int(piece)] = file
+
+ try:
+ f.close()
+ except:
+ pass
+
+ logger.info('successfully retrieved torrent ordering data for '+str(len(pieces))+' files')
+
+ return (pieces, headers)
+
+def orderpieces(fs, pieces, lengths, separate_all = 1, piece_ordering = {}, num_pieces = 0):
+ """Order the pieces appropriately in the info dictionary.
+
+ @type fs: C{dictionary}
+ @param fs: the files data for the torrent, keys are the file names from the
+ Packages file, values are the dictionaries to use for them in the torrent
+ @type pieces: C{dictionary}
+ @param pieces: piece hashes for the torrent, keys are the file names from the
+ Packages file, values are lists of the piece hashes for the file
+ @type lengths: C{dictionary}
+ @param lengths: lengths of the pieces for the torrent, keys are the file
+ names from the Packages file, values are lists of the piece lengths
+ @type separate_all: C{boolean}
+ @param separate_all: whether to separate the architecture:all packages into
+ a separate torrent (optional, defaults to True)
+ @type piece_ordering: C{dictionary}
+ @param piece_ordering: the piece ordering info to use for the torrent
+ (optional, defaults to being ordered by file name)
+ @type num_pieces: C{int}
+ @param num_pieces: the number of pieces in the piece ordering
+ (optional, but must be specified if using ordering data)
+ @rtype: C{dictionary}
+ @return: the properly ordered info section of the torrent
+
+ """
+
+ if piece_ordering and separate_all:
+ pieces_list = ['\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']*num_pieces
+ lengths_list = [0]*num_pieces
+ fs_list = []
+
+ ordered_pieces = piece_ordering.keys()
+ ordered_pieces.sort()
+ cur_piece = 0
+ for next_piece in ordered_pieces:
+ if next_piece > cur_piece:
+ fs_list.append({'length': 0, 'path': []})
+ cur_piece = next_piece
+ elif next_piece < cur_piece:
+ raise ValueError, 'piece ordering is invalid'
+
+ len_pieces = len(pieces[piece_ordering[cur_piece]])
+ pieces_list[cur_piece:(cur_piece+len_pieces)] = \
+ pieces[piece_ordering[cur_piece]]
+ lengths_list[cur_piece:(cur_piece+len_pieces)] = \
+ lengths[piece_ordering[cur_piece]]
+ fs_list.append(fs[piece_ordering[cur_piece]])
+ cur_piece = cur_piece + len_pieces
+ else:
+ pieces_list = []
+ lengths_list = []
+ fs_list = []
+
+ files = fs.keys()
+ files.sort()
+ for file in files:
+ pieces_list.extend(pieces[file])
+ lengths_list.extend(lengths[file])
+ fs_list.append(fs[file])
+
+ return {'pieces': ''.join(pieces_list), 'piece lengths': lengths_list,
+ 'files': fs_list}
+
+
+def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 1,
+ sub_pieces = {}, piece_ordering = {}, piece_ordering_all = {},
+ num_pieces = 0, num_all_pieces = 0):
"""Extract the piece information from the Packages file.
@type f: C{iterable}
@@ -372,11 +552,23 @@
(optional, defaults to not printing progress updates)
@type separate_all: C{boolean}
@param separate_all: whether to separate the architecture:all packages into
- a separate torrent (optional, defaults to False)
+ a separate torrent (optional, defaults to True)
@type sub_pieces: C{dictionary}
@param sub_pieces: the sub-package piece info, keys are the file names,
values are tuples of a list of piece SHA1 hashes and a list of piece
sizes (optional, defaults to not using sub-package pieces)
+ @type piece_ordering: C{dictionary}
+ @param piece_ordering: the piece ordering info to use for the architecture
+ specific torrent (optional, defaults to being ordered by file name)
+ @type piece_ordering_all: C{dictionary}
+ @param piece_ordering_all: the piece ordering info to use for the architecture:all
+ torrent (optional, defaults to being ordered by file name)
+ @type num_pieces: C{int}
+ @param num_pieces: the number of pieces in the piece ordering for the architecture
+ specific torrent (optional, but must be specified if using ordering data)
+ @type num_all_pieces: C{int}
+ @param num_all_pieces: the number of pieces in the piece ordering for the architecture:all
+ torrent (optional, but must be specified if using ordering data)
@rtype: (C{dictionary}, C{dictionary})
@return: the two torrents, the second is the architecture:all one, if that
was requested, otherwise it is None
@@ -388,9 +580,9 @@
if not encoding:
encoding = 'ascii'
- pieces = ([], [])
- lengths = ([], [])
- fs = ([], [])
+ pieces = ({}, {})
+ lengths = ({}, {})
+ fs = ({}, {})
packages = [0, 0]
info = None
info_all = None
@@ -406,17 +598,18 @@
all = 1
if sub_pieces.has_key(p[1]):
- lengths[all].extend(sub_pieces[p[1]][1])
- pieces[all].extend(sub_pieces[p[1]][0])
+ lengths[all][p[1]] = sub_pieces[p[1]][1]
+ pieces[all][p[1]] = sub_pieces[p[1]][0]
else:
- lengths[all].append(p[0])
- pieces[all].append(p[2])
+ lengths[all][p[1]] = [p[0]]
+ pieces[all][p[1]] = [p[2]]
path = []
- while p[1]:
- p[1],d = split(p[1])
+ temp = p[1]
+ while temp:
+ temp,d = split(temp)
path.insert(0,d)
- fs[all].append({'length': p[0], 'path': uniconvertl(path, encoding)})
+ fs[all][p[1]] = {'length': p[0], 'path': uniconvertl(path, encoding)}
packages[all] += 1
progress(packages[0] + packages[1])
p = [None, None, None, None, None, None]
@@ -434,18 +627,23 @@
p[5] = line[14:]
if packages[0] > 0:
- info = {'pieces': ''.join(pieces[0]), 'piece lengths': lengths[0], 'files': fs[0]}
- logger.info('got metainfo for torrent of '+str(len(pieces[0]))+
- ' pieces for '+str(len(fs[0]))+' files')
+ info = orderpieces(fs[0], pieces[0], lengths[0], separate_all,
+ piece_ordering, num_pieces)
+ logger.info('got metainfo for torrent of '+str(len(info['piece lengths']))+
+ ' pieces for '+str(len(info['files']))+' files')
+
if packages[1] > 0:
- info_all = {'pieces': ''.join(pieces[1]), 'piece lengths': lengths[1], 'files': fs[1]}
- logger.info('got metainfo for torrent of '+str(len(pieces[0]))+
- ' pieces for '+str(len(fs[0]))+' files')
+ info_all = orderpieces(fs[1], pieces[1], lengths[1], separate_all,
+ piece_ordering_all, num_all_pieces)
+ logger.info('got metainfo for arch:all torrent of ' +
+ str(len(info_all['piece lengths'])) +
+ ' pieces for '+str(len(info_all['files']))+' files')
return (info, info_all)
-def makeinfo(file, piece_length, encoding, progress, separate_all = 0, pieces_file = ''):
- """
+def makeinfo(file, piece_length, encoding, progress, separate_all = 1,
+ pieces_file = '', torrent_file = '', torrent_all_file = ''):
+ """Open the file and pass it to the getpieces function.
@type file: C{string}
@param file: the file name of the Packages file to make into a torrent
@@ -457,24 +655,41 @@
@param progress: the method to call with updates on the progress
@type separate_all: C{boolean}
@param separate_all: whether to separate the architecture:all packages into
- a separate torrent (optional, defaults to False)
+ a separate torrent (optional, defaults to True)
@type pieces_file: C{string}
@param pieces_file: the file that contains the piece information
(optional, defaults to retrieving the info from the web)
- @rtype: (C{dictionary}, C{dictionary})
+ @type torrent_file: C{string}
+ @param torrent_file: the file that contains the piece ordering information
+ (optional, defaults to retrieving the info from the web)
+ @type torrent_all_file: C{string}
+ @param torrent_all_file: the file that contains the piece ordering information
+ for arch:all (optional, defaults to retrieving the info from the web)
+ @rtype: (C{dictionary}, C{dictionary}, C{dictionary}, C{dictionary})
@return: the two torrents, the second is the architecture:all one, if that
- was requested, otherwise it is None
+ was requested, otherwise it is None, followed by the ordering headers
+ for the torrents
"""
sub_pieces = getsubpieces(file, pieces_file)
+
+ (piece_ordering, ordering_headers) = getordering(file, torrent_file)
+ if separate_all:
+ (piece_ordering_all, ordering_all_headers) = getordering(file, torrent_all_file, True)
+ else:
+ piece_ordering_all = {}
+ ordering_all_headers = {}
file = abspath(file)
f = open(file)
- (info, info_all) = getpieces(f, encoding, progress, separate_all = separate_all, sub_pieces = sub_pieces)
+ (info, info_all) = getpieces(f, encoding, progress, separate_all, sub_pieces,
+ piece_ordering, piece_ordering_all,
+ int(ordering_headers.get('NextPiece', 0)),
+ int(ordering_all_headers.get('NextPiece', 0)))
f.close()
- return (info, info_all)
+ return (info, info_all, ordering_headers, ordering_all_headers)
def subfiles(d):
"""Process a directory structure to find all the files in it.
@@ -502,15 +717,13 @@
return r
-def completedir(dir, url, params = {}, vc = lambda x: None, fc = lambda x: None):
+def completedir(dir, params = {}, vc = lambda x: None, fc = lambda x: None):
"""Create a torrent for each file in a directory.
Does not recurse into sub-directories.
@type dir: C{string}
@param dir: the directory to find files in
- @type url: C{string}
- @param url: the announce address to use for the torrents
@type params: C{dictionary}
@param params: the configuration options (optional, defaults to None)
@type vc: C{method}
@@ -540,7 +753,7 @@
if t not in ignore and t[0] != '.':
if target != '':
params['target'] = join(target,t+ext)
- make_meta_file(i, url, params, progress = vc)
+ make_meta_file(i, params, progress = vc)
except ValueError:
print_exc()
@@ -549,7 +762,7 @@
"""
- def __init__(self, path, data, callback, sched, separate_all = 0):
+ def __init__(self, path, data, callback, sched, config):
"""Process a downloaded Packages file and start the torrent making thread.
@type path: C{list} of C{string}
@@ -560,9 +773,8 @@
@param callback: the method to call with the torrent when it has been created
@type sched: C{method}
@param sched: the method to call to schedule future invocation of a function
- @type separate_all: C{boolean}
- @param separate_all: whether to separate the architecture:all packages into
- a separate torrent (optional, defaults to False)
+ @type config: C{dictionary}
+ @param config: the configuration parameters
"""
@@ -570,7 +782,7 @@
self.data = data
self.callback = callback
self.sched = sched
- self.separate_all = separate_all
+ self.config = config
self.name = '_'.join(self.path[:-1])
self.responses = []
@@ -606,30 +818,63 @@
logger.debug('Packages file successfully decompressed')
sub_pieces = getsubpieces('_'.join(self.path))
- (info, info_all) = getpieces(h, separate_all = self.separate_all, sub_pieces = sub_pieces)
+ (piece_ordering, ordering_headers) = getordering('_'.join(path))
+ if self.config['separate_all']:
+ (piece_ordering_all, ordering_all_headers) = getordering('_'.join(path), all = True)
+ else:
+ piece_ordering_all = {}
+ ordering_all_headers = {}
+
+ (info, info_all) = getpieces(h, separate_all = self.config['separate_all'],
+ sub_pieces = sub_pieces,
+ piece_ordering = piece_ordering,
+ piece_ordering_all = piece_ordering_all,
+ num_pieces = int(ordering_headers.get('NextPiece', 0)),
+ num_all_pieces = int(ordering_all_headers.get('NextPiece', 0)))
del h[:]
- mirror = []
+ name = self.name
+ if info and self.config['separate_all'] in (0, 2, 3):
+ self.responses.append(self._create_response(info, ordering_headers, name))
+
+ name = convert_all(self.name)
+ if info_all and self.config['separate_all'] in (1, 3):
+ self.responses.append(self._create_response(info_all, ordering_all_headers, name))
+
+ self.sched(self._finished)
+
+ def _create_response(self, info, headers, name):
+ """Create the response dictionary for the torrent.
+
+ @type info: C{dictionary}
+ @param info: the info dictionary to use for the torrent
+ @type headers: C{dictionary}
+ @param headers: the headers from the torrent file
+ @type name: C{string}
+ @param name: the name to use for the torrent
+
+ """
+
+ response = {'info': info,
+ 'announce': self.config['default_tracker'],
+ 'name': uniconvert(name)}
+ if "Tracker" in headers:
+ response['announce'] = headers["Tracker"].strip()
+ del headers["Tracker"]
+ if "Torrent" in headers:
+ response['identifier'] = a2b_hex(headers["Torrent"].strip())
+ del headers["Torrent"]
+ for header, value in headers.items():
+ response[header] = value.strip()
+
+ if self.config["force_tracker"]:
+ response['announce'] = self.config["force_tracker"]
+
if self.path.count('dists'):
- mirror.append('http://' + '/'.join(self.path[:self.path.index('dists')]) + '/')
+ mirror = 'http://' + '/'.join(self.path[:self.path.index('dists')]) + '/'
+ response.setdefault('deb_mirrors', []).append(mirror)
- name = self.name
- if info and self.separate_all in (0, 2, 3):
- response = {'info': info,
- 'name': uniconvert(name)}
- if mirror:
- response['deb_mirrors'] = mirror
- self.responses.append((response, name))
-
- name = convert_all(self.name)
- if info_all and self.separate_all in (1, 3):
- response = {'info': info,
- 'name': uniconvert(name)}
- if mirror:
- response['deb_mirrors'] = mirror
- self.responses.append((response, name))
-
- self.sched(self._finished)
+ return (response, name)
def _finished(self):
"""Wrap up the creation and call the callback function."""
Modified: debtorrent/trunk/DebTorrent/bitfield.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/bitfield.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/bitfield.py (original)
+++ debtorrent/trunk/DebTorrent/bitfield.py Sat Jan 19 21:32:11 2008
@@ -79,17 +79,22 @@
raise ValueError, "length must be provided unless copying from another array"
self.length = length
if bitstring is not None:
- extra = len(bitstring) * 8 - length
- if extra < 0 or extra >= 8:
- raise ValueError
+ # Construct the bitfield
t = lookup_table
r = []
for c in bitstring:
r.extend(t[ord(c)])
- if extra > 0:
- if r[-extra:] != [0] * extra:
- raise ValueError
- del r[-extra:]
+ if len(r) > length:
+ # Stop if enough bits have been found
+ break
+
+ if len(r) > length:
+ # Remove any extra bits that were added
+ del r[length:]
+ elif len(r) < length:
+ # Add any missing bits as all False
+ r.extend([False]*(length - len(r)))
+
self.array = r
self.numfalse = negsum(r)
else:
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Sat Jan 19 21:32:11 2008
@@ -147,10 +147,10 @@
('disable_http_downloader', 0,
'(for testing purposes only) whether to disable the backup HTTP downloader'),
# Other Things
- ('separate_all',0, 'whether to separate the architecture:all packages, ' +
+ ('separate_all', 3, 'whether to separate the architecture:all packages, ' +
'0 = don\'t separate, 1 = separate and run architecture:all, ' +
'2 = separate and run all architectures but all, ' +
- '3 = separate and run both (not for btdownloadheadless)'),
+ '3 = separate and run both'),
# End of Normal Options
# BitTorrent Options
('keepalive_interval', 120.0,
@@ -187,6 +187,8 @@
# Tracker Connections
('default_tracker', 'http://dttracker.debian.net:6969/announce',
'the default tracker address to use for new torrents'),
+ ('force_tracker', '',
+ 'set this to the tracker address to use for all torrents'),
('rerequest_interval', 5 * 60,
'time to wait between requesting more peers'),
('min_peers', 20,
@@ -426,8 +428,19 @@
return (None, None)
sub_pieces = getsubpieces(name)
-
- (info, info_all) = getpieces(h, separate_all = separate_all, sub_pieces = sub_pieces)
+
+ (piece_ordering, ordering_headers) = getordering(name)
+ if separate_all:
+ (piece_ordering_all, ordering_all_headers) = getordering(name, all = True)
+ else:
+ piece_ordering_all = {}
+ ordering_all_headers = {}
+
+ (info, info_all) = getpieces(h, separate_all = separate_all, sub_pieces = sub_pieces,
+ piece_ordering = piece_ordering,
+ piece_ordering_all = piece_ordering_all,
+ num_pieces = int(ordering_headers.get('NextPiece', 0)),
+ num_all_pieces = int(ordering_all_headers.get('NextPiece', 0)))
response = None
response_all = None
@@ -436,11 +449,29 @@
'announce': default_tracker,
'name': uniconvert(name)}
+ if "Tracker" in ordering_headers:
+ response['announce'] = ordering_headers["Tracker"].strip()
+ del ordering_headers["Tracker"]
+ if "Torrent" in ordering_headers:
+ response['identifier'] = binascii.a2b_hex(ordering_headers["Torrent"].strip())
+ del ordering_headers["Torrent"]
+ for header, value in ordering_headers.items():
+ response[header] = value.strip()
+
if info_all:
response_all = {'info': info_all,
'announce': default_tracker,
'name': uniconvert(convert_all(name))}
+ if "Tracker" in ordering_all_headers:
+ response_all['announce'] = ordering_all_headers["Tracker"].strip()
+ del ordering_all_headers["Tracker"]
+ if "Torrent" in ordering_all_headers:
+ response_all['identifier'] = binascii.a2b_hex(ordering_all_headers["Torrent"].strip())
+ del ordering_all_headers["Torrent"]
+ for header, value in ordering_all_headers.items():
+ response_all[header] = value.strip()
+
except IOError, e:
logger.exception('problem getting Packages info')
return (None, None)
Modified: debtorrent/trunk/DebTorrent/launchmanycore.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/launchmanycore.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/launchmanycore.py (original)
+++ debtorrent/trunk/DebTorrent/launchmanycore.py Sat Jan 19 21:32:11 2008
@@ -67,6 +67,8 @@
@ivar controller: the manager for all torrent downloads
@type hash: C{string}
@ivar hash: the info hash of the torrent
+ @type identifier: C{string}
+ @ivar identifier: the identifier of the torrent
@type response: C{dictionary}
@ivar response: the meta info for the torrent
@type config: C{dictionary}
@@ -103,13 +105,15 @@
"""
- def __init__(self, controller, hash, response, config, myid):
+ def __init__(self, controller, hash, identifier, response, config, myid):
"""Initialize the instance and start a new downloader.
@type controller: L{LaunchMany}
@param controller: the manager for all torrent downloads
@type hash: C{string}
@param hash: the info hash of the torrent
+ @type identifier: C{string}
+ @param identifier: the identifier of the torrent
@type response: C{dictionary}
@param response: the meta info for the torrent
@type config: C{dictionary}
@@ -121,6 +125,7 @@
self.controller = controller
self.hash = hash
+ self.identifier = identifier
self.response = response
self.config = config
@@ -136,11 +141,11 @@
self.status_errtime = 0
self.status_done = 0.0
- self.rawserver = controller.handler.newRawServer(hash, self.doneflag)
+ self.rawserver = controller.handler.newRawServer(identifier, self.doneflag)
d = BT1Download(self.update_status, self.finished, self.error,
- self.doneflag, config, response,
- hash, myid, self.rawserver, controller.listen_port,
+ self.doneflag, config, response, identifier,
+ myid, self.rawserver, controller.listen_port,
self.controller.configdir)
self.d = d
@@ -537,7 +542,9 @@
# Stop any running previous versions of the same torrent
same_names = []
for old_hash in self.torrent_list:
- if self.torrent_cache[old_hash]['name'] == data['name']:
+ if (self.torrent_cache[old_hash]['name'] == data['name'] or
+ self.torrent_cache[old_hash]['metainfo'].get('identifier', old_hash) ==
+ data['metainfo'].get('identifier', hash)):
same_names.append(old_hash)
for old_hash in same_names:
self.remove(old_hash)
@@ -556,7 +563,15 @@
logger.error('Asked to start a torrent that is not in the cache')
return
- logger.info('Starting torrent: '+str(binascii.b2a_hex(hash)))
+ # Assign the torrent identifier from the metainfo data
+ if "identifier" in self.torrent_cache[hash]['metainfo']:
+ id = self.torrent_cache[hash]['metainfo']['identifier']
+ logger.info('Starting torrent: '+str(binascii.b2a_hex(hash)) +
+ ' identified by: '+str(binascii.b2a_hex(id)))
+ else:
+ id = hash
+ logger.info('Starting torrent: '+str(binascii.b2a_hex(hash)))
+
c = self.counter
self.counter += 1
x = ''
@@ -564,7 +579,8 @@
x = mapbase64[c & 0x3F]+x
c >>= 6
peer_id = createPeerID(x)
- d = SingleDownload(self, hash, self.torrent_cache[hash]['metainfo'], self.config, peer_id)
+ d = SingleDownload(self, hash, id, self.torrent_cache[hash]['metainfo'],
+ self.config, peer_id)
self.torrent_list.append(hash)
self.downloads[hash] = d
d.start()
@@ -648,7 +664,7 @@
file_num += 1
# Check that the file ends with the desired file name
- if file.endswith('/'.join(f['path'])):
+ if f['path'] and file.endswith('/'.join(f['path'])):
logger.debug('Found file in: '+str(binascii.b2a_hex(hash)))
found_torrents.append((hash, file_num))
Modified: debtorrent/trunk/btcompletedir.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btcompletedir.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/btcompletedir.py (original)
+++ debtorrent/trunk/btcompletedir.py Sat Jan 19 21:32:11 2008
@@ -48,9 +48,9 @@
print "\nProcessing file: %s" % file
-if len(argv) < 3:
+if len(argv) < 2:
a,b = split(argv[0])
- print 'Usage: ' + b + ' <trackerurl> <dir> [dir...] [params...]'
+ print 'Usage: ' + b + ' <dir> [dir...] [params...]'
print 'makes a .dtorrent file for every Packages file present in each dir.'
print
print formatDefinitions(defaults, 80)
@@ -59,9 +59,9 @@
exit(2)
try:
- config, args = parseargs(argv[1:], defaults, 2, None)
- for dir in args[1:]:
- completedir(dir, args[0], config, vc = prog, fc = next_file)
+ config, args = parseargs(argv[1:], defaults, 1, None)
+ for dir in args:
+ completedir(dir, config, vc = prog, fc = next_file)
except ValueError, e:
print 'error: ' + str(e)
print 'run with no args for parameter explanations'
Modified: debtorrent/trunk/btmakemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btmakemetafile.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/btmakemetafile.py (original)
+++ debtorrent/trunk/btmakemetafile.py Sat Jan 19 21:32:11 2008
@@ -40,9 +40,9 @@
print '%d packages found\r' % amount,
-if len(argv) < 3:
+if len(argv) < 2:
a,b = split(argv[0])
- print 'Usage: ' + b + ' <trackerurl> <Packages file> [Packages file...] [params...]'
+ print 'Usage: ' + b + ' <Packages file> [Packages file...] [params...]'
print
print formatDefinitions(defaults, 80)
print_announcelist_details()
@@ -50,9 +50,9 @@
exit(2)
try:
- config, args = parseargs(argv[1:], defaults, 2, None)
- for file in args[1:]:
- make_meta_file(file, args[0], config, progress = prog)
+ config, args = parseargs(argv[1:], defaults, 1, None)
+ for file in args:
+ make_meta_file(file, config, progress = prog)
print ''
except ValueError, e:
print 'error: ' + str(e)
Modified: debtorrent/trunk/btshowmetainfo.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btshowmetainfo.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/btshowmetainfo.py (original)
+++ debtorrent/trunk/btshowmetainfo.py Sat Jan 19 21:32:11 2008
@@ -37,10 +37,11 @@
# print metainfo
info = metainfo['info']
- info_hash = sha(bencode(info))
+ info_hash = sha(bencode(info)).hexdigest()
print 'metainfo file.: %s' % basename(metainfo_name)
- print 'info hash.....: %s' % info_hash.hexdigest()
+ print 'identifier....: %s' % metainfo.get("identifier", info_hash)
+ print 'info hash.....: %s' % info_hash
piece_lengths = info['piece lengths']
print 'directory name: %s' % metainfo['name']
print 'files.........: '
Modified: debtorrent/trunk/debtorrent-client.conf
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/debtorrent-client.conf?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/debtorrent-client.conf (original)
+++ debtorrent/trunk/debtorrent-client.conf Sat Jan 19 21:32:11 2008
@@ -612,6 +612,19 @@
# default_tracker = "http://dttracker.debian.net:6969/announce"
#
+# Force Tracker
+#
+# If set, this will force all new torrents to use this address for the tracker,
+# regardless of the one specified for the torrent.
+#
+# WARNING: This may seriously limit the number of peers you can find, since
+# most will be using the address specified in the torrent. This option
+# is only intended for testing purposes.
+#
+
+# force_tracker = ""
+
+#
# Rerequest Interval
#
# The time to wait between requesting more peers from the tracker (in seconds).
Modified: debtorrent/trunk/test.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/test.py?rev=333&op=diff
==============================================================================
--- debtorrent/trunk/test.py (original)
+++ debtorrent/trunk/test.py Sat Jan 19 21:32:11 2008
@@ -641,7 +641,7 @@
sleep(5)
for k, v in downloaders.items():
- opts = v[1] + ['--default_tracker', tracker_address(v[0])]
+ opts = v[1] + ['--force_tracker', tracker_address(v[0])]
running_downloaders[k] = start_downloader(k, opts, **v[2])
sleep(10)
Copied: debtorrent/trunk/uniquely.py (from r213, debtorrent/branches/unique/uniquely.py)
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/uniquely.py?rev=333&op=diff
==============================================================================
--- debtorrent/branches/unique/uniquely.py (original)
+++ debtorrent/trunk/uniquely.py Sat Jan 19 21:32:11 2008
@@ -2,34 +2,93 @@
"""Process a Release file, creating, finding and updating any torrent files."""
-import bsddb, sha, binascii
+import sha
import sys
import gzip
from bz2 import BZ2File
from math import ceil
from os import remove
from os.path import exists
-
-# Some default values
-default_piecesize = 512*1024
-extension = ".gz"
-# can not contain Date, Infohash, NextPiece or OriginalPieces
-default_hash_fields = ["Codename", "Suite", "Component", "Architecture",
+from time import strftime, gmtime
+
+# The piece size to use (must match the '-extrapieces' file's piece size)
+DEFAULT_PIECESIZE = 512*1024
+
+# The Packages files to read
+EXTENSION = ".gz"
+
+# The fields to hash to determine the torrent identifier
+# (can not contain Date, Infohash, NextPiece or OriginalPieces)
+DEFAULT_HASH_FIELDS = ["Codename", "Suite", "Component", "Architecture",
"PieceSize", "OriginalDate"]
-header_order = ["Torrent", "Infohash", "OriginalDate", "Date", "PieceSize",
- "NextPiece", "OriginalPieces", "Codename", "Suite",
- "Component", "Architecture", "TorrentHashFields"]
+
+# The tracker announce URL to use
+DEFAULT_TRACKER = "http://dttracker.debian.net:6969/announce"
+
+# The order to write the headers in (headers not listed won't be written)
+HEADER_ORDER = ["Torrent", "Infohash", "InfohashArchs", "OriginalDate", "Date",
+ "PieceSize", "NextPiece", "OriginalPieces", "Codename", "Suite",
+ "Component", "Architecture", "Tracker", "TorrentHashFields"]
+
+def read_release(filename):
+ """Read the headers and Packages file names from a Release file.
+
+ @type filename: C[string}
+ @param filename: the Release file to read
+ @rtype: C{dictionary}, C{list} of C{string}
+ @return: the headers and full file names of Packages files
+
+ """
+
+ # Initialize the Release file variables
+ release_dir = filename.rsplit('/', 1)[0]
+ read_packages = False
+ headers = {}
+ packages = []
+
+ f = open(filename, 'r')
+
+ for line in f:
+ line = line.rstrip()
+
+ if line[:1] != " ":
+ read_packages = False
+ try:
+ # Read the various headers from the file
+ h, v = line.split(":", 1)
+ if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
+ read_packages = True
+ elif len(v) > 0:
+ headers[h] = v[1:]
+ except:
+ # Bad header line, just ignore it
+ print "WARNING: Ignoring badly formatted Release line:", line
+
+ # Skip to the next line
+ continue
+
+ # Read file names from the multiple hash sections of the file
+ if read_packages:
+ p = line.split()
+ if len(p) == 3 and p[2].endswith("Packages"+EXTENSION):
+ if release_dir + "/" + p[2] not in packages:
+ packages.append(release_dir + "/" + p[2])
+
+ f.close()
+
+ return headers, packages
def get_old(old_file):
"""Read the headers and piece ordering data from an old file.
@type old_file: C[string}
@param old_file: the old piece ordering file to open
- @rtype: (C{dictionary}, C{dictionary})
+ @rtype: C{dictionary}, C{dictionary}
@return: the old piece ordering (keys are the file names, values are the
starting piece number) and headers
"""
+
pieces = {}
headers = {}
@@ -61,12 +120,64 @@
# Delete the file and return empty variables to create a new torrent
if exists(old_file):
remove(old_file)
- pass
return pieces, headers
-def get_new(filename, old_files, headers, old_all_files, all_pieces,
- all_new_pieces):
+def update_headers(headers, release_headers, component, arch):
+ """Update the headers with new fields from the Release file.
+
+ @type headers: C{dictionary}
+ @param headers: the headers from the piece ordering file
+ @type release_headers: C{dictionary}
+ @param release_headers: the headers from the Release file
+ @type component: C{string}
+ @param component: the component name (e.g. main, contrib, non-free)
+ @type arch: C{string}
+ @param arch: the architecture name (e.g. i386, amd64, all)
+ @rtype: C{boolean}
+ @return: whether a new torrent has been created
+
+ """
+
+ # Set any required Release headers
+ if len(release_headers.get("Date", "")) == 0:
+ # Use today's date
+ release_headers["Date"] = strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime())
+
+ # Create/update the headers
+ headers.setdefault("OriginalDate", release_headers["Date"])
+ headers["Date"] = release_headers["Date"]
+ headers.setdefault("PieceSize", str(DEFAULT_PIECESIZE))
+ headers.setdefault("NextPiece", str(0))
+ headers["Codename"] = release_headers.get("Codename", "")
+ headers["Suite"] = release_headers.get("Suite", "")
+ headers["Component"] = component
+ headers["Architecture"] = arch
+ headers.setdefault("Tracker", DEFAULT_TRACKER)
+ headers.setdefault("TorrentHashFields", " ".join(DEFAULT_HASH_FIELDS))
+
+ # Calculate the new hash
+ sha1 = sha.new()
+ for header in headers["TorrentHashFields"].split():
+ sha1.update(headers[header])
+ new_hash = sha1.hexdigest()
+
+ # Check if the hash has changed
+ if headers.get("Torrent", "") == new_hash:
+ return False
+ else:
+ # If it has, then reset the torrent to create a new one
+ headers["OriginalDate"] = release_headers["Date"]
+ headers["NextPiece"] = str(0)
+ headers.pop("OriginalPieces", "")
+ sha1 = sha.new()
+ for header in headers["TorrentHashFields"].split():
+ sha1.update(headers[header])
+ headers["Torrent"] = sha1.hexdigest()
+
+ return True
+
+def get_new(filename, old_files, old_all_files, all_pieces, all_new_pieces):
"""Read the new piece data from a Packages file.
Reads the Packages file, finding old files in it and copying their data to
@@ -83,8 +194,6 @@
@type old_files: C{dictionary}
@param old_files: the original piece ordering, keys are the file names,
values are the starting piece number
- @type headers: C{dictionary}
- @param headers: the original headers
@type old_all_files: C{dictionary}
@param old_all_files: the original piece ordering for architecture:all
files, keys are the file names, values are the starting piece number
@@ -100,10 +209,6 @@
"""
- # Get the needed header information
- next_piece = int(headers["NextPiece"])
- piece_size = int(headers["PieceSize"])
-
# Open the possibly compressed file
if filename.endswith(".gz"):
f = gzip.open(filename, 'r')
@@ -113,6 +218,7 @@
f = open(filename, 'r')
pieces = {}
+ new_pieces = []
p = [None, None, None]
for line in f:
@@ -137,9 +243,8 @@
pieces[old_files[p[0]]] = p[0]
del old_files[p[0]]
else:
- # Add new file to the end of the torrent
- pieces[next_piece] = p[0]
- next_piece += int(ceil(p[1]/float(piece_size)))
+ # Found new file, save it for later processing
+ new_pieces.append((p[0], p[1]))
p = [None, None, None]
if line[:9] == "Filename:":
@@ -151,256 +256,186 @@
f.close()
+ return pieces, new_pieces
+
+def add_new(pieces, new_pieces, headers):
+ """Read the new piece data from a Packages file.
+
+ Adds new files to the end of the piece ordering. The 'pieces' input is
+ modified by having the new pieces added to it. The 'new_pieces' input
+ list is sorted. The 'NextPiece' header in the input 'headers' is updated.
+
+ @type pieces: C{dictionary}
+ @param pieces: the current piece ordering, keys are the starting piece
+ numbers, values are the file names
+ @type new_pieces: C{list} of (C{string}, C{long})
+ @param new_pieces: the file name and file size of the new files that have
+ been found and are to be added to the pirce ordering
+ @type headers: C{dictionary}
+ @param headers: the headers from the piece ordering file
+
+ """
+
+ # Get the needed header information
+ next_piece = int(headers["NextPiece"])
+ piece_size = int(headers["PieceSize"])
+
+ new_pieces.sort()
+ old_file = ""
+ old_size = 0L
+ for (file, size) in new_pieces:
+ if file == old_file:
+ if size != old_size:
+ print "WARNING: multiple files with different size:", file
+ else:
+ pieces[next_piece] = file
+ next_piece += int(ceil(size/float(piece_size)))
+
+ old_file = file
+ old_size = size
+
+ # Set the final header values
headers["NextPiece"] = str(next_piece)
-
- return pieces
-
-#cache_file = sys.argv[1]
-#cache = bsddb.btopen(cache_file, "w")
-
-# The only input is the Release file to process
-releasefile = sys.argv[1]
-print "Processing: %s" % releasefile
-
-# Initialize the Release file variables
-release_dir = releasefile.rsplit('/', 1)[0]
-origin = ""
-label = ""
-suite = ""
-codename = ""
-date = ""
-components = []
-archs = []
-read_files = False
-packages = []
-packages_sha1 = {}
-packages_size = {}
-
-f = open(releasefile, 'r')
-
-for line in f:
- line = line.rstrip()
-
- # Read the various headers from the file
- if line[:7] == "Origin:":
- origin = line[8:]
- if line[:6] == "Label:":
- label = line[7:]
- if line[:6] == "Suite:":
- suite = line[7:]
- if line[:9] == "Codename:":
- codename = line[10:]
- if line[:5] == "Date:":
- date = line[6:]
- if line[:11] == "Components:":
- components = line[12:].split()
- if line[:14] == "Architectures:":
- archs = line[15:].split()
-
- # Read multiple lines from the SHA1 section of the file
- if line[:1] != " ":
- read_files = False
- if read_files:
- p = line.split()
- if len(p) == 3 and p[2].endswith("Packages"+extension):
- packages.append(release_dir + "/" + p[2])
- packages_sha1[p[2]] = binascii.a2b_hex(p[0])
- packages_size[p[2]] = long(p[1])
- if line[:7] == "MD5Sum:":
- read_files = True
-
-f.close()
-
-torrent_prefix = "dists_" + codename + "_"
-torrent_suffix = "_Packages-torrent.gz"
-
-for component in components:
- # Get the old 'all' data
- all_file = torrent_prefix + component + "_binary-all" + torrent_suffix
- old_all_files, all_headers = get_old(all_file)
- all_pieces = {}
- all_new_pieces = []
- new_all_torrent = False
-
- # Create the all headers
- all_headers.setdefault("OriginalDate", date)
- all_headers["Date"] = date
- all_headers.setdefault("PieceSize", str(default_piecesize))
- all_headers.setdefault("NextPiece", str(0))
- all_headers["Codename"] = codename
- all_headers["Suite"] = suite
- all_headers["Component"] = component
- all_headers["Architecture"] = "all"
- all_headers.setdefault("TorrentHashFields", " ".join(default_hash_fields))
-
- # Calculate the new hash
- sha1 = sha.new()
- for header in all_headers["TorrentHashFields"].split():
- sha1.update(all_headers[header])
- new_hash = sha1.hexdigest()
-
- # Check if the hash has changed
- if all_headers.get("Torrent", "") != new_hash:
- # If it has, then reset the torrent
- new_all_torrent = True
- old_all_files = {}
- all_headers["OriginalDate"] = date
- all_headers["NextPiece"] = str(0)
- all_headers.pop("OriginalPieces", "")
- sha1 = sha.new()
- for header in all_headers["TorrentHashFields"].split():
- sha1.update(all_headers[header])
- all_headers["Torrent"] = sha1.hexdigest()
-
- for arch in archs:
- torrent_file = torrent_prefix + component + "_binary-" + arch + torrent_suffix
-
- # Find the Packages file that will be parsed
- found = False
- for filename in packages:
- if (filename.find(component) >= 0 and
- filename.find("binary-"+arch) >= 0):
- found = True
- break
- if not found:
- print "WARNING: no matching Packages file for component %s, arch %s" % (component, arch)
- if exists(torrent_file):
- remove(torrent_file)
- continue
- packages.pop(packages.index(filename))
-
- # Get the old data for this torrent, if any existed
- print torrent_file + ": reading ...",
+ headers.setdefault("OriginalPieces", headers["NextPiece"])
+
+def write_file(filename, pieces, headers):
+ """Print the new data to the file.
+
+ @type filename: C[string}
+ @param filename: the file to write to
+ @type pieces: C{dictionary}
+ @param pieces: the current piece ordering, keys are the starting piece
+ numbers, values are the file names
+ @type headers: C{dictionary}
+ @param headers: the headers from the piece ordering file
+
+ """
+
+ f = gzip.open(filename, 'w')
+
+ # Write the headers
+ for header in HEADER_ORDER:
+ if header in headers:
+ f.write("%s: %s\n" % (header, headers[header]))
+ f.write("PieceNumbers:\n")
+
+ # Write the starting piece numbers
+ ps = pieces.keys()
+ ps.sort()
+ format_string = " %"+str(len(str(max(ps))))+"d %s\n"
+ for p in ps:
+ f.write(format_string % (p, pieces[p]))
+
+ f.close()
+
+def run(releasefile):
+ """Process a single Release file.
+
+ @type releasefile: C[string}
+ @param releasefile: the Release file to process
+
+ """
+
+ # Process the Release file
+ print "Processing: %s" % releasefile
+ release_headers, packages = read_release(releasefile)
+
+ torrent_prefix = "dists_" + release_headers.get("Codename", "") + "_"
+ torrent_suffix = "_Packages-torrent.gz"
+
+ for component in release_headers.get("Components", "").split():
+ # Get the old 'all' data
+ all_file = torrent_prefix + component + "_binary-all" + torrent_suffix
+ old_all_pieces, all_headers = get_old(all_file)
+ all_pieces = {}
+ all_new_pieces = []
+ new_all_torrent = False
+
+ # First update the 'all' headers
+ if update_headers(all_headers, release_headers, component, "all"):
+ # If it has, then reset the torrent
+ new_all_torrent = True
+ old_all_pieces = {}
+
+ for arch in release_headers.get("Architectures", "").split():
+ torrent_file = torrent_prefix + component + "_binary-" + arch + torrent_suffix
+
+ # Find the Packages file that will be parsed
+ found = False
+ for filename in packages:
+ if (filename.find(component) >= 0 and
+ filename.find("binary-"+arch) >= 0):
+ found = True
+ break
+ if not found:
+ print "WARNING: no matching Packages file for component %s, arch %s" % (component, arch)
+ if exists(torrent_file):
+ remove(torrent_file)
+ continue
+ packages.pop(packages.index(filename))
+
+ # Get the old data for this torrent, if any existed
+ print torrent_file + ": reading ...",
+ sys.stdout.flush()
+ old_pieces, headers = get_old(torrent_file)
+
+ # Update the headers from the Release file ones
+ if update_headers(headers, release_headers, component, arch):
+ print "new torrent created ...",
+ sys.stdout.flush()
+ old_pieces = {}
+
+ # Parse the Packages file for the new data
+ print "updating ...",
+ sys.stdout.flush()
+ pieces, new_pieces = get_new(filename, old_pieces, old_all_pieces,
+ all_pieces, all_new_pieces)
+
+ if pieces or new_pieces:
+ # Add any new pieces to the end of pieces
+ add_new(pieces, new_pieces, headers)
+
+ # Write the headers
+ print "writing ...",
+ sys.stdout.flush()
+ write_file(torrent_file, pieces, headers)
+ else:
+ print "empty ...",
+ if exists(torrent_file):
+ remove(torrent_file)
+
+ print "done."
+
+ print all_file + ": reading ...",
+ if new_all_torrent:
+ print "new torrent created ...",
sys.stdout.flush()
- old_files, headers = get_old(torrent_file)
-
- # Create the headers
- headers.setdefault("OriginalDate", date)
- headers["Date"] = date
- headers.setdefault("PieceSize", str(default_piecesize))
- headers.setdefault("NextPiece", str(0))
- headers["Codename"] = codename
- headers["Suite"] = suite
- headers["Component"] = component
- headers["Architecture"] = arch
- headers.setdefault("TorrentHashFields", " ".join(default_hash_fields))
+ # If there were 'all' files found
+ if all_pieces or all_new_pieces:
+ # Process the new 'all' files found
+ print "updating ...",
+ sys.stdout.flush()
+ add_new(all_pieces, all_new_pieces, all_headers)
- # Calculate the new hash
- sha1 = sha.new()
- for header in headers["TorrentHashFields"].split():
- sha1.update(headers[header])
- new_hash = sha1.hexdigest()
-
- # Check if the hash has changed
- if headers.get("Torrent", "") != new_hash:
- # If it has, then reset the torrent
- print "new torrent created ...",
- sys.stdout.flush()
- old_files = {}
- headers["OriginalDate"] = date
- headers["NextPiece"] = str(0)
- headers.pop("OriginalPieces", "")
- sha1 = sha.new()
- for header in headers["TorrentHashFields"].split():
- sha1.update(headers[header])
- headers["Torrent"] = sha1.hexdigest()
-
- # Parse the Packages file for the new data
- print "updating ...",
- sys.stdout.flush()
- new_pieces = get_new(filename, old_files, headers, old_all_files,
- all_pieces, all_new_pieces)
-
- # Set the final header values
- headers.setdefault("OriginalPieces", headers["NextPiece"])
-
- if new_pieces:
- # Write the headers
+ # Write the all_headers
print "writing ...",
sys.stdout.flush()
- f = gzip.open(torrent_file, 'w')
- for header in header_order:
- if header in headers:
- f.write("%s: %s\n" % (header, headers[header]))
- f.write("PieceNumbers:\n")
-
- # Write the starting piece numbers
- pieces = new_pieces.keys()
- pieces.sort()
- format_string = " %"+str(len(str(max(pieces))))+"d %s\n"
- for piece in pieces:
- f.write(format_string % (piece, new_pieces[piece]))
-
- f.close()
+ write_file(all_file, all_pieces, all_headers)
else:
print "empty ...",
- if exists(torrent_file):
- remove(torrent_file)
-
+ if exists(all_file):
+ remove(all_file)
+
print "done."
-
- print all_file + ": reading ...",
- if new_all_torrent:
- print "new torrent created ...",
- sys.stdout.flush()
- # If there were 'all' files found
- if all_pieces or all_new_pieces:
- # Process the new 'all' files found
- print "updating ...",
- sys.stdout.flush()
- next_piece = int(all_headers["NextPiece"])
- piece_size = int(all_headers["PieceSize"])
- all_new_pieces.sort()
- old_file = ""
- old_size = 0L
- for (file, size) in all_new_pieces:
- if file == old_file:
- if size != old_size:
- print "WARNING: multiple architecture:all files with different size:", file
- else:
- all_pieces[next_piece] = file
- next_piece += int(ceil(size/float(piece_size)))
-
- old_file = file
- old_size = size
-
- # Set the final header values
- all_headers["NextPiece"] = str(next_piece)
- all_headers.setdefault("OriginalPieces", all_headers["NextPiece"])
-
- # Write the all_headers
- print "writing ...",
- sys.stdout.flush()
- f = gzip.open(all_file, 'w')
- for header in header_order:
- if header in all_headers:
- f.write("%s: %s\n" % (header, all_headers[header]))
- f.write("PieceNumbers:\n")
-
- # Write the all starting piece numbers
- pieces = all_pieces.keys()
- pieces.sort()
- format_string = " %"+str(len(str(max(pieces))))+"d %s\n"
- for piece in pieces:
- f.write(format_string % (piece, all_pieces[piece]))
-
- f.close()
+
+ if packages:
+ print "The following packages files were not used:"
+ for package in packages:
+ print " %s" % package
+
+if __name__ == '__main__':
+ if len(sys.argv) >= 2:
+ for file in sys.argv[1:]:
+ run(file)
else:
- print "empty ...",
- if exists(all_file):
- remove(all_file)
-
- print "done."
-
-if packages:
- print "The following packages files were not used:"
- for package in packages:
- print " %s" % package
-
-
-# fnkey = filename + ":pc"
-# if cache.has_key(fnkey):
-# sha1, result = str2hash(cache[fnkey])
-# cache[fnkey] = values
-#cache.sync()
-#cache.close()
+ print "Usage: " + sys.argv[0] + " Releasefile [Releasefile ...]"
More information about the Debtorrent-commits
mailing list