r335 - in /debtorrent/trunk: ./ DebTorrent/ DebTorrent/BT1/
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Sat Jan 19 22:05:46 UTC 2008
Author: camrdale-guest
Date: Sat Jan 19 22:05:46 2008
New Revision: 335
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=335
Log:
Merged revisions 229-236 via svnmerge from
svn+ssh://camrdale-guest@svn.debian.org/svn/debtorrent/debtorrent/branches/unique
........
r229 | camrdale-guest | 2007-08-13 12:36:36 -0700 (Mon, 13 Aug 2007) | 2 lines
Fix bug that didn't add empty file to end of torrent if there were empty pieces there.
btmakemetafile works with gzipped files now.
........
r230 | camrdale-guest | 2007-08-13 12:37:34 -0700 (Mon, 13 Aug 2007) | 1 line
Hexify btshowmetainfo's printing of the new identifier field.
........
r231 | camrdale-guest | 2007-08-13 15:08:25 -0700 (Mon, 13 Aug 2007) | 1 line
Add better error checking when creating torrents.
........
r232 | camrdale-guest | 2007-08-13 15:12:05 -0700 (Mon, 13 Aug 2007) | 1 line
Add better logging to some of the scripts.
........
r233 | camrdale-guest | 2007-08-13 15:17:46 -0700 (Mon, 13 Aug 2007) | 1 line
Update modules that deal with files/pieces to ignore zero length ones.
........
r234 | camrdale-guest | 2007-08-13 17:51:19 -0700 (Mon, 13 Aug 2007) | 1 line
Keep old unique piece orderings so out-of-date mirrors will work.
........
r235 | camrdale-guest | 2007-08-13 21:17:09 -0700 (Mon, 13 Aug 2007) | 2 lines
Rewrite hippy for readability.
Calculate rather than simulate optimal piece size.
........
r236 | camrdale-guest | 2007-08-13 22:14:24 -0700 (Mon, 13 Aug 2007) | 1 line
Various minor fixes.
........
Modified:
debtorrent/trunk/ (props changed)
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/FileSelector.py
debtorrent/trunk/DebTorrent/BT1/Storage.py
debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py
debtorrent/trunk/DebTorrent/BT1/btformats.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/btcompletedir.py
debtorrent/trunk/btmakemetafile.py
debtorrent/trunk/btshowmetainfo.py
debtorrent/trunk/hippy.py
debtorrent/trunk/test.py
debtorrent/trunk/uniquely.py
Propchange: debtorrent/trunk/
------------------------------------------------------------------------------
--- svnmerge-integrated (original)
+++ svnmerge-integrated Sat Jan 19 22:05:46 2008
@@ -1,1 +1,1 @@
-/debtorrent/branches/http1.1:1-257 /debtorrent/branches/unique:1-204,209-213,216-217,219,222-225,227
+/debtorrent/branches/http1.1:1-257 /debtorrent/branches/unique:1-204,209-213,216-217,219,222-225,227,229-236
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Sat Jan 19 22:05:46 2008
@@ -441,6 +441,11 @@
logger.warning('Unable to find the file in any torrents: '+'/'.join(path))
return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+ # Check if the torrent is running
+ if d.doneflag.isSet():
+ logger.error('The needed torrent is not running')
+ return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
+
# Check if the file has already been downloaded
data = ''
pieces_needed = []
@@ -456,11 +461,6 @@
if not pieces_needed:
return (200, 'OK', {'Server': VERSION, 'Content-Type': 'text/plain'}, data)
- # Check if the torrent is running/not paused
- if d.doneflag.isSet():
- logger.error('The needed torrent is not running')
- return (404, 'Not Found', {'Server': VERSION, 'Content-Type': 'text/plain', 'Pragma': 'no-cache'}, alas)
-
if not d.unpauseflag.isSet():
d.Unpause()
Modified: debtorrent/trunk/DebTorrent/BT1/FileSelector.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/FileSelector.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/FileSelector.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/FileSelector.py Sat Jan 19 22:05:46 2008
@@ -111,7 +111,17 @@
cur_piece = 0
for file, length in files:
if not length:
- self.filepieces.append(())
+ # Assign any zero-length pieces to the empty file
+ start_piece = cur_piece
+ for cur_piece in xrange(start_piece,len(piece_lengths)):
+ if piece_lengths[cur_piece] > 0:
+ break
+ end_piece = cur_piece-1
+ if piece_lengths[cur_piece] == 0:
+ # Special case if the last file in a torrent is empty
+ end_piece = cur_piece
+ pieces = range(start_piece,end_piece+1)
+ self.filepieces.append(tuple(pieces))
else:
total += length
start_piece = cur_piece
@@ -153,7 +163,7 @@
for f in xrange(self.numfiles):
if init_priority[f] < 0:
self.storage.disable_file(f)
- else:
+ elif self.files[f][1] > 0:
self.storage.enable_file(f)
self.storage.reset_file_status()
self.init_priority = init_priority
@@ -268,7 +278,7 @@
new_disabled = [p == -1 for p in new_priority]
data_to_update = []
for f in xrange(self.numfiles):
- if new_disabled[f] != old_disabled[f]:
+ if new_disabled[f] != old_disabled[f] and self.files[f][1] > 0:
data_to_update.extend(self.storage.get_piece_update_list(f))
buffer = []
for piece, start, length in data_to_update:
@@ -284,7 +294,7 @@
if new_disabled[f] and not old_disabled[f]:
self.storage.disable_file(f)
files_updated = True
- if old_disabled[f] and not new_disabled[f]:
+ if old_disabled[f] and not new_disabled[f] and self.files[f][1] > 0:
self.storage.enable_file(f)
files_updated = True
except (IOError, OSError), e:
@@ -320,7 +330,7 @@
"""
l = [-1] * self.numpieces
for f in xrange(self.numfiles):
- if file_priority_list[f] == -1:
+ if file_priority_list[f] == -1 or self.files[f][1] == 0:
continue
for i in self.filepieces[f]:
if l[i] == -1:
Modified: debtorrent/trunk/DebTorrent/BT1/Storage.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Storage.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Storage.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Storage.py Sat Jan 19 22:05:46 2008
@@ -181,10 +181,18 @@
if length == 0:
self.file_ranges.append(None)
self.working_ranges.append([])
- if total == piece_total and cur_piece > 0:
- self.file_pieces.append((cur_piece-1, cur_piece-1))
- else:
- self.file_pieces.append((cur_piece, cur_piece))
+
+ # Assign any zero-length pieces to the empty file
+ start_piece = cur_piece
+ for cur_piece in xrange(start_piece,len(self.piece_lengths)):
+ if self.piece_lengths[cur_piece] > 0:
+ break
+ self.piece_files[cur_piece] = (0L, 0L, 0L, '')
+ end_piece = cur_piece-1
+ if self.piece_lengths[cur_piece] == 0:
+ # Special case if the last file in a torrent is empty
+ end_piece = cur_piece
+ self.file_pieces.append((start_piece, end_piece))
else:
range = (total, total + length, 0, file)
self.file_ranges.append(range)
Modified: debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py Sat Jan 19 22:05:46 2008
@@ -396,7 +396,7 @@
self.bgalloc_active = False
self.total_length = storage.get_total_length()
self.amount_left = self.total_length
- if self.total_length <= self.datalength - self.piece_sizes[-1]:
+ if self.total_length < self.datalength:
raise ValueError, 'bad data in responsefile - total too small'
if self.total_length > self.datalength:
raise ValueError, 'bad data in responsefile - total too big'
@@ -532,6 +532,8 @@
assert not got.has_key(v)
got[v] = 1
for i in xrange(len(self.hashes)):
+ if self.piece_sizes[i] == 0:
+ continue
if self.places.has_key(i): # restored from pickled
self.check_targets[self.hashes[i]] = []
if self.places[i] == i:
@@ -864,6 +866,7 @@
"""
length = self._piecelen(index)
+ assert length > 0
l = []
x = 0
while x + self.request_size < length:
@@ -929,7 +932,7 @@
Random bits are removed from the bitfield and added to the list of haves.
@rtype: (C{string}, C{list} of C{int})
- @return: the incomplete bitfiled as a binary string, and the haves to
+ @return: the incomplete bitfield as a binary string, and the haves to
complete it
"""
@@ -1263,6 +1266,7 @@
"""
assert not self.have[index]
+ assert self.piece_sizes[index] > 0
if not self.places.has_key(index):
while self._clear_space(index):
@@ -1567,10 +1571,7 @@
Pickled data format::
- d['pieces'] = either a string containing a bitfield of complete pieces,
- or the numeric value "1" signifying a seed. If it is
- a seed, d['places'] and d['partials'] should be empty
- and needn't even exist.
+ d['pieces'] = a string containing a bitfield of complete pieces
d['partials'] = [ piece, [ offset, length... ]... ]
a list of partial data that had been previously
downloaded, plus the given offsets. Adjacent partials
@@ -1592,8 +1593,6 @@
"""
- if self.have.complete():
- return {'pieces': 1}
pieces = Bitfield(len(self.hashes))
places = []
partials = []
@@ -1650,26 +1649,20 @@
restored_partials = []
try:
- if data['pieces'] == 1: # a seed
- assert not data.get('places',None)
- assert not data.get('partials',None)
- have = Bitfield(len(self.hashes))
- for i in xrange(len(self.hashes)):
+ have = Bitfield(len(self.hashes))
+ old_have = Bitfield(len(self.hashes), data['pieces'])
+ for i in xrange(len(self.hashes)):
+ if old_have[i] and self.piece_sizes[i] > 0:
have[i] = True
- assert have.complete()
- _places = []
- _partials = []
- else:
- have = Bitfield(len(self.hashes), data['pieces'])
- _places = data['places']
- assert len(_places) % 2 == 0
- _places = [_places[x:x+2] for x in xrange(0,len(_places),2)]
- _partials = data['partials']
- assert len(_partials) % 2 == 0
- _partials = [_partials[x:x+2] for x in xrange(0,len(_partials),2)]
+ _places = data['places']
+ assert len(_places) % 2 == 0
+ _places = [_places[x:x+2] for x in xrange(0,len(_places),2)]
+ _partials = data['partials']
+ assert len(_partials) % 2 == 0
+ _partials = [_partials[x:x+2] for x in xrange(0,len(_partials),2)]
for index, place in _places:
- if place not in valid_places:
+ if place not in valid_places or self.piece_sizes[index] == 0:
continue
assert not got.has_key(index)
assert not got.has_key(place)
@@ -1696,6 +1689,8 @@
for index, plist in _partials:
assert not dirty.has_key(index)
assert not have[index]
+ if self.piece_sizes[index] == 0:
+ continue
if not places.has_key(index):
if index not in valid_places:
continue
Modified: debtorrent/trunk/DebTorrent/BT1/btformats.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/btformats.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/btformats.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/btformats.py Sat Jan 19 22:05:46 2008
@@ -44,41 +44,45 @@
piecelengths = info.get('piece lengths')
if type(piecelengths) != ListType:
raise ValueError, 'bad metainfo - bad piece lengths list'
- total_length = 0L
- piece_bounds = [0L]
- for length in piecelengths:
- if type(length) not in ints or length < 0:
- raise ValueError, 'bad metainfo - bad piece length'
- total_length += length
- piece_bounds.append(total_length)
- if info.has_key('files') == info.has_key('length'):
- raise ValueError, 'single/multiple file mix'
files = info.get('files')
if type(files) != ListType:
raise ValueError, 'bad metainfo - bad files list'
- total_length = 0L
- for f in files:
- if type(f) != DictType:
- raise ValueError, 'bad metainfo - bad file value'
- length = f.get('length')
- if type(length) not in ints or length < 0:
- raise ValueError, 'bad metainfo - bad length'
- total_length += length
- if piece_bounds[bisect(piece_bounds,total_length)-1] != total_length:
+ total_piece_length = 0L
+ total_file_length = 0L
+ file_length = -1L
+ cur_file = -1
+ for piece_length in piecelengths:
+ if type(piece_length) not in ints or piece_length < 0:
+ raise ValueError, 'bad metainfo - bad piece length'
+
+ if total_piece_length == total_file_length and (piece_length != 0 or file_length != 0):
+ cur_file += 1
+ f = files[cur_file]
+ if type(f) != DictType:
+ raise ValueError, 'bad metainfo - bad file value'
+ old_file_length = file_length
+ file_length = f.get('length')
+ if type(file_length) not in ints or file_length < 0:
+ raise ValueError, 'bad metainfo - bad length'
+ if old_file_length == 0 and file_length == 0:
+ raise ValueError, 'bad metainfo - cannot have sequential files of length 0'
+ total_file_length += file_length
+ path = f.get('path')
+ if type(path) != ListType:
+ raise ValueError, 'bad metainfo - bad path'
+ for p in path:
+ if type(p) != StringType:
+ raise ValueError, 'bad metainfo - bad path dir'
+ if not reg.match(p):
+ raise ValueError, 'path %s disallowed for security reasons' % p
+
+ total_piece_length += piece_length
+ if piece_length > 0 and file_length == 0:
+ raise ValueError, 'bad metainfo - piece length should be 0 for file of length 0'
+ if file_length > 0 and piece_length == 0:
+ raise ValueError, 'bad metainfo - file of length greater than 0 can not contain pieces of length 0'
+ if total_piece_length > total_file_length:
raise ValueError, 'bad metainfo - file does not end on piece boundary'
- path = f.get('path')
- if type(path) != ListType:
- raise ValueError, 'bad metainfo - bad path'
- for p in path:
- if type(p) != StringType:
- raise ValueError, 'bad metainfo - bad path dir'
- if not reg.match(p):
- raise ValueError, 'path %s disallowed for security reasons' % p
-# Removed to speed up checking of large files
-# for i in xrange(len(files)):
-# for j in xrange(i):
-# if files[i]['path'] == files[j]['path']:
-# raise ValueError, 'bad metainfo - duplicate path'
def check_message(message):
"""Checks the metainfo dictionary for conformance.
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Sat Jan 19 22:05:46 2008
@@ -24,12 +24,12 @@
from copy import copy
from string import strip
from DebTorrent.bencode import bencode
-from btformats import check_info
+from btformats import check_info, check_message
from threading import Event, Thread
from time import time
from traceback import print_exc
from DebTorrent.zurllib import urlopen
-from gzip import GzipFile
+import gzip
from bz2 import decompress
from StringIO import StringIO
from re import subn
@@ -309,7 +309,10 @@
if pieces_file:
try:
- f = open(pieces_file)
+ if torrent_file.endswith('.gz'):
+ f = gzip.open(pieces_file)
+ else:
+ f = open(pieces_file)
except:
logger.exception('sub-pieces file not found: '+pieces_file)
return {}
@@ -402,7 +405,10 @@
if torrent_file:
try:
- f = open(torrent_file)
+ if torrent_file.endswith('.gz'):
+ f = gzip.open(torrent_file)
+ else:
+ f = open(torrent_file)
except:
logger.exception('torrent ordering file not found: '+torrent_file)
return (pieces, headers)
@@ -508,6 +514,10 @@
ordered_pieces.sort()
cur_piece = 0
for next_piece in ordered_pieces:
+ if piece_ordering[next_piece] not in pieces:
+ # Skip files we don't know about
+ continue
+
if next_piece > cur_piece:
fs_list.append({'length': 0, 'path': []})
cur_piece = next_piece
@@ -521,6 +531,9 @@
lengths[piece_ordering[cur_piece]]
fs_list.append(fs[piece_ordering[cur_piece]])
cur_piece = cur_piece + len_pieces
+ if num_pieces > cur_piece:
+ fs_list.append({'length': 0, 'path': []})
+ cur_piece = num_pieces
else:
pieces_list = []
lengths_list = []
@@ -682,7 +695,10 @@
ordering_all_headers = {}
file = abspath(file)
- f = open(file)
+ if file.endswith('.gz'):
+ f = gzip.open(file)
+ else:
+ f = open(file)
(info, info_all) = getpieces(f, encoding, progress, separate_all, sub_pieces,
piece_ordering, piece_ordering_all,
int(ordering_headers.get('NextPiece', 0)),
@@ -816,22 +832,27 @@
return
logger.debug('Packages file successfully decompressed')
- sub_pieces = getsubpieces('_'.join(self.path))
-
- (piece_ordering, ordering_headers) = getordering('_'.join(path))
- if self.config['separate_all']:
- (piece_ordering_all, ordering_all_headers) = getordering('_'.join(path), all = True)
- else:
- piece_ordering_all = {}
- ordering_all_headers = {}
-
- (info, info_all) = getpieces(h, separate_all = self.config['separate_all'],
- sub_pieces = sub_pieces,
- piece_ordering = piece_ordering,
- piece_ordering_all = piece_ordering_all,
- num_pieces = int(ordering_headers.get('NextPiece', 0)),
- num_all_pieces = int(ordering_all_headers.get('NextPiece', 0)))
- del h[:]
+ try:
+ sub_pieces = getsubpieces('_'.join(self.path))
+
+ (piece_ordering, ordering_headers) = getordering('_'.join(path))
+ if self.config['separate_all']:
+ (piece_ordering_all, ordering_all_headers) = getordering('_'.join(path), all = True)
+ else:
+ piece_ordering_all = {}
+ ordering_all_headers = {}
+
+ (info, info_all) = getpieces(h, separate_all = self.config['separate_all'],
+ sub_pieces = sub_pieces,
+ piece_ordering = piece_ordering,
+ piece_ordering_all = piece_ordering_all,
+ num_pieces = int(ordering_headers.get('NextPiece', 0)),
+ num_all_pieces = int(ordering_all_headers.get('NextPiece', 0)))
+ del h[:]
+ except:
+ logger.exception('Failed to create torrent for: %s', self.name)
+ del h[:]
+ return
name = self.name
if info and self.config['separate_all'] in (0, 2, 3):
@@ -880,6 +901,11 @@
"""Wrap up the creation and call the callback function."""
for (response, name) in self.responses:
- self.callback(response, name, self.path)
+ try:
+ check_message(response)
+ except:
+ logger.exception('Poorly formatted torrent, not starting')
+ else:
+ self.callback(response, name, self.path)
del self.responses[:]
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Sat Jan 19 22:05:46 2008
@@ -755,10 +755,13 @@
files = []
for x in self.info['files']:
- n = file
- for i in x['path']:
- n = path.join(n, i)
- files.append((n, x['length']))
+ if x['path']:
+ n = file
+ for i in x['path']:
+ n = path.join(n, i)
+ files.append((n, x['length']))
+ else:
+ files.append(('', 0L))
#Move directory creation to Storage
#make(n)
except OSError, e:
Modified: debtorrent/trunk/btcompletedir.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btcompletedir.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/btcompletedir.py (original)
+++ debtorrent/trunk/btcompletedir.py Sat Jan 19 22:05:46 2008
@@ -26,6 +26,10 @@
from os.path import split
from DebTorrent.BT1.makemetafile import defaults, completedir, print_announcelist_details
from DebTorrent.parseargs import parseargs, formatDefinitions
+import logging
+
+logging.basicConfig()
+logger = logging.getLogger()
def prog(amount):
"""Display the current status of the file scan.
@@ -63,5 +67,6 @@
for dir in args:
completedir(dir, config, vc = prog, fc = next_file)
except ValueError, e:
+ logger.exception(str(e))
print 'error: ' + str(e)
print 'run with no args for parameter explanations'
Modified: debtorrent/trunk/btmakemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btmakemetafile.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/btmakemetafile.py (original)
+++ debtorrent/trunk/btmakemetafile.py Sat Jan 19 22:05:46 2008
@@ -28,6 +28,10 @@
assert version_info >= (2,3), 'Requires Python 2.3 or better'
from DebTorrent.BT1.makemetafile import make_meta_file, defaults, print_announcelist_details
from DebTorrent.parseargs import parseargs, formatDefinitions
+import logging
+
+logging.basicConfig()
+logger = logging.getLogger()
def prog(amount):
"""Display the current status of the file scan.
@@ -55,5 +59,6 @@
make_meta_file(file, config, progress = prog)
print ''
except ValueError, e:
+ logger.exception(str(e))
print 'error: ' + str(e)
print 'run with no args for parameter explanations'
Modified: debtorrent/trunk/btshowmetainfo.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btshowmetainfo.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/btshowmetainfo.py (original)
+++ debtorrent/trunk/btshowmetainfo.py Sat Jan 19 22:05:46 2008
@@ -12,10 +12,16 @@
from sys import *
from os.path import *
from sha import *
+from binascii import b2a_hex
+from DebTorrent.BT1.btformats import check_message
from DebTorrent.bencode import *
from DebTorrent.download_bt1 import get_packages
+import logging
assert version_info >= (2,3), 'Requires Python 2.3 or better'
+
+logging.basicConfig()
+logger = logging.getLogger()
NAME, EXT = splitext(basename(argv[0]))
VERSION = '20030621'
@@ -34,13 +40,22 @@
else:
metainfo_file = open(metainfo_name, 'rb')
metainfo = bdecode(metainfo_file.read())
-
+
+ try:
+ check_message(metainfo)
+ except:
+ logger.exception('Error found in metainfo of: %s', metainfo_name)
+ continue
+
# print metainfo
info = metainfo['info']
info_hash = sha(bencode(info)).hexdigest()
+ identifier = b2a_hex(metainfo.get('identifier', ''))
+ if not identifier:
+ identifier = info_hash
print 'metainfo file.: %s' % basename(metainfo_name)
- print 'identifier....: %s' % metainfo.get("identifier", info_hash)
+ print 'identifier....: %s' % identifier
print 'info hash.....: %s' % info_hash
piece_lengths = info['piece lengths']
print 'directory name: %s' % metainfo['name']
Modified: debtorrent/trunk/hippy.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/hippy.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/hippy.py (original)
+++ debtorrent/trunk/hippy.py Sat Jan 19 22:05:46 2008
@@ -1,83 +1,177 @@
#!/usr/bin/env python
+
+"""Calculate the sub-piece hashes for large package files."""
import bsddb, sha, binascii
import os, sys
-from gzip import GzipFile
-from StringIO import StringIO
+import struct
+from math import ceil
-def hash(file, piecesize):
- h = []
- fullh = sha.new()
- while 1:
- x = file.read(piecesize)
- if x == "": break
- h.append((sha.new(x).hexdigest(), len(x)))
- fullh.update(x)
- return (fullh.hexdigest(), h)
+MAX_PIECE_SIZE = 512*1024
+CHUNK_SIZE = 16*1024
-piecesize = 512*1024
-chunksize = 16*1024
+def hash(file, piece_size):
+ """Read a file and hash it's sub-pieces.
-def optimalpiecesize(size):
- def eval(s,c,m):
- b = m/c
- return [ i*c for i in range(int(b/2), b+1) if s - i*c*int(s/m) <= i*c ]
+ @type file: C{file}
+ @param file: an already opened file-like object to read from
+ @type piece_size: C{int}
+ @param piece_size: the piece size to divide the file into
+ @rtype: C{string}, C{list} of (C{string}, C{int})
+ @return: the 40-byte hex representation of the SHA1 hash of the file, and
+ the 40-byte hex representation of the SHA1 hash of the piece and the
+ length of the piece, for each sub-piece of the file
+
+ """
+
+ hashes = []
+ file_hash = sha.new()
- def score(s,c,m):
- l = int(s/m)
- return [ (abs(i - (s - l*i)), i) for i in eval(s,c,m) ]
+ while 1:
+ data = file.read(piece_size)
+ if data == "":
+ break
+
+ hashes.append((sha.new(data).hexdigest(), len(data)))
+ file_hash.update(data)
- def bestest(s,c,m):
- return min( score(s,c,m) )
+ return file_hash.hexdigest(), hashes
- return bestest(size,chunksize,piecesize)[1]
-cache_file = sys.argv[1]
-pieces = {}
+def optimal_piece_size(size):
+ """Calculate the optimal piece size to use for a file.
+
+ The optimal piece size is the largest possible piece size such that the
+ piece size is larger than the extra piece, the piece size is a multiple of
+ the chunk size, and the difference between the piece size and the extra
+ piece size is a minimum.
-cache = bsddb.btopen(cache_file, "w")
+ This function currently contains an error, as it returns a non-optimal
+ piece size when the size is a multiple of the maximum piece size. This
+ error is kept for backwards compatibility with previous versions. To
+ correct it::
+
+ n = 1 + (size-1) / MAX_PIECE_SIZE
+
+ @type size: C{long}
+ @param size: the file size
+ @rtype: C{int}
+ @return: the optimal piece size
+
+ """
+
+ n = 1 + size / MAX_PIECE_SIZE
+ return max(MAX_PIECE_SIZE/2, int(ceil((float(size)/n)/CHUNK_SIZE))*CHUNK_SIZE)
-def str2hash(s):
- r = []
- if s == "": return None, []
+def cache2hash(cache_value):
+ """Convert a list of sub-piece hashes to a cacheable value.
+
+ The cache is stored as a string. The first 20 bytes are the SHA1 hash of
+ the entire file. Then there are repeating 24 byte sequences, the first 4
+ bytes being the length of the piece in network (big-endian) order, the
+ next 20 bytes being the SHA1 hash of the piece. If there are no sub-pieces
+ for the file, the cached string is empty.
+
+ @type cache_value: C{string}
+ @param cache_value: the cached value for this file
+ @rtype: C{string}, C{list} of (C{string}, C{int})
+ @return: the 40-byte hex representation of the SHA1 hash of the file, and
+ the 40-byte hex representation of the SHA1 hash of the piece and the
+ length of the piece, for each sub-piece of the file
+
+ """
- fh,s = binascii.b2a_hex(s[:20]), s[20:]
- while len(s) > 0:
- (l,h,s) = s[:4], s[4:24], s[24:]
- r.append( (binascii.b2a_hex(h), long(binascii.b2a_hex(l), 16)) )
- return fh,r
+ if cache_value == "":
+ return None, []
-def hash2str(fh, hs):
- s = binascii.a2b_hex(fh)
- for (h, l) in hs:
- s += binascii.a2b_hex("%08x" % l) + binascii.a2b_hex(h)
- return s
+ piece_list = []
+ file_hash = binascii.b2a_hex(cache_value[:20])
+ cache_value = cache_value[20:]
+
+ while len(cache_value) > 0:
+ length = struct.unpack(">i", cache_value[:4])[0]
+ hash = binascii.b2a_hex(cache_value[4:24])
+ cache_value = cache_value[24:]
+ piece_list.append((hash, length))
+
+ return file_hash, piece_list
-for filename in sys.stdin:
+def hash2cache(sha1, piece_list):
+ """Convert a list of sub-piece hashes to a cacheable value.
+
+ @type sha1: C{string}
+ @param sha1: the 40-byte hex representation of the SHA1 hash of the file
+ @type piece_list: C{list} of (C{string}, C{int})
+ @param piece_list: for each sub-piece of the file, the 40-byte hex
+ representation of the SHA1 hash and the length of the piece
+ @rtype: C{string}
+ @return: the cacheable string
+
+ """
+
+ if not piece_list:
+ return ""
+
+ cache_value = binascii.a2b_hex(sha1)
+ for (hash, length) in piece_list:
+ cache_value += struct.pack(">i", length) + binascii.a2b_hex(hash)
+ return cache_value
+
+def sub_piece(cache, filename):
+ """Calculate and print the sub-pieces for a single file.
+
+ @type cache: C{bsddb.BTree}
+ @param cache: an already opened bDB b-tree
+ @type filename: C{String}
+ @param filename: the file to calculate sub pieces for
+
+ """
+
filename = filename.rstrip()
+
+ # Check if this file's sub-pieces are already known
fnkey = filename + ":pc"
if cache.has_key(fnkey):
- sha1, result = str2hash(cache[fnkey])
+ # Use the cached result
+ sha1, piece_list = cache2hash(cache[fnkey])
else:
- size = os.stat(filename).st_size
- if size <= piecesize:
- values = ""
- result = []
- else:
- ps = optimalpiecesize(size)
- file = open(filename)
- sha1, result = hash(file, ps)
- values = hash2str(sha1, result)
- file.close()
- cache[fnkey] = values
+ # Get the size fo the file
+ size = os.stat(filename).st_size
+
+ if size <= MAX_PIECE_SIZE:
+ # No sub-pieces are needed for this file
+ cache_value = ""
+ piece_list = []
+ else:
+ # Calculate all the sub-piece hashes
+ piece_size = optimal_piece_size(size)
+ file = open(filename)
+ sha1, piece_list = hash(file, piece_size)
+ cache_value = hash2cache(sha1, piece_list)
+ file.close()
+
+ # Save the result for next time
+ cache[fnkey] = cache_value
- if result:
- print "Filename: %s" % (filename)
- print "SHA1: %s" % (sha1)
- print "SHA1-Pieces:"
- for x in result:
+ if piece_list:
+ # Print the resulting sub-piece hashes
+ print "Filename: %s" % (filename)
+ print "SHA1: %s" % (sha1)
+ print "SHA1-Pieces:"
+ for x in piece_list:
print " %s %d" % x
- print ""
+ print ""
-cache.sync()
-cache.close()
+if __name__ == '__main__':
+
+ # Open the cache file specified on the command line
+ cache_file = sys.argv[1]
+ cache = bsddb.btopen(cache_file, "w")
+
+ # Read files to sub-piece from standard in
+ for filename in sys.stdin:
+ sub_piece(cache, filename)
+
+ # Close the cache file
+ cache.sync()
+ cache.close()
Modified: debtorrent/trunk/test.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/test.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/test.py (original)
+++ debtorrent/trunk/test.py Sat Jan 19 22:05:46 2008
@@ -267,6 +267,13 @@
(1, ['install', 'doc-iana']),
]),
+ '10': ('Test updating to a (possibly) out-of-date mirror.',
+ {1: []},
+ {1: (1, [], {'mirror': 'debian.mirror.iweb.ca/debian'})},
+ [(1, ['update']),
+ (1, ['install', 'aboot-base']),
+ ]),
+
}
assert 'all' not in tests
Modified: debtorrent/trunk/uniquely.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/uniquely.py?rev=335&op=diff
==============================================================================
--- debtorrent/trunk/uniquely.py (original)
+++ debtorrent/trunk/uniquely.py Sat Jan 19 22:05:46 2008
@@ -390,6 +390,10 @@
sys.stdout.flush()
pieces, new_pieces = get_new(filename, old_pieces, old_all_pieces,
all_pieces, all_new_pieces)
+
+ # Add the old removed pieces so out-of-date mirrors will work too
+ for file in old_pieces:
+ pieces[old_pieces[file]] = file
if pieces or new_pieces:
# Add any new pieces to the end of pieces
@@ -406,6 +410,10 @@
print "done."
+ # Add the old removed pieces so out-of-date mirrors will work too
+ for file in old_all_pieces:
+ all_pieces[old_all_pieces[file]] = file
+
print all_file + ": reading ...",
if new_all_torrent:
print "new torrent created ...",
More information about the Debtorrent-commits
mailing list