[Debtorrent-commits] r11 - in /debtorrent/trunk: ./
DebTorrent/BT1/FileSelector.py
DebTorrent/BT1/Statistics.py DebTorrent/BT1/Storage.py
DebTorrent/BT1/StorageWrapper.py DebTorrent/download_bt1.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Thu Apr 26 06:10:21 UTC 2007
Author: camrdale-guest
Date: Thu Apr 26 06:10:21 2007
New Revision: 11
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=11
Log:
Add some variable-size piece stuff (see TODOs for work still needed)
Modified:
debtorrent/trunk/ (props changed)
debtorrent/trunk/DebTorrent/BT1/FileSelector.py
debtorrent/trunk/DebTorrent/BT1/Statistics.py
debtorrent/trunk/DebTorrent/BT1/Storage.py
debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py
debtorrent/trunk/DebTorrent/download_bt1.py
Propchange: debtorrent/trunk/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Thu Apr 26 06:10:21 2007
@@ -1,0 +1,1 @@
+tags
Modified: debtorrent/trunk/DebTorrent/BT1/FileSelector.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/FileSelector.py?rev=11&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/FileSelector.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/FileSelector.py Thu Apr 26 06:10:21 2007
@@ -10,7 +10,7 @@
False = 0
-class FileSelector:
+class FileSelector: # TODO change piece_length to piece_lengths array
def __init__(self, files, piece_length, bufferdir,
storage, storagewrapper, sched, failfunc):
self.files = files
Modified: debtorrent/trunk/DebTorrent/BT1/Statistics.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Statistics.py?rev=11&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Statistics.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Statistics.py Thu Apr 26 06:10:21 2007
@@ -31,7 +31,7 @@
self.placesopen = None
self.storage_totalpieces = len(self.storage.hashes)
-
+ # TODO change piece_length to piece_lengths array
def set_dirstats(self, files, piece_length):
self.piecescomplete = 0
self.placesopen = 0
Modified: debtorrent/trunk/DebTorrent/BT1/Storage.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Storage.py?rev=11&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Storage.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Storage.py Thu Apr 26 06:10:21 2007
@@ -32,19 +32,22 @@
pass
class Storage:
- def __init__(self, files, piece_length, doneflag, config,
+ def __init__(self, files, piece_lengths, datalength, doneflag, config,
disabled_files = None):
# can raise IOError and ValueError
self.files = files
- self.piece_length = piece_length
+ self.piece_lengths = piece_lengths
self.doneflag = doneflag
self.disabled = [False] * len(files)
self.file_ranges = []
+ self.file_pieces = []
self.disabled_ranges = []
self.working_ranges = []
numfiles = 0
total = 0l
so_far = 0l
+ cur_piece = 0l
+ piece_total = 0l
self.handles = {}
self.whandles = {}
self.tops = {}
@@ -68,12 +71,26 @@
if length == 0:
self.file_ranges.append(None)
self.working_ranges.append([])
+ if total == piece_total and cur_piece > 0:
+ self.file_pieces.append((cur_piece-1, cur_piece-1))
+ else:
+ self.file_pieces.append((cur_piece, cur_piece))
else:
range = (total, total + length, 0, file)
self.file_ranges.append(range)
self.working_ranges.append([range])
numfiles += 1
total += length
+ start_piece = cur_piece
+ for cur_piece in xrange(start_piece,len(self.piece_lengths)+1):
+ if piece_total >= total:
+ break
+ piece_total += self.piece_lengths[cur_piece]
+ end_piece = cur_piece-1
+ if piece_total > total:
+ cur_piece -= 1
+ piece_total -= self.piece_lengths[cur_piece]
+ self.file_pieces.append((start_piece, end_piece))
if disabled_files[i]:
l = 0
else:
@@ -353,61 +370,17 @@
if r:
return r
start, end, offset, file = self.file_ranges[f]
+ start_piece, end_piece = self.file_pieces[f]
if DEBUG:
print 'calculating disabled range for '+self.files[f][0]
print 'bytes: '+str(start)+'-'+str(end)
- print 'file spans pieces '+str(int(start/self.piece_length))+'-'+str(int((end-1)/self.piece_length)+1)
- pieces = range( int(start/self.piece_length),
- int((end-1)/self.piece_length)+1 )
+ print 'file spans pieces '+str(start_piece)+'-'+str(end_piece)
+ pieces = range(start_piece, end_piece+1)
offset = 0
disabled_files = []
- if len(pieces) == 1:
- if ( start % self.piece_length == 0
- and end % self.piece_length == 0 ): # happens to be a single,
- # perfect piece
- working_range = [(start, end, offset, file)]
- update_pieces = []
- else:
- midfile = os.path.join(self.bufferdir,str(f))
- working_range = [(start, end, 0, midfile)]
- disabled_files.append((midfile, start, end))
- length = end - start
- self.sizes[midfile] = length
- piece = pieces[0]
- update_pieces = [(piece, start-(piece*self.piece_length), length)]
- else:
- update_pieces = []
- if start % self.piece_length != 0: # doesn't begin on an even piece boundary
- end_b = pieces[1]*self.piece_length
- startfile = os.path.join(self.bufferdir,str(f)+'b')
- working_range_b = [ ( start, end_b, 0, startfile ) ]
- disabled_files.append((startfile, start, end_b))
- length = end_b - start
- self.sizes[startfile] = length
- offset = length
- piece = pieces.pop(0)
- update_pieces.append((piece, start-(piece*self.piece_length), length))
- else:
- working_range_b = []
- if f != len(self.files)-1 and end % self.piece_length != 0:
- # doesn't end on an even piece boundary
- start_e = pieces[-1] * self.piece_length
- endfile = os.path.join(self.bufferdir,str(f)+'e')
- working_range_e = [ ( start_e, end, 0, endfile ) ]
- disabled_files.append((endfile, start_e, end))
- length = end - start_e
- self.sizes[endfile] = length
- piece = pieces.pop(-1)
- update_pieces.append((piece, 0, length))
- else:
- working_range_e = []
- if pieces:
- working_range_m = [ ( pieces[0]*self.piece_length,
- (pieces[-1]+1)*self.piece_length,
- offset, file ) ]
- else:
- working_range_m = []
- working_range = working_range_b + working_range_m + working_range_e
+ # Dramatically simplified since files always start and end on piece boundaries
+ working_range = [(start, end, offset, file)]
+ update_pieces = []
if DEBUG:
print str(working_range)
@@ -529,10 +502,10 @@
if not r:
continue
start, end, offset, file =r
+ start_piece, end_piece = self.file_pieces[i]
if DEBUG:
print 'adding '+file
- for p in xrange( int(start/self.piece_length),
- int((end-1)/self.piece_length)+1 ):
+ for p in xrange(start_piece, end_piece+1):
valid_pieces[p] = 1
if DEBUG:
@@ -550,16 +523,7 @@
for i in xrange(len(self.files)):
if self.disabled[i]:
- for file, start, end in self._get_disabled_ranges(i)[2]:
- f1 = basename(file)
- if ( not pfiles.has_key(f1)
- or not test(pfiles[f1],getsize(file),getmtime(file)) ):
- if DEBUG:
- print 'removing '+file
- for p in xrange( int(start/self.piece_length),
- int((end-1)/self.piece_length)+1 ):
- if valid_pieces.has_key(p):
- del valid_pieces[p]
+ # Simplified due to file boundaries being piece boundaries
continue
file, size = self.files[i]
if not size:
@@ -567,10 +531,10 @@
if ( not files.has_key(i)
or not test(files[i],getsize(file),getmtime(file)) ):
start, end, offset, file = self.file_ranges[i]
+ start_piece, end_piece = self.file_pieces[i]
if DEBUG:
print 'removing '+file
- for p in xrange( int(start/self.piece_length),
- int((end-1)/self.piece_length)+1 ):
+ for p in xrange(start_piece, end_piece+1):
if valid_pieces.has_key(p):
del valid_pieces[p]
except:
Modified: debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py?rev=11&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py Thu Apr 26 06:10:21 2007
@@ -64,7 +64,7 @@
return self.state
-class StorageWrapper:
+class StorageWrapper: # TODO change piece_size to piece_lengths array
def __init__(self, storage, request_size, hashes,
piece_size, finished, failed,
statusfunc = dummy_status, flag = fakeflag(), check_hashes = True,
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=11&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Thu Apr 26 06:10:21 2007
@@ -336,6 +336,7 @@
self.info = self.response['info']
self.pieces = [self.info['pieces'][x:x+20]
for x in xrange(0, len(self.info['pieces']), 20)]
+ self.piece_lengths = self.info['piece lengths']
self.len_pieces = len(self.pieces)
self.argslistheader = argslistheader
self.unpauseflag = Event()
@@ -463,7 +464,7 @@
self.choker.set_round_robin_period(
max( self.config['round_robin_period'],
self.config['round_robin_period'] *
- self.info['piece length'] / 200000 ) )
+ (self.datalength / len(self.piece_lengths)) / 200000 ) )
self.rerequest_complete()
self.finfunc()
@@ -513,7 +514,7 @@
try:
try:
- self.storage = Storage(self.files, self.info['piece length'],
+ self.storage = Storage(self.files, self.piece_lengths, self.datalength
self.doneflag, self.config, disabled_files)
except IOError, e:
self.errorfunc('trouble accessing files - ' + str(e))
@@ -522,7 +523,7 @@
return None
self.storagewrapper = StorageWrapper(self.storage, self.config['download_slice_size'],
- self.pieces, self.info['piece length'], self._finished, self._failed,
+ self.pieces, self.piece_lengths, self.datalength, self._finished, self._failed,
statusfunc, self.doneflag, self.config['check_hashes'],
self._data_flunked, self.rawserver.add_task,
self.config, self.unpauseflag)
@@ -535,7 +536,7 @@
return None
if self.selector_enabled:
- self.fileselector = FileSelector(self.files, self.info['piece length'],
+ self.fileselector = FileSelector(self.files, self.piece_lengths, self.datalength,
self.appdataobj.getPieceDir(self.infohash),
self.storage, self.storagewrapper,
self.rawserver.add_task,
@@ -701,7 +702,7 @@
self.connecter, self.httpdownloader, self.ratelimiter,
self.rerequest_lastfailed, self.filedatflag)
if self.info.has_key('files'):
- self.statistics.set_dirstats(self.files, self.info['piece length'])
+ self.statistics.set_dirstats(self.files, self.piece_lengths, self.datalength)
if self.config['spew']:
self.spewflag.set()
More information about the Debtorrent-commits
mailing list