r190 - in /debtorrent/trunk/DebTorrent: BT1/Rerequester.py BT1/Statistics.py BT1/StorageWrapper.py BT1/StreamCheck.py BT1/T2T.py BT1/Uploader.py download_bt1.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Sun Jul 29 09:23:29 UTC 2007
Author: camrdale-guest
Date: Sun Jul 29 09:23:29 2007
New Revision: 190
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=190
Log:
More documentation, almost done!
Modified:
debtorrent/trunk/DebTorrent/BT1/Rerequester.py
debtorrent/trunk/DebTorrent/BT1/Statistics.py
debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py
debtorrent/trunk/DebTorrent/BT1/StreamCheck.py
debtorrent/trunk/DebTorrent/BT1/T2T.py
debtorrent/trunk/DebTorrent/BT1/Uploader.py
debtorrent/trunk/DebTorrent/download_bt1.py
Modified: debtorrent/trunk/DebTorrent/BT1/Rerequester.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Rerequester.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Rerequester.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Rerequester.py Sun Jul 29 09:23:29 2007
@@ -221,6 +221,7 @@
@param doneflag: the flag that indicates when the program is to be shutdown
@type unpauseflag: C{threading.Event}
@param unpauseflag: the flag to unset to pause the download
+ (optional, defaults to an always True dummy flag)
@type seededfunc: C{method}
@param seededfunc: method to call if the tracker reports the torrent
is seeded (optional, defaults to not checking)
@@ -705,9 +706,9 @@
class SuccessLock:
"""Locks to synchronize threaded requests to trackers.
- @type lock: C{threading.Lock)
+ @type lock: C{threading.Lock}
@ivar lock: lock to ensure no concurrent access to this object
- @type pause: C{threading.Lock)
+ @type pause: C{threading.Lock}
@ivar pause: lock to synchronize sending requests to trackers
@type code: C{long}
@ivar code: a unique code sent to each setter of the lock
Modified: debtorrent/trunk/DebTorrent/BT1/Statistics.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Statistics.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Statistics.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Statistics.py Sun Jul 29 09:23:29 2007
@@ -1,8 +1,10 @@
# Written by Edward Keyes
# Modified by Cameron Dale
# see LICENSE.txt for license information
-
+#
# $Id$
+
+"""Generate statistics for the swarm."""
from threading import Event
try:
@@ -12,12 +14,84 @@
False = 0
class Statistics_Response:
- pass # empty class
+ """Empty class to add arbitrary variables to."""
+ pass
class Statistics:
+ """Generate statistics for the swarm.
+
+ @type upmeasure: L{CurrentRateMeasure.Measure}
+ @ivar upmeasure: the measure of the upload rate
+ @type downmeasure: L{CurrentRateMeasure.Measure}
+ @ivar downmeasure: the measure of the download rate
+ @type connecter: L{Connecter.Connecter}
+ @ivar connecter: the Connecter instance that manages all the connections
+ @type httpdl: L{BT1.HTTPDownloader.HTTPDownloader}
+ @ivar httpdl: the backup HTTP downloader
+ @type ratelimiter: L{RateLimiter.RateLimiter}
+ @ivar ratelimiter: the RateLimiter instance to limit the upload rate
+ @type downloader: L{Downloader.Downloader}
+ @ivar downloader: the Downloader instance to use
+ @type picker: L{PiecePicker.PiecePicker}
+ @ivar picker: the PiecePicker instance
+ @type storage: L{StorageWrapper.StorageWrapper}
+ @ivar storage: the StorageWrapper instance
+ @type torrentmeasure: L{DebTorrent.CurrentRateMeasure.Measure}
+ @ivar torrentmeasure: for measuring the total download rate from all peers
+ @type rerequest_lastfailed: C{method}
+ @ivar rerequest_lastfailed: method to call to check if the last
+ tracker request failed
+ @type fdatflag: C{threading.Event}
+ @ivar fdatflag: unknown
+ @type fdatactive: C{boolean}
+ @ivar fdatactive: unknown
+ @type piecescomplete: C{int}
+ @ivar piecescomplete: the number of complete pieces the last time the
+ statistics were generated
+ @type placesopen: C{int}
+ @ivar placesopen: the number of out of place pieces there were the last
+ time the statistics were generated
+ @type storage_totalpieces: C{int}
+ @ivar storage_totalpieces: the total number of pieces in the download
+ @type filelistupdated: C{threading.Event}
+ @ivar filelistupdated: whether the file list has been updated
+ @type filepieces: C{list} of C{list} of C{int}
+ @ivar filepieces: for each file, a list of pieces that are not yet downloaded
+ @type filepieces2: C{list} of C{list} of C{int}
+ @ivar filepieces2: for each file, a list of pieces that are not yet in place
+ @type fileamtdone: C{list} of C{float}
+ @ivar fileamtdone: for each file, the fraction of the file that is downloaded
+ and in place
+ @type filecomplete: C{list} of C{boolean}
+ @ivar filecomplete: whether each file has been completely downloaded
+ @type fileinplace: C{list} of C{boolean}
+ @ivar fileinplace: whether each file has been put in its proper place
+
+ """
+
def __init__(self, upmeasure, downmeasure, connecter, httpdl,
ratelimiter, rerequest_lastfailed, fdatflag):
+ """Initialize the instance.
+
+ @type upmeasure: L{CurrentRateMeasure.Measure}
+ @param upmeasure: the measure of the upload rate
+ @type downmeasure: L{CurrentRateMeasure.Measure}
+ @param downmeasure: the measure of the download rate
+ @type connecter: L{BT1.Connecter.Connecter}
+ @param connecter: the Connecter instance that manages all the connections
+ @type httpdl: L{BT1.HTTPDownloader.HTTPDownloader}
+ @param httpdl: the backup HTTP downloader
+ @type ratelimiter: L{RateLimiter.RateLimiter}
+ @param ratelimiter: the RateLimiter instance to limit the upload rate
+ @type rerequest_lastfailed: C{method}
+ @param rerequest_lastfailed: method to call to check if the last
+ tracker request failed
+ @type fdatflag: C{threading.Event}
+ @param fdatflag: unknown
+
+ """
+
self.upmeasure = upmeasure
self.downmeasure = downmeasure
self.connecter = connecter
@@ -35,6 +109,15 @@
self.storage_totalpieces = len(self.storage.hashes)
def set_dirstats(self, files, piece_lengths):
+ """Initialize the statistics for the files in the download.
+
+ @type files: C{list} of (C{string}, C{long})
+ @param files: the full file names and lengths of all the files in the download
+ @type piece_lengths: C{list} of C{int}
+ @param piece_lengths: the lengths of the pieces
+
+ """
+
self.piecescomplete = 0
self.placesopen = 0
self.filelistupdated = Event()
@@ -73,6 +156,14 @@
def update(self):
+ """Generate the statistics response.
+
+ @rtype: L{Statistics_Response}
+ @return: the response containing instance variables that are the
+ generated statistics
+
+ """
+
s = Statistics_Response()
s.upTotal = self.upmeasure.get_total()
s.downTotal = self.downmeasure.get_total()
Modified: debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/StorageWrapper.py Sun Jul 29 09:23:29 2007
@@ -1,8 +1,15 @@
# Written by Bram Cohen
# Modified by Cameron Dale
# see LICENSE.txt for license information
-
+#
# $Id$
+
+"""Manage the storage of data at the piece level.
+
+ at type STATS_INTERVAL: C{float}
+ at var STATS_INTERVAL: how often to print a status update during an old-style init
+
+"""
from DebTorrent.bitfield import Bitfield
from sha import sha
@@ -24,23 +31,86 @@
STATS_INTERVAL = 0.2
def dummy_status(fractionDone = None, activity = None):
+ """Dummy function to do nothing with the status info."""
pass
class Olist:
+ """A list that will be popped in order.
+
+ @type d: C{dictionary}
+ @ivar d: the ordered list, keys are the list entries
+
+ """
+
def __init__(self, l = []):
+ """Initialize the instance.
+
+ @type l: C{list}
+ @param l: the list to initialize the instance with
+ (optional, defaults to an empty initialization)
+
+ """
+
self.d = {}
for i in l:
self.d[i] = 1
+
def __len__(self):
+ """Get the length of the list.
+
+ @rtype: C{int}
+ @return: the length of the list
+
+ """
+
return len(self.d)
+
def includes(self, i):
+ """Check if the list includes a value.
+
+ @type i: unknown
+ @param i: the value to check the list for
+ @rtype: C{boolean}
+ @return: whether the list already includes the value
+
+ """
+
return self.d.has_key(i)
+
def add(self, i):
+ """Add a value to the list.
+
+ @type i: unknown
+ @param i: the value to add to the list
+
+ """
+
self.d[i] = 1
+
def extend(self, l):
+ """Extend the list with multiple new values.
+
+ @type l: C{list}
+ @param l: the list of values to add to the list
+
+ """
+
for i in l:
self.d[i] = 1
+
def pop(self, n=0):
+ """Remove a value from the list and return it.
+
+ @type n: C{int}
+ @param n: the list item to pop::
+ -1 -- the smallest item is popped
+ 0 -- the largest item is popped (default)
+ n -- the nth item in the list is popped
+ @rtype: unknown
+ @return: the value popped from the list
+
+ """
+
# assert self.d
k = self.d.keys()
if n == 0:
@@ -52,25 +122,265 @@
i = k[n]
del self.d[i]
return i
+
def remove(self, i):
+ """Remove a value from the list.
+
+ @type i: unknown
+ @param i: the value to remove from the list
+
+ """
+
if self.d.has_key(i):
del self.d[i]
class fakeflag:
+ """A fake flag to use if one is not supplied.
+
+ @type state: C{boolean}
+ @ivar state: the current state of the flag
+
+ """
+
def __init__(self, state=False):
+ """Set the new flag.
+
+ @type state: C{boolean}
+ @param state: the initial state of the flag
+
+ """
+
self.state = state
+
def wait(self):
+ """Do nothing."""
pass
+
def isSet(self):
+ """Check if the current state is set.
+
+ @rtype: C{boolean}
+ @return: whether the flag is set
+
+ """
+
return self.state
class StorageWrapper:
+ """Manage the storage of data at the piece level.
+
+ @type storage: L{BT1.Storage.Storage}
+ @ivar storage: the file storage instance
+ @type request_size: C{int}
+ @ivar request_size: the number of bytes to query for per request
+ @type hashes: C{list} of C{string}
+ @ivar hashes: the hashes of the pieces
+ @type piece_sizes: C{list} of C{int}
+ @ivar piece_sizes: the lengths of the pieces
+ @type piece_lengths: C{list} of C{int}
+ @ivar piece_lengths: the lengths of the pieces
+ @type piece_begins: C{list} of C{long}
+ @ivar piece_begins: the beginning offset of the pieces in the total download
+ @type datalength: C{long}
+ @ivar datalength: the total length of the download
+ @type finished: C{method}
+ @ivar finished: method to call when the download is complete
+ @type failed: C{method}
+ @ivar failed: method to call when a critical failure occurs
+ @type statusfunc: C{method}
+ @ivar statusfunc: the method to use to diplay status updates
+ @type flag: C{threading.Event}
+ @ivar flag: the flag that indicates when the program is to be shutdown
+ @type check_hashes: C{boolean}
+ @ivar check_hashes: whether to re-check hashes of data already
+ downloaded, during startup
+ @type data_flunked: C{method}
+ @ivar data_flunked: method to call when a hash check fails for a piece
+ @type backfunc: C{method}
+ @ivar backfunc: method to call to schedule future invocation of requester functions
+ @type config: C{dictionary}
+ @ivar config: the configuration parameters
+ @type unpauseflag: C{threading.Event}
+ @ivar unpauseflag: the flag to unset to pause the download
+ @type alloc_type: C{string}
+ @ivar alloc_type: the type of allocation to do, always 'pre-allocate'
+ @type double_check: C{boolean}
+ @ivar double_check: whether to double-check data being written to the disk for errors
+ @type triple_check: C{boolean}
+ @ivar triple_check: whether to thoroughly check data being written to the disk
+ @type bgalloc_enabled: C{boolean}
+ @ivar bgalloc_enabled: unknown
+ @type bgalloc_active: C{boolean}
+ @ivar bgalloc_active: unknown
+ @type total_length: C{long}
+ @ivar total_length: the total length of the download
+ @type amount_left: unknown
+ @ivar amount_left: unknown
+ @type numactive: unknown
+ @ivar numactive: unknown
+ @type inactive_requests: unknown
+ @ivar inactive_requests: unknown
+ @type amount_inactive: unknown
+ @ivar amount_inactive: unknown
+ @type amount_obtained: unknown
+ @ivar amount_obtained: unknown
+ @type amount_desired: unknown
+ @ivar amount_desired: unknown
+ @type have: unknown
+ @ivar have: unknown
+ @type have_cloaked_data: unknown
+ @ivar have_cloaked_data: unknown
+ @type blocked: unknown
+ @ivar blocked: unknown
+ @type blocked_holes: unknown
+ @ivar blocked_holes: unknown
+ @type blocked_movein: unknown
+ @ivar blocked_movein: unknown
+ @type blocked_moveout: unknown
+ @ivar blocked_moveout: unknown
+ @type waschecked: unknown
+ @ivar waschecked: unknown
+ @type places: unknown
+ @ivar places: unknown
+ @type holes: unknown
+ @ivar holes: unknown
+ @type stat_active: unknown
+ @ivar stat_active: unknown
+ @type stat_new: unknown
+ @ivar stat_new: unknown
+ @type dirty: unknown
+ @ivar dirty: unknown
+ @type stat_numflunked: unknown
+ @ivar stat_numflunked: unknown
+ @type stat_numdownloaded: unknown
+ @ivar stat_numdownloaded: unknown
+ @type stat_numfound: unknown
+ @ivar stat_numfound: unknown
+ @type download_history: unknown
+ @ivar download_history: unknown
+ @type failed_pieces: unknown
+ @ivar failed_pieces: unknown
+ @type out_of_place: unknown
+ @ivar out_of_place: unknown
+ @type write_buf_max: unknown
+ @ivar write_buf_max: unknown
+ @type write_buf_size: unknown
+ @ivar write_buf_size: unknown
+ @type write_buf: unknown
+ @ivar write_buf: unknown
+ @type write_buf_list: unknown
+ @ivar write_buf_list: unknown
+ @type initialize_tasks: C{list} of [C{string}, C{int}, C{method, C{method}]
+ @ivar initialize_tasks: the initialization tasks to perform, the status message,
+ initial fraction done, method to call to initialize the task, and the task
+ @type initialize_done: C{method}
+ @ivar initialize_done: method to call when the background initialization is complete
+ @type initialize_status: C{method}
+ @ivar initialize_status: method to call to report the status of the initialization
+ @type initialize_next: C{method}
+ @ivar initialize_next: current method to call in the initialization tasks
+ @type check_list: unknown
+ @ivar check_list: unknown
+ @type check_total: unknown
+ @ivar check_total: unknown
+ @type check_targets: unknown
+ @ivar check_targets: unknown
+ @type check_numchecked: unknown
+ @ivar check_numchecked: unknown
+ @type numchecked: unknown
+ @ivar numchecked: unknown
+ @type movelist: unknown
+ @ivar movelist: unknown
+ @type tomove: unknown
+ @ivar tomove: unknown
+ @type numholes: unknown
+ @ivar numholes: unknown
+ @type alloc_buf: unknown
+ @ivar alloc_buf: unknown
+
+ @group Initialization: __init__, _bgsync, old_style_init, initialize,
+ _initialize, init_hashcheck, _markgot, hashcheckfunc, init_movedata,
+ movedatafunc, init_alloc, _allocfunc, allocfunc, bgalloc, _bgalloc
+_waspre
+_piecelen
+get_amount_left
+do_I_have_anything
+_make_inactive
+is_endgame
+am_I_complete
+reset_endgame
+get_have_list
+get_have_list_cloaked
+do_I_have
+do_I_have_requests
+is_unstarted
+get_hash
+get_stats
+new_request
+write_raw
+_write_to_buffer
+_flush_buffer
+sync
+_move_piece
+_clear_space
+piece_came_in
+request_lost
+get_piece
+read_raw
+set_file_readonly
+has_data
+doublecheck_data
+reblock
+pickle
+unpickle
+
+
+ """
+
def __init__(self, storage, request_size, hashes,
piece_sizes, datalength, finished, failed,
statusfunc = dummy_status, flag = fakeflag(), check_hashes = True,
data_flunked = lambda x: None, backfunc = None,
config = {}, unpauseflag = fakeflag(True) ):
+ """Initialize the instance and begin the background initialization tasks.
+
+ @type storage: L{BT1.Storage.Storage}
+ @param storage: the file storage instance
+ @type request_size: C{int}
+ @param request_size: the number of bytes to query for per request
+ @type hashes: C{list} of C{string}
+ @param hashes: the hashes of the pieces
+ @type piece_sizes: C{list} of C{int}
+ @param piece_sizes: the lengths of the pieces
+ @type datalength: C{long}
+ @param datalength: the total length of the download
+ @type finished: C{method}
+ @param finished: method to call when the download is complete
+ @type failed: C{method}
+ @param failed: method to call when a critical failure occurs
+ @type statusfunc: C{method}
+ @param statusfunc: the method to use to diplay status updates
+ (optional, defaults to not displaying status updates)
+ @type flag: C{threading.Event}
+ @param flag: the flag that indicates when the program is to be shutdown
+ (optional, defaults to using a dummy flag)
+ @type check_hashes: C{boolean}
+ @param check_hashes: whether to re-check hashes of data already
+ downloaded, during startup (optional, defaults to True)
+ @type data_flunked: C{method}
+ @param data_flunked: method to call when a hash check fails for a piece
+ (optional, defaults to doing nothing)
+ @type backfunc: C{method}
+ @param backfunc: method to call to schedule future invocation of requester functions
+ @type config: C{dictionary}
+ @param config: the configuration parameters
+ @type unpauseflag: C{threading.Event}
+ @param unpauseflag: the flag to unset to pause the download
+ (optional, defaults to using a dummy flag)
+
+ """
+
self.storage = storage
self.request_size = long(request_size)
self.hashes = hashes
@@ -140,12 +450,20 @@
self.backfunc(self._bgsync,max(self.config['auto_flush']*60,60))
def _bgsync(self):
+ """Periodically flush the files to disk."""
if self.config['auto_flush']:
self.sync()
self.backfunc(self._bgsync,max(self.config['auto_flush']*60,60))
def old_style_init(self):
+ """Initialize the storage in the foreground.
+
+ @rtype: C{boolean}
+ @return: whether the init completed
+
+ """
+
while self.initialize_tasks:
msg, done, init, next = self.initialize_tasks.pop(0)
if init():
@@ -167,6 +485,16 @@
def initialize(self, donefunc, statusfunc = None):
+ """Schedule the background initialization of the storage.
+
+ @type donefunc: C{method}
+ @param donefunc: method to call when the initialization is complete
+ @type statusfunc: C{method}
+ @param statusfunc: method to call to report the status of the initialization
+ (optional, defaults to using L{StorageWrapper.statusfunc})
+
+ """
+
self.initialize_done = donefunc
if statusfunc is None:
statusfunc = self.statusfunc
@@ -176,6 +504,7 @@
self.backfunc(self._initialize)
def _initialize(self):
+ """Perform the background initialization of the storage."""
if not self.unpauseflag.isSet():
self.backfunc(self._initialize, 1)
return
@@ -200,6 +529,13 @@
def init_hashcheck(self):
+ """Initialize the hash check initialization task.
+
+ @rtype: C{boolean}
+ @return: whether the task should proceed
+
+ """
+
if self.flag.isSet():
return False
self.check_list = []
@@ -244,6 +580,15 @@
return self.check_total > 0
def _markgot(self, piece, pos):
+ """Mark a piece as complete during the initial hash check.
+
+ @type piece: C{int}
+ @param piece: the piece to mark as complete
+ @type pos: C{int}
+ @param pos: the place the piece currently occupies
+
+ """
+
logging.debug(str(piece)+' at '+str(pos))
self.places[piece] = pos
self.have[piece] = True
@@ -256,6 +601,14 @@
self.stat_numfound += 1
def hashcheckfunc(self):
+ """Hash check the current data as part of the initialization tasks.
+
+ @rtype: C{float}
+ @return: the fraction of the task that is complete, or None if the
+ task is complete
+
+ """
+
if self.flag.isSet():
return None
if not self.check_list:
@@ -297,6 +650,13 @@
def init_movedata(self):
+ """Initialize the move data initialization task.
+
+ @rtype: C{boolean}
+ @return: whether the task should proceed
+
+ """
+
if self.flag.isSet():
return False
if self.alloc_type != 'sparse':
@@ -318,6 +678,14 @@
return True
def movedatafunc(self):
+ """Move pieces around as part of the initialization tasks.
+
+ @rtype: C{float}
+ @return: the fraction of the task that is complete, or None if the
+ task is complete
+
+ """
+
if self.flag.isSet():
return None
if not self.movelist:
@@ -346,6 +714,13 @@
def init_alloc(self):
+ """Initialize the allocation initialization task.
+
+ @rtype: C{boolean}
+ @return: whether the task should proceed
+
+ """
+
if self.flag.isSet():
return False
if not self.holes:
@@ -363,6 +738,13 @@
def _allocfunc(self):
+ """Find the next location that needs allocating
+
+ @rtype: C{int}
+ @return: the next piece hole to allocate, or None if there are none
+
+ """
+
while self.holes:
n = self.holes.pop(0)
if self.blocked[n]: # assume not self.blocked[index]
@@ -382,6 +764,14 @@
return None
def allocfunc(self):
+ """Allocate space for files as part of the initialization tasks.
+
+ @rtype: C{float}
+ @return: the fraction of the task that is complete, or None if the
+ task is complete
+
+ """
+
if self.flag.isSet():
return None
@@ -410,6 +800,13 @@
return None
def bgalloc(self):
+ """Enable the backround allocation.
+
+ @rtype: C{boolean}
+ @return: False
+
+ """
+
if self.bgalloc_enabled:
if not self.holes and not self.blocked_moveout and self.backfunc:
self.backfunc(self.storage.flush)
@@ -418,6 +815,7 @@
return False
def _bgalloc(self):
+ """Allocate the file storage in the background."""
self.allocfunc()
if self.config.get('alloc_rate',0) < 0.1:
self.config['alloc_rate'] = 2.0
Modified: debtorrent/trunk/DebTorrent/BT1/StreamCheck.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/StreamCheck.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/StreamCheck.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/StreamCheck.py Sun Jul 29 09:23:29 2007
@@ -3,6 +3,8 @@
# see LICENSE.txt for license information
# $Id$
+
+"""Not used."""
from cStringIO import StringIO
from binascii import b2a_hex
Modified: debtorrent/trunk/DebTorrent/BT1/T2T.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/T2T.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/T2T.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/T2T.py Sun Jul 29 09:23:29 2007
@@ -1,8 +1,17 @@
# Written by John Hoffman
# Modified by Cameron Dale
# see LICENSE.txt for license information
-
+#
# $Id$
+
+"""Tracker to tracker connection management.
+
+ at type R_0: C{function}
+ at var R_0: a function that always returns 0
+ at type R_1: C{function}
+ at var R_1: a function that always returns 0
+
+"""
from Rerequester import Rerequester
from urllib import quote
@@ -19,14 +28,89 @@
def excfunc(x):
+ """Log an exception that occurred.
+
+ @type x: C{string}
+ @param x: the error message to log
+
+ """
+
logging.error(x)
R_0 = lambda: 0
R_1 = lambda: 1
class T2TConnection:
+ """A single tracker to tracker connection for a single torrent.
+
+ Creates a L{Rerequester.Rerequester} object but does not start it running,
+ instead all requesting is controlled from this class.
+
+ @type tracker: C{string}
+ @ivar tracker: the tracker address to contact
+ @type interval: C{int}
+ @ivar interval: original seconds between outgoing tracker announces
+ @type hash: C{string}
+ @ivar hash: the info hash of the torrent
+ @type operatinginterval: C{int}
+ @ivar operatinginterval: current seconds between outgoing tracker announces
+ @type peers: C{int}
+ @ivar peers: number of peers to get in a tracker announce
+ @type rawserver: L{DebTorrent.RawServer.RawServer}
+ @ivar rawserver: the server instance to use
+ @type disallow: C{method}
+ @ivar disallow: method to call to disallow a tracker address
+ @type isdisallowed: C{method}
+ @ivar isdisallowed: method to call to check if a tracker address is disallowed
+ @type active: C{boolean}
+ @ivar active: whether the connection is active
+ @type busy: C{boolean}
+ @ivar busy: not used
+ @type errors: C{int}
+ @ivar errors: the number of errors that have occurred since the last
+ successful request
+ @type rejected: C{int}
+ @ivar rejected: the number of rejections that have occurred since the last
+ successful request
+ @type trackererror: C{boolean}
+ @ivar trackererror: not used
+ @type peerlists: C{list} of C{list} of (C{string}, C{string}, C{int})
+ @ivar peerlists: the last 10 announce peer data of peer ID, IP address, and port
+ @type rerequester: L{Rerequester.Rerequester}
+ @ivar rerequester: the tracker rerequester instance
+ @type lastsuccessful: C{boolean}
+ @ivar lastsuccessful: whether the last request was successful
+ @type newpeerdata: C{list} of (C{string}, C{string}, C{int})
+ @ivar newpeerdata: the list of peer data of peer ID, IP address, and port
+
+ """
+
def __init__(self, myid, tracker, hash, interval, peers, timeout,
rawserver, disallow, isdisallowed):
+ """Initialize the instance and schedule a request.
+
+ @type myid: C{string}
+ @param myid: the peer ID to send to the tracker
+ @type tracker: C{string}
+ @param tracker: the tracker address to contact
+ @type hash: C{string}
+ @param hash: the info hash of the torrent
+ @type interval: C{int}
+ @param interval: seconds between outgoing tracker announces
+ @type peers: C{int}
+ @param peers: number of peers to get in a tracker announce
+ @type timeout: C{int}
+ @param timeout: number of seconds to wait before assuming that a
+ tracker connection has timed out
+ @type rawserver: L{DebTorrent.RawServer.RawServer}
+ @param rawserver: the server instance to use
+ @type disallow: C{method}
+ @param disallow: method to call to disallow a tracker address
+ @type isdisallowed: C{method}
+ @param isdisallowed: method to call to check if a tracker address is disallowed
+
+ """
+
self.tracker = tracker
self.interval = interval
self.hash = hash
@@ -55,14 +139,23 @@
# stagger announces
def isactive(self):
+ """Check if the tracker connection has been disallowed.
+
+ @rtype: C{boolean}
+ @return: whether the connection is active
+
+ """
+
if self.isdisallowed(self.tracker): # whoops!
self.deactivate()
return self.active
def deactivate(self):
+ """Deactive the connection."""
self.active = False
def refresh(self):
+ """Request new peer data from the tracker."""
if not self.isactive():
return
self.lastsuccessful = True
@@ -71,6 +164,7 @@
self.rerequester.snoop(self.peers, self.callback)
def callback(self):
+ """Process the returned peer data from the tracker."""
self.busy = False
if self.lastsuccessful:
self.errors = 0
@@ -86,10 +180,24 @@
self.rawserver.add_task(self.refresh, self.operatinginterval)
def addtolist(self, peers):
+ """Save the returned peer data from the tracker for later processing.
+
+ @type peers: C{list} of ((C{string}, C{int}), C{string}, C{boolean})
+ @param peers: the list of IP address, port, peer ID, and whether to encrypt
+
+ """
+
for peer in peers:
self.newpeerdata.append((peer[1],peer[0][0],peer[0][1]))
def errorfunc(self, r):
+ """Process an error that occurred.
+
+ @type r: C{string}
+ @param r: the error message
+
+ """
+
self.lastsuccessful = False
logging.info(self.tracker+' with info_hash='+quote(self.hash)+' gives error: "'+r+'"')
if r == self.rerequester.rejectedmessage + 'disallowed': # whoops!
@@ -109,6 +217,13 @@
logging.info(' -- lengthening interval to '+str(self.operatinginterval)+' seconds')
def harvest(self):
+ """Retrieve the saved list of peers from this tracker connection.
+
+ @rtype: C{list} of (C{string}, C{string}, C{int})
+ @return: the list of peer data of peer ID, IP address, and port
+
+ """
+
x = []
for list in self.peerlists:
x += list
@@ -117,7 +232,56 @@
class T2TList:
+ """A list of all tracker to tracker connections.
+
+ @type enabled: C{boolean}
+ @ivar enabled: whether to enable multitracker operation
+ @type trackerid: C{string}
+ @ivar trackerid: this tracker's ID
+ @type interval: C{int}
+ @ivar interval: seconds between outgoing tracker announces
+ @type maxpeers: C{int}
+ @ivar maxpeers: number of peers to get in a tracker announce
+ @type timeout: C{int}
+ @ivar timeout: number of seconds to wait before assuming that a
+ tracker connection has timed out
+ @type rawserver: L{DebTorrent.RawServer.RawServer}
+ @ivar rawserver: the server instance to use
+ @type list: {C{string}: {C{string}: L{T2TConnection}, ...}, ...}
+ @ivar list: keys are the tracker addresses, values are dictionaries with
+ torrent info hashes as keys and the connection to that tracker for
+ that torrent as values
+ @type torrents: {C{string}: [L{T2TConnection}, ...], ...}
+ @ivar torrents: keys are the info hashes, values are a list of the
+ tracker connections for that torrent
+ @type disallowed: {C{string}: C{boolean}, ...}
+ @ivar disallowed: keys are the tracker addresses, values are True if the
+ tracker is disallowing this tracker
+ @type oldtorrents: C{list} of L{T2TConnection}
+ @ivar oldtorrents: deactivated connections that are kept in case threads
+ try to access them
+
+ """
+
def __init__(self, enabled, trackerid, interval, maxpeers, timeout, rawserver):
+ """Initialize the instance.
+
+ @type enabled: C{boolean}
+ @param enabled: whether to enable multitracker operation
+ @type trackerid: C{string}
+ @param trackerid: this tracker's ID
+ @type interval: C{int}
+ @param interval: seconds between outgoing tracker announces
+ @type maxpeers: C{int}
+ @param maxpeers: number of peers to get in a tracker announce
+ @type timeout: C{int}
+ @param timeout: number of seconds to wait before assuming that a
+ tracker connection has timed out
+ @type rawserver: L{DebTorrent.RawServer.RawServer}
+ @param rawserver: the server instance to use
+
+ """
+
self.enabled = enabled
self.trackerid = trackerid
self.interval = interval
@@ -130,6 +294,13 @@
self.oldtorrents = []
def parse(self, allowed_list):
+ """Parse a list of allowed torrents and enable any new ones.
+
+ @type allowed_list: C{dictionary}
+ @param allowed_list: keys are info hashes, values are the torrent data
+
+ """
+
if not self.enabled:
return
@@ -172,19 +343,38 @@
self.torrents = newtorrents
- # structures:
- # list = {tracker: {hash: T2TConnection, ...}, ...}
- # torrents = {hash: [T2TConnection, ...]}
- # disallowed = {tracker: flag, ...}
- # oldtorrents = [T2TConnection, ...]
-
def _disallow(self,tracker):
+ """Disallow all connections from contacting a tracker.
+
+ @type tracker: C{string}
+ @param tracker: the tracker address to disallow
+
+ """
+
self.disallowed[tracker] = True
def _isdisallowed(self,tracker):
+ """Check if a tracker has been disallowed.
+
+ @type tracker: C{string}
+ @param tracker: the tracker address to check
+ @rtype: C{boolean}
+ @return: whether the tracker has been disallowed
+
+ """
+
return self.disallowed[tracker]
def harvest(self,hash):
+ """Harvest a list of peers from all tracker's for a torrent.
+
+ @type hash: C{string}
+ @param hash: the info hash of the torrent to get peers for
+ @rtype: C{list} of (C{string}, C{string}, C{int})
+ @return: the list of peer data of peer ID, IP address, and port
+
+ """
+
harvest = []
if self.enabled:
for t2t in self.torrents[hash]:
Modified: debtorrent/trunk/DebTorrent/BT1/Uploader.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/Uploader.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/Uploader.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/Uploader.py Sun Jul 29 09:23:29 2007
@@ -1,8 +1,10 @@
# Written by Bram Cohen
# Modified by Cameron Dale
# see LICENSE.txt for license information
-
+#
# $Id$
+
+"""Manage uploading to a single peer."""
from DebTorrent.CurrentRateMeasure import Measure
@@ -13,8 +15,74 @@
False = 0
class Upload:
+ """Manage uploading to a single peer.
+
+ @type connection: L{DebTorrent.SocketHandler.SingleSocket}
+ @ivar connection: the connection to the peer
+ @type ratelimiter: L{RateLimiter.RateLimiter}
+ @ivar ratelimiter: the RateLimiter instance to use
+ @type totalup: L{Debtorrent.CurrentRateMeasure.Measure}
+ @ivar totalup: the Measure instance to use
+ @type choker: L{Choker.Choker}
+ @ivar choker: the Choker instance to use
+ @type storage: L{BT1.StorageWrapper.StorageWrapper}
+ @ivar storage: the StorageWrapper instance
+ @type picker: L{BT1.PiecePicker.PiecePicker}
+ @ivar picker: the PiecePicker instance
+ @type config: C{dictionary}
+ @ivar config: the configration information
+ @type max_slice_length: C{int}
+ @ivar max_slice_length: maximum length chunk to send to peers
+ @type choked: C{boolean}
+ @ivar choked: whether we are choking the connection
+ @type cleared: C{boolean}
+ @ivar cleared: whether requests are allowed to be appended to the buffer
+ @type interested: C{boolean}
+ @ivar interested: whether the peer is interested
+ @type super_seeding: C{boolean}
+ @ivar super_seeding: whether we are in super-seed mode
+ @type buffer: C{list} of (C{int}, C{int}, C{int})
+ @ivar buffer: the pending requests for the peer, the piece index, offset
+ within the piece, and chunk length requested
+ @type measure: L{DebTorrent.CurrentRateMeasure.Measure}
+ @ivar measure: for measuring the upload rate to the peer
+ @type was_ever_interested: C{boolean}
+ @ivar was_ever_interested: whether the peer has ever been interested
+ @type seed_have_list: C{list} of C{int}
+ @ivar seed_have_list: the list of pieces the peer is allowed to request
+ in super-seed mode
+ @type skipped_count: C{int}
+ @ivar skipped_count: the number of pieces the peer has refused to request
+ from us in super-seed mode
+ @type piecedl: C{int}
+ @ivar piecedl: the current piece being downloaded by the peer
+ @type piecebuf: L{DebTorrent.piecebuffer.SingleBuffer}
+ @ivar piecebuf: the buffer containing the entire piece currently being
+ downloaded by the peer
+
+ """
+
def __init__(self, connection, ratelimiter, totalup, choker, storage,
picker, config):
+ """Initialize the instance and send the initial bitfield.
+
+ @type connection: L{DebTorrent.SocketHandler.SingleSocket}
+ @param connection: the connection to the peer
+ @type ratelimiter: L{RateLimiter.RateLimiter}
+ @param ratelimiter: the RateLimiter instance to use
+ @type totalup: L{Debtorrent.CurrentRateMeasure.Measure}
+ @param totalup: the Measure instance to use
+ @type choker: L{Choker.Choker}
+ @param choker: the Choker instance to use
+ @type storage: L{BT1.StorageWrapper.StorageWrapper}
+ @param storage: the StorageWrapper instance
+ @type picker: L{BT1.PiecePicker.PiecePicker}
+ @param picker: the PiecePicker instance
+ @type config: C{dictionary}
+ @param config: the configration information
+
+ """
+
self.connection = connection
self.ratelimiter = ratelimiter
self.totalup = totalup
@@ -50,6 +118,7 @@
self.piecebuf = None
def got_not_interested(self):
+ """Process a received not interested message."""
if self.interested:
self.interested = False
del self.buffer[:]
@@ -60,12 +129,20 @@
self.choker.not_interested(self.connection)
def got_interested(self):
+ """Process a received interested message."""
if not self.interested:
self.interested = True
self.was_ever_interested = True
self.choker.interested(self.connection)
def get_upload_chunk(self):
+ """Get a chunk to upload to the peer.
+
+ @rtype: (C{int}, C{int}, C{string})
+ @return: the piece index, offset within the piece, and the chunk
+
+ """
+
if self.choked or not self.buffer:
return None
index, begin, length = self.buffer.pop(0)
@@ -94,6 +171,17 @@
return (index, begin, piece)
def got_request(self, index, begin, length):
+ """Add a received request for a chunk to the buffer.
+
+ @type index: C{int}
+ @param index: the piece index
+ @type begin: C{int}
+ @param begin: the offset within the piece
+ @type length: C{int}
+ @param length: the amount of data to return
+
+ """
+
if ( (self.super_seeding and not index in self.seed_have_list)
or not self.interested or length > self.max_slice_length ):
self.connection.close()
@@ -105,12 +193,24 @@
def got_cancel(self, index, begin, length):
+ """Cancel a request for a chunk.
+
+ @type index: C{int}
+ @param index: the piece index
+ @type begin: C{int}
+ @param begin: the offset within the piece
+ @type length: C{int}
+ @param length: the amount of data to return
+
+ """
+
try:
self.buffer.remove((index, begin, length))
except ValueError:
pass
def choke(self):
+ """Start choking the connection."""
if not self.choked:
self.choked = True
self.connection.send_choke()
@@ -120,29 +220,60 @@
self.piecebuf = None
def choke_sent(self):
+ """Remove all requests after a choke is sent."""
del self.buffer[:]
self.cleared = True
def unchoke(self):
+ """Unchoke the connection."""
if self.choked:
self.choked = False
self.cleared = False
self.connection.send_unchoke()
def disconnected(self):
+ """Clean up for disconnection from the peer."""
if self.piecebuf:
self.piecebuf.release()
self.piecebuf = None
def is_choked(self):
+ """Check whether we are choking the connection.
+
+ @rtype: C{boolean}
+ @return: whether the connection is being choked
+
+ """
+
return self.choked
def is_interested(self):
+ """Check whether the peer is interested in downloading.
+
+ @rtype: C{boolean}
+ @return: whether the connected peer is interested
+
+ """
+
return self.interested
def has_queries(self):
+ """Check whether we have pending requests for chunks from the peer.
+
+ @rtype: C{boolean}
+ @return: whether there are requests pending
+
+ """
+
return not self.choked and len(self.buffer) > 0
def get_rate(self):
+ """Get the current upload rate to the peer.
+
+ @rtype: C{float}
+ @return: the current upload rate
+
+ """
+
return self.measure.get_rate()
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=190&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Sun Jul 29 09:23:29 2007
@@ -197,6 +197,7 @@
('http_timeout', 60,
'number of seconds to wait before assuming that an http connection ' +
'has timed out'),
+ # File System
('buffer_reads', 1,
'whether to buffer disk reads'),
('write_buffer_size', 4,
More information about the Debtorrent-commits
mailing list