r227 - in /debtorrent/branches/unique: DebTorrent/BT1/FileSelector.py DebTorrent/BT1/PiecePicker.py DebTorrent/BT1/Storage.py DebTorrent/download_bt1.py test.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Mon Aug 13 03:04:19 UTC 2007
Author: camrdale-guest
Date: Mon Aug 13 03:04:18 2007
New Revision: 227
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=227
Log:
Modify the pickling of torrent data to work with unique piece numbers.
Modified:
debtorrent/branches/unique/DebTorrent/BT1/FileSelector.py
debtorrent/branches/unique/DebTorrent/BT1/PiecePicker.py
debtorrent/branches/unique/DebTorrent/BT1/Storage.py
debtorrent/branches/unique/DebTorrent/download_bt1.py
debtorrent/branches/unique/test.py
Modified: debtorrent/branches/unique/DebTorrent/BT1/FileSelector.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/FileSelector.py?rev=227&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/FileSelector.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/FileSelector.py Mon Aug 13 03:04:18 2007
@@ -168,14 +168,18 @@
"""Unpickle the previously saved state.
Data is in the format::
- d['priority'] = [file #1 priority, file #2 priority, ...]
-
- It is a list of download priorities for each file. The priority may be::
+ d['priority'] = [file name, priority, {file name, priority, ...}]
+
+ It is a list of file names and their corrseponding download priorities
+ for each file. The priority must be::
-1 -- download disabled
0 -- highest
1 -- normal
2 -- lowest
+ Any files not listed are assumed to have a priority of -1 (download
+ disabled).
+
Also see Storage.pickle and StorageWrapper.pickle for additional keys.
@type d: C{dictionary}
@@ -184,8 +188,30 @@
"""
if d.has_key('priority'):
- if not self.init_priority(d['priority']):
+ try:
+ # Create a dictionary of the saved file priorities
+ saved_files = {}
+ l = d['priority']
+ assert len(l) % 2 == 0
+ l = [l[x:x+2] for x in xrange(0,len(l),2)]
+ for file, priority in l:
+ saved_files[file] = priority
+
+ # Create the new file priorities list from the saved ones
+ new_priority = []
+ for file, length in self.files:
+ if not length or not file:
+ # Empty files are not downloaded
+ new_priority.append(-1)
+ else:
+ new_priority.append(saved_files.get(file, -1))
+
+ if not self.init_priority(new_priority):
+ return
+ except:
+ logger.exception('Error unpickling file priority cache')
return
+
pieces = self.storage.unpickle(d)
if not pieces: # don't bother, nothing restoreable
return
@@ -510,7 +536,15 @@
"""
- d = {'priority': self.priority}
+ files = []
+ for i in xrange(len(self.priority)):
+ if not self.files[i][1]: # length == 0
+ continue
+ if self.priority[i] == -1:
+ continue
+ files.extend([self.files[i][0], self.priority[i]])
+
+ d = {'priority': files}
try:
s = self.storage.pickle()
sw = self.storagewrapper.pickle()
Modified: debtorrent/branches/unique/DebTorrent/BT1/PiecePicker.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/PiecePicker.py?rev=227&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/PiecePicker.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/PiecePicker.py Mon Aug 13 03:04:18 2007
@@ -289,6 +289,7 @@
l = self.interests[self.level_in_interests[piece]]
p = self.pos_in_interests[piece]
+ logger.info('removing from position ' + str(p) + ' of interests level ' + str(self.level_in_interests[piece]) + ' of length ' + str(len(l)) + ' piece ' + str(piece))
assert l[p] == piece
q = l[-1]
l[p] = q
Modified: debtorrent/branches/unique/DebTorrent/BT1/Storage.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/BT1/Storage.py?rev=227&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/BT1/Storage.py (original)
+++ debtorrent/branches/unique/DebTorrent/BT1/Storage.py Mon Aug 13 03:04:18 2007
@@ -818,20 +818,10 @@
Pickled data format::
- d['files'] = [ file #, size, mtime {, file #, size, mtime...} ]
- file # in torrent, and the size and last modification
+ d['files'] = [ file name, size, mtime {, file name, size, mtime...} ]
+ file name in torrent, and the size and last modification
time for those files. Missing files are either empty
or disabled.
- d['partial files'] = [ name, size, mtime... ]
- Names, sizes and last modification times of files containing
- partial piece data. Filenames go by the following convention:
- {file #, 0-based}{nothing, "b" or "e"}
- eg: "0e" "3" "4b" "4e"
- Where "b" specifies the partial data for the first piece in
- the file, "e" the last piece, and no letter signifying that
- the file is disabled but is smaller than one piece, and that
- all the data is cached inside so adjacent files may be
- verified.
@rtype: C{dictionary}
@return: the pickled current status of the download
@@ -839,18 +829,14 @@
"""
files = []
- pfiles = []
for i in xrange(len(self.files)):
if not self.files[i][1]: # length == 0
continue
if self.disabled[i]:
- # Removed due to files always ending on pieces
- #for start, end, offset, file in self._get_disabled_ranges(i)[2]:
- # pfiles.extend([basename(file),getsize(file),int(getmtime(file))])
continue
file = self.files[i][0]
- files.extend([i,getsize(file),int(getmtime(file))])
- return {'files': files, 'partial files': pfiles}
+ files.extend([file,getsize(file),int(getmtime(file))])
+ return {'files': files}
def unpickle(self, data):
@@ -867,18 +853,13 @@
try:
files = {}
- pfiles = {}
l = data['files']
assert len(l) % 3 == 0
l = [l[x:x+3] for x in xrange(0,len(l),3)]
- for f, size, mtime in l:
- files[f] = (size, mtime)
- l = data.get('partial files',[])
- assert len(l) % 3 == 0
- l = [l[x:x+3] for x in xrange(0,len(l),3)]
for file, size, mtime in l:
- pfiles[file] = (size, mtime)
-
+ files[file] = (size, mtime)
+ logger.debug('saved file characteristics: %r', files)
+
valid_pieces = {}
for i in xrange(len(self.files)):
if self.disabled[i]:
@@ -910,22 +891,24 @@
oldsize, oldmtime = old
if size != oldsize:
+ logger.debug('Size of file has changed: %d to %d', oldsize, size)
return False
if mtime > oldmtime+1:
+ logger.debug('Time of file has changed: %d to %d', oldmtime, mtime)
return False
if mtime < oldmtime-1:
+ logger.debug('Time of file has changed: %d to %d', oldmtime, mtime)
return False
return True
for i in xrange(len(self.files)):
if self.disabled[i]:
- # Simplified due to file boundaries being piece boundaries
continue
file, size = self.files[i]
if not size:
continue
- if ( not files.has_key(i)
- or not test(files[i],getsize(file),getmtime(file)) ):
+ if ( not files.has_key(file)
+ or not test(files[file],getsize(file),getmtime(file)) ):
start, end, offset, file = self.file_ranges[i]
start_piece, end_piece = self.file_pieces[i]
logger.debug('removing '+file)
Modified: debtorrent/branches/unique/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/DebTorrent/download_bt1.py?rev=227&op=diff
==============================================================================
--- debtorrent/branches/unique/DebTorrent/download_bt1.py (original)
+++ debtorrent/branches/unique/DebTorrent/download_bt1.py Mon Aug 13 03:04:18 2007
@@ -907,17 +907,6 @@
return self.storagewrapper.initialize
- def getCachedTorrentData(self):
- """Get the cached torrent data from the cache directory.
-
- @rtype: C{dictionary}
- @return: the bdecoded cached data
-
- """
-
- return self.appdataobj.getTorrentData(self.infohash)
-
-
def _make_upload(self, connection, ratelimiter, totalup):
"""Create a new Upload instance
@@ -1054,6 +1043,7 @@
for i in xrange(self.len_pieces):
if self.storagewrapper.do_I_have(i):
+ logger.info('informing picker I have piece: ' + str(i))
self.picker.complete(i)
self.upmeasure = Measure(self.config['max_rate_period'],
self.config['upload_rate_fudge'])
Modified: debtorrent/branches/unique/test.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/branches/unique/test.py?rev=227&op=diff
==============================================================================
--- debtorrent/branches/unique/test.py (original)
+++ debtorrent/branches/unique/test.py Mon Aug 13 03:04:18 2007
@@ -41,6 +41,7 @@
from DebTorrent.__init__ import resetPeerIDs
import sys, os, signal
from traceback import print_exc
+from os.path import exists
DebTorrentClient = __import__('debtorrent-client', globals(), locals(), [])
@@ -172,7 +173,23 @@
(1, ['update']),
]),
- }
+ '7': ('Run this test multiple times to test restarting the downloader.',
+ {1: []},
+ {1: (1, [], {'clean': False})},
+ [(1, ['update']),
+ (1, ['install', 'aboot-base']),
+ (1, ['install', 'aap-doc']),
+ (1, ['install', 'ada-reference-manual']),
+ (1, ['install', 'aspectj-doc']),
+ (1, ['install', 'fop-doc']),
+ (1, ['install', 'jswat-doc']),
+ (1, ['install', 'asis-doc']),
+ (1, ['install', 'bison-doc']),
+ (1, ['install', 'crash-whitepaper']),
+ (1, ['install', 'doc-iana']),
+ ]),
+
+ }
assert 'all' not in tests
assert 'help' not in tests
@@ -250,11 +267,12 @@
"""
- for root, dirs, files in os.walk(top, topdown=False):
- for name in files:
- os.remove(os.path.join(root, name))
- for name in dirs:
- os.rmdir(os.path.join(root, name))
+ if exists(top):
+ for root, dirs, files in os.walk(top, topdown=False):
+ for name in files:
+ os.remove(os.path.join(root, name))
+ for name in dirs:
+ os.rmdir(os.path.join(root, name))
def join(dir):
"""Join together a list of directories into a path string.
@@ -279,7 +297,8 @@
"""
- os.makedirs(join(dir))
+ if not exists(join(dir)):
+ os.makedirs(join(dir))
def touch(path):
"""Create an empty file.
@@ -288,9 +307,10 @@
@param path: the path to create
"""
-
- f = open(join(path), 'w')
- f.close()
+
+ if not exists(join(path)):
+ f = open(join(path), 'w')
+ f.close()
def start(func, args, work_dir = None):
"""Fork and start a background process running.
@@ -456,21 +476,24 @@
except:
pass
- # Create the directory structure needed by apt
- makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d'])
- makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial'])
- makedirs([downloader_dir, 'var', 'lib', 'dpkg'])
- makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial'])
- touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock'])
- touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock'])
- touch([downloader_dir, 'var', 'lib', 'dpkg', 'status'])
- touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock'])
-
+ # Create the directory structure needed by apt
+ makedirs([downloader_dir, 'etc', 'apt', 'apt.conf.d'])
+ makedirs([downloader_dir, 'var', 'lib', 'apt', 'lists', 'partial'])
+ makedirs([downloader_dir, 'var', 'lib', 'dpkg'])
+ rmrf(join([downloader_dir, 'var', 'cache', 'apt', 'archives']))
+ makedirs([downloader_dir, 'var', 'cache', 'apt', 'archives', 'partial'])
+ touch([downloader_dir, 'var', 'lib', 'apt', 'lists', 'lock'])
+ touch([downloader_dir, 'var', 'lib', 'dpkg', 'lock'])
+ touch([downloader_dir, 'var', 'lib', 'dpkg', 'status'])
+ touch([downloader_dir, 'var', 'cache', 'apt', 'archives', 'lock'])
+
+ if not exists(join([downloader_dir, 'etc', 'apt', 'sources.list'])):
# Create apt's config files
f = open(join([downloader_dir, 'etc', 'apt', 'sources.list']), 'w')
f.write('deb http://localhost:' + str(num_down) + '988/' + mirror + '/ unstable ' + suites + '\n')
f.close()
+ if not exists(join([downloader_dir, 'etc', 'apt', 'apt.conf'])):
f = open(join([downloader_dir, 'etc', 'apt', 'apt.conf']), 'w')
f.write('Dir "' + downloader_dir + '"')
f.write(apt_conf_template)
More information about the Debtorrent-commits
mailing list