r359 - in /debtorrent/trunk/DebTorrent/BT1: AptListener.py makemetafile.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Sun Jan 27 02:07:39 UTC 2008
Author: camrdale-guest
Date: Sun Jan 27 02:07:38 2008
New Revision: 359
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=359
Log:
TorrentCreator uses the HTTPCache to download files.
Modified:
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=359&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Sun Jan 27 02:07:38 2008
@@ -369,10 +369,11 @@
if path[-1] in ('Packages', 'Packages.gz', 'Packages.bz2'):
TorrentCreator(path, filename, self.start_torrent,
- self.rawserver.add_task, self.config)
+ self.rawserver.add_task, self.Cache, self.config)
# Returning a file, so open the file to be returned
- r = r[0:3] + (open(filename, 'rb'), )
+ if r[0] != 304:
+ r = r[0:3] + (open(filename, 'rb'), )
return r
@@ -407,9 +408,9 @@
# If it's a torrent file, start it
if r[0] == 200 and path[-1] in ('Packages', 'Packages.gz', 'Packages.bz2'):
TorrentCreator(path, filename, self.start_torrent,
- self.rawserver.add_task, self.config)
-
- if filename:
+ self.rawserver.add_task, self.Cache, self.config)
+
+ if filename and r[0] != 304:
# Returning a file, so open the file to be returned
r = r[0:3] + (open(filename, 'rb'), )
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=359&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Sun Jan 27 02:07:38 2008
@@ -296,14 +296,11 @@
"sha1-pieces": [ "SHA1", "size" ],
}
-def getsubpieces(file, pieces_file = ''):
- """Retrieve the sub-package piece information for the Packages file.
-
- @type file: C{string}
- @param file: the Packages file name to retrieve piece information for
+def getsubpieces(pieces_file):
+ """Process the sub-package piece information for the Packages file.
+
@type pieces_file: C{string}
@param pieces_file: the file that contains the piece information
- (optional, defaults to retrieving the info from the web)
@rtype: C{dictionary}
@return: the piece info, keys are the file names, values are tuples of
a list of piece SHA1 hashes and a list of piece sizes
@@ -314,49 +311,13 @@
packages = 0
piece_url = ''
- if pieces_file:
- try:
- if torrent_file.endswith('.gz'):
- f = gzip.open(pieces_file)
- else:
- f = open(pieces_file)
- except:
- logger.exception('sub-pieces file not found: '+pieces_file)
- return {}
- elif 'dists' in file.split('_'):
- try:
- parts = file.split('_')
- try:
- parts[parts.index('stable', parts.index('dists'))] = 'etch'
- except:
- pass
- try:
- parts[parts.index('testing', parts.index('dists'))] = 'lenny'
- except:
- pass
- try:
- parts[parts.index('unstable', parts.index('dists'))] = 'sid'
- except:
- pass
- piece_url = 'http://merkel.debian.org/~ajt/extrapieces/dists_'
- piece_url += '_'.join(parts[parts.index('dists')+1:])
- if piece_url.endswith('.gz'):
- piece_url = piece_url[:-3]
- if piece_url.endswith('.bz2'):
- piece_url = piece_url[:-4]
- piece_url += '-extrapieces.gz'
- piece_file = urlopen(piece_url)
- piece_data = piece_file.read()
- try:
- piece_file.close()
- except:
- pass
- f = piece_data.split('\n')
- except:
- logger.exception('sub-pieces URL not working: '+piece_url)
- return {}
- else:
- logger.warning('unable to find sub-pieces data')
+ try:
+ if pieces_file.endswith('.gz'):
+ f = gzip.open(pieces_file)
+ else:
+ f = open(pieces_file)
+ except:
+ logger.exception('sub-pieces file not found: '+pieces_file)
return {}
for pkg in ExtraPieces.iter_paragraphs(f):
@@ -382,17 +343,11 @@
"piecenumbers": [ "number", "file" ],
}
-def getordering(file, torrent_file = '', all = False):
+def getordering(torrent_file):
"""Retrieve unique piece piece ordering information for the Packages file.
- @type file: C{string}
- @param file: the Packages file name to retrieve piece ordering information for
@type torrent_file: C{string}
@param torrent_file: the file that contains the piece ordering information
- (optional, defaults to retrieving the info from the web)
- @type all: C{boolean}
- @param all: whether to get it for the architecture, or for arch:all
- (optional, defaults to the specific architecture)
@rtype: C{dictionary}
@return: the piece ordering info, keys are the starting piece numbers to
use for the files, values are the file names
@@ -403,51 +358,13 @@
headers = {}
piece_url = ''
- if torrent_file:
- try:
- if torrent_file.endswith('.gz'):
- f = gzip.open(torrent_file)
- else:
- f = open(torrent_file)
- except:
- logger.exception('torrent ordering file not found: '+torrent_file)
- return (pieces, headers)
- elif 'dists' in file.split('_'):
- try:
- parts = file.split('_')
- try:
- parts[parts.index('stable', parts.index('dists'))] = 'etch'
- except:
- pass
- try:
- parts[parts.index('testing', parts.index('dists'))] = 'lenny'
- except:
- pass
- try:
- parts[parts.index('unstable', parts.index('dists'))] = 'sid'
- except:
- pass
- piece_url = 'http://debian.camrdale.org/debtorrent/dists_'
- piece_url += '_'.join(parts[parts.index('dists')+1:])
- if piece_url.endswith('.gz'):
- piece_url = piece_url[:-3]
- if piece_url.endswith('.bz2'):
- piece_url = piece_url[:-4]
- piece_url += '-torrent.gz'
- if all:
- piece_url = convert_all(piece_url)
- piece_file = urlopen(piece_url)
- piece_data = piece_file.read()
- try:
- piece_file.close()
- except:
- pass
- f = piece_data.split('\n')
- except:
- logger.exception('torrent ordering URL not working: '+piece_url)
- return (pieces, headers)
- else:
- logger.warning('unable to find torrent ordering data')
+ try:
+ if torrent_file.endswith('.gz'):
+ f = gzip.open(torrent_file)
+ else:
+ f = open(torrent_file)
+ except:
+ logger.exception('torrent ordering file not found: '+torrent_file)
return (pieces, headers)
tor = Torrent(f)
@@ -641,7 +558,7 @@
a separate torrent (optional, defaults to True)
@type pieces_file: C{string}
@param pieces_file: the file that contains the piece information
- (optional, defaults to retrieving the info from the web)
+ (optional, defaults to not using sub pieces)
@type torrent_file: C{string}
@param torrent_file: the file that contains the piece ordering information
(optional, defaults to retrieving the info from the web)
@@ -655,15 +572,19 @@
"""
- sub_pieces = getsubpieces(file, pieces_file)
+ sub_pieces = {}
+ if pieces_file:
+ sub_pieces = getsubpieces(pieces_file)
piece_ordering_all = {}
ordering_all_headers = {}
piece_ordering = {}
ordering_headers = {}
if separate_all:
- (piece_ordering, ordering_headers) = getordering(file, torrent_file)
- (piece_ordering_all, ordering_all_headers) = getordering(file, torrent_all_file, True)
+ if torrent_file:
+ (piece_ordering, ordering_headers) = getordering(torrent_file)
+ if torrent_all_file:
+ (piece_ordering_all, ordering_all_headers) = getordering(torrent_all_file)
file = abspath(file)
if file.endswith('.gz'):
@@ -749,7 +670,7 @@
"""
- def __init__(self, path, filename, callback, sched, config):
+ def __init__(self, path, filename, callback, sched, cache, config):
"""Process a downloaded Packages file and start the torrent making thread.
@type path: C{list} of C{string}
@@ -760,6 +681,8 @@
@param callback: the method to call with the torrent when it has been created
@type sched: C{method}
@param sched: the method to call to schedule future invocation of a function
+ @type cache: L{DebTorrent.HTTPCache.HTTPCache}
+ @param cache: the cache of downloaded files
@type config: C{dictionary}
@param config: the configuration parameters
@@ -769,11 +692,91 @@
self.filename = filename
self.callback = callback
self.sched = sched
+ self.cache = cache
self.config = config
self.name = '_'.join(self.path[:-1])
self.responses = []
-
- # Create and start the thread to create the torrent metainfo
+ self.download = []
+ self.subpieces_url = None
+ self.subpieces_file = None
+ self.torrent_url = None
+ self.torrent_all_url = None
+ self.torrent_file = None
+ self.torrent_all_file = None
+
+ if 'dists' in self.path:
+ parts = self.path[:]
+ try:
+ parts[parts.index('stable', parts.index('dists'))] = 'etch'
+ except:
+ pass
+ try:
+ parts[parts.index('testing', parts.index('dists'))] = 'lenny'
+ except:
+ pass
+ try:
+ parts[parts.index('unstable', parts.index('dists'))] = 'sid'
+ except:
+ pass
+ url = 'dists_'
+ url += '_'.join(parts[parts.index('dists')+1:])
+ if url.endswith('.gz'):
+ url = url[:-3]
+ if url.endswith('.bz2'):
+ url = url[:-4]
+ self.subpieces_url = 'merkel.debian.org/~ajt/extrapieces/' + url + '-extrapieces.gz'
+ self.download.append('subpieces')
+ if self.config['separate_all']:
+ self.torrent_url = 'debian.camrdale.org/debtorrent/' + url + '-torrent.gz'
+ self.download.append('torrent')
+ if self.config['separate_all'] in [1, 3]:
+ self.torrent_all_url = convert_all(self.torrent_url)
+ self.download.append('torrent_all')
+
+ for down in self.download[:]:
+ down_path = getattr(self, down + '_url').split('/')
+ r, filename = self.cache.cache_get(down_path, True)
+ if filename:
+ setattr(self, down + '_file', filename)
+ self.download.remove(down)
+ else:
+ self.cache.download_get(down_path, self._download_callback)
+
+ if not self.download:
+ self._start()
+
+ def _download_callback(self, path, r, filename):
+ """Save the newly downloaded file's name and move on to creation if downloading is complete.
+
+ @type path: C{list} of C{string}
+ @param path: the path of the file to download, starting with the mirror name
+ @type r: (C{int}, C{string}, C{dictionary}, C{string})
+ @param r: the HTTP status code, status message, headers, and cached data
+ @type filename: C{string}
+ @param filename: the file containing the successfully downloaded file
+
+ """
+
+ found = False
+ for down in self.download:
+ if getattr(self, down + '_url') == '/'.join(path):
+ if filename:
+ setattr(self, down + '_file', filename)
+ else:
+ logger.warning('Failed to download file: %s', '/'.join(path))
+ found = True
+ self.download.remove(down)
+ break
+
+ if not found:
+ logger.warning('Got response for unrequested download: %s', '/'.join(path))
+
+ if not self.download:
+ self._start()
+
+ def _start(self):
+ """Create and start the thread to create the torrent metainfo."""
+
logger.debug('starting thread to create torrent for: '+self.name)
rq = Thread(target = self._create, name = 'TorrentCreator('+self.name+')')
rq.setDaemon(False)
@@ -797,16 +800,18 @@
logger.debug('Packages file successfully opened')
try:
- sub_pieces = getsubpieces('_'.join(self.path))
+ sub_pieces = {}
+ if self.subpieces_file:
+ sub_pieces = getsubpieces(self.subpieces_file)
piece_ordering_all = {}
ordering_all_headers = {}
piece_ordering = {}
ordering_headers = {}
- if self.config['separate_all']:
- (piece_ordering, ordering_headers) = getordering('_'.join(self.path))
- if self.config['separate_all'] in [1, 3]:
- (piece_ordering_all, ordering_all_headers) = getordering('_'.join(self.path), all = True)
+ if self.torrent_file:
+ (piece_ordering, ordering_headers) = getordering(self.torrent_file)
+ if self.torrent_all_file:
+ (piece_ordering_all, ordering_all_headers) = getordering(self.torrent_all_file)
(info, info_all) = getpieces(f, separate_all = self.config['separate_all'],
sub_pieces = sub_pieces,
More information about the Debtorrent-commits
mailing list