r112 - in /debtorrent/trunk: DebTorrent/BT1/AptListener.py DebTorrent/BT1/makemetafile.py DebTorrent/download_bt1.py DebTorrent/launchmanycore.py TODO btlaunchmany.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Fri Jun 15 04:52:06 UTC 2007
Author: camrdale-guest
Date: Fri Jun 15 04:52:05 2007
New Revision: 112
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=112
Log:
Change the naming scheme for the download directory.
Remove the need to run btlaunchmany with a directory (it just listens).
Modified:
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/DebTorrent/launchmanycore.py
debtorrent/trunk/TODO
debtorrent/trunk/btlaunchmany.py
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=112&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Fri Jun 15 04:52:05 2007
@@ -25,7 +25,7 @@
from Filter import Filter
from urlparse import urlparse
from os import rename, getpid
-from os.path import exists, isfile
+from os.path import exists, isfile, join
from cStringIO import StringIO
from gzip import GzipFile
from bz2 import decompress
@@ -488,6 +488,7 @@
# First check the cache for the file
data = self.Cache.cache_get(path, uptodate)
+ # TODO: watch out for getting cached deb's that haven't yet downloaded (with saveas_style 2)
# If the cache doesn't have it
if data is None:
@@ -648,7 +649,7 @@
elif path[-1].endswith('.bz2'):
data = decompress(data)
- name = "dt_" + '_'.join(path)
+ name = '_'.join(path[:-1])
assert data[:8] == "Package:"
h = data.split('\n')
@@ -657,7 +658,7 @@
print 'ERROR: Packages file could not be converted to a torrent'
return
- sub_pieces = getsubpieces(name)
+ sub_pieces = getsubpieces('_'.join(path))
info = getpieces(h, sub_pieces = sub_pieces)
@@ -675,6 +676,10 @@
a['path'] = '/'.join(path)
a['file'] = name
a['type'] = path[-1]
+ mirror = path[0]
+ for i in path[1:path.index('dists')]:
+ mirror = join(mirror, i)
+ a['mirror'] = mirror
i = response['info']
l = 0
nf = 0
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=112&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Fri Jun 15 04:52:05 2007
@@ -128,7 +128,7 @@
check_info(info)
h = open(f, 'wb')
data = {'info': info, 'announce': strip(url),
- 'name': uniconvert("dt_" + split(file)[1], encoding),
+ 'name': uniconvert(split(file)[1], encoding),
'creation date': long(time())}
if params.has_key('comment') and params['comment']:
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=112&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Fri Jun 15 04:52:05 2007
@@ -461,7 +461,7 @@
try:
if file:
- name = "dt_" + path.split(file)[1]
+ name = path.split(file)[1]
h = open(file, 'r')
try:
for line in h: # quick test to see if packages file is correct
@@ -481,14 +481,14 @@
h = open(file, 'r')
else:
urlp = urlparse(url)
- name = "dt_" + urlp[1] + urlp[2].replace('/','_')
+ name = urlp[1] + urlp[2].replace('/','_')
try:
h = urlopen(url)
file = url
data = h.read()
assert len(data) > 8
if url[-3:] == ".gz":
- name = "dt_" + urlp[1] + urlp[2][:-3].replace('/','_')
+ name = urlp[1] + urlp[2][:-3].replace('/','_')
if data[:8] != "Package:":
compressed = StringIO(data)
f = GzipFile(fileobj = compressed)
@@ -785,18 +785,19 @@
if not path.isdir(file):
self.errorfunc(file + 'is not a dir')
return None
- if len(listdir(file)) > 0: # if it's not empty
- for x in self.info['files']:
- if path.exists(path.join(file, x['path'][0])):
- existing = 1
- if not existing:
- file = path.join(file, self.response['name'])
- if path.exists(file) and not path.isdir(file):
- if file[-8:] == '.dtorrent':
- file = file[:-8]
- if path.exists(file) and not path.isdir(file):
- self.errorfunc("Can't create dir - " + self.response['name'])
- return None
+# Don't create a directory with the default name within it
+# if len(listdir(file)) > 0: # if it's not empty
+# for x in self.info['files']:
+# if path.exists(path.join(file, x['path'][0])):
+# existing = 1
+# if not existing:
+# file = path.join(file, self.response['name'])
+# if path.exists(file) and not path.isdir(file):
+# if file[-8:] == '.dtorrent':
+# file = file[:-8]
+# if path.exists(file) and not path.isdir(file):
+# self.errorfunc("Can't create dir - " + self.response['name'])
+# return None
make(file, True)
# alert the UI to any possible change in path
@@ -818,7 +819,6 @@
self.filename = file
self.files = files
self.datalength = file_length
-
return file
Modified: debtorrent/trunk/DebTorrent/launchmanycore.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/launchmanycore.py?rev=112&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/launchmanycore.py (original)
+++ debtorrent/trunk/DebTorrent/launchmanycore.py Fri Jun 15 04:52:05 2007
@@ -290,16 +290,12 @@
@ivar Output: the displayer instance to use
@type configdir: L{ConfigDir.ConfigDir}
@ivar configdir: the configuration and cache directory manager
- @type torrent_dir: C{string}
- @ivar torrent_dir: the directory to parse for torrent files
@type torrent_cache: C{dictionary}
@ivar torrent_cache: the cache of known torrents, keys are info hashes
@type file_cache: C{dictionary}
@ivar file_cache: the files found in the parsing of the torrent directory
@type blocked_files: C{dictionary}
@ivar blocked_files: the torrents in the torrent directory that will not be run
- @type scan_period: C{int}
- @ivar scan_period: the number of seconds between scans of L{torrent_dir}
@type stats_period: C{int}
@ivar stats_period: the number of seconds between printing the stats for the user
@type torrent_list: C{list} of C{string}
@@ -344,11 +340,9 @@
self.Output = Output
self.configdir = configdir
- self.torrent_dir = config['torrent_dir']
self.torrent_cache = {}
self.file_cache = {}
self.blocked_files = {}
- self.scan_period = config['parse_dir_interval']
self.stats_period = config['display_interval']
self.torrent_list = []
@@ -391,7 +385,6 @@
self.handler = MultiHandler(self.rawserver, self.doneflag, config)
seed(createPeerID())
-# self.rawserver.add_task(self.scan, 0)
self.rawserver.add_task(self.stats, 0)
# Restore the previous state of the downloads
@@ -414,24 +407,6 @@
print_exc(file = data)
Output.exception(data.getvalue())
-
- def scan(self):
- """Scan the torrent directory for changes."""
- self.rawserver.add_task(self.scan, self.scan_period)
-
- r = parsedir(self.torrent_dir, self.torrent_cache,
- self.file_cache, self.blocked_files,
- return_metainfo = True, errfunc = self.Output.message)
-
- ( self.torrent_cache, self.file_cache, self.blocked_files,
- added, removed ) = r
-
- for hash, data in removed.items():
- self.Output.message('dropped "'+data['path']+'"')
- self.remove(hash)
- for hash, data in added.items():
- self.Output.message('added "'+data['path']+'"')
- self.add(hash, data)
def stats(self):
"""Call the Output display with the currently running torrents' statistics."""
@@ -536,7 +511,7 @@
# Save the new torrent file to the cache
if save_cache:
- self.configdir.writeTorrent(self.torrent_cache[hash], hash)
+ self.configdir.writeTorrent(self.torrent_cache[hash]['metainfo'], hash)
if hash in self.torrent_list:
for u in new_debmirrors:
@@ -573,32 +548,33 @@
x = self.torrent_cache[hash]
style = self.config['saveas_style']
- if style == 1 or style == 3:
- if saveas:
- saveas = os.path.join(saveas,x['file'][:-1-len(x['type'])])
- else:
- saveas = x['path'][:-1-len(x['type'])]
- if style == 3:
- if not os.path.isdir(saveas):
- try:
- os.mkdir(saveas)
- except:
- raise OSError("couldn't create directory for "+x['path']
- +" ("+saveas+")")
- if not isdir:
- saveas = os.path.join(saveas, name)
- else:
+ if style == 1:
if saveas:
saveas = os.path.join(saveas, name)
else:
- saveas = os.path.join(os.path.split(x['path'])[0], name)
+ saveas = name
+ else:
+ if saveas:
+ saveas = os.path.join(saveas, x['mirror'])
+ else:
+ saveas = x['mirror']
- if isdir and not os.path.isdir(saveas):
- try:
- os.mkdir(saveas)
- except:
- raise OSError("couldn't create directory for "+x['path']
- +" ("+saveas+")")
+ if isdir:
+ if not os.path.exists(saveas):
+ try:
+ os.makedirs(saveas)
+ except:
+ print_exc()
+ raise OSError("couldn't create directory for "+x['path']
+ +" ("+saveas+")")
+ else:
+ if not os.path.exists(os.path.split(saveas)[0]):
+ try:
+ os.makedirs(os.path.split(saveas)[0])
+ except:
+ raise OSError("couldn't create directory for "+x['path']
+ +" ("+os.path.split(saveas)[0]+")")
+
return saveas
@@ -625,8 +601,7 @@
# Check each torrent in the cache
for hash, data in self.torrent_cache.items():
# Make sure this torrent is from the mirror in question
- # (TODO: later make this more certain by not prepending 'dt_' to the name)
- if data['metainfo']['name'].find(mirror) == -1:
+ if not data['metainfo']['name'].startswith(mirror):
continue
file_num = -1
Modified: debtorrent/trunk/TODO
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/TODO?rev=112&op=diff
==============================================================================
--- debtorrent/trunk/TODO (original)
+++ debtorrent/trunk/TODO Fri Jun 15 04:52:05 2007
@@ -89,20 +89,6 @@
how much computation will be required to generate it.
-Determine how to name the directory for downloaded files
-
-A policy is needed for determing what name to give the directory that debtorrent
-downloads files into. Using the full Packages file name from /var/lib/apt/lists
-means the directory will need to be changed/redownloaded if a mirror is changed.
-Using the origin field from the Release file could cause problems if another
-archive uses the same origin, and means that multiple torrents will download to
-the same directory. Something like "<mirror>_<codename>_<section>_binary-<arch>"
-might work. That will result in duplicate downloads, if someone tracks both
-sid and testing, for example. However, before downloading, the file could be
-searched for in other directories, and hardlinked if found (and SHA1 sum
-matches.
-
-
Consider Sources
Sources files contain 2 or 3 files for every source package, but no SHA1 sums
Modified: debtorrent/trunk/btlaunchmany.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btlaunchmany.py?rev=112&op=diff
==============================================================================
--- debtorrent/trunk/btlaunchmany.py (original)
+++ debtorrent/trunk/btlaunchmany.py Fri Jun 15 04:52:05 2007
@@ -135,11 +135,10 @@
"""
defaults.extend( [
- ( 'parse_dir_interval', 60,
- "how often to rescan the torrent directory, in seconds" ),
( 'saveas_style', 1,
- "How to name torrent downloads (1 = rename to torrent name, " +
- "2 = save under name in torrent, 3 = save in directory under torrent name)" ),
+ "How to name torrent download directory " +
+ "(1 = <mirror>_dists_<suite>_<section>_binary-<arch>, " +
+ "2 = <mirror>)" ),
( 'display_path', 1,
"whether to display the full path or the torrent contents for each torrent" ),
] )
@@ -151,21 +150,14 @@
defaults.append(('save_options',0,
"whether to save the current options as the new default configuration " +
"(only for btlaunchmany.py)"))
- if len(argv) < 2:
- print "Usage: btlaunchmany.py <directory> <global options>\n"
- print "<directory> - directory to look for .dtorrent files (semi-recursive)"
- print get_usage(defaults, 80, configdefaults)
- exit(1)
- # TODO: remove the need for a directory to parse (and options)
- config, args = parseargs(params, defaults, 1, 1, configdefaults)
+ config, args = parseargs(params, defaults, 0, 0, configdefaults)
if config['save_options']:
configdir.saveConfig(config)
configdir.deleteOldCacheData(config['expire_cache_data'])
- if not os.path.isdir(args[0]):
- raise ValueError("Warning: "+args[0]+" is not a directory")
- config['torrent_dir'] = args[0]
except ValueError, e:
- print 'error: ' + str(e) + '\nrun with no args for parameter explanations'
+ print 'error: ' + str(e) + '\n'
+ print "Usage: btlaunchmany.py <global options>\n"
+ print get_usage(defaults, 80, configdefaults)
exit(1)
LaunchMany(config, HeadlessDisplayer(), configdir)
More information about the Debtorrent-commits
mailing list