r119 - in /debtorrent/trunk: DebTorrent/BT1/AptListener.py DebTorrent/BT1/makemetafile.py DebTorrent/download_bt1.py DebTorrent/launchmanycore.py DebTorrent/parseargs.py DebTorrent/parsedir.py btdownloadheadless.py btlaunchmany.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Wed Jun 20 07:40:59 UTC 2007
Author: camrdale-guest
Date: Wed Jun 20 07:40:58 2007
New Revision: 119
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=119
Log:
Add the option to split torrents into two, arch:all and the rest.
parsedir no longer supports Packages files (though it's not used much so it should not matter).
Modified:
debtorrent/trunk/DebTorrent/BT1/AptListener.py
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/DebTorrent/download_bt1.py
debtorrent/trunk/DebTorrent/launchmanycore.py
debtorrent/trunk/DebTorrent/parseargs.py
debtorrent/trunk/DebTorrent/parsedir.py
debtorrent/trunk/btdownloadheadless.py
debtorrent/trunk/btlaunchmany.py
Modified: debtorrent/trunk/DebTorrent/BT1/AptListener.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/AptListener.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/AptListener.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/AptListener.py Wed Jun 20 07:40:58 2007
@@ -37,7 +37,7 @@
from types import StringType, IntType, LongType, ListType, DictType
from binascii import b2a_hex, a2b_hex, a2b_base64
from string import lower
-from makemetafile import getpieces, getsubpieces, uniconvert
+from makemetafile import getpieces, getsubpieces, uniconvert, convert_all
from DebTorrent.HTTPCache import HTTPCache
import sys, os
import signal
@@ -654,8 +654,15 @@
sub_pieces = getsubpieces('_'.join(path))
- info = getpieces(h, sub_pieces = sub_pieces)
-
+ (info, info_all) = getpieces(h, separate_all = self.config['separate_all'], sub_pieces = sub_pieces)
+
+ if info and self.config['separate_all'] in (0, 2, 3):
+ self.start_torrent(info, name, path)
+
+ if info_all and self.config['separate_all'] in (1, 3):
+ self.start_torrent(info_all, convert_all(name), path)
+
+ def start_torrent(self, info, name, path):
response = {'info': info,
'announce': 'http://dttracker.debian.net:6969/announce',
'name': uniconvert(name)}
@@ -698,7 +705,6 @@
self.handler.add(infohash, a)
-
def get(self, connection, path, headers):
"""Respond to a GET request.
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Wed Jun 20 07:40:58 2007
@@ -18,6 +18,7 @@
from DebTorrent.zurllib import urlopen
from gzip import GzipFile
from StringIO import StringIO
+from re import subn
import binascii
try:
from sys import getfilesystemencoding
@@ -41,6 +42,7 @@
('target', '',
"optional target file for the torrent"),
('pieces_file', '', 'the file that contains the sub-package piece information'),
+ ('separate_all', 0, 'create a separate torrent for the architecture:all packages'),
]
default_piece_len_exp = 18
@@ -87,6 +89,17 @@
raise UnicodeError('bad filename: '+s)
return s.encode('utf-8')
+def convert_all(f):
+ # Find the architecture and replace it with 'all'
+ (f_all, n) = subn(r'binary-[a-zA-Z0-9]+([^a-zA-Z0-9]?)', r'binary-all\1', f)
+ if n == 0:
+ # Otherwise add '-all' before the extension
+ (f_all, n) = subn(r'\.([^.]*)$', r'-all.\1', f)
+ if n == 0:
+ # Otherwise add '-all' to the end
+ f_all = f + '-all'
+ return f_all
+
def make_meta_file(file, url, params = {}, progress = lambda x: None):
if params.has_key('piece_size_pow2'):
piece_len_exp = params['piece_size_pow2']
@@ -98,8 +111,10 @@
a, b = split(file)
if b == '':
f = a + '.dtorrent'
+ name = a
else:
f = join(a, b + '.dtorrent')
+ name = b
if piece_len_exp == 0: # automatic
size = calcsize(file)
@@ -123,12 +138,20 @@
if params.has_key('filesystem_encoding'):
encoding = params['filesystem_encoding']
- info = makeinfo(file, piece_length, encoding, progress, params['pieces_file'])
-
+ (info, info_all) = makeinfo(file, piece_length, encoding, progress,
+ params['separate_all'], params['pieces_file'])
+
+ if info:
+ create_file(f, info, url, uniconvert(name, encoding), params)
+
+ if info_all:
+ create_file(convert_all(f), info_all, url, uniconvert(convert_all(name), encoding), params)
+
+def create_file(f, info, url, name, params):
check_info(info)
h = open(f, 'wb')
data = {'info': info, 'announce': strip(url),
- 'name': uniconvert(split(file)[1], encoding),
+ 'name': name,
'creation date': long(time())}
if params.has_key('comment') and params['comment']:
@@ -229,38 +252,45 @@
return pieces
-def getpieces(f, encoding = None, progress = lambda x: None, sub_pieces = {}):
+def getpieces(f, encoding = None, progress = lambda x: None, separate_all = 0, sub_pieces = {}):
if not encoding:
encoding = ENCODING
if not encoding:
encoding = 'ascii'
- pieces = []
- lengths = []
- fs = []
- packages = 0
-
- p = [None, None, None, None, None]
+ pieces = ([], [])
+ lengths = ([], [])
+ fs = ([], [])
+ packages = [0, 0]
+ info = None
+ info_all = None
+
+ p = [None, None, None, None, None, None]
for line in f:
line = line.rstrip()
if line == "":
if (p[0] and p[1] and p[2]):
+ # Check which torrent to add the info to
+ all = 0
+ if (separate_all and p[5] == 'all'):
+ all = 1
+
if sub_pieces.has_key(p[1]):
- lengths.extend(sub_pieces[p[1]][1])
- pieces.extend(sub_pieces[p[1]][0])
+ lengths[all].extend(sub_pieces[p[1]][1])
+ pieces[all].extend(sub_pieces[p[1]][0])
else:
- lengths.append(p[0])
- pieces.append(p[2])
+ lengths[all].append(p[0])
+ pieces[all].append(p[2])
path = []
while p[1]:
p[1],d = split(p[1])
path.insert(0,d)
- fs.append({'length': p[0], 'path': uniconvertl(path, encoding)})
- packages += 1
- progress(packages)
- p = [None, None, None, None, None]
+ fs[all].append({'length': p[0], 'path': uniconvertl(path, encoding)})
+ packages[all] += 1
+ progress(packages[0] + packages[1])
+ p = [None, None, None, None, None, None]
if line[:9] == "Filename:":
p[1] = line[10:]
if line[:5] == "Size:":
@@ -271,10 +301,17 @@
p[3] = line[9:]
if line[:8] == "Version:":
p[4] = line[9:]
-
- return {'pieces': ''.join(pieces), 'piece lengths': lengths, 'files': fs}
-
-def makeinfo(file, piece_length, encoding, progress, pieces_file = ''):
+ if line[:13] == "Architecture:":
+ p[5] = line[14:]
+
+ if packages[0] > 0:
+ info = {'pieces': ''.join(pieces[0]), 'piece lengths': lengths[0], 'files': fs[0]}
+ if packages[1] > 0:
+ info_all = {'pieces': ''.join(pieces[1]), 'piece lengths': lengths[1], 'files': fs[1]}
+
+ return (info, info_all)
+
+def makeinfo(file, piece_length, encoding, progress, separate_all = 0, pieces_file = ''):
sub_pieces = getsubpieces(file, pieces_file)
@@ -283,10 +320,10 @@
file = abspath(file)
f = open(file)
- info = getpieces(f, encoding, progress, sub_pieces = sub_pieces)
+ (info, info_all) = getpieces(f, encoding, progress, separate_all = separate_all, sub_pieces = sub_pieces)
f.close()
- return info
+ return (info, info_all)
def subfiles(d):
r = []
Modified: debtorrent/trunk/DebTorrent/download_bt1.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/download_bt1.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/download_bt1.py (original)
+++ debtorrent/trunk/DebTorrent/download_bt1.py Wed Jun 20 07:40:58 2007
@@ -47,7 +47,7 @@
from clock import clock
from BTcrypto import CRYPTO_OK
from __init__ import createPeerID
-from BT1.makemetafile import getpieces, getsubpieces, uniconvert
+from BT1.makemetafile import getpieces, getsubpieces, uniconvert, convert_all
from gzip import GzipFile
from StringIO import StringIO
import binascii
@@ -372,7 +372,7 @@
return (argslistheader + formatDefinitions(defaults, cols, presets))
-def get_response(file, url, errorfunc):
+def get_response(file, url, errorfunc, separate_all = 0):
"""Get the response data from a metainfo or Packages file.
First checks to see if the data is in the Packages file format, and
@@ -386,19 +386,29 @@
@param url: the URL to download the metainfo file from
@type errorfunc: C{function}
@param errorfunc: the function to use to print any error messages
- @rtype: C{dictionary}
+ @type separate_all: C{int}
+ @param separate_all: whether to separate the architecture:all packages into
+ a separate torrent (optional, defaults to no)
+ @rtype: C{dictionary}, C{dictionary}
@return: the metainfo data
"""
- response = get_packages(file, url)
+ (response, response_all) = get_packages(file, url, separate_all)
if response:
try:
check_message(response)
except ValueError, e:
errorfunc("got bad file info - " + str(e))
- return None
- return response
+ return (None, None)
+ return (response, response_all)
+ if response_all:
+ try:
+ check_message(response_all)
+ except ValueError, e:
+ errorfunc("got bad file info - " + str(e))
+ return (None, None)
+ return (response, response_all)
try:
if file:
h = open(file, 'rb')
@@ -409,7 +419,7 @@
int(front[1:])
except:
errorfunc(file+' is not a valid responsefile')
- return None
+ return (None, None)
try:
h.seek(0)
except:
@@ -423,12 +433,12 @@
h = urlopen(url)
except:
errorfunc(url+' bad url')
- return None
+ return (None, None)
response = h.read()
except IOError, e:
errorfunc('problem getting response info - ' + str(e))
- return None
+ return (None, None)
try:
h.close()
except:
@@ -442,11 +452,11 @@
check_message(response)
except ValueError, e:
errorfunc("got bad file info - " + str(e))
- return None
-
- return response
-
-def get_packages(file, url):
+ return (None, None)
+
+ return (response, None)
+
+def get_packages(file, url, separate_all = 0):
"""Extract the response data from a Packages file.
@type file: C{string}
@@ -454,7 +464,10 @@
to be used
@type url: C{string}
@param url: the URL to download the metainfo file from
- @rtype: C{dictionary}
+ @type separate_all: C{int}
+ @param separate_all: whether to separate the architecture:all packages into
+ a separate torrent (optional, defaults to no)
+ @rtype: C{dictionary}, C{dictionary}
@return: the metainfo data
"""
@@ -470,7 +483,7 @@
assert line[:8] == "Package:"
except:
# errorfunc(file+' is not a valid Packages file')
- return None
+ return (None, None)
try:
h.seek(0)
except:
@@ -497,25 +510,29 @@
h = data.split('\n')
except:
# errorfunc(url+' bad url')
- return None
+ return (None, None)
sub_pieces = getsubpieces(name)
- info = getpieces(h, sub_pieces = sub_pieces)
+ (info, info_all) = getpieces(h, separate_all = separate_all, sub_pieces = sub_pieces)
response = {'info': info,
'announce': 'http://dttracker.debian.net:6969/announce',
'name': uniconvert(name)}
+
+ response_all = {'info': info_all,
+ 'announce': 'http://dttracker.debian.net:6969/announce',
+ 'name': uniconvert(convert_all(name))}
except IOError, e:
# errorfunc('problem getting Packages info - ' + str(e))
- return None
+ return (None, None)
try:
h.close()
except:
pass
- return response
+ return (response, response_all)
class BT1Download:
Modified: debtorrent/trunk/DebTorrent/launchmanycore.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/launchmanycore.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/launchmanycore.py (original)
+++ debtorrent/trunk/DebTorrent/launchmanycore.py Wed Jun 20 07:40:58 2007
@@ -21,7 +21,6 @@
from RawServer import RawServer, UPnP_ERROR
from RateLimiter import RateLimiter
from ServerPortHandler import MultiHandler
-from parsedir import parsedir
from natpunch import UPnP_test
from random import seed
from socket import error as socketerror
Modified: debtorrent/trunk/DebTorrent/parseargs.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/parseargs.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/parseargs.py (original)
+++ debtorrent/trunk/DebTorrent/parseargs.py Wed Jun 20 07:40:58 2007
@@ -111,7 +111,7 @@
"""Parse a list of input arguments and set the configuration options variable.
All configuration options must begin with '--', must be followed by a space
- and then the new setting to use, and must appear in the options variable
+ and then the new setting to use, and must appear in the L{options} variable
keys. Non-options can appear anywhere in the arguments (except between an
option and it's setting), and there must be between minargs and maxargs of
them.
Modified: debtorrent/trunk/DebTorrent/parsedir.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/parsedir.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/parsedir.py (original)
+++ debtorrent/trunk/DebTorrent/parsedir.py Wed Jun 20 07:40:58 2007
@@ -36,9 +36,12 @@
print ":: "+x
def parsedir(directory, parsed, files, blocked,
- exts = ['dtorrent','Packages'],
+ exts = ['dtorrent'],
return_metainfo = False, errfunc = _errfunc):
"""Parse a directory for torrent files.
+
+ Only works for .dtorrent files, it will not find or properly parse
+ Packages files to extract the torrent from them.
@type directory: C{string}
@param directory: the directory to parse (somewhat recursively)
@@ -50,7 +53,7 @@
@param blocked: the files that were previously blocked
@type exts: C{list} of C{string}
@param exts: the extensions to look for torrent files in
- (optional, defaults to 'dtorrent' and 'Packages')
+ (optional, defaults to 'dtorrent')
@type return_metainfo: C{boolean}
@param return_metainfo: whether to return the torrent metainfo
(optional, defaults to False)
@@ -132,7 +135,7 @@
if NOISY:
errfunc('adding '+p)
try:
- d = get_response(p, '', errfunc)
+ (d, garbage) = get_response(p, '', errfunc)
h = sha(bencode(d['info'])).digest()
new_file[1] = h
if new_parsed.has_key(h):
Modified: debtorrent/trunk/btdownloadheadless.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btdownloadheadless.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/btdownloadheadless.py (original)
+++ debtorrent/trunk/btdownloadheadless.py Wed Jun 20 07:40:58 2007
@@ -256,9 +256,12 @@
defaultsToIgnore = ['responsefile', 'url', 'priority']
configdir.setDefaults(defaults,defaultsToIgnore)
configdefaults = configdir.loadConfig()
- defaults.append(('save_options',0,
- "whether to save the current options as the new default configuration " +
- "(only for btdownloadheadless.py)"))
+ defaults.extend([('save_options',0,
+ "whether to save the current options as the new default configuration " +
+ "(only for btdownloadheadless.py)"),
+ ('separate_all',0, 'whether to separate the architecture:all packages, ' +
+ "0 = don't separate, 1 = separate and run architecture:all, " +
+ "2 = separate and run all architectures but all")])
try:
config = parse_params(params, configdefaults)
except ValueError, e:
@@ -304,7 +307,11 @@
config['min_time_between_log_flushes']),
config['port'])
- response = get_response(config['responsefile'], config['url'], h.error)
+ (response, response_all) = get_response(config['responsefile'], config['url'], h.error, config['separate_all'])
+
+ if config['separate_all'] == 1:
+ response = response_all
+
if not response:
break
Modified: debtorrent/trunk/btlaunchmany.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/btlaunchmany.py?rev=119&op=diff
==============================================================================
--- debtorrent/trunk/btlaunchmany.py (original)
+++ debtorrent/trunk/btlaunchmany.py Wed Jun 20 07:40:58 2007
@@ -139,8 +139,11 @@
"How to name torrent download directory " +
"(1 = <mirror>_dists_<suite>_<section>_binary-<arch>, " +
"2 = <mirror>)" ),
- ( 'display_path', 1,
+ ( 'display_path', 0,
"whether to display the full path or the torrent contents for each torrent" ),
+ ('separate_all',0, 'whether to separate the architecture:all packages, ' +
+ "0 = don't separate, 1 = separate and run architecture:all, " +
+ "2 = separate and run all architectures but all, " + "3 = separate and run both"),
] )
try:
configdir = ConfigDir('launchmany')
More information about the Debtorrent-commits
mailing list