r355 - in /debtorrent/trunk: DebTorrent/BT1/makemetafile.py debian/changelog debian/control hippy.py uniquely.py
camrdale-guest at users.alioth.debian.org
camrdale-guest at users.alioth.debian.org
Fri Jan 25 08:17:31 UTC 2008
Author: camrdale-guest
Date: Fri Jan 25 08:17:31 2008
New Revision: 355
URL: http://svn.debian.org/wsvn/debtorrent/?sc=1&rev=355
Log:
Use python-debian routines for parsing all RFC 822 format files.
Modified:
debtorrent/trunk/DebTorrent/BT1/makemetafile.py
debtorrent/trunk/debian/changelog
debtorrent/trunk/debian/control
debtorrent/trunk/hippy.py
debtorrent/trunk/uniquely.py
Modified: debtorrent/trunk/DebTorrent/BT1/makemetafile.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/DebTorrent/BT1/makemetafile.py?rev=355&op=diff
==============================================================================
--- debtorrent/trunk/DebTorrent/BT1/makemetafile.py (original)
+++ debtorrent/trunk/DebTorrent/BT1/makemetafile.py Fri Jan 25 08:17:31 2008
@@ -33,6 +33,7 @@
from bz2 import decompress
from StringIO import StringIO
from re import subn
+from debian_bundle import deb822
import binascii, logging
try:
from sys import getfilesystemencoding
@@ -289,6 +290,12 @@
total += getsize(s[1])
return total
+class ExtraPieces(deb822._multivalued):
+ """For reading sub-piece data from an extrapieces file."""
+ _multivalued_fields = {
+ "sha1-pieces": [ "SHA1", "size" ],
+ }
+
def getsubpieces(file, pieces_file = ''):
"""Retrieve the sub-package piece information for the Packages file.
@@ -352,26 +359,12 @@
logger.warning('unable to find sub-pieces data')
return {}
- p = [None, [], []]
- read_data = False
- for line in f:
- line = line.rstrip()
- if line == "":
- if (p[0] and p[1] and p[2]):
- pieces[p[0]] = (p[1], p[2])
- packages += 1
- #progress(packages)
- p = [None, [], []]
- read_data = False
- if read_data == True and line[:1] != " ":
- read_data = False
- if line[:9] == "Filename:":
- p[0] = line[10:]
- if line == "SHA1-Pieces:":
- read_data = True
- if read_data == True and line[:1] == " ":
- p[1].append(binascii.a2b_hex(line[1:41]))
- p[2].append(int(line[42:]))
+ for pkg in ExtraPieces.iter_paragraphs(f):
+ if len(pkg.get('Filename', '')) > 0 and len(pkg.get('SHA1-pieces', [])) > 0:
+ pieces[pkg['Filename']] = ([], [])
+ for piece in pkg['SHA1-pieces']:
+ pieces[pkg['Filename']][0].append(binascii.a2b_hex(piece['SHA1']))
+ pieces[pkg['Filename']][1].append(int(piece['size']))
try:
f.close()
@@ -381,6 +374,12 @@
logger.info('successfully retrieved sub-piece data for '+str(len(pieces))+' files')
return pieces
+
+class Torrent(deb822._multivalued):
+ """For reading piece numbers from a unique piece number torrent file."""
+ _multivalued_fields = {
+ "piecenumbers": [ "number", "file" ],
+ }
def getordering(file, torrent_file = '', all = False):
"""Retrieve unique piece piece ordering information for the Packages file.
@@ -450,33 +449,18 @@
logger.warning('unable to find torrent ordering data')
return (pieces, headers)
- headers_done = False
- for line in f:
- line = line.rstrip()
-
- if not headers_done:
+ tor = Torrent(f)
+ for header in tor:
+ if header.lower() != 'piecenumbers':
# Read the headers from the file
- h, v = line.split(":", 1)
- if h == "PieceNumbers":
- headers_done = True
- continue
+ headers[header] = tor[header]
- headers[h] = v[1:]
- else:
- # Read the piece ordering data from the file
- if line[:1] != " ":
- break
-
- piece, file = line.split()
- pieces[int(piece)] = file
-
- try:
- f.close()
- except:
- pass
-
+ # Read the piece ordering data from the file
+ for piece in tor['PieceNumbers']:
+ pieces[int(piece['number'])] = piece['file']
+
logger.info('successfully retrieved torrent ordering data for '+str(len(pieces))+' files')
-
+ logger.debug('headers: %r', headers)
return (pieces, headers)
def orderpieces(fs, pieces, lengths, separate_all = 1, piece_ordering = {}, num_pieces = 0):
@@ -600,45 +584,29 @@
info = None
info_all = None
- p = [None, None, None, None, None, None]
- for line in f:
- line = line.rstrip()
- if line == "":
- if (p[0] and p[1] and p[2]):
- # Check which torrent to add the info to
- all = 0
- if (separate_all and p[5] == 'all'):
- all = 1
-
- if sub_pieces.has_key(p[1]):
- lengths[all][p[1]] = sub_pieces[p[1]][1]
- pieces[all][p[1]] = sub_pieces[p[1]][0]
- else:
- lengths[all][p[1]] = [p[0]]
- pieces[all][p[1]] = [p[2]]
-
- path = []
- temp = p[1]
- while temp:
- temp,d = split(temp)
- path.insert(0,d)
- fs[all][p[1]] = {'length': p[0], 'path': uniconvertl(path, encoding)}
- packages[all] += 1
- progress(packages[0] + packages[1])
- p = [None, None, None, None, None, None]
- if line[:9] == "Filename:":
- p[1] = line[10:]
- if line[:5] == "Size:":
- p[0] = long(line[6:])
- if line[:5] == "SHA1:":
- p[2] = binascii.a2b_hex(line[6:])
- if line[:8] == "Package:":
- p[3] = line[9:]
- if line[:8] == "Version:":
- p[4] = line[9:]
- if line[:13] == "Architecture:":
- p[5] = line[14:]
-
+ for pkg in deb822.Packages.iter_paragraphs(f, fields = ['Filename', 'Size', 'SHA1', 'Architecture']):
+ if pkg.get('Size', '') and pkg.get('SHA1', '') and pkg.get('Filename', ''):
+ # Check which torrent to add the info to
+ all = 0
+ if (separate_all and pkg.get('Architecture', '') == 'all'):
+ all = 1
+
+ if sub_pieces.has_key(pkg['Filename']):
+ lengths[all][pkg['Filename']] = sub_pieces[pkg['Filename']][1]
+ pieces[all][pkg['Filename']] = sub_pieces[pkg['Filename']][0]
+ else:
+ lengths[all][pkg['Filename']] = [long(pkg['Size'])]
+ pieces[all][pkg['Filename']] = [binascii.a2b_hex(pkg['SHA1'])]
+
+ path = []
+ temp = pkg['Filename']
+ while temp:
+ temp,d = split(temp)
+ path.insert(0,d)
+ fs[all][pkg['Filename']] = {'length': long(pkg['Size']), 'path': uniconvertl(path, encoding)}
+ packages[all] += 1
+ progress(packages[0] + packages[1])
+
if packages[0] > 0:
info = orderpieces(fs[0], pieces[0], lengths[0], separate_all,
piece_ordering, num_pieces)
Modified: debtorrent/trunk/debian/changelog
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/debian/changelog?rev=355&op=diff
==============================================================================
--- debtorrent/trunk/debian/changelog (original)
+++ debtorrent/trunk/debian/changelog Fri Jan 25 08:17:31 2008
@@ -8,12 +8,12 @@
* Switch to using 2 torrents per Packages file: one for architecture-
specific files, and one for architecture-independent files
- also added a new script split_cache_for_all to ease the upgrade
- (which is why python-debian is now required)
+ * Use python-debian for all reading of RFC 822 type files
* Add torrent names to the tracker display
* Make the download/upload statistics persist over restarts
* Report more and better statistics on the tracker's info page
- -- Cameron Dale <camrdale at gmail.com> Tue, 22 Jan 2008 19:19:35 -0800
+ -- Cameron Dale <camrdale at gmail.com> Thu, 24 Jan 2008 19:19:35 -0800
debtorrent (0.1.5) unstable; urgency=low
Modified: debtorrent/trunk/debian/control
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/debian/control?rev=355&op=diff
==============================================================================
--- debtorrent/trunk/debian/control (original)
+++ debtorrent/trunk/debian/control Fri Jan 25 08:17:31 2008
@@ -12,7 +12,7 @@
Package: debtorrent
Architecture: all
-Depends: ${python:Depends}, adduser, python-debian
+Depends: ${python:Depends}, adduser, python-debian (>= 0.1.4)
Suggests: python-psyco
Recommends: python-crypto, apt-transport-debtorrent (>= 0.2)
Provides: python-debtorrent
Modified: debtorrent/trunk/hippy.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/hippy.py?rev=355&op=diff
==============================================================================
--- debtorrent/trunk/hippy.py (original)
+++ debtorrent/trunk/hippy.py Fri Jan 25 08:17:31 2008
@@ -18,6 +18,7 @@
import struct
from bz2 import BZ2File
from math import ceil
+from debian_bundle import deb822
MAX_PIECE_SIZE = 512*1024
CHUNK_SIZE = 16*1024
@@ -42,31 +43,22 @@
f = open(filename, 'r')
- for line in f:
- line = line.rstrip()
-
- if line[:1] != " ":
- read_packages = False
- try:
- # Read the various headers from the file
- h, v = line.split(":", 1)
- if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
- read_packages = True
- elif len(v) > 0:
- headers[h] = v[1:]
- except:
- # Bad header line, just ignore it
- print "WARNING: Ignoring badly formatted Release line:", line
-
- # Skip to the next line
- continue
-
- # Read file names from the multiple hash sections of the file
- if read_packages:
- p = line.split()
- if len(p) == 3 and p[2].endswith("Packages"+EXTENSION):
- if p[2] not in packages:
- packages.append(p[2])
+ rel = deb822.Release(f)
+ for header in rel:
+ if header.lower() not in ["md5sum", "sha1", "sha256"]:
+ # Read the headers from the file
+ headers[header] = rel[header]
+
+ # Read the Packages file names
+ for file in rel.get('MD5Sum', []):
+ if file['name'].endswith("Packages"+EXTENSION) and file['name'] not in packages:
+ packages.append(file['name'])
+ for file in rel.get('SHA1', []):
+ if file['name'].endswith("Packages"+EXTENSION) and file['name'] not in packages:
+ packages.append(file['name'])
+ for file in rel.get('SHA256', []):
+ if file['name'].endswith("Packages"+EXTENSION) and file['name'] not in packages:
+ packages.append(file['name'])
f.close()
@@ -270,16 +262,9 @@
debs = []
- p = [None]
- for line in f:
- line = line.rstrip()
-
- if line == "":
- if p[0]:
- debs.append(p[0])
- p = [None]
- if line[:9] == "Filename:":
- p[0] = line[10:]
+ for pkg in deb822.Packages.iter_paragraphs(f, fields = ['Filename']):
+ if pkg.get('Filename', ''):
+ debs.append(pkg['Filename'])
f.close()
Modified: debtorrent/trunk/uniquely.py
URL: http://svn.debian.org/wsvn/debtorrent/debtorrent/trunk/uniquely.py?rev=355&op=diff
==============================================================================
--- debtorrent/trunk/uniquely.py (original)
+++ debtorrent/trunk/uniquely.py Fri Jan 25 08:17:31 2008
@@ -30,6 +30,12 @@
"PieceSize", "NextPiece", "OriginalPieces", "Codename", "Suite",
"Component", "Architecture", "Tracker", "TorrentHashFields"]
+class Torrent(deb822._multivalued):
+ """For reading piece numbers from a unique piece number torrent file."""
+ _multivalued_fields = {
+ "piecenumbers": [ "number", "file" ],
+ }
+
def read_release(filename):
"""Read the headers and Packages file names from a Release file.
@@ -48,31 +54,22 @@
f = open(filename, 'r')
- for line in f:
- line = line.rstrip()
-
- if line[:1] != " ":
- read_packages = False
- try:
- # Read the various headers from the file
- h, v = line.split(":", 1)
- if h == "MD5Sum" or h == "SHA1" or h == "SHA256":
- read_packages = True
- elif len(v) > 0:
- headers[h] = v[1:]
- except:
- # Bad header line, just ignore it
- print "WARNING: Ignoring badly formatted Release line:", line
-
- # Skip to the next line
- continue
-
- # Read file names from the multiple hash sections of the file
- if read_packages:
- p = line.split()
- if len(p) == 3 and p[2].endswith("Packages"+EXTENSION):
- if release_dir + "/" + p[2] not in packages:
- packages.append(release_dir + "/" + p[2])
+ rel = deb822.Release(f)
+ for header in rel:
+ if header.lower() not in ["md5sum", "sha1", "sha256"]:
+ # Read the headers from the file
+ headers[header] = rel[header]
+
+ # Read the Packages file names
+ for file in rel.get('MD5Sum', []):
+ if file['name'].endswith("Packages"+EXTENSION) and file['name'] not in packages:
+ packages.append(file['name'])
+ for file in rel.get('SHA1', []):
+ if file['name'].endswith("Packages"+EXTENSION) and file['name'] not in packages:
+ packages.append(file['name'])
+ for file in rel.get('SHA256', []):
+ if file['name'].endswith("Packages"+EXTENSION) and file['name'] not in packages:
+ packages.append(file['name'])
f.close()
@@ -95,26 +92,16 @@
try:
f = gzip.open(old_file, 'r')
- # Read the headers from the file
- for line in f:
- line = line.rstrip()
-
- h, v = line.split(":", 1)
- if h == "PieceNumbers":
- break
-
- headers[h] = v[1:]
-
- # Read the piece ordering from the file
- for line in f:
- line = line.rstrip()
-
- if line[:1] != " ":
- break
-
- piece, file = line.split()
- pieces[file] = int(piece)
-
+ tor = Torrent(f)
+ for header in tor:
+ if header.lower() != 'piecenumbers':
+ # Read the headers from the file
+ headers[header] = tor[header]
+
+ # Read the piece ordering data from the file
+ for piece in tor['PieceNumbers']:
+ pieces[piece['file']] = int(piece['number'])
+
f.close()
except:
# Delete the file and return empty variables to create a new torrent
@@ -220,40 +207,29 @@
pieces = {}
new_pieces = []
- p = [None, None, None]
- for line in f:
- line = line.rstrip()
- if line == "":
- if (p[0] and p[1] and p[2]):
- # Check which torrent to add the info to
- if p[2] == 'all':
- if p[0] in all_pieces.values():
- # Already found the old file
- pass
- elif p[0] in old_all_files:
- # Found old file, so add it
- all_pieces[old_all_files[p[0]]] = p[0]
- del old_all_files[p[0]]
- elif (p[0], p[1]) not in all_new_pieces:
- # Found new file, save it for later processing
- all_new_pieces.append((p[0], p[1]))
+ for pkg in deb822.Packages.iter_paragraphs(f, fields = ['Filename', 'Size', 'Architecture']):
+ if pkg.get('Size', '') and pkg.get('Filename', ''):
+ # Check which torrent to add the info to
+ if pkg.get('Architecture', '') == 'all':
+ if pkg['Filename'] in all_pieces.values():
+ # Already found the old file
+ pass
+ elif pkg['Filename'] in old_all_files:
+ # Found old file, so add it
+ all_pieces[old_all_files[pkg['Filename']]] = pkg['Filename']
+ del old_all_files[pkg['Filename']]
+ elif (pkg['Filename'], long(pkg['Size'])) not in all_new_pieces:
+ # Found new file, save it for later processing
+ all_new_pieces.append((pkg['Filename'], long(pkg['Size'])))
+ else:
+ if pkg['Filename'] in old_files:
+ # Found old file, so add it
+ pieces[old_files[pkg['Filename']]] = pkg['Filename']
+ del old_files[pkg['Filename']]
else:
- if p[0] in old_files:
- # Found old file, so add it
- pieces[old_files[p[0]]] = p[0]
- del old_files[p[0]]
- else:
- # Found new file, save it for later processing
- new_pieces.append((p[0], p[1]))
-
- p = [None, None, None]
- if line[:9] == "Filename:":
- p[0] = line[10:]
- if line[:5] == "Size:":
- p[1] = long(line[6:])
- if line[:13] == "Architecture:":
- p[2] = line[14:]
-
+ # Found new file, save it for later processing
+ new_pieces.append((pkg['Filename'], long(pkg['Size'])))
+
f.close()
return pieces, new_pieces
More information about the Debtorrent-commits
mailing list