r424 - in branches/rewrite: . src
Otavio Salvador
partial-mirror-devel@lists.alioth.debian.org
Sun, 19 Dec 2004 14:16:15 -0700
Author: otavio
Date: Sun Dec 19 14:16:13 2004
New Revision: 424
Added:
branches/rewrite/src/Files.py (contents, props changed)
Modified:
branches/rewrite/ (props changed)
branches/rewrite/src/Backend.py
branches/rewrite/src/Config.py
branches/rewrite/src/Download.py
Log:
r1386@nurf: otavio | 2004-12-19T21:15:25.114469Z
Add suport to get files.
Modified: branches/rewrite/src/Backend.py
==============================================================================
--- branches/rewrite/src/Backend.py (original)
+++ branches/rewrite/src/Backend.py Sun Dec 19 14:16:13 2004
@@ -19,6 +19,7 @@
import Config
import Dists
+import Files
import Pool
class Backend:
@@ -39,9 +40,11 @@
if isinstance(self._cfg.getBackend(self._name), Config.ConfigBackendMirror):
self._dists = Dists.RemoteDists(self)
self._pool = Pool.RemotePool(self)
+ self._files = Files.RemoteFiles(self)
elif isinstance(self._cfg.getBackend(self._name), Config.ConfigBackendMerge):
self._dists = Dists.LocalDists(self)
self._pool = Pool.Pool(self)
+ self._files = Files.Files(self)
def __getitem__ (self, key):
try:
@@ -69,6 +72,7 @@
def upgrade (self):
self._dists.upgrade()
+ self._files.upgrade()
self._pool.upgrade()
def remove (self):
Modified: branches/rewrite/src/Config.py
==============================================================================
--- branches/rewrite/src/Config.py (original)
+++ branches/rewrite/src/Config.py Sun Dec 19 14:16:13 2004
@@ -197,6 +197,7 @@
'components',
'distributions',
'filter',
+ 'files',
'get_suggests',
'get_recommends',
'get_provides',
@@ -209,6 +210,7 @@
'architectures': 'list',
'distributions': 'list',
'filter': 'filter',
+ 'files': 'list',
'get_provides': 'boolean',
'get_recommends': 'boolean',
'get_suggests': 'boolean',
Modified: branches/rewrite/src/Download.py
==============================================================================
--- branches/rewrite/src/Download.py (original)
+++ branches/rewrite/src/Download.py Sun Dec 19 14:16:13 2004
@@ -16,8 +16,12 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# $Id$
+import StringIO
import threading
import Queue
+import re
+import os
+import os.path
import pycurl
@@ -45,24 +49,46 @@
self._curl.parent = self
self.DisplayStatus = DisplayStatus
+
+ def __processDir(self):
+ # in case of it to be a directory, we process to find all
+ # needed objects and queue all.
+ if self._fp != None and isinstance(self._fp, StringIO.StringIO):
+ self._fp.seek(0)
+ matches = re.compile('<a href="([a-zA-Z0-9_][a-zA-Z0-9\-\./_]+)">',
+ re.IGNORECASE | re.MULTILINE ).findall(self._fp.read())
+ for f in matches:
+ try:
+ os.makedirs(os.path.dirname(self._curl.filename))
+ except OSError, (errno, msg):
+ if errno != 17:
+ print "ERROR:", msg
+ Download().get(os.path.dirname(self._curl.url) + '/' + f, os.path.dirname(self._curl.filename) + '/' + f)
- def setUrl(self, url):
+ def setTarget(self, url, filename):
+ self.__processDir()
self._curl.setopt(pycurl.URL, url)
self._curl.url = url
- def setFilename(self, filename):
if self._fp is not None:
try:
self._fp.close()
except IOError:
self._fp = None
- self._fp = open(filename, "wb")
+
+ if filename[-1] == '/':
+ self._fp = StringIO.StringIO()
+ else:
+ self._fp = open(filename, "wb")
+
+ self._curl.filename = filename
self._curl.setopt(pycurl.WRITEFUNCTION, self._fp.write)
def perform(self):
self._curl.perform()
def close(self):
+ self.__processDir()
if self._fp != None:
self._fp.close()
self._fp = None
@@ -70,12 +96,15 @@
def progress(self, download_t, download_d, upload_t, upload_d):
# If we doesn't know how much data we will receive, return.
- if download_t == 0.0:
+ if download_t == 0.0 or download_d == 0.0:
return
- if self.DisplayStatus[self._curl.url] == None:
+ if self.DisplayStatus[self._curl.url] == None \
+ and self._curl.url[-1] != '/':
self.DisplayStatus.start(self._curl.url, download_t)
- self.DisplayStatus.update(self._curl.url, download_d)
+
+ if self._curl.url[-1] != '/':
+ self.DisplayStatus.update(self._curl.url, download_d)
class DownloadFetcher(threading.Thread):
def __init__(self, info = None, downloaders = 3):
@@ -116,9 +145,7 @@
break # Empty queue. Continue processing the others.
# Get a free fetcher
- fetcher.setUrl(url)
- fetcher.setFilename(filename)
-
+ fetcher.setTarget(url, filename)
self._multi.add_handle(fetcher._curl)
# Run the internal curl state machine for the multi stack
@@ -129,14 +156,12 @@
if ret != pycurl.E_CALL_MULTI_PERFORM:
# If we already tranfered all files, stop.
if num_handles == 0:
- self.running = False
while len(self._multi.handles) > 0:
c = self._multi.handles.pop()
c.close()
del c
- break
- else:
- break
+ self.running = False
+ break
# Check for curl objects which have terminated, and add them to the freelist
while 1:
@@ -151,6 +176,8 @@
if num_q == 0:
break
+ Download.fetcher = None
+
class Download:
""" Download queue """
queue = DownloadQueue()
@@ -158,15 +185,16 @@
fetcher = None
def __init__(self, max=3):
+ self.__max = max
+
+ def get(self, uri, destine):
+ self.queue.put((uri, destine))
# Create the needed fetcher.
if Download.fetcher == None:
- Download.fetcher = DownloadFetcher(None, max)
+ Download.fetcher = DownloadFetcher(None, self.__max)
Download.fetcher.start()
-
- def get(self, uri, destine):
- self.queue.put((uri, destine))
def join(self):
self.fetcher.join(1.0)
- while self.fetcher.running:
+ while self.fetcher and self.fetcher.running:
self.fetcher.join(1.0)
Added: branches/rewrite/src/Files.py
==============================================================================
--- (empty file)
+++ branches/rewrite/src/Files.py Sun Dec 19 14:16:13 2004
@@ -0,0 +1,51 @@
+# debpartial-mirror - partial debian mirror package tool
+# (c) 2004 Otavio Salvador <otavio@debian.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+# $Id$
+
+# TODO: Build two classes to be used. One for remote files and other
+# for local files.
+
+import os.path
+
+import pycurl
+
+import Download
+import DisplayStatus
+import FileSystem
+
+class Files:
+ """
+ This class provides methods to manage normal files of partial-mirrors.
+
+ It can be used to get files based on regexp and full paths.
+ """
+
+ def __init__ (self, backend):
+ self._backend = backend
+ self._files = []
+ self._fs = FileSystem.FileSystem(backend["mirror_dir"], backend["name"])
+
+ def upgrade(self):
+ pass
+
+class RemoteFiles(Files):
+ def upgrade(self):
+ d = Download.Download()
+ for f in self._backend['files']:
+ d.get(self._backend['server'] + '/' + f,
+ self._backend['mirror_dir'] + self._backend['name'] + '/' + f)
+ d.join()