r233 - branches/rewrite/src

[Otavio Salvador]

Author: otavio
Date: Tue Sep 21 17:56:01 2004
New Revision: 233

Modified:
   branches/rewrite/src/Download.py
Log:
Worked. Need to provide more useful information but it's not works :-D

Modified: branches/rewrite/src/Download.py
==============================================================================
--- branches/rewrite/src/Download.py	(original)
+++ branches/rewrite/src/Download.py	Tue Sep 21 17:56:01 2004
@@ -37,79 +37,52 @@
     print "Total to upload", upload_t
     print "Total uploaded", upload_d
 
-class DownloadFetcher(threading.Thread):
-    def __init__(self, max_connections = 2):
-        """ Make the needed objects to handle the connections."""
-        self.__objs = pycurl.CurlMulti()
-        self.__handles = []
-        self.__threads = []
-        for i in range(max_connections):
-            c = pycurl.Curl()
-            c.fp = None
-            c.setopt(pycurl.HTTPHEADER, ["User-Agent: debpartial-mirror"])
-            c.setopt(pycurl.FOLLOWLOCATION, 1)
-            c.setopt(pycurl.MAXREDIRS, 5)
-            c.setopt(pycurl.CONNECTTIMEOUT, 30)
-            c.setopt(pycurl.TIMEOUT, 300)
-            c.setopt(pycurl.PROGRESSFUNCTION, progress)
-            c.setopt(pycurl.NOPROGRESS, 0)
-            self.__handles.append(c)
-
-        threading.Thread.__init__(self)
-        self.setName('DownloadFetcher Thread')
-        self.start()
-
+class DownloadThread(threading.Thread):
     def run(self):
         while 1:
-            while Download.queue and self.__handles:
-                url, filename = Download.queue.get()
-                c = self.__handles.pop()
-                c.fp = open(filename, "wb")
-                c.setopt(pycurl.URL, url)
-                c.setopt(pycurl.WRITEDATA, c.fp)
-                self.__objs.add_handle(c)
-                # store some info
-                c.filename = filename
-                c.url = url
-
-            # Run the internal curl state machine for the multi stack
-            while 1:
-                ret, num_handles = self.__objs.perform()
-                if ret != pycurl.E_CALL_MULTI_PERFORM:
-                    break
-
-            while 1:
-                num_q, ok_list, err_list = self.__objs.info_read()
-                for c in ok_list:
-                    c.fp.close()
-                    c.fp = None
-                    self.__objs.remove_handle(c)
-                    print "Success:", c.filename, c.url, c.getinfo(pycurl.EFFECTIVE_URL)
-                    self.__handles.append(c)
-                for c, errno, errmsg in err_list:
-                    c.fp.close()
-                    c.fp = None
-                    self.__objs.remove_handle(c)
-                    print "Failed: ", c.filename, c.url, errno, errmsg
-                    self.__handles.append(c)
-                    num_processed = num_processed + len(ok_list) + len(err_list)
-                if num_q == 0:
-                    break
-
-            # Currently no more I/O is pending, could do something in the meantime
-            # (display a progress bar, etc.).
-            # We just use select() to wait until some more data is available.
-            self.__objs.select()
+            url, filename = Download.queue.get()
+            f = open(filename, "wb")
+            curl = pycurl.Curl()
+            curl.setopt(pycurl.HTTPHEADER, ["User-Agent: PycURL"])
+            curl.setopt(pycurl.FOLLOWLOCATION, 1)
+            curl.setopt(pycurl.MAXREDIRS, 5)
+            curl.setopt(pycurl.URL, url)
+            curl.setopt(pycurl.WRITEDATA, f)
+            curl.setopt(pycurl.NOSIGNAL, 1)
+            curl.setopt(pycurl.CONNECTTIMEOUT, 30)
+            curl.setopt(pycurl.PROGRESSFUNCTION, progress)
+            curl.setopt(pycurl.NOPROGRESS, 0)
+            curl.setopt(pycurl.TIMEOUT, 300)
+            
+            try:
+                curl.perform()
+            except:
+                import traceback
+                traceback.print_exc(file=sys.stderr)
+                sys.stderr.flush()
+            curl.close()
+            f.close()
+            sys.stdout.write(".")
+            sys.stdout.flush()
 
 class Download:
     """ Download queue """
     queue = DownloadQueue()
     """ Fetcher to use """
-    fetcher = DownloadFetcher(2)
+    fetchers = []
 
-    def __init__(self, uri, destine):
+    def __init__(self, uri, destine, max_threads=2):
         self.queue.put((uri, destine))
-        print str(self.queue.qsize()) + " / " + str(self.fetcher)
+
+        # Alloc all needed threads.
+        if len(self.fetchers) < max_threads:
+            for i in range(max_threads - len(self.fetchers)):
+                t = DownloadThread()
+                self.fetchers.append(t)
+                t.start()
+
+        # Information ... should be removed
+        print str(self.queue.qsize()) + " / " + str(len(self.fetchers))
 
 urls = open(sys.argv[1]).readlines()
 fileno = 1