[Collab-qa-commits] r897 - upload-history

[filippo at alioth.debian.org]

Author: filippo
Date: 2008-06-24 18:56:48 +0000 (Tue, 24 Jun 2008)
New Revision: 897

Modified:
   upload-history/munge_ddc.py
Log:
keep a cache of msgids to avoid rescanning

Modified: upload-history/munge_ddc.py
===================================================================
--- upload-history/munge_ddc.py	2008-06-24 17:59:26 UTC (rev 896)
+++ upload-history/munge_ddc.py	2008-06-24 18:56:48 UTC (rev 897)
@@ -8,9 +8,12 @@
 import subprocess
 import sys
 import os
+import cPickle
 
 from debian_bundle import deb822
 
+msgid_cache = "msgid_cache.db"
+
 # XXX gives false positives for example with -x+y.z.w, mitigated by checking Changes also
 # XXX might change in the future, see DEP1 http://wiki.debian.org/NmuDep
 nmu_version_RE = re.compile("(-\S+\.\d+|\+nmu\d+)$")
@@ -64,9 +67,23 @@
     
     mb = mailbox.PortableUnixMailbox(file(mb_file), factory=email.message_from_file)
 
+    if os.path.exists(msgid_cache):
+        m_cache = cPickle.load(open(msgid_cache))
+    else:
+        m_cache = {}
+
     for msg in mb:
         if msg.is_multipart():
             continue
+        
+        msgid = msg.get('message-id', 0)
+        if msgid:
+            if m_cache.has_key(msgid):
+                continue
+            else:
+                # store the filename as value if purging is needed
+                m_cache[msgid] = mb_file 
+        
         body = msg.get_payload(decode=True).split('\n')
         
 # XXX work around Hash: sha1(\n)+ at the beginning of signed body
@@ -104,6 +121,7 @@
         # separate different packages
         print
 
+    cPickle.dump(m_cache, open(msgid_cache, "w"), 2)
 
 if __name__ == '__main__':
     if len(sys.argv) < 2: