[misc] 02/02: fetch-logs: store the mtimes so we might be able to delete oudated files automatically

[Chris West]

This is an automated email from the git hooks/post-receive script.

faux-guest pushed a commit to branch master
in repository misc.

commit 33ddaec76c5374c42180d9c79a1badd6d5c375cf
Author: Chris West (Faux) <git at goeswhere.com>
Date:   Sat Apr 22 12:14:30 2017 +0100

    fetch-logs: store the mtimes so we might be able to delete oudated files automatically
---
 fetch-logs | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/fetch-logs b/fetch-logs
index d77a451..aeb9faf 100755
--- a/fetch-logs
+++ b/fetch-logs
@@ -10,7 +10,7 @@ import sys
 import tempfile
 import urllib.request
 
-from email.utils import format_datetime
+import email.utils as email_utils
 from typing import List, Optional, Iterable
 
 
@@ -180,6 +180,9 @@ class Store:
                 print("done.")
         self.files = parsey.files
 
+    def exists_locally(self, file: File):
+        return os.path.exists(self.file_named(file.name))
+
     def up_to_date(self, file: File) -> bool:
         mtime = self.mtime(self.file_named(file.name))
         if not mtime:
@@ -208,9 +211,13 @@ class Store:
                 if 200 != resp.status:
                     raise Exception('fetching {} failed ({}): {}'.format(path, resp.status, resp))
 
-                with Sponge(self.to_local_path(path)) as dest:
+                written_file = self.to_local_path(path)
+                with Sponge(written_file) as dest:
                     shutil.copyfileobj(resp, dest.fd)
 
+                mtime = email_utils.parsedate_to_datetime(resp.headers['Last-Modified']).timestamp()
+                os.utime(written_file, times=(mtime, mtime))
+
                 sys.stdout.write('.')
                 sys.stdout.flush()
 
@@ -219,6 +226,15 @@ class Store:
 
         print()
 
+    def remove_unmatched_files(self):
+        allowed_names = {file.name for file in self.files}
+        for name in os.listdir(self.to_local_path(self.folder)):
+            if name not in allowed_names:
+                self.delete_local_file(name)
+
+    def delete_local_file(self, name: str):
+        os.unlink(self.to_local_path(self.file_named(name)))
+
 
 def package_name(name: str) -> str:
     return name[0:name.index('_')]
@@ -241,6 +257,8 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-v', '--verbose', action='store_true')
     parser.add_argument('-d', '--download', action='store_true')
+    parser.add_argument('-u', '--delete-outdated', action='store_true',
+                        help='remove *all* files which are older than the server versions')
     parser.add_argument('-t', '--type', type=str, help='|'.join(allowed_types), default='rbuild')
     parser.add_argument('-s', '--suite', type=str, help='|'.join(allowed_suites), default='unstable')
     parser.add_argument('-a', '--arch', type=str, help='|'.join(allowed_arches), default='amd64')
@@ -286,6 +304,10 @@ def main():
     up_to_date = set()
     for file in store.files:
         name = package_name(file.name)
+
+        if args.delete_outdated and store.exists_locally(file) and not store.up_to_date(file):
+            store.delete_local_file(file.name)
+
         if name not in wanted:
             continue
         if file.size > max_size:
@@ -296,6 +318,9 @@ def main():
             continue
         to_fetch.add(file)
 
+    if args.delete_outdated:
+        store.remove_unmatched_files()
+
     if args.verbose:
         print_package_set(too_big, "too big")
         print_package_set(up_to_date, "up to date")

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git