[misc] 01/01: cache reproducible.json.bz2 for 20 minutes (..)

[Chris West]

This is an automated email from the git hooks/post-receive script.

faux-guest pushed a commit to branch master
in repository misc.

commit a974e2a7c50923243a4a1601aceca6ed49dc5afe
Author: Chris West (Faux) <git at goeswhere.com>
Date:   Fri Apr 14 13:22:41 2017 +0100

    cache reproducible.json.bz2 for 20 minutes (..)
    
    I thought this would make edit-notes faster, but the profiler lies and it's actually
    dominated by loading YAML. 4.1s to load the YAML, 0.7s to load a file off a remote
    machine. THE FUTURE.
---
 rblib/remote.py | 35 +++++++++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/rblib/remote.py b/rblib/remote.py
index 6647042..aaaad16 100644
--- a/rblib/remote.py
+++ b/rblib/remote.py
@@ -1,27 +1,50 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
+import bz2
 import json
+import os
+import tempfile
+import time
+import sys
 
 import psycopg2
 import requests
 
 from . import logger
 
-RB_SITE = 'https://reproducible.debian.net'
-REPRODUCIBLE_JSON = '{}/reproducible.json'.format(RB_SITE)
+RB_SITE = 'https://tests.reproducible-builds.org'
+REPRODUCIBLE_JSON = '{}/reproducible.json.bz2'.format(RB_SITE)
+JSON_CACHE_FILE = 'reproducible.json'
 # [{package: xxx, suite: sid, version: 0.0.0, status: reproducible}, {...}]
 
 log = logger.setup_logging(__name__)
 
+def load_json_cache():
+    with open(JSON_CACHE_FILE) as f:
+        return json.load(f)
 
 def load_json():
+    CACHE_SECONDS = 10 * 60
     try:
-        with open('reproducible.json') as f:
-            rstatus = json.load(f)
+        mtime = os.path.getmtime(JSON_CACHE_FILE)
     except FileNotFoundError:
-        rstatus = requests.get(REPRODUCIBLE_JSON).json()
-    return rstatus
+        mtime = None
+
+    if mtime and time.time() - mtime < CACHE_SECONDS:
+        return load_json_cache()
+
+    try:
+        content = bz2.decompress(requests.get(REPRODUCIBLE_JSON).content)
+        fd, temp_name = tempfile.mkstemp(prefix='.', suffix='.json', dir='.')
+        with open(fd, 'wb') as f:
+            f.write(content)
+        os.rename(temp_name, JSON_CACHE_FILE)
+        os.chmod(JSON_CACHE_FILE, 0o644)
+    except Exception as e:
+        sys.stderr.write('downloading cache file failed ({}), trying cached...\n'.format(e))
+
+    return load_json_cache()
 
 
 def load_reproducible_status():

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git