From f0fa70bfbdb6526ccad391aefa20c5bc2f03c772 Mon Sep 17 00:00:00 2001
From: Chris Dent <cdent@anticdent.org>
Date: Mon, 21 Jan 2019 21:05:47 +0000
Subject: [PATCH] Correct caching of disk images

The cachecontrol library can't cache large images [1], which we
work around here by using our own serializer to allow msgpage
to load big files.

It's quite likely this caching is the wrong way to go, and
missing some important details, but it is a useful way to
speed up the experimentation.

[1] https://github.com/ionrock/cachecontrol/issues/200
---
 compute.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/compute.py b/compute.py
index 5501d43..c5fd107 100644
--- a/compute.py
+++ b/compute.py
@@ -13,8 +13,10 @@
 
 import cachecontrol
 from cachecontrol.caches import file_cache
+from cachecontrol import serialize
 import etcd3
 import libvirt
+import msgpack
 import psutil
 import requests
 import yaml
@@ -22,12 +24,28 @@
 from ecomp import clients
 from ecomp import conf
 
+
+# deal with size limitations in CacheControl
+class MySerializer(serialize.Serializer):
+
+    def _loads_v4(self, request, data):
+        try:
+            cached = msgpack.loads(
+                data, encoding="utf-8", max_bin_len=2147483647)
+        except ValueError:
+            return
+
+        return self.prepare_response(request, cached)
+
+
 KEY = '/hosts'
 SLEEP = 1
 COMPUTE_UUID = str(uuid.uuid4())
 CLIENT = None
 CACHED_SESSION = cachecontrol.CacheControl(
-    requests.Session(), cache=file_cache.FileCache('.web_cache'))
+    requests.Session(),
+    cache=file_cache.FileCache('.web_cache'),
+    serializer=MySerializer())
 
 
 # default config
@@ -196,6 +214,7 @@ def _copy_image(source, instance, size):
         os.unlink(source_file)
     except FileNotFoundError:
         pass
+    _print('Fetching image from %s' % source)
     source_refresh = CACHED_SESSION.get(source, stream=True)
     with open(source_file, 'wb') as sf:
         shutil.copyfileobj(source_refresh.raw, sf)