From 78feb554a7bd4db852bedd62462b47ae928833a9 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Mon, 10 Oct 2016 00:38:57 -0400 Subject: [PATCH 1/2] Cut parallel JSON downloads to 8. Running 128 parallel HEAD requests is overkill and liable to get someone banned. --- lib/iris/tests/test_image_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/tests/test_image_json.py b/lib/iris/tests/test_image_json.py index d767245a84..3ba0ed3727 100644 --- a/lib/iris/tests/test_image_json.py +++ b/lib/iris/tests/test_image_json.py @@ -35,7 +35,7 @@ from threading import Thread # maximum number of threads for multi-threading code -MAXTHREADS = 128 +MAXTHREADS = 8 # Turn down requests logging logging.getLogger("requests").setLevel(logging.CRITICAL) From 87d1cf84d46e50a05bcfd5a9b3bcc3b79ae106b9 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Mon, 10 Oct 2016 00:45:38 -0400 Subject: [PATCH 2/2] TST: Check for valid URI before making HEAD request. --- lib/iris/tests/test_image_json.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/test_image_json.py b/lib/iris/tests/test_image_json.py index 3ba0ed3727..ae144ce3f6 100644 --- a/lib/iris/tests/test_image_json.py +++ b/lib/iris/tests/test_image_json.py @@ -58,8 +58,7 @@ def run(self): resource = self.queue.get() try: result = requests.head(resource) - if (result.status_code == 200 and - resource.startswith('https://scitools.github.io')): + if result.status_code == 200: self.deque.append(resource) else: msg = '{} is not resolving correctly.'.format(resource) @@ -80,7 +79,11 @@ def test_resolve(self): exceptions = deque() uri_queue = Queue() for uri in uris: - uri_queue.put(uri) + if uri.startswith('https://scitools.github.io'): + uri_queue.put(uri) + else: + msg = '{} is not a valid resource.'.format(uri) + exceptions.append(ValueError(msg)) for i in range(MAXTHREADS): _ResolveWorkerThread(uri_queue, uri_list, exceptions).start()