test: local http server for tests (#1010)

* pytest file server * range request handler * use local server on some tests * skip format * variable name * use local http server * fix issue with parametrize indirect * remove network marks * attempt to fix tests * attempt to fix tests * attempt to fix ci * attempt to properly close * work on windows please * attempt to close properly * Revert "attempt to fix tests" This reverts commit 5d721df. * Revert "attempt to fix tests" This reverts commit 5254c24. * update fixture scope * do not run problematic test in local server * rename test * rename tests * reenabled example.com tests * download all files (only once) * download files on demand
scikit-hep · Oct 26, 2023 · 3341ce0 · 3341ce0
1 parent b4209bf
commit 3341ce0
Show file tree

Hide file tree

Showing 6 changed files with 143 additions and 100 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -68,7 +68,8 @@ test = [
   "pytest-rerunfailures",
   "requests",
   "scikit-hep-testdata",
-  "xxhash"
+  "xxhash",
+  "rangehttpserver"
 ]
 
 [project.urls]

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,6 +1,14 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
 
 import pytest
+import threading
+import contextlib
+import skhep_testdata
+from functools import partial
+
+# The base http server does not support range requests. Watch https://github.com/python/cpython/issues/86809 for updates
+from http.server import HTTPServer
+from RangeHTTPServer import RangeRequestHandler
 
 import uproot
 
@@ -9,3 +17,55 @@
 def reset_classes():
     uproot.model.reset_classes()
     return
+
+
+@contextlib.contextmanager
+def serve():
+    # serve files from the skhep_testdata cache directory.
+    # This directory is initially empty and files are downloaded on demand
+    class Handler(RangeRequestHandler):
+        def _cache_file(self, path: str):
+            path = path.lstrip("/")
+            if path in skhep_testdata.known_files:
+                return skhep_testdata.data_path(path)
+            else:
+                raise FileNotFoundError(
+                    f"File '{path}' not available in skhep_testdata"
+                )
+
+        def do_HEAD(self):
+            self._cache_file(self.path)
+            return super().do_HEAD()
+
+        def do_GET(self):
+            self._cache_file(self.path)
+            return super().do_GET()
+
+    server = HTTPServer(
+        server_address=("localhost", 0),
+        RequestHandlerClass=partial(
+            Handler, directory=skhep_testdata.local_files._cache_path()
+        ),
+    )
+    server.server_activate()
+
+    def serve_forever(httpd=server):
+        with httpd:
+            httpd.serve_forever()
+
+    thread = threading.Thread(target=serve_forever, daemon=True)
+
+    try:
+        thread.start()
+        address, port = server.server_address
+        yield f"http://{address}:{port}"
+    finally:
+        # stop the server
+        server.shutdown()
+        thread.join()
+
+
+@pytest.fixture(scope="module")
+def server():
+    with serve() as server_url:
+        yield server_url
diff --git a/tests/test_0001-source-class.py → tests/test_0001_source_class.py b/tests/test_0001-source-class.py → tests/test_0001_source_class.py
@@ -1,11 +1,7 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
 
-import os
-import platform
 import queue
-import sys
 from io import StringIO
-import contextlib
 
 import numpy
 import pytest
@@ -33,9 +29,8 @@ def use_threads(request):
 
 
 @pytest.mark.parametrize(
-    "use_threads,num_workers",
+    "use_threads, num_workers",
     [(True, 1), (True, 2), (False, 0)],
-    indirect=["use_threads"],
 )
 def test_file(use_threads, num_workers, tmp_path):
     filename = tmp_path / "tmp.raw"
@@ -63,9 +58,8 @@ def test_file(use_threads, num_workers, tmp_path):
 
 
 @pytest.mark.parametrize(
-    "use_threads,num_workers",
+    "use_threads, num_workers",
     [(True, 1), (True, 2), (False, 0)],
-    indirect=["use_threads"],
 )
 def test_file_fail(use_threads, num_workers, tmp_path):
     filename = tmp_path / "tmp.raw"
@@ -123,30 +117,30 @@ def test_memmap_fail(use_threads, tmp_path):
             ...
 
 
-@pytest.mark.skip(reason="RECHECK: example.com is flaky, too")
-@pytest.mark.parametrize("use_threads", [True, False], indirect=True)
+@pytest.mark.parametrize("use_threads", [True, False])
 @pytest.mark.network
-def test_http(use_threads):
+def test_http(server, use_threads):
+    url = "https://example.com"
     with uproot.source.http.HTTPSource(
-        "https://example.com",
+        url,
         timeout=10,
         num_fallback_workers=1,
         use_threads=use_threads,
-    ) as tmp:
+    ) as source:
         notifications = queue.Queue()
-        chunks = tmp.chunks([(0, 100), (50, 55), (200, 400)], notifications)
+        chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications)
         one, two, three = (tobytes(chunk.raw_data) for chunk in chunks)
         assert len(one) == 100
         assert len(two) == 5
         assert len(three) == 200
-        assert tmp.fallback is None
+        assert source.fallback is None
 
     with uproot.source.http.MultithreadedHTTPSource(
-        "https://example.com", num_workers=1, timeout=10, use_threads=use_threads
-    ) as tmp:
+        url, num_workers=1, timeout=10, use_threads=use_threads
+    ) as source:
         notifications = queue.Queue()
-        chunks = tmp.chunks([(0, 100), (50, 55), (200, 400)], notifications)
-        assert [tobytes(x.raw_data) for x in chunks] == [one, two, three]
+        chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications)
+        assert [tobytes(chunk.raw_data) for chunk in chunks] == [one, two, three]
 
 
 def test_colons_and_ports():
@@ -163,7 +157,6 @@ def test_colons_and_ports():
     ) == ("https://example.com:443/something", "else")
 
 
-@pytest.mark.skip(reason="RECHECK: example.com is flaky, too")
 @pytest.mark.parametrize("use_threads", [True, False], indirect=True)
 @pytest.mark.network
 def test_http_port(use_threads):
@@ -190,19 +183,19 @@ def test_http_port(use_threads):
         assert [tobytes(x.raw_data) for x in chunks] == [one, two, three]
 
 
-@pytest.mark.parametrize("use_threads", [True, False], indirect=True)
-@pytest.mark.network
-def test_http_size(use_threads):
+@pytest.mark.parametrize("use_threads", [True, False])
+def test_http_size(server, use_threads):
+    url = f"{server}/uproot-issue121.root"
     with uproot.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
+        url,
         timeout=10,
         num_fallback_workers=1,
         use_threads=use_threads,
     ) as source:
         size1 = source.num_bytes
 
     with uproot.source.http.MultithreadedHTTPSource(
-        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
+        url,
         num_workers=1,
         timeout=10,
         use_threads=use_threads,
@@ -243,16 +236,15 @@ def test_http_fail(use_threads):
         num_fallback_workers=1,
         use_threads=use_threads,
     )
-    with pytest.raises(Exception) as err:
+    with pytest.raises(Exception):
         notifications = queue.Queue()
         chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications)
         chunks[0].raw_data
 
 
 @pytest.mark.parametrize(
-    "use_threads,num_workers",
+    "use_threads, num_workers",
     [(True, 1), (True, 2), (False, 0)],
-    indirect=["use_threads"],
 )
 @pytest.mark.network
 def test_no_multipart(use_threads, num_workers):
@@ -272,9 +264,8 @@ def test_no_multipart(use_threads, num_workers):
 
 
 @pytest.mark.parametrize(
-    "use_threads,num_workers",
+    "use_threads, num_workers",
     [(True, 1), (True, 2), (False, 0)],
-    indirect=["use_threads"],
 )
 @pytest.mark.network
 def test_no_multipart_fail(use_threads, num_workers):
@@ -284,21 +275,17 @@ def test_no_multipart_fail(use_threads, num_workers):
         timeout=0.1,
         use_threads=use_threads,
     )
-    with pytest.raises(Exception) as err:
+    with pytest.raises(Exception):
         notifications = queue.Queue()
         chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications)
         chunks[0].raw_data
 
 
-@pytest.mark.parametrize(
-    "use_threads,num_workers",
-    [(True, 1), (True, 2), (False, 0)],
-    indirect=["use_threads"],
-)
-@pytest.mark.network
-def test_fallback(use_threads, num_workers):
+@pytest.mark.parametrize("use_threads, num_workers", [(True, 1), (True, 2), (False, 0)])
+def test_fallback(server, use_threads, num_workers):
+    url = f"{server}/uproot-issue121.root"
     with uproot.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
+        url,
         timeout=10,
         num_fallback_workers=num_workers,
         use_threads=use_threads,
@@ -343,7 +330,7 @@ def test_xrootd(use_threads):
 @pytest.mark.parametrize("use_threads", [True, False], indirect=True)
 def test_xrootd_deadlock(use_threads):
     pytest.importorskip("XRootD")
-    # Attach this file to the "test_xrootd_deadlock" function so it leaks
+    # Attach this file to the "test_xrootd_deadlock" function, so it leaks
     pytest.uproot_test_xrootd_deadlock_f = uproot.source.xrootd.XRootDResource(
         "root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/Run2012B_DoubleMuParked.root",
         timeout=20,
@@ -356,7 +343,7 @@ def test_xrootd_deadlock(use_threads):
 @pytest.mark.parametrize("use_threads", [True, False], indirect=True)
 def test_xrootd_fail(use_threads):
     pytest.importorskip("XRootD")
-    with pytest.raises(Exception) as err:
+    with pytest.raises(Exception):
         uproot.source.xrootd.MultithreadedXRootDSource(
             "root://wonky.cern/does-not-exist",
             num_workers=1,
@@ -447,7 +434,7 @@ def get_chunk(Source, **kwargs):
 @pytest.mark.parametrize("use_threads", [True, False], indirect=True)
 def test_xrootd_vectorread_fail(use_threads):
     pytest.importorskip("XRootD")
-    with pytest.raises(Exception) as err:
+    with pytest.raises(Exception):
         uproot.source.xrootd.XRootDSource(
             "root://wonky.cern/does-not-exist",
             timeout=1,

diff --git a/tests/test_0006-notify-when-downloaded.py → tests/test_0006_notify_when_downloaded.py b/tests/test_0006-notify-when-downloaded.py → tests/test_0006_notify_when_downloaded.py
@@ -2,10 +2,6 @@
 
 import os
 import queue
-import sys
-from io import StringIO
-
-import numpy
 import pytest
 
 import uproot
@@ -68,12 +64,11 @@ def test_memmap(tmpdir):
             expected.pop((chunk.start, chunk.stop))
 
 
-@pytest.mark.skip(reason="RECHECK: example.com is flaky, too")
-@pytest.mark.network
-def test_http_multipart():
+def test_http_multipart(server):
+    url = f"{server}/uproot-issue121.root"
     notifications = queue.Queue()
     with uproot.source.http.HTTPSource(
-        "https://example.com", timeout=10, num_fallback_workers=1, use_threads=True
+        url, timeout=10, num_fallback_workers=1, use_threads=True
     ) as source:
         chunks = source.chunks(
             [(0, 100), (50, 55), (200, 400)], notifications=notifications
@@ -84,12 +79,11 @@ def test_http_multipart():
             expected.pop((chunk.start, chunk.stop))
 
 
-@pytest.mark.skip(reason="RECHECK: example.com is flaky, too")
-@pytest.mark.network
-def test_http():
+def test_http(server):
+    url = f"{server}/uproot-issue121.root"
     notifications = queue.Queue()
     with uproot.source.http.MultithreadedHTTPSource(
-        "https://example.com", timeout=10, num_workers=1, use_threads=True
+        url, timeout=10, num_workers=1, use_threads=True
     ) as source:
         chunks = source.chunks(
             [(0, 100), (50, 55), (200, 400)], notifications=notifications
@@ -100,12 +94,11 @@ def test_http():
             expected.pop((chunk.start, chunk.stop))
 
 
-@pytest.mark.skip(reason="RECHECK: example.com is flaky, too")
-@pytest.mark.network
-def test_http_workers():
+def test_http_workers(server):
+    url = f"{server}/uproot-issue121.root"
     notifications = queue.Queue()
     with uproot.source.http.MultithreadedHTTPSource(
-        "https://example.com", timeout=10, num_workers=2, use_threads=True
+        url, timeout=10, num_workers=2, use_threads=True
     ) as source:
         chunks = source.chunks(
             [(0, 100), (50, 55), (200, 400)], notifications=notifications
@@ -116,11 +109,11 @@ def test_http_workers():
             expected.pop((chunk.start, chunk.stop))
 
 
-@pytest.mark.network
-def test_http_fallback():
+def test_http_fallback(server):
+    url = f"{server}/uproot-issue121.root"
     notifications = queue.Queue()
     with uproot.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
+        url,
         timeout=10,
         num_fallback_workers=1,
         use_threads=True,
@@ -134,11 +127,11 @@ def test_http_fallback():
             expected.pop((chunk.start, chunk.stop))
 
 
-@pytest.mark.network
-def test_http_fallback_workers():
+def test_http_fallback_workers(server):
+    url = f"{server}/uproot-issue121.root"
     notifications = queue.Queue()
     with uproot.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
+        url,
         timeout=10,
         num_fallback_workers=5,
         use_threads=True,