From 3341ce06ed271825697d66c2c255c3a990350aa8 Mon Sep 17 00:00:00 2001 From: Luis Antonio Obis Aparicio <35803280+lobis@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:58:47 -0500 Subject: [PATCH] test: local http server for tests (#1010) * pytest file server * range request handler * use local server on some tests * skip format * variable name * use local http server * fix issue with parametrize indirect * remove network marks * attempt to fix tests * attempt to fix tests * attempt to fix ci * attempt to properly close * work on windows please * attempt to close properly * Revert "attempt to fix tests" This reverts commit 5d721dfcbe31ad595e40994c399769cbbb25d5ac. * Revert "attempt to fix tests" This reverts commit 5254c24a6fa6ddc6ad33a61ec28f2d75866ac797. * update fixture scope * do not run problematic test in local server * rename test * rename tests * reenabled example.com tests * download all files (only once) * download files on demand --- pyproject.toml | 3 +- tests/conftest.py | 60 ++++++++++++++++ ...rce-class.py => test_0001_source_class.py} | 71 ++++++++----------- ...py => test_0006_notify_when_downloaded.py} | 37 ++++------ ...py => test_0007_single_chunk_interface.py} | 54 +++++++------- tests/test_0692_fsspec.py | 18 ++--- 6 files changed, 143 insertions(+), 100 deletions(-) rename tests/{test_0001-source-class.py => test_0001_source_class.py} (91%) rename tests/{test_0006-notify-when-downloaded.py => test_0006_notify_when_downloaded.py} (89%) rename tests/{test_0007-single-chunk-interface.py => test_0007_single_chunk_interface.py} (82%) diff --git a/pyproject.toml b/pyproject.toml index a28d05c97..7e76e7561 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,8 @@ test = [ "pytest-rerunfailures", "requests", "scikit-hep-testdata", - "xxhash" + "xxhash", + "rangehttpserver" ] [project.urls] diff --git a/tests/conftest.py b/tests/conftest.py index a249eacc7..86683a71d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,14 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE import pytest +import threading +import contextlib +import skhep_testdata +from functools import partial + +# The base http server does not support range requests. Watch https://github.com/python/cpython/issues/86809 for updates +from http.server import HTTPServer +from RangeHTTPServer import RangeRequestHandler import uproot @@ -9,3 +17,55 @@ def reset_classes(): uproot.model.reset_classes() return + + +@contextlib.contextmanager +def serve(): + # serve files from the skhep_testdata cache directory. + # This directory is initially empty and files are downloaded on demand + class Handler(RangeRequestHandler): + def _cache_file(self, path: str): + path = path.lstrip("/") + if path in skhep_testdata.known_files: + return skhep_testdata.data_path(path) + else: + raise FileNotFoundError( + f"File '{path}' not available in skhep_testdata" + ) + + def do_HEAD(self): + self._cache_file(self.path) + return super().do_HEAD() + + def do_GET(self): + self._cache_file(self.path) + return super().do_GET() + + server = HTTPServer( + server_address=("localhost", 0), + RequestHandlerClass=partial( + Handler, directory=skhep_testdata.local_files._cache_path() + ), + ) + server.server_activate() + + def serve_forever(httpd=server): + with httpd: + httpd.serve_forever() + + thread = threading.Thread(target=serve_forever, daemon=True) + + try: + thread.start() + address, port = server.server_address + yield f"http://{address}:{port}" + finally: + # stop the server + server.shutdown() + thread.join() + + +@pytest.fixture(scope="module") +def server(): + with serve() as server_url: + yield server_url diff --git a/tests/test_0001-source-class.py b/tests/test_0001_source_class.py similarity index 91% rename from tests/test_0001-source-class.py rename to tests/test_0001_source_class.py index 1c8366d46..5f0cb1e86 100644 --- a/tests/test_0001-source-class.py +++ b/tests/test_0001_source_class.py @@ -1,11 +1,7 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE -import os -import platform import queue -import sys from io import StringIO -import contextlib import numpy import pytest @@ -33,9 +29,8 @@ def use_threads(request): @pytest.mark.parametrize( - "use_threads,num_workers", + "use_threads, num_workers", [(True, 1), (True, 2), (False, 0)], - indirect=["use_threads"], ) def test_file(use_threads, num_workers, tmp_path): filename = tmp_path / "tmp.raw" @@ -63,9 +58,8 @@ def test_file(use_threads, num_workers, tmp_path): @pytest.mark.parametrize( - "use_threads,num_workers", + "use_threads, num_workers", [(True, 1), (True, 2), (False, 0)], - indirect=["use_threads"], ) def test_file_fail(use_threads, num_workers, tmp_path): filename = tmp_path / "tmp.raw" @@ -123,30 +117,30 @@ def test_memmap_fail(use_threads, tmp_path): ... -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") -@pytest.mark.parametrize("use_threads", [True, False], indirect=True) +@pytest.mark.parametrize("use_threads", [True, False]) @pytest.mark.network -def test_http(use_threads): +def test_http(server, use_threads): + url = "https://example.com" with uproot.source.http.HTTPSource( - "https://example.com", + url, timeout=10, num_fallback_workers=1, use_threads=use_threads, - ) as tmp: + ) as source: notifications = queue.Queue() - chunks = tmp.chunks([(0, 100), (50, 55), (200, 400)], notifications) + chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications) one, two, three = (tobytes(chunk.raw_data) for chunk in chunks) assert len(one) == 100 assert len(two) == 5 assert len(three) == 200 - assert tmp.fallback is None + assert source.fallback is None with uproot.source.http.MultithreadedHTTPSource( - "https://example.com", num_workers=1, timeout=10, use_threads=use_threads - ) as tmp: + url, num_workers=1, timeout=10, use_threads=use_threads + ) as source: notifications = queue.Queue() - chunks = tmp.chunks([(0, 100), (50, 55), (200, 400)], notifications) - assert [tobytes(x.raw_data) for x in chunks] == [one, two, three] + chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications) + assert [tobytes(chunk.raw_data) for chunk in chunks] == [one, two, three] def test_colons_and_ports(): @@ -163,7 +157,6 @@ def test_colons_and_ports(): ) == ("https://example.com:443/something", "else") -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") @pytest.mark.parametrize("use_threads", [True, False], indirect=True) @pytest.mark.network def test_http_port(use_threads): @@ -190,11 +183,11 @@ def test_http_port(use_threads): assert [tobytes(x.raw_data) for x in chunks] == [one, two, three] -@pytest.mark.parametrize("use_threads", [True, False], indirect=True) -@pytest.mark.network -def test_http_size(use_threads): +@pytest.mark.parametrize("use_threads", [True, False]) +def test_http_size(server, use_threads): + url = f"{server}/uproot-issue121.root" with uproot.source.http.HTTPSource( - "https://scikit-hep.org/uproot3/examples/Zmumu.root", + url, timeout=10, num_fallback_workers=1, use_threads=use_threads, @@ -202,7 +195,7 @@ def test_http_size(use_threads): size1 = source.num_bytes with uproot.source.http.MultithreadedHTTPSource( - "https://scikit-hep.org/uproot3/examples/Zmumu.root", + url, num_workers=1, timeout=10, use_threads=use_threads, @@ -243,16 +236,15 @@ def test_http_fail(use_threads): num_fallback_workers=1, use_threads=use_threads, ) - with pytest.raises(Exception) as err: + with pytest.raises(Exception): notifications = queue.Queue() chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications) chunks[0].raw_data @pytest.mark.parametrize( - "use_threads,num_workers", + "use_threads, num_workers", [(True, 1), (True, 2), (False, 0)], - indirect=["use_threads"], ) @pytest.mark.network def test_no_multipart(use_threads, num_workers): @@ -272,9 +264,8 @@ def test_no_multipart(use_threads, num_workers): @pytest.mark.parametrize( - "use_threads,num_workers", + "use_threads, num_workers", [(True, 1), (True, 2), (False, 0)], - indirect=["use_threads"], ) @pytest.mark.network def test_no_multipart_fail(use_threads, num_workers): @@ -284,21 +275,17 @@ def test_no_multipart_fail(use_threads, num_workers): timeout=0.1, use_threads=use_threads, ) - with pytest.raises(Exception) as err: + with pytest.raises(Exception): notifications = queue.Queue() chunks = source.chunks([(0, 100), (50, 55), (200, 400)], notifications) chunks[0].raw_data -@pytest.mark.parametrize( - "use_threads,num_workers", - [(True, 1), (True, 2), (False, 0)], - indirect=["use_threads"], -) -@pytest.mark.network -def test_fallback(use_threads, num_workers): +@pytest.mark.parametrize("use_threads, num_workers", [(True, 1), (True, 2), (False, 0)]) +def test_fallback(server, use_threads, num_workers): + url = f"{server}/uproot-issue121.root" with uproot.source.http.HTTPSource( - "https://scikit-hep.org/uproot3/examples/Zmumu.root", + url, timeout=10, num_fallback_workers=num_workers, use_threads=use_threads, @@ -343,7 +330,7 @@ def test_xrootd(use_threads): @pytest.mark.parametrize("use_threads", [True, False], indirect=True) def test_xrootd_deadlock(use_threads): pytest.importorskip("XRootD") - # Attach this file to the "test_xrootd_deadlock" function so it leaks + # Attach this file to the "test_xrootd_deadlock" function, so it leaks pytest.uproot_test_xrootd_deadlock_f = uproot.source.xrootd.XRootDResource( "root://eospublic.cern.ch//eos/root-eos/cms_opendata_2012_nanoaod/Run2012B_DoubleMuParked.root", timeout=20, @@ -356,7 +343,7 @@ def test_xrootd_deadlock(use_threads): @pytest.mark.parametrize("use_threads", [True, False], indirect=True) def test_xrootd_fail(use_threads): pytest.importorskip("XRootD") - with pytest.raises(Exception) as err: + with pytest.raises(Exception): uproot.source.xrootd.MultithreadedXRootDSource( "root://wonky.cern/does-not-exist", num_workers=1, @@ -447,7 +434,7 @@ def get_chunk(Source, **kwargs): @pytest.mark.parametrize("use_threads", [True, False], indirect=True) def test_xrootd_vectorread_fail(use_threads): pytest.importorskip("XRootD") - with pytest.raises(Exception) as err: + with pytest.raises(Exception): uproot.source.xrootd.XRootDSource( "root://wonky.cern/does-not-exist", timeout=1, diff --git a/tests/test_0006-notify-when-downloaded.py b/tests/test_0006_notify_when_downloaded.py similarity index 89% rename from tests/test_0006-notify-when-downloaded.py rename to tests/test_0006_notify_when_downloaded.py index 527983642..aea1a96da 100644 --- a/tests/test_0006-notify-when-downloaded.py +++ b/tests/test_0006_notify_when_downloaded.py @@ -2,10 +2,6 @@ import os import queue -import sys -from io import StringIO - -import numpy import pytest import uproot @@ -68,12 +64,11 @@ def test_memmap(tmpdir): expected.pop((chunk.start, chunk.stop)) -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") -@pytest.mark.network -def test_http_multipart(): +def test_http_multipart(server): + url = f"{server}/uproot-issue121.root" notifications = queue.Queue() with uproot.source.http.HTTPSource( - "https://example.com", timeout=10, num_fallback_workers=1, use_threads=True + url, timeout=10, num_fallback_workers=1, use_threads=True ) as source: chunks = source.chunks( [(0, 100), (50, 55), (200, 400)], notifications=notifications @@ -84,12 +79,11 @@ def test_http_multipart(): expected.pop((chunk.start, chunk.stop)) -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") -@pytest.mark.network -def test_http(): +def test_http(server): + url = f"{server}/uproot-issue121.root" notifications = queue.Queue() with uproot.source.http.MultithreadedHTTPSource( - "https://example.com", timeout=10, num_workers=1, use_threads=True + url, timeout=10, num_workers=1, use_threads=True ) as source: chunks = source.chunks( [(0, 100), (50, 55), (200, 400)], notifications=notifications @@ -100,12 +94,11 @@ def test_http(): expected.pop((chunk.start, chunk.stop)) -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") -@pytest.mark.network -def test_http_workers(): +def test_http_workers(server): + url = f"{server}/uproot-issue121.root" notifications = queue.Queue() with uproot.source.http.MultithreadedHTTPSource( - "https://example.com", timeout=10, num_workers=2, use_threads=True + url, timeout=10, num_workers=2, use_threads=True ) as source: chunks = source.chunks( [(0, 100), (50, 55), (200, 400)], notifications=notifications @@ -116,11 +109,11 @@ def test_http_workers(): expected.pop((chunk.start, chunk.stop)) -@pytest.mark.network -def test_http_fallback(): +def test_http_fallback(server): + url = f"{server}/uproot-issue121.root" notifications = queue.Queue() with uproot.source.http.HTTPSource( - "https://scikit-hep.org/uproot3/examples/Zmumu.root", + url, timeout=10, num_fallback_workers=1, use_threads=True, @@ -134,11 +127,11 @@ def test_http_fallback(): expected.pop((chunk.start, chunk.stop)) -@pytest.mark.network -def test_http_fallback_workers(): +def test_http_fallback_workers(server): + url = f"{server}/uproot-issue121.root" notifications = queue.Queue() with uproot.source.http.HTTPSource( - "https://scikit-hep.org/uproot3/examples/Zmumu.root", + url, timeout=10, num_fallback_workers=5, use_threads=True, diff --git a/tests/test_0007-single-chunk-interface.py b/tests/test_0007_single_chunk_interface.py similarity index 82% rename from tests/test_0007-single-chunk-interface.py rename to tests/test_0007_single_chunk_interface.py index 359e796c2..5f3223a7a 100644 --- a/tests/test_0007-single-chunk-interface.py +++ b/tests/test_0007_single_chunk_interface.py @@ -1,9 +1,7 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE import os -import sys -import numpy import pytest import uproot @@ -75,36 +73,40 @@ def test_memmap(tmpdir): ) -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") -@pytest.mark.network -def test_http(): - for num_workers in [1, 2]: - with uproot.source.http.MultithreadedHTTPSource( - "https://example.com", num_workers=num_workers, timeout=10, use_threads=True - ) as source: - for start, stop in [(0, 100), (50, 55), (200, 400)]: - chunk = source.chunk(start, stop) - assert len(tobytes(chunk.raw_data)) == stop - start +@pytest.mark.parametrize( + "num_workers", + [1, 2], +) +def test_http(server, num_workers): + url = f"{server}/uproot-issue121.root" + with uproot.source.http.MultithreadedHTTPSource( + url, num_workers=num_workers, timeout=10, use_threads=True + ) as source: + for start, stop in [(0, 100), (50, 55), (200, 400)]: + chunk = source.chunk(start, stop) + assert len(tobytes(chunk.raw_data)) == stop - start +@pytest.mark.parametrize( + "num_workers", + [1, 2], +) @pytest.mark.network -def test_http_fail(): - for num_workers in [1, 2]: - with pytest.raises(Exception): - with uproot.source.http.MultithreadedHTTPSource( - "https://wonky.cern/does-not-exist", - num_workers=num_workers, - timeout=0.1, - use_threads=True, - ) as source: - source.chunk(0, 100) +def test_http_fail(num_workers): + with pytest.raises(Exception): + with uproot.source.http.MultithreadedHTTPSource( + "https://wonky.cern/does-not-exist", + num_workers=num_workers, + timeout=0.1, + use_threads=True, + ) as source: + source.chunk(0, 100) -@pytest.mark.skip(reason="RECHECK: example.com is flaky, too") -@pytest.mark.network -def test_http_multipart(): +def test_http_multipart(server): + url = f"{server}/uproot-issue121.root" with uproot.source.http.HTTPSource( - "https://example.com", timeout=10, num_fallback_workers=1, use_threads=True + url, timeout=10, num_fallback_workers=1, use_threads=True ) as source: for start, stop in [(0, 100), (50, 55), (200, 400)]: chunk = source.chunk(start, stop) diff --git a/tests/test_0692_fsspec.py b/tests/test_0692_fsspec.py index baed29466..206750f51 100644 --- a/tests/test_0692_fsspec.py +++ b/tests/test_0692_fsspec.py @@ -8,13 +8,14 @@ import queue -@pytest.mark.network @pytest.mark.parametrize("use_threads", [True, False]) -def test_open_fsspec_http(use_threads): +def test_open_fsspec_http(server, use_threads): pytest.importorskip("aiohttp") + url = f"{server}/uproot-issue121.root" + with uproot.open( - "https://github.com/scikit-hep/scikit-hep-testdata/raw/v0.4.33/src/skhep_testdata/data/uproot-issue121.root", + url, handler=uproot.source.fsspec.FSSpecSource, use_threads=use_threads, ) as f: @@ -23,10 +24,10 @@ def test_open_fsspec_http(use_threads): @pytest.mark.network +@pytest.mark.skip( + reason="skipping due to GitHub API rate limitations - this should work fine - see https://github.com/scikit-hep/uproot5/pull/973 for details" +) def test_open_fsspec_github(): - pytest.skip( - "skipping due to GitHub API rate limitations - this should work fine - see https://github.com/scikit-hep/uproot5/pull/973 for details" - ) with uproot.open( "github://scikit-hep:scikit-hep-testdata@v0.4.33/src/skhep_testdata/data/uproot-issue121.root", handler=uproot.source.fsspec.FSSpecSource, @@ -86,11 +87,10 @@ def test_open_fsspec_xrootd(handler, use_threads): assert (data == 194778).all() -@pytest.mark.network -def test_fsspec_chunks(): +def test_fsspec_chunks(server): pytest.importorskip("aiohttp") - url = "https://github.com/scikit-hep/scikit-hep-testdata/raw/v0.4.33/src/skhep_testdata/data/uproot-issue121.root" + url = f"{server}/uproot-issue121.root" notifications = queue.Queue() with uproot.source.fsspec.FSSpecSource(url) as source: