diff --git a/numba_cuda/numba/cuda/testing.py b/numba_cuda/numba/cuda/testing.py index 8fdbc38f8..646ef5bf9 100644 --- a/numba_cuda/numba/cuda/testing.py +++ b/numba_cuda/numba/cuda/testing.py @@ -7,7 +7,7 @@ from numba.cuda.cudadrv import driver, devices, libs from numba.cuda.dispatcher import CUDADispatcher from numba.core import config -from numba.tests.support import TestCase +from numba.cuda.tests.support import TestCase from pathlib import Path from typing import Union from io import StringIO diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_cuda_ndarray.py b/numba_cuda/numba/cuda/tests/cudadrv/test_cuda_ndarray.py index 398211d6a..638676128 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_cuda_ndarray.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_cuda_ndarray.py @@ -4,7 +4,7 @@ from numba import cuda from numba.cuda.testing import unittest, CUDATestCase from numba.cuda.testing import skip_on_cudasim -from numba.tests.support import IS_NUMPY_2 +from numba.cuda.tests.support import IS_NUMPY_2 class TestCudaNDArray(CUDATestCase): diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_deallocations.py b/numba_cuda/numba/cuda/tests/cudadrv/test_deallocations.py index 7f03912c1..d39980c9e 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_deallocations.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_deallocations.py @@ -9,7 +9,7 @@ skip_if_external_memmgr, CUDATestCase, ) -from numba.tests.support import captured_stderr +from numba.cuda.tests.support import captured_stderr from numba.core import config diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_detect.py b/numba_cuda/numba/cuda/tests/cudadrv/test_detect.py index d70b6776e..03e2e0b8c 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_detect.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_detect.py @@ -9,7 +9,7 @@ skip_on_cudasim, skip_under_cuda_memcheck, ) -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout class TestCudaDetect(CUDATestCase): diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_emm_plugins.py b/numba_cuda/numba/cuda/tests/cudadrv/test_emm_plugins.py index be77a40ad..29993f19c 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_emm_plugins.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_emm_plugins.py @@ -5,7 +5,7 @@ from numba import cuda from numba.core import config from numba.cuda.testing import unittest, CUDATestCase, skip_on_cudasim -from numba.tests.support import linux_only +from numba.cuda.tests.support import linux_only if not config.ENABLE_CUDASIM: diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_linker.py b/numba_cuda/numba/cuda/tests/cudadrv/test_linker.py index ed503dab6..9cb096ece 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_linker.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_linker.py @@ -6,7 +6,7 @@ from numba.cuda.testing import CUDATestCase, test_data_dir from numba.cuda.cudadrv.driver import CudaAPIError, _Linker, LinkerError from numba.cuda import require_context -from numba.tests.support import ignore_internal_warnings +from numba.cuda.tests.support import ignore_internal_warnings from numba import cuda, void, float64, int64, int32, typeof, float32 from numba.cuda.cudadrv.error import NvrtcError diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_managed_alloc.py b/numba_cuda/numba/cuda/tests/cudadrv/test_managed_alloc.py index 69a85a9b6..357e74981 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_managed_alloc.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_managed_alloc.py @@ -4,7 +4,7 @@ from numba import cuda from numba.cuda.testing import unittest, ContextResettingTestCase from numba.cuda.testing import skip_on_cudasim, skip_on_arm -from numba.tests.support import linux_only +from numba.cuda.tests.support import linux_only @skip_on_cudasim("CUDA Driver API unsupported in the simulator") diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_mvc.py b/numba_cuda/numba/cuda/tests/cudadrv/test_mvc.py index 4da56e009..e4a19f6d5 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_mvc.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_mvc.py @@ -6,7 +6,7 @@ skip_under_cuda_memcheck, skip_if_mvc_libraries_unavailable, ) -from numba.tests.support import linux_only +from numba.cuda.tests.support import linux_only def child_test(): diff --git a/numba_cuda/numba/cuda/tests/cudadrv/test_ptds.py b/numba_cuda/numba/cuda/tests/cudadrv/test_ptds.py index a532f8c28..6bce378b3 100644 --- a/numba_cuda/numba/cuda/tests/cudadrv/test_ptds.py +++ b/numba_cuda/numba/cuda/tests/cudadrv/test_ptds.py @@ -7,7 +7,7 @@ skip_with_cuda_python, skip_under_cuda_memcheck, ) -from numba.tests.support import linux_only +from numba.cuda.tests.support import linux_only def child_test(): diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_caching.py b/numba_cuda/numba/cuda/tests/cudapy/test_caching.py index 2f3726628..c9b5c6b10 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_caching.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_caching.py @@ -3,6 +3,9 @@ import shutil import unittest import warnings +import sys +import stat +import subprocess from numba import cuda from numba.core.errors import NumbaWarning @@ -14,12 +17,152 @@ skip_if_mvc_enabled, test_data_dir, ) -from numba.tests.test_caching import ( - DispatcherCacheUsecasesTest, - skip_bad_access, +from numba.cuda.tests.support import ( + TestCase, + temp_directory, + import_dynamic, ) +class BaseCacheTest(TestCase): + # The source file that will be copied + usecases_file = None + # Make sure this doesn't conflict with another module + modname = None + + def setUp(self): + self.tempdir = temp_directory("test_cache") + sys.path.insert(0, self.tempdir) + self.modfile = os.path.join(self.tempdir, self.modname + ".py") + self.cache_dir = os.path.join(self.tempdir, "__pycache__") + shutil.copy(self.usecases_file, self.modfile) + os.chmod(self.modfile, stat.S_IREAD | stat.S_IWRITE) + self.maxDiff = None + + def tearDown(self): + sys.modules.pop(self.modname, None) + sys.path.remove(self.tempdir) + + def import_module(self): + # Import a fresh version of the test module. All jitted functions + # in the test module will start anew and load overloads from + # the on-disk cache if possible. + old = sys.modules.pop(self.modname, None) + if old is not None: + # Make sure cached bytecode is removed + cached = [old.__cached__] + for fn in cached: + try: + os.unlink(fn) + except FileNotFoundError: + pass + mod = import_dynamic(self.modname) + self.assertEqual(mod.__file__.rstrip("co"), self.modfile) + return mod + + def cache_contents(self): + try: + return [ + fn + for fn in os.listdir(self.cache_dir) + if not fn.endswith((".pyc", ".pyo")) + ] + except FileNotFoundError: + return [] + + def get_cache_mtimes(self): + return dict( + (fn, os.path.getmtime(os.path.join(self.cache_dir, fn))) + for fn in sorted(self.cache_contents()) + ) + + def check_pycache(self, n): + c = self.cache_contents() + self.assertEqual(len(c), n, c) + + def dummy_test(self): + pass + + +class DispatcherCacheUsecasesTest(BaseCacheTest): + here = os.path.dirname(__file__) + usecases_file = os.path.join(here, "cache_usecases.py") + modname = "dispatcher_caching_test_fodder" + + def run_in_separate_process(self, *, envvars={}): + # Cached functions can be run from a distinct process. + # Also stresses issue #1603: uncached function calling cached function + # shouldn't fail compiling. + code = """if 1: + import sys + + sys.path.insert(0, %(tempdir)r) + mod = __import__(%(modname)r) + mod.self_test() + """ % dict(tempdir=self.tempdir, modname=self.modname) + + subp_env = os.environ.copy() + subp_env.update(envvars) + popen = subprocess.Popen( + [sys.executable, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=subp_env, + ) + out, err = popen.communicate() + if popen.returncode != 0: + raise AssertionError( + "process failed with code %s: \n" + "stdout follows\n%s\n" + "stderr follows\n%s\n" + % (popen.returncode, out.decode(), err.decode()), + ) + + def check_hits(self, func, hits, misses=None): + st = func.stats + self.assertEqual(sum(st.cache_hits.values()), hits, st.cache_hits) + if misses is not None: + self.assertEqual( + sum(st.cache_misses.values()), misses, st.cache_misses + ) + + +def check_access_is_preventable(): + # This exists to check whether it is possible to prevent access to + # a file/directory through the use of `chmod 500`. If a user has + # elevated rights (e.g. root) then writes are likely to be possible + # anyway. Tests that require functioning access prevention are + # therefore skipped based on the result of this check. + tempdir = temp_directory("test_cache") + test_dir = os.path.join(tempdir, "writable_test") + os.mkdir(test_dir) + # check a write is possible + with open(os.path.join(test_dir, "write_ok"), "wt") as f: + f.write("check1") + # now forbid access + os.chmod(test_dir, 0o500) + try: + with open(os.path.join(test_dir, "write_forbidden"), "wt") as f: + f.write("check2") + # access prevention is not possible + return False + except PermissionError: + # Check that the cause of the exception is due to access/permission + # as per + # https://github.com/conda/conda/blob/4.5.0/conda/gateways/disk/permissions.py#L35-L37 # noqa: E501 + # errno reports access/perm fail so access prevention via + # `chmod 500` works for this user. + return True + finally: + os.chmod(test_dir, 0o775) + shutil.rmtree(test_dir) + + +_access_preventable = check_access_is_preventable() +_access_msg = "Cannot create a directory to which writes are preventable" +skip_bad_access = unittest.skipUnless(_access_preventable, _access_msg) + + @skip_on_cudasim("Simulator does not implement caching") class CUDACachingTest(DispatcherCacheUsecasesTest): here = os.path.dirname(__file__) diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_cffi.py b/numba_cuda/numba/cuda/tests/cudapy/test_cffi.py index 57f3efdb8..caa6cc6a8 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_cffi.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_cffi.py @@ -7,7 +7,7 @@ unittest, CUDATestCase, ) -from numba.tests.support import skip_unless_cffi +from numba.cuda.tests.support import skip_unless_cffi @skip_unless_cffi diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_cuda_array_interface.py b/numba_cuda/numba/cuda/tests/cudapy/test_cuda_array_interface.py index 39f8b73ba..f26efc5e8 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_cuda_array_interface.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_cuda_array_interface.py @@ -4,7 +4,7 @@ from numba import cuda from numba.cuda.testing import unittest, ContextResettingTestCase, ForeignArray from numba.cuda.testing import skip_on_cudasim, skip_if_external_memmgr -from numba.tests.support import linux_only, override_config +from numba.cuda.tests.support import linux_only, override_config from unittest.mock import call, patch diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_cuda_jit_no_types.py b/numba_cuda/numba/cuda/tests/cudapy/test_cuda_jit_no_types.py index 99f614677..78a93430b 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_cuda_jit_no_types.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_cuda_jit_no_types.py @@ -1,7 +1,7 @@ from numba import cuda import numpy as np from numba.cuda.testing import CUDATestCase -from numba.tests.support import override_config +from numba.cuda.tests.support import override_config import unittest diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_debug.py b/numba_cuda/numba/cuda/tests/cudapy/test_debug.py index 00fb70c06..432aeb7c5 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_debug.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_debug.py @@ -2,7 +2,7 @@ from numba.core.utils import PYVERSION from numba.cuda.testing import skip_on_cudasim, CUDATestCase -from numba.tests.support import ( +from numba.cuda.tests.support import ( override_config, captured_stderr, captured_stdout, diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_debuginfo.py b/numba_cuda/numba/cuda/tests/cudapy/test_debuginfo.py index 5bf1829a0..d9917ae4b 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_debuginfo.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_debuginfo.py @@ -1,4 +1,4 @@ -from numba.tests.support import override_config, captured_stdout +from numba.cuda.tests.support import override_config, captured_stdout from numba.cuda.testing import skip_on_cudasim from numba import cuda from numba.core import types diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_device_func.py b/numba_cuda/numba/cuda/tests/cudapy/test_device_func.py index 4ff973baa..49746a00e 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_device_func.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_device_func.py @@ -12,7 +12,7 @@ ) from numba import cuda, jit, float32, int32, types from numba.core.errors import TypingError -from numba.tests.support import skip_unless_cffi +from numba.cuda.tests.support import skip_unless_cffi from types import ModuleType diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_gufunc.py b/numba_cuda/numba/cuda/tests/cudapy/test_gufunc.py index 954ed635d..62d17fdf5 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_gufunc.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_gufunc.py @@ -8,7 +8,7 @@ import unittest import warnings from numba.core.errors import NumbaPerformanceWarning, TypingError -from numba.tests.support import override_config +from numba.cuda.tests.support import override_config def _get_matmulcore_gufunc(dtype=float32): diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_ipc.py b/numba_cuda/numba/cuda/tests/cudapy/test_ipc.py index 4a6083cd2..52f6cfd29 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_ipc.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_ipc.py @@ -14,7 +14,7 @@ ContextResettingTestCase, ForeignArray, ) -from numba.tests.support import linux_only, windows_only +from numba.cuda.tests.support import linux_only, windows_only import unittest diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_lineinfo.py b/numba_cuda/numba/cuda/tests/cudapy/test_lineinfo.py index 5ff55f7bb..17457be33 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_lineinfo.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_lineinfo.py @@ -1,7 +1,7 @@ from numba import cuda, float32, int32 from numba.core.errors import NumbaInvalidConfigWarning from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import ignore_internal_warnings +from numba.cuda.tests.support import ignore_internal_warnings import re import unittest import warnings diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_ufuncs.py b/numba_cuda/numba/cuda/tests/cudapy/test_ufuncs.py index 22aa83590..6d3f11372 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_ufuncs.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_ufuncs.py @@ -3,7 +3,7 @@ import unittest from numba import config, cuda, types -from numba.tests.support import TestCase +from numba.cuda.tests.support import TestCase from numba.tests.test_ufuncs import BasicUFuncTest diff --git a/numba_cuda/numba/cuda/tests/cudapy/test_warning.py b/numba_cuda/numba/cuda/tests/cudapy/test_warning.py index 5d1ef05d6..1cc40d5e6 100644 --- a/numba_cuda/numba/cuda/tests/cudapy/test_warning.py +++ b/numba_cuda/numba/cuda/tests/cudapy/test_warning.py @@ -6,7 +6,11 @@ CUDATestCase, skip_on_cudasim, ) -from numba.tests.support import linux_only, override_config, run_in_subprocess +from numba.cuda.tests.support import ( + linux_only, + override_config, + run_in_subprocess, +) from numba.core.errors import NumbaPerformanceWarning from numba.core import config import warnings diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_cpointer.py b/numba_cuda/numba/cuda/tests/doc_examples/test_cpointer.py index 8b204cd1a..159e44e5d 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_cpointer.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_cpointer.py @@ -1,7 +1,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_on_cudasim("cudasim doesn't support cuda import at non-top-level") diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_cpu_gpu_compat.py b/numba_cuda/numba/cuda/tests/doc_examples/test_cpu_gpu_compat.py index f8ec6f51f..a04cd39ce 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_cpu_gpu_compat.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_cpu_gpu_compat.py @@ -1,7 +1,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout import numpy as np diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_ffi.py b/numba_cuda/numba/cuda/tests/doc_examples/test_ffi.py index cc0122f4f..acbffd40b 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_ffi.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_ffi.py @@ -3,7 +3,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import skip_unless_cffi, override_config +from numba.cuda.tests.support import skip_unless_cffi, override_config @skip_unless_cffi diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_laplace.py b/numba_cuda/numba/cuda/tests/doc_examples/test_laplace.py index 75f38446a..c044ec578 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_laplace.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_laplace.py @@ -7,7 +7,7 @@ skip_unless_cc_60, skip_if_mvc_enabled, ) -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_if_cudadevrt_missing diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_matmul.py b/numba_cuda/numba/cuda/tests/doc_examples/test_matmul.py index 9633954f0..3dc7e9aff 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_matmul.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_matmul.py @@ -9,7 +9,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_on_cudasim("cudasim doesn't support cuda import at non-top-level") diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_montecarlo.py b/numba_cuda/numba/cuda/tests/doc_examples/test_montecarlo.py index 8a5d9f46f..5432c8947 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_montecarlo.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_montecarlo.py @@ -1,7 +1,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_on_cudasim("cudasim doesn't support cuda import at non-top-level") diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_reduction.py b/numba_cuda/numba/cuda/tests/doc_examples/test_reduction.py index 92a0e6ade..d572eb9c6 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_reduction.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_reduction.py @@ -1,7 +1,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_on_cudasim("cudasim doesn't support cuda import at non-top-level") diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_sessionize.py b/numba_cuda/numba/cuda/tests/doc_examples/test_sessionize.py index c3a23471a..d7b82f766 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_sessionize.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_sessionize.py @@ -7,7 +7,7 @@ skip_unless_cc_60, skip_if_mvc_enabled, ) -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_if_cudadevrt_missing diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_ufunc.py b/numba_cuda/numba/cuda/tests/doc_examples/test_ufunc.py index c1f56b07e..f038530f6 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_ufunc.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_ufunc.py @@ -1,7 +1,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_on_cudasim("cudasim doesn't support cuda import at non-top-level") diff --git a/numba_cuda/numba/cuda/tests/doc_examples/test_vecadd.py b/numba_cuda/numba/cuda/tests/doc_examples/test_vecadd.py index 64131f0a7..a120874a0 100644 --- a/numba_cuda/numba/cuda/tests/doc_examples/test_vecadd.py +++ b/numba_cuda/numba/cuda/tests/doc_examples/test_vecadd.py @@ -1,7 +1,7 @@ import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import captured_stdout +from numba.cuda.tests.support import captured_stdout @skip_on_cudasim("cudasim doesn't support cuda import at non-top-level") diff --git a/numba_cuda/numba/cuda/tests/nocuda/test_import.py b/numba_cuda/numba/cuda/tests/nocuda/test_import.py index b44ccbc95..fdedfb02b 100644 --- a/numba_cuda/numba/cuda/tests/nocuda/test_import.py +++ b/numba_cuda/numba/cuda/tests/nocuda/test_import.py @@ -1,4 +1,4 @@ -from numba.tests.support import run_in_subprocess +from numba.cuda.tests.support import run_in_subprocess import unittest diff --git a/numba_cuda/numba/cuda/tests/nrt/test_nrt.py b/numba_cuda/numba/cuda/tests/nrt/test_nrt.py index 2e14549cc..bf995d9dc 100644 --- a/numba_cuda/numba/cuda/tests/nrt/test_nrt.py +++ b/numba_cuda/numba/cuda/tests/nrt/test_nrt.py @@ -4,7 +4,7 @@ import numpy as np import unittest from numba.cuda.testing import CUDATestCase, skip_on_cudasim -from numba.tests.support import run_in_subprocess, override_config +from numba.cuda.tests.support import run_in_subprocess, override_config from numba.cuda import get_current_device from numba.cuda.cudadrv.nvrtc import compile from numba import config, types diff --git a/numba_cuda/numba/cuda/tests/nrt/test_nrt_refct.py b/numba_cuda/numba/cuda/tests/nrt/test_nrt_refct.py index 41d1dc4dd..a1cabdea1 100644 --- a/numba_cuda/numba/cuda/tests/nrt/test_nrt_refct.py +++ b/numba_cuda/numba/cuda/tests/nrt/test_nrt_refct.py @@ -1,6 +1,6 @@ import numpy as np import unittest -from numba.tests.support import override_config +from numba.cuda.tests.support import override_config from numba.cuda.memory_management import rtsys from numba.cuda.tests.support import EnableNRTStatsMixin from numba.cuda.testing import CUDATestCase, skip_on_cudasim diff --git a/numba_cuda/numba/cuda/tests/support.py b/numba_cuda/numba/cuda/tests/support.py index 3050bcd52..f66e8dcef 100644 --- a/numba_cuda/numba/cuda/tests/support.py +++ b/numba_cuda/numba/cuda/tests/support.py @@ -1,4 +1,34 @@ +import cmath +import contextlib +import enum +import gc +import math +import unittest +import os +import io +import subprocess +import sys +import shutil +import warnings +import tempfile +import time +import types as pytypes +from functools import cached_property + +import numpy as np + +from numba import types +from numba.core import errors, config +from numba.core.typing import cffi_utils from numba.cuda.memory_management.nrt import rtsys +from numba.core.extending import ( + typeof_impl, + register_model, + unbox, + NativeValue, +) +from numba.core.datamodel.models import OpaqueModel +from numba.np import numpy_support class EnableNRTStatsMixin(object): @@ -9,3 +39,725 @@ def setUp(self): def tearDown(self): rtsys.memsys_disable_stats() + + +skip_unless_cffi = unittest.skipUnless(cffi_utils.SUPPORTED, "requires cffi") + +_lnx_reason = "linux only test" +linux_only = unittest.skipIf(not sys.platform.startswith("linux"), _lnx_reason) + +_win_reason = "Windows only test" +windows_only = unittest.skipIf(not sys.platform.startswith("win"), _win_reason) + +IS_NUMPY_2 = numpy_support.numpy_version >= (2, 0) +skip_if_numpy_2 = unittest.skipIf(IS_NUMPY_2, "Not supported on numpy 2.0+") + +_trashcan_dir = "numba-cuda-tests" + +if os.name == "nt": + # Under Windows, gettempdir() points to the user-local temp dir + _trashcan_dir = os.path.join(tempfile.gettempdir(), _trashcan_dir) +else: + # Mix the UID into the directory name to allow different users to + # run the test suite without permission errors (issue #1586) + _trashcan_dir = os.path.join( + tempfile.gettempdir(), "%s.%s" % (_trashcan_dir, os.getuid()) + ) + +# Stale temporary directories are deleted after they are older than this value. +# The test suite probably won't ever take longer than this... +_trashcan_timeout = 24 * 3600 # 1 day + + +def _create_trashcan_dir(): + try: + os.mkdir(_trashcan_dir) + except FileExistsError: + pass + + +def _purge_trashcan_dir(): + freshness_threshold = time.time() - _trashcan_timeout + for fn in sorted(os.listdir(_trashcan_dir)): + fn = os.path.join(_trashcan_dir, fn) + try: + st = os.stat(fn) + if st.st_mtime < freshness_threshold: + shutil.rmtree(fn, ignore_errors=True) + except OSError: + # In parallel testing, several processes can attempt to + # remove the same entry at once, ignore. + pass + + +def _create_trashcan_subdir(prefix): + _purge_trashcan_dir() + path = tempfile.mkdtemp(prefix=prefix + "-", dir=_trashcan_dir) + return path + + +def temp_directory(prefix): + """ + Create a temporary directory with the given *prefix* that will survive + at least as long as this process invocation. The temporary directory + will be eventually deleted when it becomes stale enough. + + This is necessary because a DLL file can't be deleted while in use + under Windows. + + An interesting side-effect is to be able to inspect the test files + shortly after a test suite run. + """ + _create_trashcan_dir() + return _create_trashcan_subdir(prefix) + + +def import_dynamic(modname): + """ + Import and return a module of the given name. Care is taken to + avoid issues due to Python's internal directory caching. + """ + import importlib + + importlib.invalidate_caches() + __import__(modname) + return sys.modules[modname] + + +def ignore_internal_warnings(): + """Use in testing within a ` warnings.catch_warnings` block to filter out + warnings that are unrelated/internally generated by Numba. + """ + # Filter out warnings from typeguard + warnings.filterwarnings("ignore", module="typeguard") + # Filter out warnings about TBB interface mismatch + warnings.filterwarnings( + action="ignore", + message=r".*TBB_INTERFACE_VERSION.*", + category=errors.NumbaWarning, + module=r"numba\.np\.ufunc\.parallel.*", + ) + + +@contextlib.contextmanager +def override_config(name, value): + """ + Return a context manager that temporarily sets Numba config variable + *name* to *value*. *name* must be the name of an existing variable + in numba.config. + """ + old_value = getattr(config, name) + setattr(config, name, value) + try: + yield + finally: + setattr(config, name, old_value) + + +def run_in_subprocess(code, flags=None, env=None, timeout=30): + """Run a snippet of Python code in a subprocess with flags, if any are + given. 'env' is passed to subprocess.Popen(). 'timeout' is passed to + popen.communicate(). + + Returns the stdout and stderr of the subprocess after its termination. + """ + if flags is None: + flags = [] + cmd = ( + [ + sys.executable, + ] + + flags + + ["-c", code] + ) + popen = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env + ) + out, err = popen.communicate(timeout=timeout) + if popen.returncode != 0: + msg = "process failed with code %s: stderr follows\n%s\n" + raise AssertionError(msg % (popen.returncode, err.decode())) + return out, err + + +@contextlib.contextmanager +def captured_output(stream_name): + """Return a context manager used by captured_stdout/stdin/stderr + that temporarily replaces the sys stream *stream_name* with a StringIO.""" + orig_stdout = getattr(sys, stream_name) + setattr(sys, stream_name, io.StringIO()) + try: + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stdout) + + +def captured_stdout(): + """Capture the output of sys.stdout: + + with captured_stdout() as stdout: + print("hello") + self.assertEqual(stdout.getvalue(), "hello\n") + """ + return captured_output("stdout") + + +def captured_stderr(): + """Capture the output of sys.stderr: + + with captured_stderr() as stderr: + print("hello", file=sys.stderr) + self.assertEqual(stderr.getvalue(), "hello\n") + """ + return captured_output("stderr") + + +class TestCase(unittest.TestCase): + longMessage = True + + # A random state yielding the same random numbers for any test case. + # Use as `self.random.` + @cached_property + def random(self): + return np.random.RandomState(42) + + def reset_module_warnings(self, module): + """ + Reset the warnings registry of a module. This can be necessary + as the warnings module is buggy in that regard. + See http://bugs.python.org/issue4180 + """ + if isinstance(module, str): + module = sys.modules[module] + try: + del module.__warningregistry__ + except AttributeError: + pass + + @contextlib.contextmanager + def assertTypingError(self): + """ + A context manager that asserts the enclosed code block fails + compiling in nopython mode. + """ + _accepted_errors = ( + errors.LoweringError, + errors.TypingError, + TypeError, + NotImplementedError, + ) + with self.assertRaises(_accepted_errors) as cm: + yield cm + + @contextlib.contextmanager + def assertRefCount(self, *objects): + """ + A context manager that asserts the given objects have the + same reference counts before and after executing the + enclosed block. + """ + old_refcounts = [sys.getrefcount(x) for x in objects] + yield + gc.collect() + new_refcounts = [sys.getrefcount(x) for x in objects] + for old, new, obj in zip(old_refcounts, new_refcounts, objects): + if old != new: + self.fail( + "Refcount changed from %d to %d for object: %r" + % (old, new, obj) + ) + + def assertRefCountEqual(self, *objects): + gc.collect() + rc = [sys.getrefcount(x) for x in objects] + rc_0 = rc[0] + for i in range(len(objects))[1:]: + rc_i = rc[i] + if rc_0 != rc_i: + self.fail( + f"Refcount for objects does not match. " + f"#0({rc_0}) != #{i}({rc_i}) does not match." + ) + + @contextlib.contextmanager + def assertNoNRTLeak(self): + """ + A context manager that asserts no NRT leak was created during + the execution of the enclosed block. + """ + old = rtsys.get_allocation_stats() + yield + new = rtsys.get_allocation_stats() + total_alloc = new.alloc - old.alloc + total_free = new.free - old.free + total_mi_alloc = new.mi_alloc - old.mi_alloc + total_mi_free = new.mi_free - old.mi_free + self.assertEqual( + total_alloc, + total_free, + "number of data allocs != number of data frees", + ) + self.assertEqual( + total_mi_alloc, + total_mi_free, + "number of meminfo allocs != number of meminfo frees", + ) + + _bool_types = (bool, np.bool_) + _exact_typesets = [ + _bool_types, + (int,), + (str,), + (np.integer,), + (bytes, np.bytes_), + ] + _approx_typesets = [(float,), (complex,), (np.inexact)] + _sequence_typesets = [(tuple, list)] + _float_types = (float, np.floating) + _complex_types = (complex, np.complexfloating) + + def _detect_family(self, numeric_object): + """ + This function returns a string description of the type family + that the object in question belongs to. Possible return values + are: "exact", "complex", "approximate", "sequence", and "unknown" + """ + if isinstance(numeric_object, np.ndarray): + return "ndarray" + + if isinstance(numeric_object, enum.Enum): + return "enum" + + for tp in self._sequence_typesets: + if isinstance(numeric_object, tp): + return "sequence" + + for tp in self._exact_typesets: + if isinstance(numeric_object, tp): + return "exact" + + for tp in self._complex_types: + if isinstance(numeric_object, tp): + return "complex" + + for tp in self._approx_typesets: + if isinstance(numeric_object, tp): + return "approximate" + + return "unknown" + + def _fix_dtype(self, dtype): + """ + Fix the given *dtype* for comparison. + """ + # Under 64-bit Windows, Numpy may return either int32 or int64 + # arrays depending on the function. + if ( + sys.platform == "win32" + and sys.maxsize > 2**32 + and dtype == np.dtype("int32") + ): + return np.dtype("int64") + else: + return dtype + + def _fix_strides(self, arr): + """ + Return the strides of the given array, fixed for comparison. + Strides for 0- or 1-sized dimensions are ignored. + """ + if arr.size == 0: + return [0] * arr.ndim + else: + return [ + stride / arr.itemsize + for (stride, shape) in zip(arr.strides, arr.shape) + if shape > 1 + ] + + def assertStridesEqual(self, first, second): + """ + Test that two arrays have the same shape and strides. + """ + self.assertEqual(first.shape, second.shape, "shapes differ") + self.assertEqual(first.itemsize, second.itemsize, "itemsizes differ") + self.assertEqual( + self._fix_strides(first), + self._fix_strides(second), + "strides differ", + ) + + def assertPreciseEqual( + self, + first, + second, + prec="exact", + ulps=1, + msg=None, + ignore_sign_on_zero=False, + abs_tol=None, + ): + """ + Versatile equality testing function with more built-in checks than + standard assertEqual(). + + For arrays, test that layout, dtype, shape are identical, and + recursively call assertPreciseEqual() on the contents. + + For other sequences, recursively call assertPreciseEqual() on + the contents. + + For scalars, test that two scalars or have similar types and are + equal up to a computed precision. + If the scalars are instances of exact types or if *prec* is + 'exact', they are compared exactly. + If the scalars are instances of inexact types (float, complex) + and *prec* is not 'exact', then the number of significant bits + is computed according to the value of *prec*: 53 bits if *prec* + is 'double', 24 bits if *prec* is single. This number of bits + can be lowered by raising the *ulps* value. + ignore_sign_on_zero can be set to True if zeros are to be considered + equal regardless of their sign bit. + abs_tol if this is set to a float value its value is used in the + following. If, however, this is set to the string "eps" then machine + precision of the type(first) is used in the following instead. This + kwarg is used to check if the absolute difference in value between first + and second is less than the value set, if so the numbers being compared + are considered equal. (This is to handle small numbers typically of + magnitude less than machine precision). + + Any value of *prec* other than 'exact', 'single' or 'double' + will raise an error. + """ + try: + self._assertPreciseEqual( + first, second, prec, ulps, msg, ignore_sign_on_zero, abs_tol + ) + except AssertionError as exc: + failure_msg = str(exc) + # Fall off of the 'except' scope to avoid Python 3 exception + # chaining. + else: + return + # Decorate the failure message with more information + self.fail("when comparing %s and %s: %s" % (first, second, failure_msg)) + + def _assertPreciseEqual( + self, + first, + second, + prec="exact", + ulps=1, + msg=None, + ignore_sign_on_zero=False, + abs_tol=None, + ): + """Recursive workhorse for assertPreciseEqual().""" + + def _assertNumberEqual(first, second, delta=None): + if ( + delta is None + or first == second == 0.0 + or math.isinf(first) + or math.isinf(second) + ): + self.assertEqual(first, second, msg=msg) + # For signed zeros + if not ignore_sign_on_zero: + try: + if math.copysign(1, first) != math.copysign(1, second): + self.fail( + self._formatMessage( + msg, "%s != %s" % (first, second) + ) + ) + except TypeError: + pass + else: + self.assertAlmostEqual(first, second, delta=delta, msg=msg) + + first_family = self._detect_family(first) + second_family = self._detect_family(second) + + assertion_message = "Type Family mismatch. (%s != %s)" % ( + first_family, + second_family, + ) + if msg: + assertion_message += ": %s" % (msg,) + self.assertEqual(first_family, second_family, msg=assertion_message) + + # We now know they are in the same comparison family + compare_family = first_family + + # For recognized sequences, recurse + if compare_family == "ndarray": + dtype = self._fix_dtype(first.dtype) + self.assertEqual(dtype, self._fix_dtype(second.dtype)) + self.assertEqual( + first.ndim, second.ndim, "different number of dimensions" + ) + self.assertEqual(first.shape, second.shape, "different shapes") + self.assertEqual( + first.flags.writeable, + second.flags.writeable, + "different mutability", + ) + # itemsize is already checked by the dtype test above + self.assertEqual( + self._fix_strides(first), + self._fix_strides(second), + "different strides", + ) + if first.dtype != dtype: + first = first.astype(dtype) + if second.dtype != dtype: + second = second.astype(dtype) + for a, b in zip(first.flat, second.flat): + self._assertPreciseEqual( + a, b, prec, ulps, msg, ignore_sign_on_zero, abs_tol + ) + return + + elif compare_family == "sequence": + self.assertEqual(len(first), len(second), msg=msg) + for a, b in zip(first, second): + self._assertPreciseEqual( + a, b, prec, ulps, msg, ignore_sign_on_zero, abs_tol + ) + return + + elif compare_family == "exact": + exact_comparison = True + + elif compare_family in ["complex", "approximate"]: + exact_comparison = False + + elif compare_family == "enum": + self.assertIs(first.__class__, second.__class__) + self._assertPreciseEqual( + first.value, + second.value, + prec, + ulps, + msg, + ignore_sign_on_zero, + abs_tol, + ) + return + + elif compare_family == "unknown": + # Assume these are non-numeric types: we will fall back + # on regular unittest comparison. + self.assertIs(first.__class__, second.__class__) + exact_comparison = True + + else: + assert 0, "unexpected family" + + # If a Numpy scalar, check the dtype is exactly the same too + # (required for datetime64 and timedelta64). + if hasattr(first, "dtype") and hasattr(second, "dtype"): + self.assertEqual(first.dtype, second.dtype) + + # Mixing bools and non-bools should always fail + if isinstance(first, self._bool_types) != isinstance( + second, self._bool_types + ): + assertion_message = "Mismatching return types (%s vs. %s)" % ( + first.__class__, + second.__class__, + ) + if msg: + assertion_message += ": %s" % (msg,) + self.fail(assertion_message) + + try: + if cmath.isnan(first) and cmath.isnan(second): + # The NaNs will compare unequal, skip regular comparison + return + except TypeError: + # Not floats. + pass + + # if absolute comparison is set, use it + if abs_tol is not None: + if abs_tol == "eps": + rtol = np.finfo(type(first)).eps + elif isinstance(abs_tol, float): + rtol = abs_tol + else: + raise ValueError( + 'abs_tol is not "eps" or a float, found %s' % abs_tol + ) + if abs(first - second) < rtol: + return + + exact_comparison = exact_comparison or prec == "exact" + + if not exact_comparison and prec != "exact": + if prec == "single": + bits = 24 + elif prec == "double": + bits = 53 + else: + raise ValueError("unsupported precision %r" % (prec,)) + k = 2 ** (ulps - bits - 1) + delta = k * (abs(first) + abs(second)) + else: + delta = None + if isinstance(first, self._complex_types): + _assertNumberEqual(first.real, second.real, delta) + _assertNumberEqual(first.imag, second.imag, delta) + elif isinstance(first, (np.timedelta64, np.datetime64)): + # Since Np 1.16 NaT == NaT is False, so special comparison needed + if np.isnat(first): + self.assertEqual(np.isnat(first), np.isnat(second)) + else: + _assertNumberEqual(first, second, delta) + else: + _assertNumberEqual(first, second, delta) + + def subprocess_test_runner( + self, + test_module, + test_class=None, + test_name=None, + envvars=None, + timeout=60, + flags=None, + _subproc_test_env="1", + ): + """ + Runs named unit test(s) as specified in the arguments as: + test_module.test_class.test_name. test_module must always be supplied + and if no further refinement is made with test_class and test_name then + all tests in the module will be run. The tests will be run in a + subprocess with environment variables specified in `envvars`. + If given, envvars must be a map of form: + environment variable name (str) -> value (str) + If given, flags must be a map of form: + flag including the `-` (str) -> value (str) + It is most convenient to use this method in conjunction with + @needs_subprocess as the decorator will cause the decorated test to be + skipped unless the `SUBPROC_TEST` environment variable is set to + the same value of ``_subproc_test_env`` + (this special environment variable is set by this method such that the + specified test(s) will not be skipped in the subprocess). + + + Following execution in the subprocess this method will check the test(s) + executed without error. The timeout kwarg can be used to allow more time + for longer running tests, it defaults to 60 seconds. + """ + parts = (test_module, test_class, test_name) + fully_qualified_test = ".".join(x for x in parts if x is not None) + flags_args = [] + if flags is not None: + for flag, value in flags.items(): + flags_args.append(f"{flag}") + flags_args.append(f"{value}") + cmd = [ + sys.executable, + *flags_args, + "-m", + "numba.runtests", + fully_qualified_test, + ] + env_copy = os.environ.copy() + env_copy["SUBPROC_TEST"] = _subproc_test_env + try: + env_copy["COVERAGE_PROCESS_START"] = os.environ["COVERAGE_RCFILE"] + except KeyError: + pass # ignored + envvars = pytypes.MappingProxyType({} if envvars is None else envvars) + env_copy.update(envvars) + status = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + env=env_copy, + universal_newlines=True, + ) + streams = ( + f"\ncaptured stdout: {status.stdout}\n" + f"captured stderr: {status.stderr}" + ) + self.assertEqual(status.returncode, 0, streams) + # Python 3.12.1 report + no_tests_ran = "NO TESTS RAN" + if no_tests_ran in status.stderr: + self.skipTest(no_tests_ran) + else: + self.assertIn("OK", status.stderr) + return status + + def run_test_in_subprocess(maybefunc=None, timeout=60, envvars=None): + """Runs the decorated test in a subprocess via invoking numba's test + runner. kwargs timeout and envvars are passed through to + subprocess_test_runner.""" + + def wrapper(func): + def inner(self, *args, **kwargs): + if os.environ.get("SUBPROC_TEST", None) != func.__name__: + # Not in a subprocess test env, so stage the call to run the + # test in a subprocess which will set the env var. + class_name = self.__class__.__name__ + self.subprocess_test_runner( + test_module=self.__module__, + test_class=class_name, + test_name=func.__name__, + timeout=timeout, + envvars=envvars, + _subproc_test_env=func.__name__, + ) + else: + # env var is set, so we're in the subprocess, run the + # actual test. + func(self) + + return inner + + if isinstance(maybefunc, pytypes.FunctionType): + return wrapper(maybefunc) + else: + return wrapper + + def make_dummy_type(self): + """Use to generate a dummy type unique to this test. Returns a python + Dummy class and a corresponding Numba type DummyType.""" + + # Use test_id to make sure no collision is possible. + test_id = self.id() + DummyType = type("DummyTypeFor{}".format(test_id), (types.Opaque,), {}) + + dummy_type = DummyType("my_dummy") + register_model(DummyType)(OpaqueModel) + + class Dummy(object): + pass + + @typeof_impl.register(Dummy) + def typeof_dummy(val, c): + return dummy_type + + @unbox(DummyType) + def unbox_dummy(typ, obj, c): + return NativeValue(c.context.get_dummy_value()) + + return Dummy, DummyType + + def skip_if_no_external_compiler(self): + """ + Call this to ensure the test is skipped if no suitable external compiler + is found. This is a method on the TestCase opposed to a stand-alone + decorator so as to make it "lazy" via runtime evaluation opposed to + running at test-discovery time. + """ + # This is a local import to avoid deprecation warnings being generated + # through the use of the numba.pycc module. + from numba.pycc.platform import external_compiler_works + + if not external_compiler_works(): + self.skipTest("No suitable external compiler was found.")