Parameterize CTS tests across all available adapters and devices.

scripts/core/INTRO.rst

@@ -430,6 +430,13 @@ no valid platforms, then the tests will fail. Command line arguments take priori
 
     A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
 
+.. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES
+
+   A boolean option to enable running tests which have been marked as known
+   failures using the :c:macro:`UUR_KNOWN_FAILURE_ON` macro. Enabled when the
+   environment variable is set to any of the following values: ``1``, ``on``,
+   ``ON``, ``yes``, ``YES``, ``true``, ``TRUE``.
+
 Service identifiers
 ---------------------
 

source/adapters/hip/device.cpp

@@ -903,7 +903,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
   case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT:
     return ReturnValue(ur_bool_t{false});
   case UR_DEVICE_INFO_USM_POOL_SUPPORT:
+#ifdef UMF_ENABLE_POOL_TRACKING
     return ReturnValue(ur_bool_t{true});
+#else
+    return ReturnValue(ur_bool_t{false});
+#endif
   case UR_DEVICE_INFO_BFLOAT16:
     return ReturnValue(true);
   case UR_DEVICE_INFO_ASYNC_BARRIER:

test/adapters/cuda/urDeviceCreateWithNativeHandle.cpp

@@ -5,9 +5,10 @@
 
 #include "fixtures.h"
 
-using urCudaDeviceCreateWithNativeHandle = uur::urSelectedPlatformTest;
+using urCudaDeviceCreateWithNativeHandle = uur::urPlatformTest;
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urCudaDeviceCreateWithNativeHandle);
 
-TEST_F(urCudaDeviceCreateWithNativeHandle, Success) {
+TEST_P(urCudaDeviceCreateWithNativeHandle, Success) {
     // get a device from cuda
     int nCudaDevices;
     ASSERT_SUCCESS_CUDA(cuDeviceGetCount(&nCudaDevices));
@@ -16,7 +17,12 @@ TEST_F(urCudaDeviceCreateWithNativeHandle, Success) {
     ASSERT_SUCCESS_CUDA(cuDeviceGet(&cudaDevice, 0));
 
     ur_native_handle_t nativeCuda = static_cast<ur_native_handle_t>(cudaDevice);
-    ur_device_handle_t urDevice;
+
+    ur_adapter_handle_t adapter = nullptr;
+    ASSERT_SUCCESS(urPlatformGetInfo(platform, UR_PLATFORM_INFO_ADAPTER,
+                                     sizeof(adapter), &adapter, nullptr));
+
+    ur_device_handle_t urDevice = nullptr;
     ASSERT_SUCCESS(urDeviceCreateWithNativeHandle(nativeCuda, adapter, nullptr,
                                                   &urDevice));
 }

test/adapters/level_zero/event_cache_tests.cpp

@@ -167,7 +167,7 @@ TEST_P(urEventCacheTest, eventsReuseWithVisibleEventAndWait) {
 template <typename T>
 inline std::string
 printFlags(const testing::TestParamInfo<typename T::ParamType> &info) {
-    const auto device_handle = std::get<0>(info.param);
+    const auto device_handle = std::get<0>(info.param).device;
     const auto platform_device_name =
         uur::GetPlatformAndDeviceName(device_handle);
     auto flags = combineFlags(std::get<1>(info.param));
@@ -181,7 +181,7 @@ printFlags(const testing::TestParamInfo<typename T::ParamType> &info) {
     return platform_device_name + "__" + str;
 }
 
-UUR_TEST_SUITE_P(
+UUR_DEVICE_TEST_SUITE_P(
     urEventCacheTest,
     ::testing::Combine(
         testing::Values(0, UR_QUEUE_FLAG_DISCARD_EVENTS),

test/adapters/level_zero/multi_device_event_cache_tests.cpp

@@ -3,13 +3,9 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "ur_print.hpp"
 #include "uur/fixtures.h"
 #include "uur/raii.h"
 
-#include <map>
-#include <string>
-
 #include "ze_tracer_common.hpp"
 
 size_t zeCommandListAppendWaitOnEventsCount = 0;
@@ -27,7 +23,9 @@ static std::shared_ptr<_zel_tracer_handle_t> tracer = [] {
 }();
 
 using urMultiQueueMultiDeviceEventCacheTest = uur::urAllDevicesTest;
-TEST_F(urMultiQueueMultiDeviceEventCacheTest,
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMultiQueueMultiDeviceEventCacheTest);
+
+TEST_P(urMultiQueueMultiDeviceEventCacheTest,
        GivenMultiSubDeviceWithQueuePerSubDeviceThenEventIsSharedBetweenQueues) {
     uint32_t max_sub_devices = 0;
     ASSERT_SUCCESS(
@@ -83,7 +81,7 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest,
     ASSERT_SUCCESS(urQueueRelease(queue1));
 }
 
-TEST_F(urMultiQueueMultiDeviceEventCacheTest,
+TEST_P(urMultiQueueMultiDeviceEventCacheTest,
        GivenMultiDeviceWithQueuePerDeviceThenMultiDeviceEventIsCreated) {
     if (devices.size() < 2) {
         GTEST_SKIP();

test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp

@@ -19,7 +19,7 @@ TEST_P(urLevelZeroKernelNativeHandleTest, OwnedHandleRelease) {
     urDeviceGetNativeHandle(device, (ur_native_handle_t *)&native_device);
 
     std::shared_ptr<std::vector<char>> il_binary;
-    uur::KernelsEnvironment::instance->LoadSource("foo", il_binary);
+    uur::KernelsEnvironment::instance->LoadSource("foo", platform, il_binary);
 
     auto kernel_name =
         uur::KernelsEnvironment::instance->GetEntryPointNames("foo")[0];
@@ -79,7 +79,7 @@ TEST_P(urLevelZeroKernelNativeHandleTest, NullProgram) {
     urDeviceGetNativeHandle(device, (ur_native_handle_t *)&native_device);
 
     std::shared_ptr<std::vector<char>> il_binary;
-    uur::KernelsEnvironment::instance->LoadSource("foo", il_binary);
+    uur::KernelsEnvironment::instance->LoadSource("foo", platform, il_binary);
 
     auto kernel_name =
         uur::KernelsEnvironment::instance->GetEntryPointNames("foo")[0];

test/adapters/level_zero/v2/event_pool_test.cpp

@@ -88,7 +88,7 @@ struct ProviderParams {
 template <typename T>
 inline std::string
 printParams(const testing::TestParamInfo<typename T::ParamType> &info) {
-    const auto device_handle = std::get<0>(info.param);
+    const auto device_handle = std::get<0>(info.param).device;
     const auto platform_device_name =
         uur::GetPlatformAndDeviceName(device_handle);
     auto params = std::get<1>(info.param);
@@ -144,8 +144,8 @@ static ProviderParams test_cases[] = {
     //{TEST_PROVIDER_COUNTER, EVENT_COUNTER, QUEUE_IMMEDIATE},
 };
 
-UUR_TEST_SUITE_P(EventPoolTest, testing::ValuesIn(test_cases),
-                 printParams<EventPoolTest>);
+UUR_DEVICE_TEST_SUITE_P(EventPoolTest, testing::ValuesIn(test_cases),
+                        printParams<EventPoolTest>);
 
 TEST_P(EventPoolTest, InvalidDevice) {
     auto pool = cache->borrow(MAX_DEVICES, getParam().flags);
@@ -237,8 +237,8 @@ TEST_P(EventPoolTest, ProviderNormalUseMostFreePool) {
 
 using EventPoolTestWithQueue = uur::urQueueTestWithParam<ProviderParams>;
 
-UUR_TEST_SUITE_P(EventPoolTestWithQueue, testing::ValuesIn(test_cases),
-                 printParams<EventPoolTest>);
+UUR_DEVICE_TEST_SUITE_P(EventPoolTestWithQueue, testing::ValuesIn(test_cases),
+                        printParams<EventPoolTest>);
 
 // TODO: actual min version is unknown, retest after drivers on CI are
 // updated.

test/adapters/level_zero/v2/memory_residency.cpp

@@ -3,43 +3,34 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "ur_print.hpp"
 #include "uur/fixtures.h"
-#include "uur/raii.h"
 #include "uur/utils.h"
 
-#include <map>
-#include <string>
-
 using urMemoryResidencyTest = uur::urMultiDeviceContextTestTemplate<1>;
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMemoryResidencyTest);
 
-TEST_F(urMemoryResidencyTest, allocatingDeviceMemoryWillResultInOOM) {
+TEST_P(urMemoryResidencyTest, allocatingDeviceMemoryWillResultInOOM) {
     static constexpr size_t allocSize = 1024 * 1024;
 
-    if (!uur::isPVC(uur::DevicesEnvironment::instance->devices[0])) {
+    if (!uur::isPVC(devices[0])) {
         GTEST_SKIP() << "Test requires a PVC device";
     }
 
     size_t initialMemFree = 0;
-    ASSERT_SUCCESS(
-        urDeviceGetInfo(uur::DevicesEnvironment::instance->devices[0],
-                        UR_DEVICE_INFO_GLOBAL_MEM_FREE, sizeof(size_t),
-                        &initialMemFree, nullptr));
+    ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_GLOBAL_MEM_FREE,
+                                   sizeof(size_t), &initialMemFree, nullptr));
 
     if (initialMemFree < allocSize) {
         GTEST_SKIP() << "Not enough device memory available";
     }
 
     void *ptr = nullptr;
-    ASSERT_SUCCESS(
-        urUSMDeviceAlloc(context, uur::DevicesEnvironment::instance->devices[0],
-                         nullptr, nullptr, allocSize, &ptr));
+    ASSERT_SUCCESS(urUSMDeviceAlloc(context, devices[0], nullptr, nullptr,
+                                    allocSize, &ptr));
 
     size_t currentMemFree = 0;
-    ASSERT_SUCCESS(
-        urDeviceGetInfo(uur::DevicesEnvironment::instance->devices[0],
-                        UR_DEVICE_INFO_GLOBAL_MEM_FREE, sizeof(size_t),
-                        &currentMemFree, nullptr));
+    ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_GLOBAL_MEM_FREE,
+                                   sizeof(size_t), &currentMemFree, nullptr));
 
     // amount of free memory should decrease after making a memory allocation resident
     ASSERT_LE(currentMemFree, initialMemFree);

test/conformance/CMakeLists.txt

@@ -14,33 +14,16 @@ function(add_test_adapter name adapter backend)
     set(TEST_NAME ${name}-${adapter})
 
     set(TEST_COMMAND
-        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
+        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --gtest_filter=*${backend}*"
     )
-    set(MATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match")
 
     function(do_add_test tname env)
-        if(${UR_CONFORMANCE_ENABLE_MATCH_FILES} AND EXISTS ${MATCH_FILE})
-            add_test(NAME ${tname}
-                COMMAND ${Python3_EXECUTABLE} ${UR_CONFORMANCE_TEST_DIR}/cts_exe.py
-                    --failslist ${MATCH_FILE}
-                    --test_command ${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME}
-                    --
-                    --backend=${backend}
-                    --devices_count=${UR_TEST_DEVICES_COUNT}
-                    --platforms_count=${UR_TEST_PLATFORMS_COUNT}
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-            )
-        else()
-            separate_arguments(TEST_COMMAND)
-            add_test(NAME ${tname}
-                COMMAND ${TEST_COMMAND}
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-            )
-        endif()
+        separate_arguments(TEST_COMMAND)
+        add_test(NAME ${tname}
+            COMMAND ${TEST_COMMAND}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        )
 
-        if(UR_CONFORMANCE_ENABLE_MATCH_FILES)
-            list(APPEND env GTEST_COLOR=yes)
-        endif()
         set_tests_properties(${tname} PROPERTIES
             ENVIRONMENT "${env}"
             LABELS "conformance;${adapter}")
@@ -67,20 +50,24 @@ function(add_conformance_test name)
         GTest::gtest_main
         unit_tests_helpers)
 
+    if(UR_USE_CFI)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE UR_USE_CFI)
+    endif()
+
     if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
         add_test_adapter(${name} adapter_cuda CUDA)
     endif()
     if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
         add_test_adapter(${name} adapter_hip HIP)
     endif()
     if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_level_zero LEVEL_ZERO)
+        add_test_adapter(${name} adapter_level_zero Level_Zero)
     endif()
     if(UR_BUILD_ADAPTER_L0_V2)
-        add_test_adapter(${name} adapter_level_zero_v2 LEVEL_ZERO)
+        add_test_adapter(${name} adapter_level_zero_v2 Level_Zero)
     endif()
     if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_opencl OPENCL)
+        add_test_adapter(${name} adapter_opencl OpenCL)
     endif()
     if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
         add_test_adapter(${name} adapter_native_cpu NATIVE_CPU)

test/conformance/README.md

@@ -19,4 +19,54 @@ all available devices/platforms, set 0. The default value is 1.
 If you run binaries for the tests, you can use the parameter
 `--platforms_count=COUNT` or `--devices_count=COUNT`.
 To set test device/platform name you want to run the tests on, use
-parameter `--platform=NAME` or `--device=NAME`.
+parameter `--platform=NAME` or `--device=NAME`.
+
+## Known failures
+
+The `UUR_KNOWN_FAILURE_ON` macro can be used to skip tests on devices where the
+test is known to fail. This can be done in the following situations.
+
+For all devices in an adapter:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(uur::LevelZero{});
+```
+
+By substring match of the device name within and adapter:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
+```
+
+In certain test suits, where there is no access to a device, the platform name
+is used instead:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(uur::CUDA{"NVIDIA CUDA BACKEND"});
+```
+
+When neither device or platform is available in a test suite, the name is
+ignored and only the adapter backend is used to determine if the test is a
+known failure.
+
+The macro is variadic making it possible to specify known failures for multiple
+adapters in a single place and multiple names can also be provided per adapter:
+
+```cpp
+UUR_KNOWN_FAILURE_ON(
+uur::OpenCL{
+    "Intel(R) UHD Graphics 750",
+    "Intel(R) UHD Graphics 770",
+},
+uur::HIP{"Radeon RX 7700"},
+uur::NativeCPU{});
+```
+
+The following adapter matcher objects are available:
+
+* `uur::OpenCL`
+* `uur::LevelZero`
+* `uur::LevelZeroV2`
+* `uur::CUDA`
+* `uur::HIP`
+* `uur::NativeCPU`

test/conformance/context/urContextCreate.cpp

@@ -31,14 +31,11 @@ TEST_P(urContextCreateTest, InvalidNullPointerDevices) {
 }
 
 TEST_P(urContextCreateTest, InvalidNullPointerContext) {
-    auto device = GetParam();
     ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_NULL_POINTER,
                      urContextCreate(1, &device, nullptr, nullptr));
 }
 
 TEST_P(urContextCreateTest, InvalidEnumeration) {
-    auto device = GetParam();
-
     ur_context_properties_t properties{UR_STRUCTURE_TYPE_CONTEXT_PROPERTIES,
                                        nullptr, UR_CONTEXT_FLAGS_MASK};
     uur::raii::Context context = nullptr;
@@ -48,7 +45,9 @@ TEST_P(urContextCreateTest, InvalidEnumeration) {
 }
 
 using urContextCreateMultiDeviceTest = uur::urAllDevicesTest;
-TEST_F(urContextCreateMultiDeviceTest, Success) {
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urContextCreateMultiDeviceTest);
+
+TEST_P(urContextCreateMultiDeviceTest, Success) {
     if (devices.size() < 2) {
         GTEST_SKIP();
     }

test/conformance/context/urContextCreateWithNativeHandle.cpp

@@ -3,7 +3,6 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "uur/environment.h"
 #include <uur/fixtures.h>
 
 using urContextCreateWithNativeHandleTest = uur::urContextTest;