Port remaining tests and replace match files with KNOWN_FAILURE macro

Author: aarongreig

source/adapters/hip/device.cpp

@@ -903,7 +903,11 @@ UR_APIEXPORT ur_result_t UR_APICALL urDeviceGetInfo(ur_device_handle_t hDevice,
   case UR_DEVICE_INFO_GLOBAL_VARIABLE_SUPPORT:
     return ReturnValue(ur_bool_t{false});
   case UR_DEVICE_INFO_USM_POOL_SUPPORT:
+#ifdef UMF_ENABLE_POOL_TRACKING
     return ReturnValue(ur_bool_t{true});
+#else
+    return ReturnValue(ur_bool_t{false});
+#endif
   case UR_DEVICE_INFO_BFLOAT16:
     return ReturnValue(true);
   case UR_DEVICE_INFO_ASYNC_BARRIER:

test/adapters/cuda/urDeviceCreateWithNativeHandle.cpp

@@ -5,9 +5,10 @@
 
 #include "fixtures.h"
 
-using urCudaDeviceCreateWithNativeHandle = uur::urSelectedPlatformTest;
+using urCudaDeviceCreateWithNativeHandle = uur::urPlatformTest;
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urCudaDeviceCreateWithNativeHandle);
 
-TEST_F(urCudaDeviceCreateWithNativeHandle, Success) {
+TEST_P(urCudaDeviceCreateWithNativeHandle, Success) {
     // get a device from cuda
     int nCudaDevices;
     ASSERT_SUCCESS_CUDA(cuDeviceGetCount(&nCudaDevices));
@@ -16,7 +17,12 @@ TEST_F(urCudaDeviceCreateWithNativeHandle, Success) {
     ASSERT_SUCCESS_CUDA(cuDeviceGet(&cudaDevice, 0));
 
     ur_native_handle_t nativeCuda = static_cast<ur_native_handle_t>(cudaDevice);
-    ur_device_handle_t urDevice;
+
+    ur_adapter_handle_t adapter = nullptr;
+    ASSERT_SUCCESS(urPlatformGetInfo(platform, UR_PLATFORM_INFO_ADAPTER,
+                                     sizeof(adapter), &adapter, nullptr));
+
+    ur_device_handle_t urDevice = nullptr;
     ASSERT_SUCCESS(urDeviceCreateWithNativeHandle(nativeCuda, adapter, nullptr,
                                                   &urDevice));
 }

test/adapters/level_zero/multi_device_event_cache_tests.cpp

@@ -3,13 +3,9 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "ur_print.hpp"
 #include "uur/fixtures.h"
 #include "uur/raii.h"
 
-#include <map>
-#include <string>
-
 #include "ze_tracer_common.hpp"
 
 size_t zeCommandListAppendWaitOnEventsCount = 0;
@@ -27,7 +23,9 @@ static std::shared_ptr<_zel_tracer_handle_t> tracer = [] {
 }();
 
 using urMultiQueueMultiDeviceEventCacheTest = uur::urAllDevicesTest;
-TEST_F(urMultiQueueMultiDeviceEventCacheTest,
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMultiQueueMultiDeviceEventCacheTest);
+
+TEST_P(urMultiQueueMultiDeviceEventCacheTest,
        GivenMultiSubDeviceWithQueuePerSubDeviceThenEventIsSharedBetweenQueues) {
     uint32_t max_sub_devices = 0;
     ASSERT_SUCCESS(
@@ -83,7 +81,7 @@ TEST_F(urMultiQueueMultiDeviceEventCacheTest,
     ASSERT_SUCCESS(urQueueRelease(queue1));
 }
 
-TEST_F(urMultiQueueMultiDeviceEventCacheTest,
+TEST_P(urMultiQueueMultiDeviceEventCacheTest,
        GivenMultiDeviceWithQueuePerDeviceThenMultiDeviceEventIsCreated) {
     if (devices.size() < 2) {
         GTEST_SKIP();

test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp

@@ -19,7 +19,7 @@ TEST_P(urLevelZeroKernelNativeHandleTest, OwnedHandleRelease) {
     urDeviceGetNativeHandle(device, (ur_native_handle_t *)&native_device);
 
     std::shared_ptr<std::vector<char>> il_binary;
-    uur::KernelsEnvironment::instance->LoadSource("foo", il_binary);
+    uur::KernelsEnvironment::instance->LoadSource("foo", platform, il_binary);
 
     auto kernel_name =
         uur::KernelsEnvironment::instance->GetEntryPointNames("foo")[0];
@@ -79,7 +79,7 @@ TEST_P(urLevelZeroKernelNativeHandleTest, NullProgram) {
     urDeviceGetNativeHandle(device, (ur_native_handle_t *)&native_device);
 
     std::shared_ptr<std::vector<char>> il_binary;
-    uur::KernelsEnvironment::instance->LoadSource("foo", il_binary);
+    uur::KernelsEnvironment::instance->LoadSource("foo", platform, il_binary);
 
     auto kernel_name =
         uur::KernelsEnvironment::instance->GetEntryPointNames("foo")[0];

test/adapters/level_zero/v2/event_pool_test.cpp

@@ -88,7 +88,7 @@ struct ProviderParams {
 template <typename T>
 inline std::string
 printParams(const testing::TestParamInfo<typename T::ParamType> &info) {
-    const auto device_handle = std::get<0>(info.param);
+    const auto device_handle = std::get<0>(info.param).device;
     const auto platform_device_name =
         uur::GetPlatformAndDeviceName(device_handle);
     auto params = std::get<1>(info.param);
@@ -144,8 +144,8 @@ static ProviderParams test_cases[] = {
     //{TEST_PROVIDER_COUNTER, EVENT_COUNTER, QUEUE_IMMEDIATE},
 };
 
-UUR_TEST_SUITE_P(EventPoolTest, testing::ValuesIn(test_cases),
-                 printParams<EventPoolTest>);
+UUR_DEVICE_TEST_SUITE_P(EventPoolTest, testing::ValuesIn(test_cases),
+                        printParams<EventPoolTest>);
 
 TEST_P(EventPoolTest, InvalidDevice) {
     auto pool = cache->borrow(MAX_DEVICES, getParam().flags);
@@ -237,8 +237,8 @@ TEST_P(EventPoolTest, ProviderNormalUseMostFreePool) {
 
 using EventPoolTestWithQueue = uur::urQueueTestWithParam<ProviderParams>;
 
-UUR_TEST_SUITE_P(EventPoolTestWithQueue, testing::ValuesIn(test_cases),
-                 printParams<EventPoolTest>);
+UUR_DEVICE_TEST_SUITE_P(EventPoolTestWithQueue, testing::ValuesIn(test_cases),
+                        printParams<EventPoolTest>);
 
 // TODO: actual min version is unknown, retest after drivers on CI are
 // updated.

test/adapters/level_zero/v2/memory_residency.cpp

@@ -3,43 +3,34 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "ur_print.hpp"
 #include "uur/fixtures.h"
-#include "uur/raii.h"
 #include "uur/utils.h"
 
-#include <map>
-#include <string>
-
 using urMemoryResidencyTest = uur::urMultiDeviceContextTestTemplate<1>;
+UUR_INSTANTIATE_PLATFORM_TEST_SUITE_P(urMemoryResidencyTest);
 
-TEST_F(urMemoryResidencyTest, allocatingDeviceMemoryWillResultInOOM) {
+TEST_P(urMemoryResidencyTest, allocatingDeviceMemoryWillResultInOOM) {
     static constexpr size_t allocSize = 1024 * 1024;
 
-    if (!uur::isPVC(uur::DevicesEnvironment::instance->devices[0])) {
+    if (!uur::isPVC(devices[0])) {
         GTEST_SKIP() << "Test requires a PVC device";
     }
 
     size_t initialMemFree = 0;
-    ASSERT_SUCCESS(
-        urDeviceGetInfo(uur::DevicesEnvironment::instance->devices[0],
-                        UR_DEVICE_INFO_GLOBAL_MEM_FREE, sizeof(size_t),
-                        &initialMemFree, nullptr));
+    ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_GLOBAL_MEM_FREE,
+                                   sizeof(size_t), &initialMemFree, nullptr));
 
     if (initialMemFree < allocSize) {
         GTEST_SKIP() << "Not enough device memory available";
     }
 
     void *ptr = nullptr;
-    ASSERT_SUCCESS(
-        urUSMDeviceAlloc(context, uur::DevicesEnvironment::instance->devices[0],
-                         nullptr, nullptr, allocSize, &ptr));
+    ASSERT_SUCCESS(urUSMDeviceAlloc(context, devices[0], nullptr, nullptr,
+                                    allocSize, &ptr));
 
     size_t currentMemFree = 0;
-    ASSERT_SUCCESS(
-        urDeviceGetInfo(uur::DevicesEnvironment::instance->devices[0],
-                        UR_DEVICE_INFO_GLOBAL_MEM_FREE, sizeof(size_t),
-                        &currentMemFree, nullptr));
+    ASSERT_SUCCESS(urDeviceGetInfo(devices[0], UR_DEVICE_INFO_GLOBAL_MEM_FREE,
+                                   sizeof(size_t), &currentMemFree, nullptr));
 
     // amount of free memory should decrease after making a memory allocation resident
     ASSERT_LE(currentMemFree, initialMemFree);

test/conformance/CMakeLists.txt

@@ -14,33 +14,16 @@ function(add_test_adapter name adapter backend)
     set(TEST_NAME ${name}-${adapter})
 
     set(TEST_COMMAND
-        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --backend=${backend} --devices_count=${UR_TEST_DEVICES_COUNT} --platforms_count=${UR_TEST_PLATFORMS_COUNT}"
+        "${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME} --gtest_filter=*${backend}*"
     )
-    set(MATCH_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${name}_${adapter}.match")
 
     function(do_add_test tname env)
-        if(${UR_CONFORMANCE_ENABLE_MATCH_FILES} AND EXISTS ${MATCH_FILE})
-            add_test(NAME ${tname}
-                COMMAND ${Python3_EXECUTABLE} ${UR_CONFORMANCE_TEST_DIR}/cts_exe.py
-                    --failslist ${MATCH_FILE}
-                    --test_command ${PROJECT_BINARY_DIR}/bin/${TEST_TARGET_NAME}
-                    --
-                    --backend=${backend}
-                    --devices_count=${UR_TEST_DEVICES_COUNT}
-                    --platforms_count=${UR_TEST_PLATFORMS_COUNT}
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-            )
-        else()
-            separate_arguments(TEST_COMMAND)
-            add_test(NAME ${tname}
-                COMMAND ${TEST_COMMAND}
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-            )
-        endif()
+        separate_arguments(TEST_COMMAND)
+        add_test(NAME ${tname}
+            COMMAND ${TEST_COMMAND}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+        )
 
-        if(UR_CONFORMANCE_ENABLE_MATCH_FILES)
-            list(APPEND env GTEST_COLOR=yes)
-        endif()
         set_tests_properties(${tname} PROPERTIES
             ENVIRONMENT "${env}"
             LABELS "conformance;${adapter}")
@@ -67,20 +50,24 @@ function(add_conformance_test name)
         GTest::gtest_main
         unit_tests_helpers)
 
+    if(UR_USE_CFI)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE UR_USE_CFI)
+    endif()
+
     if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL)
         add_test_adapter(${name} adapter_cuda CUDA)
     endif()
     if(UR_BUILD_ADAPTER_HIP OR UR_BUILD_ADAPTER_ALL)
         add_test_adapter(${name} adapter_hip HIP)
     endif()
     if(UR_BUILD_ADAPTER_L0 OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_level_zero LEVEL_ZERO)
+        add_test_adapter(${name} adapter_level_zero Level_Zero)
     endif()
     if(UR_BUILD_ADAPTER_L0_V2)
-        add_test_adapter(${name} adapter_level_zero_v2 LEVEL_ZERO)
+        add_test_adapter(${name} adapter_level_zero_v2 Level_Zero)
     endif()
     if(UR_BUILD_ADAPTER_OPENCL OR UR_BUILD_ADAPTER_ALL)
-        add_test_adapter(${name} adapter_opencl OPENCL)
+        add_test_adapter(${name} adapter_opencl OpenCL)
     endif()
     if(UR_BUILD_ADAPTER_NATIVE_CPU OR UR_BUILD_ADAPTER_ALL)
         add_test_adapter(${name} adapter_native_cpu NATIVE_CPU)

test/conformance/context/context_adapter_level_zero.match

@@ -35,14 +35,6 @@ TEST_P(urContextGetInfoTest, SuccessDevices) {
     size_t devices_count = size / sizeof(ur_device_handle_t);
     ASSERT_EQ(devices_count, 1);
     ASSERT_EQ(queried_device, device);
-
-    for (uint32_t i = 0; i < devices_count; i++) {
-        auto &devices = uur::DevicesEnvironment::instance->devices;
-        auto queried_device =
-            std::find(devices.begin(), devices.end(), devices[i]);
-        EXPECT_TRUE(queried_device != devices.end())
-            << "device associated with the context is not valid";
-    }
 }
 
 TEST_P(urContextGetInfoTest, SuccessUSMMemCpy2DSupport) {

test/conformance/context/context_adapter_level_zero_v2.match

@@ -4,13 +4,17 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 #include "uur/fixtures.h"
+#include "uur/known_failure.h"
 #include "uur/raii.h"
 
 using urContextSetExtendedDeleterTest = uur::urDeviceTest;
 
 UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urContextSetExtendedDeleterTest);
 
 TEST_P(urContextSetExtendedDeleterTest, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::LevelZero{}, uur::LevelZeroV2{},
+                         uur::NativeCPU{});
+
     bool called = false;
     {
         uur::raii::Context context = nullptr;

test/conformance/context/context_adapter_native_cpu.match

@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #include <uur/fixtures.h>
+#include <uur/known_failure.h>
 
 using urDeviceCreateWithNativeHandleTest = uur::urDeviceTest;
 UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urDeviceCreateWithNativeHandleTest);
@@ -56,6 +57,8 @@ TEST_P(urDeviceCreateWithNativeHandleTest, SuccessWithUnOwnedNativeHandle) {
 }
 
 TEST_P(urDeviceCreateWithNativeHandleTest, InvalidNullHandlePlatform) {
+    UUR_KNOWN_FAILURE_ON(uur::NativeCPU{});
+
     ur_native_handle_t native_handle = 0;
     ASSERT_SUCCESS(urDeviceGetNativeHandle(device, &native_handle));
 
@@ -66,6 +69,8 @@ TEST_P(urDeviceCreateWithNativeHandleTest, InvalidNullHandlePlatform) {
 }
 
 TEST_P(urDeviceCreateWithNativeHandleTest, InvalidNullPointerDevice) {
+    UUR_KNOWN_FAILURE_ON(uur::NativeCPU{});
+
     ur_native_handle_t native_handle = 0;
     ASSERT_SUCCESS(urDeviceGetNativeHandle(device, &native_handle));
 

test/conformance/context/urContextGetInfo.cpp

@@ -7,6 +7,7 @@
 #include <thread>
 #include <type_traits>
 #include <uur/fixtures.h>
+#include <uur/known_failure.h>
 
 // WARNING  - This is the precision that is used in the OpenCL-CTS.
 //          - We might need to modify this value per-adapter.
@@ -30,6 +31,8 @@ using urDeviceGetGlobalTimestampTest = uur::urDeviceTest;
 UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urDeviceGetGlobalTimestampTest);
 
 TEST_P(urDeviceGetGlobalTimestampTest, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) FPGA"});
+
     uint64_t device_time = 0;
     uint64_t host_time = 0;
     ASSERT_SUCCESS(
@@ -39,16 +42,25 @@ TEST_P(urDeviceGetGlobalTimestampTest, Success) {
 }
 
 TEST_P(urDeviceGetGlobalTimestampTest, SuccessHostTimer) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) FPGA"});
+
     uint64_t host_time = 0;
     ASSERT_SUCCESS(urDeviceGetGlobalTimestamps(device, nullptr, &host_time));
     ASSERT_NE(host_time, 0);
 }
 
 TEST_P(urDeviceGetGlobalTimestampTest, SuccessNoTimers) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) FPGA"});
+
     ASSERT_SUCCESS(urDeviceGetGlobalTimestamps(device, nullptr, nullptr));
 }
 
 TEST_P(urDeviceGetGlobalTimestampTest, SuccessSynchronizedTime) {
+    UUR_KNOWN_FAILURE_ON(uur::CUDA{}, uur::HIP{}, uur::LevelZero{},
+                         uur::LevelZeroV2{}, uur::NativeCPU{},
+                         uur::OpenCL{"Intel(R) FPGA"},
+                         uur::OpenCL{"Intel(R) UHD Graphics 770"});
+
     // get the timer resolution of the device
     size_t deviceTimerResolutionNanoSecs = 0;
     ASSERT_SUCCESS(uur::GetDeviceProfilingTimerResolution(