Add CTS macro for marking tests as known failures in code

The new `UUR_KNOWN_FAILURE_ON` macro can be used to skip tests on
devices where the test is known to fail. This can be done for a devices
in an adapter or by substring match of the device name. For example:

```cpp
UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
```

This invocation is also used in a few places in order to have clean CTS
runs on machines with this iGPU.

Author: kbenzie

scripts/core/INTRO.rst

@@ -430,6 +430,13 @@ no valid platforms, then the tests will fail. Command line arguments take priori
 
     A (case insensitive) backend to force the test to use. For example, `opencl`, `level_zero`, `hip` and so on.
 
+.. envvar:: UR_CTS_ALSO_RUN_KNOWN_FAILURES
+
+   A boolean option to enable running tests which have been marked as known
+   failures using the :c:macro:`UUR_KNOWN_FAILURE_ON` macro. Enabled when the
+   environment variable is set to any of the following values: ``1``, ``on``,
+   ``ON``, ``yes``, ``YES``, ``true``, ``TRUE``.
+
 Service identifiers
 ---------------------
 

test/conformance/kernel/urKernelSetArgValue.cpp

@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urKernelSetArgValueTest : uur::urKernelTest {
@@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {
 }
 
 TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,
                      urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));
 }

test/conformance/memory/urMemImageCreate.cpp

@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <uur/raii.h>
 
@@ -188,6 +189,7 @@ TEST_P(urMemImageCreateTest, InvalidNullPointerImageFormat) {
 }
 
 TEST_P(urMemImageCreateTest, InvalidSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
 
     uur::raii::Mem image_handle = nullptr;
 

test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp

@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <vector>
 
@@ -90,6 +91,7 @@ UUR_TEST_SUITE_P(
     uur::deviceTestWithParamPrinter<ur_image_format_t>);
 
 TEST_P(urMemImageCreateTestWithImageFormatParam, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ur_image_channel_order_t channel_order =
         std::get<1>(GetParam()).channelOrder;
     ur_image_channel_type_t channel_type = std::get<1>(GetParam()).channelType;

test/conformance/program/urProgramGetFunctionPointer.cpp

@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urProgramGetFunctionPointerTest : uur::urProgramTest {
@@ -20,6 +21,7 @@ struct urProgramGetFunctionPointerTest : uur::urProgramTest {
 UUR_INSTANTIATE_KERNEL_TEST_SUITE_P(urProgramGetFunctionPointerTest);
 
 TEST_P(urProgramGetFunctionPointerTest, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     void *function_pointer = nullptr;
     ASSERT_SUCCESS(urProgramGetFunctionPointer(
         device, program, function_name.data(), &function_pointer));

test/conformance/testing/include/uur/known_failure.h

@@ -0,0 +1,117 @@
+// Copyright (C) 2024 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+// See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+#define UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+
+#include "uur/utils.h"
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace uur {
+struct DeviceMatcher {
+    DeviceMatcher(uint32_t adapterVersion, ur_platform_backend_t backend,
+                  std::vector<std::string> deviceNames)
+        : adapterVersion(adapterVersion), backend(backend),
+          deviceNames(deviceNames) {}
+
+    uint32_t adapterVersion;
+    ur_platform_backend_t backend;
+    std::vector<std::string> deviceNames;
+};
+
+struct OpenCL : DeviceMatcher {
+    OpenCL(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_OPENCL, {il.begin(), il.end()}) {
+    }
+};
+
+struct LevelZero : DeviceMatcher {
+    LevelZero(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_LEVEL_ZERO,
+                        {il.begin(), il.end()}) {}
+};
+
+struct LevelZeroV2 : DeviceMatcher {
+    LevelZeroV2(std::initializer_list<std::string> il)
+        : DeviceMatcher(2, UR_PLATFORM_BACKEND_LEVEL_ZERO,
+                        {il.begin(), il.end()}) {}
+};
+
+struct CUDA : DeviceMatcher {
+    CUDA(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_CUDA, {il.begin(), il.end()}) {}
+};
+
+struct HIP : DeviceMatcher {
+    HIP(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_HIP, {il.begin(), il.end()}) {}
+};
+
+struct NativeCPU : DeviceMatcher {
+    NativeCPU(std::initializer_list<std::string> il)
+        : DeviceMatcher(1, UR_PLATFORM_BACKEND_NATIVE_CPU,
+                        {il.begin(), il.end()}) {}
+};
+
+inline bool isKnownFailureOn(ur_adapter_handle_t adapter,
+                             ur_platform_handle_t platform,
+                             ur_device_handle_t device,
+                             std::vector<DeviceMatcher> deviceMatchers) {
+    for (const auto &deviceMatcher : deviceMatchers) {
+        uint32_t adapterVersion = 0;
+        urAdapterGetInfo(adapter, UR_ADAPTER_INFO_VERSION,
+                         sizeof(adapterVersion), &adapterVersion, nullptr);
+        ur_platform_backend_t backend = UR_PLATFORM_BACKEND_UNKNOWN;
+        uur::GetPlatformInfo<ur_platform_backend_t>(
+            platform, UR_PLATFORM_INFO_BACKEND, backend);
+        if (deviceMatcher.adapterVersion != adapterVersion &&
+            deviceMatcher.backend != backend) {
+            continue;
+        }
+        if (deviceMatcher.deviceNames.empty()) {
+            return true;
+        }
+        std::string deviceName;
+        uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,
+                                        deviceName);
+        for (const auto &unsupportedDeviceName : deviceMatcher.deviceNames) {
+            if (deviceName.find(unsupportedDeviceName) != std::string::npos) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+} // namespace uur
+
+#define UUR_KNOWN_FAILURE_ON(...)                                              \
+    if (uur::isKnownFailureOn(adapter, platform, device, {__VA_ARGS__})) {     \
+        using namespace std::literals;                                         \
+        std::string platformName;                                              \
+        uur::GetPlatformInfo<std::string>(platform, UR_PLATFORM_INFO_NAME,     \
+                                          platformName);                       \
+        std::string deviceName;                                                \
+        uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,           \
+                                        deviceName);                           \
+        const char *alsoRunKnownFailures =                                     \
+            std::getenv("UR_CTS_ALSO_RUN_KNOWN_FAILURES");                     \
+        if (alsoRunKnownFailures && (alsoRunKnownFailures == "1"sv ||          \
+                                     alsoRunKnownFailures == "yes"sv ||        \
+                                     alsoRunKnownFailures == "YES"sv ||        \
+                                     alsoRunKnownFailures == "on"sv ||         \
+                                     alsoRunKnownFailures == "ON"sv ||         \
+                                     alsoRunKnownFailures == "true"sv ||       \
+                                     alsoRunKnownFailures == "TRUE"sv)) {      \
+            std::cerr << "Known failure on: " << platformName << ", "          \
+                      << deviceName << "\n";                                   \
+        } else {                                                               \
+            GTEST_SKIP() << "Known failure on: " << platformName << ", "       \
+                         << deviceName;                                        \
+        }                                                                      \
+    }
+
+#endif // UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED

test/conformance/testing/include/uur/utils.h

@@ -43,12 +43,12 @@ ur_result_t GetInfo(ObjectTy object, InfoTy info, Callable cb, T &out_value) {
 
     // special case for strings
     if constexpr (std::is_same_v<std::string, T>) {
-        std::vector<char> data(size);
-        result = cb(object, info, size, data.data(), nullptr);
+        std::string value(size, '\0');
+        result = cb(object, info, size, value.data(), nullptr);
         if (result != UR_RESULT_SUCCESS) {
             return result;
         }
-        out_value = std::string(data.data(), data.size());
+        out_value = value.substr(0, value.find_last_of('\0'));
         return UR_RESULT_SUCCESS;
     } else {
         if (size != sizeof(T)) {