diff --git a/libsycl/CMakeLists.txt b/libsycl/CMakeLists.txt index f25f51def0cc7..064e1c646217f 100644 --- a/libsycl/CMakeLists.txt +++ b/libsycl/CMakeLists.txt @@ -133,5 +133,14 @@ add_custom_target(libsycl-runtime-libraries ) add_subdirectory(src) - add_subdirectory(tools) + +if(LLVM_INCLUDE_TESTS) + add_subdirectory(test_e2e) +endif() + +add_custom_target(libsycl-toolchain ALL + DEPENDS libsycl-runtime-libraries + sycl-ls + COMMENT "Building libsycl toolchain..." +) diff --git a/libsycl/test_e2e/CMakeLists.txt b/libsycl/test_e2e/CMakeLists.txt new file mode 100644 index 0000000000000..7d03688fa3538 --- /dev/null +++ b/libsycl/test_e2e/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.20.0) + +message("Configuring libsycl End-to-End Tests") + +set(LIBSYCL_CXX_COMPILER "${LLVM_BINARY_DIR}/bin/clang++") +set(LIBSYCL_E2E_CXX_FLAGS "" CACHE STRING + "Flags passed to clang++ when building SYCL end-to-end tests") + +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py + MAIN_CONFIG + ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py +) + +list(APPEND LIBSYCL_E2E_TEST_DEPS + libsycl-toolchain + FileCheck + not +) + +add_lit_testsuite(check-sycl-e2e + "Running libsycl End-to-End tests" + ${CMAKE_CURRENT_BINARY_DIR} + EXCLUDE_FROM_CHECK_ALL + DEPENDS ${LIBSYCL_E2E_TEST_DEPS}) diff --git a/libsycl/test_e2e/README.md b/libsycl/test_e2e/README.md new file mode 100644 index 0000000000000..59b33d3cf8a47 --- /dev/null +++ b/libsycl/test_e2e/README.md @@ -0,0 +1,84 @@ +## Getting Started + +This directory contains `libsycl` tests distributed in subdirectories based on +testing scope. `libsycl` uses LIT to configure and run its tests. + +Please see the [Lit Command Guide](https://llvm.org/docs/CommandGuide/lit.html) +for more information about LIT. + +## Prerequisites + +* Target runtime(s) to execute tests on devices. + TODO: add link to liboffload instruction once they add it. +* Compiler & libsycl. Can be built following these + [instructions](/libsycl/docs/index.rst). + +## Run the tests + +`libsycl` is integrated via LLVM_ENABLE_RUNTIMES and is not visible as top +level target. Same is applicable for tests. To run `check-sycl-e2e` tests you +need to prefix `/runtimes/runtimes-bins/` to the paths of all tests. +For example, to run all the libsycl end-to-end tests you can do: +```bash +/bin/llvm-lit /runtimes/runtimes-bins/libsycl/test_e2e +``` + +To run individual test, use the path to it instead. + +If you are using `ninja` as your build system, you can run all the tests in the +libsycl testsuite as: + +```bash + ninja -C /runtimes/runtimes-bins check-sycl-e2e + ``` + + +## CMake parameters + +These parameters can be used to configure tests: + +`LIBSYCL_CXX_COMPILER` - path to compiler to use it for building tests. + +`LIBSYCL_E2E_CXX_FLAGS` - flags to be passed to `LIBSYCL_CXX_COMPILER` when + building libsycl end-to-end tests. + +`LLVM_LIT` - path to llvm-lit tool. + +## Creating or modifying tests + +### LIT feature checks + +Following features can be passed to LIT via `REQUIRES`, `UNSUPPORTED`, etc. +filters to limit test execution to the specific environment. + +#### Auto-detected features + +The following features are automatically detected by `llvm-lit` by scanning the +environment: + +* `linux` - host OS; +* `any-device-is-gpu` - device type to be available; +* `any-device-is-level_zero` - backend to be available; + +Note: `sycl-ls` tool doesn't have assigned feature since it is essential for +tests configuration and is always available if test is executed. + +### llvm-lit parameters + +Following options can be passed to `llvm-lit` tool with `--param` option to +configure test execution: + +* `libsycl_compiler` - full path to compiler to use; +* `extra_environment` - comma-separated list of variables with values to be + added to test environment. Can be also set by `LIT_EXTRA_ENVIRONMENT` + variable in CMake. +* `extra_system_environment` - comma-separated list of variables to be + propagated from the host environment to test environment. Can be also set by + `LIT_EXTRA_SYSTEM_ENVIRONMENT` variable in CMake. + +Example: + +```bash +/bin/llvm-lit --param libsycl_compiler=path/to/clang++ \ + /runtimes/runtimes-bins/libsycl/test_e2e +``` \ No newline at end of file diff --git a/libsycl/test_e2e/basic/platform_get_devices.cpp b/libsycl/test_e2e/basic/platform_get_devices.cpp new file mode 100644 index 0000000000000..138688d1b0ac7 --- /dev/null +++ b/libsycl/test_e2e/basic/platform_get_devices.cpp @@ -0,0 +1,121 @@ +// RUN: %clangxx %sycl_options %s -o %t.out +// RUN: %t.out +// +// Tests platform::get_devices for each device type. + +#include + +#include +#include + +std::string BackendToString(sycl::backend Backend) { + switch (Backend) { + case sycl::backend::opencl: + return "opencl"; + case sycl::backend::level_zero: + return "level_zero"; + case sycl::backend::cuda: + return "cuda"; + case sycl::backend::hip: + return "hip"; + default: + return "unknown"; + } +} + +std::string DeviceTypeToString(sycl::info::device_type DevType) { + switch (DevType) { + case sycl::info::device_type::all: + return "device_type::all"; + case sycl::info::device_type::cpu: + return "device_type::cpu"; + case sycl::info::device_type::gpu: + return "device_type::gpu"; + case sycl::info::device_type::accelerator: + return "device_type::accelerator"; + case sycl::info::device_type::custom: + return "device_type::custom"; + case sycl::info::device_type::automatic: + return "device_type::automatic"; + case sycl::info::device_type::host: + return "device_type::host"; + default: + return "UNKNOWN"; + } +} + +std::string GenerateDeviceDescription(sycl::info::device_type DevType, + const sycl::platform &Platform) { + return std::string(DeviceTypeToString(DevType)) + " (" + + BackendToString(Platform.get_backend()) + ")"; +} + +template +int Check(const T1 &LHS, const T2 &RHS, std::string TestName) { + if (LHS != RHS) { + std::cerr << "Failed check " << LHS << " != " << RHS << ": " << TestName + << std::endl; + return 1; + } + return 0; +} + +int CheckDeviceType(const sycl::platform &P, sycl::info::device_type DevType, + std::vector &AllDevices) { + assert(DevType != sycl::info::device_type::all); + int Failures = 0; + + std::vector Devices = P.get_devices(DevType); + + if (DevType == sycl::info::device_type::automatic) { + if (AllDevices.empty()) { + Failures += Check(Devices.size(), 0, + "No devices reported for device_type::all query, but " + "device_type::automatic returns a device."); + } else { + Failures += Check(Devices.size(), 1, + "Number of devices for device_type::automatic query."); + if (Devices.size()) + Failures += Check( + std::count(AllDevices.begin(), AllDevices.end(), Devices[0]), 1, + "Device is in the set of device_type::all devices in the " + "platform."); + } + return Failures; + } + + // Count devices with the type; + size_t DevCount = 0; + for (sycl::device Device : Devices) + DevCount += (Device.get_info() == DevType); + + Failures += Check(Devices.size(), DevCount, + "Unexpected number of devices for " + + GenerateDeviceDescription(DevType, P)); + + Failures += + Check(std::all_of(Devices.begin(), Devices.end(), + [&](const auto &Dev) { + return std::count(AllDevices.begin(), + AllDevices.end(), Dev) == 1; + }), + true, + "Not all devices for " + GenerateDeviceDescription(DevType, P) + + " appear in the list of all devices"); + + return Failures; +} + +int main() { + int Failures = 0; + for (sycl::platform P : sycl::platform::get_platforms()) { + std::vector Devices = P.get_devices(); + + for (sycl::info::device_type DevType : + {sycl::info::device_type::cpu, sycl::info::device_type::gpu, + sycl::info::device_type::accelerator, sycl::info::device_type::custom, + sycl::info::device_type::automatic, sycl::info::device_type::host}) + Failures += CheckDeviceType(P, DevType, Devices); + } + return Failures; +} diff --git a/libsycl/test_e2e/lit.cfg.py b/libsycl/test_e2e/lit.cfg.py new file mode 100644 index 0000000000000..8c485eaffabf8 --- /dev/null +++ b/libsycl/test_e2e/lit.cfg.py @@ -0,0 +1,220 @@ +# -*- Python -*- + +# Configuration file for the 'lit' test runner. + +import os +import re +import subprocess +import textwrap +import shlex + +from lit.llvm import llvm_config +import lit.formats +from lit.llvm.subst import ToolSubst, FindTool + +# name: The name of this test suite. +config.name = "libsycl" + +# suffixes: A list of file extensions to treat as test files. +config.suffixes = [".cpp"] + +config.excludes = ["Inputs"] + +# test_source_root: The root path where tests are located. +config.test_source_root = os.path.dirname(__file__) + +# allow expanding substitutions that are based on other substitutions +config.recursiveExpansionLimit = 10 + +# test_exec_root: The root path where tests should be run. +config.test_exec_root = config.libsycl_obj_root + +# To be filled by lit.local.cfg files. +config.required_features = [] +config.unsupported_features = [] + +# Cleanup environment variables which may affect tests +possibly_dangerous_env_vars = [ + "COMPILER_PATH", + "RC_DEBUG_OPTIONS", + "CINDEXTEST_PREAMBLE_FILE", + "LIBRARY_PATH", + "CPATH", + "C_INCLUDE_PATH", + "CPLUS_INCLUDE_PATH", + "OBJC_INCLUDE_PATH", + "OBJCPLUS_INCLUDE_PATH", + "LIBCLANG_TIMING", + "LIBCLANG_OBJTRACKING", + "LIBCLANG_LOGGING", + "LIBCLANG_BGPRIO_INDEX", + "LIBCLANG_BGPRIO_EDIT", + "LIBCLANG_NOTHREADS", + "LIBCLANG_RESOURCE_USAGE", + "LIBCLANG_CODE_COMPLETION_LOGGING", + "INCLUDE", +] + +for name in possibly_dangerous_env_vars: + if name in llvm_config.config.environment: + del llvm_config.config.environment[name] + +# Propagate some variables from the host environment. +llvm_config.with_system_environment( + [ + "PATH", + ] +) + +# Take into account extra system environment variables if provided via parameter. +if config.extra_system_environment: + lit_config.note( + "Extra system variables to propagate value from: " + + config.extra_system_environment + ) + extra_env_vars = config.extra_system_environment.split(",") + for var in extra_env_vars: + if var in os.environ: + llvm_config.with_system_environment(var) + +llvm_config.with_environment("PATH", config.lit_tools_dir, append_path=True) + +# Configure LD_LIBRARY_PATH +config.available_features.add("linux") +llvm_config.with_system_environment( + ["LD_LIBRARY_PATH", "LIBRARY_PATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH"] +) +llvm_config.with_environment( + "LD_LIBRARY_PATH", config.libsycl_libs_dir, append_path=True +) + +llvm_config.with_environment("PATH", config.libsycl_tools_dir, append_path=True) + +if config.extra_environment: + lit_config.note("Extra environment variables") + for env_pair in config.extra_environment.split(","): + [var, val] = env_pair.split("=", 1) + if val: + llvm_config.with_environment(var, val) + lit_config.note("\t" + var + "=" + val) + else: + lit_config.note("\tUnset " + var) + llvm_config.with_environment(var, "") + + +# Temporarily modify environment to be the same that we use when running tests +class test_env: + def __enter__(self): + self.old_environ = dict(os.environ) + os.environ.clear() + os.environ.update(config.environment) + self.old_dir = os.getcwd() + os.chdir(config.libsycl_obj_root) + + def __exit__(self, exc_type, exc_value, exc_traceback): + os.environ.clear() + os.environ.update(self.old_environ) + os.chdir(self.old_dir) + + +# General substritutions +config.substitutions.append( + ( + "%sycl_options", + " -lsycl" + + " -isystem " + + config.libsycl_include + + " -isystem " + + os.path.join(config.libsycl_include, "sycl") + + " -L" + + config.libsycl_libs_dir, + ) +) +config.substitutions.append(("%sycl_libs_dir", config.libsycl_libs_dir)) +config.substitutions.append(("%sycl_static_libs_dir", config.libsycl_libs_dir)) +config.substitutions.append(("%obj_ext", ".o")) +config.substitutions.append(("%sycl_include", "-isystem " + config.libsycl_include)) +config.substitutions.append(("%include_option", "-include")) +config.substitutions.append(("%debug_option", "-g")) +config.substitutions.append(("%cxx_std_option", "-std=")) +config.substitutions.append(("%fPIC", "-fPIC")) +config.substitutions.append(("%shared_lib", "-shared")) +config.substitutions.append(("%O0", "-O0")) + +sycl_ls = FindTool("sycl-ls").resolve( + llvm_config, os.pathsep.join([config.libsycl_bin_dir, config.llvm_tools_dir]) +) +if not sycl_ls: + lit_config.fatal("can't find `sycl-ls`") + +tools = [ + ToolSubst("FileCheck", unresolved="ignore"), + # not is only substituted in certain circumstances; this is lit's default + # behaviour. + ToolSubst( + r"\| \bnot\b", command=FindTool("not"), verbatim=True, unresolved="ignore" + ), + ToolSubst("sycl-ls", command=sycl_ls, unresolved="fatal"), +] + +# Try and find each of these tools in the libsycl bin directory, in the llvm tools directory +# or the PATH, in that order. If found, they will be added as substitutions with the full path +# to the tool. +llvm_config.add_tool_substitutions( + tools, [config.libsycl_bin_dir, config.llvm_tools_dir, os.environ.get("PATH", "")] +) + +lit_config.note("Targeted devices: all") +with test_env(): + sycl_ls_output = subprocess.check_output(sycl_ls, text=True, shell=True) + + devices = set() + for line in sycl_ls_output.splitlines(): + if not line.startswith("["): + continue + backend, device = line[1:].split("]")[0].split(":") + devices.add("{}:{}".format(backend, device)) + libsycl_devices = list(devices) + +if len(libsycl_devices) == 0: + lit_config.error("No sycl devices available.") + +if len(libsycl_devices) > 1: + lit_config.note( + "Running on multiple devices, XFAIL-marked tests will be skipped on corresponding devices." + ) + +available_devices = { + "level_zero": "gpu", +} +for d in libsycl_devices: + be, dev = d.split(":") + if be not in available_devices: + lit_config.error("Unsupported device {}".format(d)) + if dev not in available_devices[be]: + lit_config.error("Unsupported device {}".format(d)) + +for sycl_device in libsycl_devices: + be, dev = sycl_device.split(":") + config.available_features.add("any-device-is-" + dev) + config.available_features.add("any-device-is-" + be) + +# Check if user passed verbose-print parameter, if yes, add VERBOSE_PRINT macro +if "verbose-print" in lit_config.params: + verbose_print = "-DVERBOSE_PRINT" +else: + verbose_print = "" + +clangxx = " " + config.libsycl_compiler + " -Werror " + config.cxx_flags + verbose_print +config.substitutions.append(("%clangxx", clangxx)) + +config.test_format = lit.formats.ShTest() + +try: + import psutil + + # Set timeout for a single test + lit_config.maxIndividualTestTime = 600 + +except ImportError: + pass diff --git a/libsycl/test_e2e/lit.site.cfg.py.in b/libsycl/test_e2e/lit.site.cfg.py.in new file mode 100644 index 0000000000000..d21cb3484592a --- /dev/null +++ b/libsycl/test_e2e/lit.site.cfg.py.in @@ -0,0 +1,34 @@ +@LIT_SITE_CFG_IN_HEADER@ + +import subprocess +import site + +site.addsitedir("@CMAKE_CURRENT_SOURCE_DIR@") + +config.libsycl_compiler = lit_config.params.get("libsycl_compiler", "@LIBSYCL_CXX_COMPILER@") +config.libsycl_root_dir= os.path.dirname(os.path.dirname(config.libsycl_compiler)) +config.libsycl_bin_dir = os.path.join(config.libsycl_root_dir, 'bin') + +config.cxx_flags = lit_config.params.get("cxx_flags", "@LIBSYCL_E2E_CLANG_CXX_FLAGS@") + +config.extra_environment = lit_config.params.get("extra_environment", "@LIT_EXTRA_ENVIRONMENT@") +config.extra_system_environment = lit_config.params.get("extra_system_environment", "@LIT_EXTRA_SYSTEM_ENVIRONMENT@") + +def get_libsycl_tool_path(name): + try: + return subprocess.check_output([config.libsycl_compiler, "-print-prog-name=" + name], text=True) + except subprocess.CalledProcessError: + return os.path.join(config.libsycl_bin_dir, name) + +config.llvm_tools_dir = os.path.dirname(get_libsycl_tool_path("llvm-config")) +config.lit_tools_dir = os.path.dirname("@TEST_SUITE_LIT@") + +config.libsycl_tools_dir = config.llvm_tools_dir +config.libsycl_include = os.path.join(config.libsycl_root_dir, 'include') +config.libsycl_obj_root = "@CMAKE_CURRENT_BINARY_DIR@" +config.libsycl_libs_dir = os.path.join(config.libsycl_root_dir, 'lib/x86_64-unknown-linux-gnu') + +import lit.llvm +lit.llvm.initialize(lit_config, config) + +lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")