From 7bf586c626e7e1f5e7fbd81cbca425127c0df82b Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Tue, 23 Nov 2021 15:51:22 +0300
Subject: [PATCH 01/18] Fix kernel program cache for multiple devices

Signed-off-by: mdimakov <maxim.dimakov@intel.com>
---
 sycl/source/detail/kernel_program_cache.cpp            | 1 +
 sycl/source/detail/program_manager/program_manager.cpp | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/sycl/source/detail/kernel_program_cache.cpp b/sycl/source/detail/kernel_program_cache.cpp
index 751e6e6e2b002..c3519b13a5b9b 100644
--- a/sycl/source/detail/kernel_program_cache.cpp
+++ b/sycl/source/detail/kernel_program_cache.cpp
@@ -33,6 +33,7 @@ KernelProgramCache::~KernelProgramCache() {
           Plugin.call<PiApiKind::piKernelRelease>(Kern);
         }
       }
+      MKernelsPerProgramCache.erase(KernIt);
     }
 
     const detail::plugin &Plugin = MParentContext->getPlugin();
diff --git a/sycl/source/detail/program_manager/program_manager.cpp b/sycl/source/detail/program_manager/program_manager.cpp
index c2d6cce38eb3e..f09a419ab841e 100644
--- a/sycl/source/detail/program_manager/program_manager.cpp
+++ b/sycl/source/detail/program_manager/program_manager.cpp
@@ -1799,7 +1799,10 @@ device_image_plain ProgramManager::build(const device_image_plain &DeviceImage,
   // Cache supports key with once device only, but here we have multiple
   // devices a program is built for, so add the program to the cache for all
   // other devices.
-  auto CacheOtherDevices = [ResProgram]() { return ResProgram; };
+  const detail::plugin &Plugin = ContextImpl->getPlugin();
+  auto CacheOtherDevices = [ResProgram, &Plugin]() { 
+    Plugin.call<PiApiKind::piProgramRetain>(ResProgram);
+    return ResProgram; };
 
   // The program for device "0" is already added to the cache during the first
   // call to getOrBuild, so starting with "1"
@@ -1817,7 +1820,6 @@ device_image_plain ProgramManager::build(const device_image_plain &DeviceImage,
   // devive_image_impl shares ownership of PIProgram with, at least, program
   // cache. The ref counter will be descremented in the destructor of
   // device_image_impl
-  const detail::plugin &Plugin = ContextImpl->getPlugin();
   Plugin.call<PiApiKind::piProgramRetain>(ResProgram);
 
   DeviceImageImplPtr ExecImpl = std::make_shared<detail::device_image_impl>(

From a0428817bd7a91aafbde40492bf5ced27b944488 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Tue, 23 Nov 2021 15:59:53 +0300
Subject: [PATCH 02/18] Clang-format fix

---
 sycl/source/detail/program_manager/program_manager.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sycl/source/detail/program_manager/program_manager.cpp b/sycl/source/detail/program_manager/program_manager.cpp
index f09a419ab841e..1dcd3560399c5 100644
--- a/sycl/source/detail/program_manager/program_manager.cpp
+++ b/sycl/source/detail/program_manager/program_manager.cpp
@@ -1800,9 +1800,10 @@ device_image_plain ProgramManager::build(const device_image_plain &DeviceImage,
   // devices a program is built for, so add the program to the cache for all
   // other devices.
   const detail::plugin &Plugin = ContextImpl->getPlugin();
-  auto CacheOtherDevices = [ResProgram, &Plugin]() { 
+  auto CacheOtherDevices = [ResProgram, &Plugin]() {
     Plugin.call<PiApiKind::piProgramRetain>(ResProgram);
-    return ResProgram; };
+    return ResProgram;
+  };
 
   // The program for device "0" is already added to the cache during the first
   // call to getOrBuild, so starting with "1"

From deaa97cc6c1b516e10fb6a91e778818d60842784 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Thu, 9 Dec 2021 16:13:19 +0300
Subject: [PATCH 03/18] Add unittest for the fix

---
 .../kernel-and-program/CMakeLists.txt         |   1 +
 sycl/unittests/kernel-and-program/Cache.cpp   |  12 +-
 .../kernel-and-program/HelperKernelInfo.hpp   |  29 +++
 .../kernel-and-program/MultipleDevsCache.cpp  | 210 ++++++++++++++++++
 4 files changed, 241 insertions(+), 11 deletions(-)
 create mode 100644 sycl/unittests/kernel-and-program/HelperKernelInfo.hpp
 create mode 100644 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp

diff --git a/sycl/unittests/kernel-and-program/CMakeLists.txt b/sycl/unittests/kernel-and-program/CMakeLists.txt
index 0527bb1be0eef..0f6349dcc8ab0 100644
--- a/sycl/unittests/kernel-and-program/CMakeLists.txt
+++ b/sycl/unittests/kernel-and-program/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_sycl_unittest(KernelAndProgramTests OBJECT
   Cache.cpp
+  MultipleDevsCache.cpp
   KernelRelease.cpp
   KernelInfo.cpp
   DeviceInfo.cpp
diff --git a/sycl/unittests/kernel-and-program/Cache.cpp b/sycl/unittests/kernel-and-program/Cache.cpp
index 655c450adf47d..5a78c048f25d4 100644
--- a/sycl/unittests/kernel-and-program/Cache.cpp
+++ b/sycl/unittests/kernel-and-program/Cache.cpp
@@ -12,6 +12,7 @@
 #include "detail/context_impl.hpp"
 #include "detail/kernel_program_cache.hpp"
 #include "detail/program_impl.hpp"
+#include "HelperKernelInfo.hpp"
 #include <CL/sycl.hpp>
 #include <helpers/CommonRedefinitions.hpp>
 #include <helpers/PiImage.hpp>
@@ -37,17 +38,6 @@ class TestKernel2 {
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 namespace detail {
-struct MockKernelInfo {
-  static constexpr unsigned getNumParams() { return 0; }
-  static const kernel_param_desc_t &getParamDesc(int) {
-    static kernel_param_desc_t Dummy;
-    return Dummy;
-  }
-  static constexpr bool isESIMD() { return false; }
-  static constexpr bool callsThisItem() { return false; }
-  static constexpr bool callsAnyThisFreeFunction() { return false; }
-};
-
 template <> struct KernelInfo<TestKernel> : public MockKernelInfo {
   static constexpr const char *getName() { return "TestKernel"; }
 };
diff --git a/sycl/unittests/kernel-and-program/HelperKernelInfo.hpp b/sycl/unittests/kernel-and-program/HelperKernelInfo.hpp
new file mode 100644
index 0000000000000..34de647cf5cfd
--- /dev/null
+++ b/sycl/unittests/kernel-and-program/HelperKernelInfo.hpp
@@ -0,0 +1,29 @@
+//==----------------------- HelperKernelInfo.hpp ---------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <CL/sycl.hpp>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace detail {
+struct MockKernelInfo {
+  static constexpr unsigned getNumParams() { return 0; }
+  static const kernel_param_desc_t &getParamDesc(int) {
+    static kernel_param_desc_t Dummy;
+    return Dummy;
+  }
+  static constexpr bool isESIMD() { return false; }
+  static constexpr bool callsThisItem() { return false; }
+  static constexpr bool callsAnyThisFreeFunction() { return false; }
+};
+
+} // namespace detail
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
new file mode 100644
index 0000000000000..69cfac91b0626
--- /dev/null
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -0,0 +1,210 @@
+//==--- KPCache.cpp --- KernelProgramCache for multiple devices unit test --==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#define SYCL2020_DISABLE_DEPRECATION_WARNINGS
+
+#include "HelperKernelInfo.hpp"
+#include "detail/kernel_bundle_impl.hpp"
+#include <CL/sycl.hpp>
+#include <helpers/CommonRedefinitions.hpp>
+#include <helpers/PiImage.hpp>
+#include <helpers/PiMock.hpp>
+
+#include <gtest/gtest.h>
+
+#include <iostream>
+
+using namespace sycl;
+
+class MultTestKernel {
+public:
+  void operator()(cl::sycl::item<1>){};
+};
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace detail {
+template <> struct KernelInfo<MultTestKernel> : public MockKernelInfo {
+  static constexpr const char *getName() { return "MultTestKernel"; }
+};
+} // namespace detail
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
+
+static sycl::unittest::PiImage generateDefaultImage() {
+  using namespace sycl::unittest;
+
+  PiPropertySet PropSet;
+
+  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
+
+  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"MultTestKernel"});
+
+  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
+              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
+              "",                                     // Compile options
+              "",                                     // Link options
+              std::move(Bin),
+              std::move(Entries),
+              std::move(PropSet)};
+
+  return Img;
+}
+
+static sycl::unittest::PiImage Img = generateDefaultImage();
+static sycl::unittest::PiImageArray<1> ImgArray{&Img};
+
+static pi_result redefinedContextCreate(
+    const pi_context_properties *properties, pi_uint32 num_devices,
+    const pi_device *devices,
+    void (*pfn_notify)(const char *errinfo, const void *private_info, size_t cb,
+                       void *user_data),
+    void *user_data, pi_context *ret_context) {
+  *ret_context = reinterpret_cast<pi_context>(123);
+  return PI_SUCCESS;
+}
+
+static pi_result redefinedContextRelease(pi_context context) {
+  return PI_SUCCESS;
+}
+
+static pi_result redefinedDevicesGet(pi_platform platform,
+                                     pi_device_type device_type,
+                                     pi_uint32 num_entries, pi_device *devices,
+                                     pi_uint32 *num_devices) {
+  if (num_devices) {
+    *num_devices = static_cast<pi_uint32>(2);
+    return PI_SUCCESS;
+  }
+
+  devices[0] = reinterpret_cast<pi_device>(1111);
+  devices[1] = reinterpret_cast<pi_device>(2222);
+  return PI_SUCCESS;
+}
+
+static pi_result redefinedDeviceGetInfo(pi_device device,
+                                        pi_device_info param_name,
+                                        size_t param_value_size,
+                                        void *param_value,
+                                        size_t *param_value_size_ret) {
+  if (param_name == PI_DEVICE_INFO_TYPE) {
+    auto *Result = reinterpret_cast<_pi_device_type *>(param_value);
+    *Result = PI_DEVICE_TYPE_GPU;
+  }
+  return PI_SUCCESS;
+}
+
+static pi_result redefinedDeviceRetain(pi_device device) { return PI_SUCCESS; }
+
+static pi_result redefinedDeviceRelease(pi_device device) { return PI_SUCCESS; }
+
+static pi_result redefinedQueueCreate(pi_context context, pi_device device,
+                                      pi_queue_properties properties,
+                                      pi_queue *queue) {
+  *queue = reinterpret_cast<pi_queue>(1234);
+  return PI_SUCCESS;
+}
+
+static pi_result redefinedQueueRelease(pi_queue command_queue) {
+  return PI_SUCCESS;
+}
+
+static int ProgramNum = 12345;
+static pi_result redefinedProgramCreate(pi_context context, const void *il,
+                                        size_t length,
+                                        pi_program *res_program) {
+  int CurrentProgram = ProgramNum;
+  *res_program = reinterpret_cast<pi_program>(CurrentProgram);
+  ++ProgramNum;
+  return PI_SUCCESS;
+}
+
+static int RetainCounter = 0;
+static pi_result redefinedProgramRetain(pi_program program) {
+  ++RetainCounter;
+  return PI_SUCCESS;
+}
+
+static int KernelReleaseCounter = 0;
+static pi_result redefinedKernelRelease(pi_kernel kernel) {
+  ++KernelReleaseCounter;
+  return PI_SUCCESS;
+}
+
+class MultipleDeviceCacheTest : public ::testing::Test {
+public:
+  MultipleDeviceCacheTest() : Plt{default_selector()} {}
+
+protected:
+  void SetUp() override {
+    if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
+      std::clog << "This test is only supported on OpenCL devices\n";
+      std::clog << "Current platform is "
+                << Plt.get_info<info::platform::name>();
+      return;
+    }
+
+    Mock = std::make_unique<unittest::PiMock>(Plt);
+
+    setupDefaultMockAPIs(*Mock);
+    Mock->redefine<detail::PiApiKind::piDevicesGet>(redefinedDevicesGet);
+    Mock->redefine<detail::PiApiKind::piDeviceGetInfo>(redefinedDeviceGetInfo);
+    Mock->redefine<detail::PiApiKind::piDeviceRetain>(redefinedDeviceRetain);
+    Mock->redefine<detail::PiApiKind::piDeviceRelease>(redefinedDeviceRelease);
+    Mock->redefine<detail::PiApiKind::piContextCreate>(redefinedContextCreate);
+    Mock->redefine<detail::PiApiKind::piContextRelease>(
+        redefinedContextRelease);
+    Mock->redefine<detail::PiApiKind::piQueueCreate>(redefinedQueueCreate);
+    Mock->redefine<detail::PiApiKind::piQueueRelease>(redefinedQueueRelease);
+    Mock->redefine<detail::PiApiKind::piProgramRetain>(redefinedProgramRetain);
+    Mock->redefine<detail::PiApiKind::piProgramCreate>(redefinedProgramCreate);
+    Mock->redefine<detail::PiApiKind::piKernelRelease>(redefinedKernelRelease);
+  }
+
+protected:
+  std::unique_ptr<unittest::PiMock> Mock;
+  platform Plt;
+};
+
+// Test that program is retained for each device
+TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
+  if (Plt.is_host()) {
+    return;
+  }
+
+  std::vector<sycl::device> Devices = Plt.get_devices(info::device_type::gpu);
+  sycl::context Context(Devices);
+  sycl::queue Queue(Context, Devices[0]);
+  assert(Devices.size() == 2);
+
+  auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
+      Queue.get_context());
+
+  Queue.submit([&](cl::sycl::handler &cgh) {
+    cgh.parallel_for<MultTestKernel>(cl::sycl::nd_range<1>(10, 10),
+                                     MultTestKernel{});
+  });
+
+  auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
+  auto KernelID = cl::sycl::get_kernel_id<MultTestKernel>();
+  auto Kernel = BundleObject.get_kernel(KernelID);
+  auto BundleImpl = getSyclObjImpl(Bundle);
+  int NumRetains = BundleImpl->size() * 2;
+
+  EXPECT_EQ(RetainCounter, NumRetains)
+      << "Expect " << NumRetains << " piProgramRetain calls";
+}
+
+// Test that each kernel released only 1 time in ~KernelProgramCache()
+TEST_F(MultipleDeviceCacheTest, KernelRelease) {
+  platform Plt{default_selector()};
+  if (Plt.is_host()) {
+    return;
+  }
+  EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";
+}

From ab22f7ce8141edb8b7e8751086811c91198a30b1 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Thu, 9 Dec 2021 16:20:26 +0300
Subject: [PATCH 04/18] Clang-format fix

---
 sycl/unittests/kernel-and-program/Cache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sycl/unittests/kernel-and-program/Cache.cpp b/sycl/unittests/kernel-and-program/Cache.cpp
index 5a78c048f25d4..116faff524d12 100644
--- a/sycl/unittests/kernel-and-program/Cache.cpp
+++ b/sycl/unittests/kernel-and-program/Cache.cpp
@@ -9,10 +9,10 @@
 #define SYCL2020_DISABLE_DEPRECATION_WARNINGS
 
 #include "CL/sycl/detail/pi.h"
+#include "HelperKernelInfo.hpp"
 #include "detail/context_impl.hpp"
 #include "detail/kernel_program_cache.hpp"
 #include "detail/program_impl.hpp"
-#include "HelperKernelInfo.hpp"
 #include <CL/sycl.hpp>
 #include <helpers/CommonRedefinitions.hpp>
 #include <helpers/PiImage.hpp>

From c248f7d526336ea0446aafe7f504e8a1ae65e67b Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Thu, 9 Dec 2021 17:50:58 +0300
Subject: [PATCH 05/18] Make test only for opencl backend

---
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 69cfac91b0626..1e7a540838e7e 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -143,9 +143,6 @@ class MultipleDeviceCacheTest : public ::testing::Test {
 protected:
   void SetUp() override {
     if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
-      std::clog << "This test is only supported on OpenCL devices\n";
-      std::clog << "Current platform is "
-                << Plt.get_info<info::platform::name>();
       return;
     }
 
@@ -173,7 +170,7 @@ class MultipleDeviceCacheTest : public ::testing::Test {
 
 // Test that program is retained for each device
 TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
-  if (Plt.is_host()) {
+  if (Plt.is_host()|| Plt.get_backend() != backend::opencl) {
     return;
   }
 
@@ -202,8 +199,7 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
 
 // Test that each kernel released only 1 time in ~KernelProgramCache()
 TEST_F(MultipleDeviceCacheTest, KernelRelease) {
-  platform Plt{default_selector()};
-  if (Plt.is_host()) {
+  if (Plt.is_host()|| Plt.get_backend() != backend::opencl) {
     return;
   }
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";

From ca3c7dda2390a5fa079e5788caae59b898b3ca12 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Thu, 9 Dec 2021 17:54:58 +0300
Subject: [PATCH 06/18] Clang-format fix

---
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 1e7a540838e7e..a53af5e6100e8 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -170,7 +170,7 @@ class MultipleDeviceCacheTest : public ::testing::Test {
 
 // Test that program is retained for each device
 TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
-  if (Plt.is_host()|| Plt.get_backend() != backend::opencl) {
+  if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
     return;
   }
 
@@ -199,7 +199,7 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
 
 // Test that each kernel released only 1 time in ~KernelProgramCache()
 TEST_F(MultipleDeviceCacheTest, KernelRelease) {
-  if (Plt.is_host()|| Plt.get_backend() != backend::opencl) {
+  if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
     return;
   }
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";

From 1aa4a39271d77d5024ca0aa12a666192746de4b7 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Fri, 10 Dec 2021 11:11:13 +0300
Subject: [PATCH 07/18] Fixed problem with kernel release

---
 .../kernel-and-program/MultipleDevsCache.cpp  | 62 ++++++++++---------
 1 file changed, 33 insertions(+), 29 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index a53af5e6100e8..161484dadabf0 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -9,7 +9,9 @@
 #define SYCL2020_DISABLE_DEPRECATION_WARNINGS
 
 #include "HelperKernelInfo.hpp"
+#include "detail/context_impl.hpp"
 #include "detail/kernel_bundle_impl.hpp"
+#include "detail/kernel_program_cache.hpp"
 #include <CL/sycl.hpp>
 #include <helpers/CommonRedefinitions.hpp>
 #include <helpers/PiImage.hpp>
@@ -168,39 +170,41 @@ class MultipleDeviceCacheTest : public ::testing::Test {
   platform Plt;
 };
 
-// Test that program is retained for each device
+// Test that program is retained for each device and each kernel is released
+// once
 TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
   if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
     return;
   }
-
-  std::vector<sycl::device> Devices = Plt.get_devices(info::device_type::gpu);
-  sycl::context Context(Devices);
-  sycl::queue Queue(Context, Devices[0]);
-  assert(Devices.size() == 2);
-
-  auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
-      Queue.get_context());
-
-  Queue.submit([&](cl::sycl::handler &cgh) {
-    cgh.parallel_for<MultTestKernel>(cl::sycl::nd_range<1>(10, 10),
-                                     MultTestKernel{});
-  });
-
-  auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
-  auto KernelID = cl::sycl::get_kernel_id<MultTestKernel>();
-  auto Kernel = BundleObject.get_kernel(KernelID);
-  auto BundleImpl = getSyclObjImpl(Bundle);
-  int NumRetains = BundleImpl->size() * 2;
-
-  EXPECT_EQ(RetainCounter, NumRetains)
-      << "Expect " << NumRetains << " piProgramRetain calls";
-}
-
-// Test that each kernel released only 1 time in ~KernelProgramCache()
-TEST_F(MultipleDeviceCacheTest, KernelRelease) {
-  if (Plt.is_host() || Plt.get_backend() != backend::opencl) {
-    return;
+  {
+    std::vector<sycl::device> Devices = Plt.get_devices(info::device_type::gpu);
+    sycl::context Context(Devices);
+    sycl::queue Queue(Context, Devices[0]);
+    assert(Devices.size() == 2);
+
+    auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
+        Queue.get_context());
+
+    Queue.submit([&](cl::sycl::handler &cgh) {
+      cgh.parallel_for<MultTestKernel>(cl::sycl::nd_range<1>(10, 10),
+                                       MultTestKernel{});
+    });
+
+    auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
+    auto KernelID = cl::sycl::get_kernel_id<MultTestKernel>();
+    auto Kernel = BundleObject.get_kernel(KernelID);
+    auto BundleImpl = getSyclObjImpl(Bundle);
+    int NumRetains = BundleImpl->size() * 2;
+
+    EXPECT_EQ(RetainCounter, NumRetains)
+        << "Expect " << NumRetains << " piProgramRetain calls";
+
+    auto CtxImpl = detail::getSyclObjImpl(Context);
+    detail::KernelProgramCache::KernelCacheT &KernelCache =
+        CtxImpl->getKernelProgramCache().acquireKernelsPerProgramCache().get();
+
+    EXPECT_EQ(KernelCache.size(), (size_t)2) << "Expect 2 kernels in cache";
   }
+  // Cache is cleared here, check kernel release
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";
 }

From 5674a12b5c64c00fa2c4d50e23364f1acc004c1a Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Fri, 10 Dec 2021 14:51:52 +0300
Subject: [PATCH 08/18] Use size_t for reinterpret_cast

---
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 161484dadabf0..79c7b7b0c0bf7 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -116,11 +116,11 @@ static pi_result redefinedQueueRelease(pi_queue command_queue) {
   return PI_SUCCESS;
 }
 
-static int ProgramNum = 12345;
+static size_t ProgramNum = 12345;
 static pi_result redefinedProgramCreate(pi_context context, const void *il,
                                         size_t length,
                                         pi_program *res_program) {
-  int CurrentProgram = ProgramNum;
+  size_t CurrentProgram = ProgramNum;
   *res_program = reinterpret_cast<pi_program>(CurrentProgram);
   ++ProgramNum;
   return PI_SUCCESS;

From a718e3f4237e74dfb562a33b699437b7717e2fc8 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Tue, 14 Dec 2021 14:53:42 +0300
Subject: [PATCH 09/18] Fix fail on windows

---
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 79c7b7b0c0bf7..087095b43576b 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -98,6 +98,10 @@ static pi_result redefinedDeviceGetInfo(pi_device device,
     auto *Result = reinterpret_cast<_pi_device_type *>(param_value);
     *Result = PI_DEVICE_TYPE_GPU;
   }
+  if (param_name == PI_DEVICE_INFO_COMPILER_AVAILABLE) {
+		auto *Result = reinterpret_cast<pi_bool *>(param_value);
+		*Result = true;
+	}
   return PI_SUCCESS;
 }
 

From 30db9431c6dfa295cd45a9a691ba6412d65f09b5 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Tue, 14 Dec 2021 15:02:47 +0300
Subject: [PATCH 10/18] Clang-format

---
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 087095b43576b..d6178dcc397ef 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -99,9 +99,9 @@ static pi_result redefinedDeviceGetInfo(pi_device device,
     *Result = PI_DEVICE_TYPE_GPU;
   }
   if (param_name == PI_DEVICE_INFO_COMPILER_AVAILABLE) {
-		auto *Result = reinterpret_cast<pi_bool *>(param_value);
-		*Result = true;
-	}
+    auto *Result = reinterpret_cast<pi_bool *>(param_value);
+    *Result = true;
+  }
   return PI_SUCCESS;
 }
 

From 004c9d59a643a94fce9f8a3636a1228feba0d340 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Thu, 16 Dec 2021 17:15:46 +0300
Subject: [PATCH 11/18] Addressed review comments

---
 .../kernel-and-program/MultipleDevsCache.cpp  | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index d6178dcc397ef..0f38ad95ea206 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -23,10 +23,7 @@
 
 using namespace sycl;
 
-class MultTestKernel {
-public:
-  void operator()(cl::sycl::item<1>){};
-};
+class MultTestKernel;
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
@@ -84,8 +81,10 @@ static pi_result redefinedDevicesGet(pi_platform platform,
     return PI_SUCCESS;
   }
 
-  devices[0] = reinterpret_cast<pi_device>(1111);
-  devices[1] = reinterpret_cast<pi_device>(2222);
+  if (num_entries == 2 && devices) {
+    devices[0] = reinterpret_cast<pi_device>(1111);
+    devices[1] = reinterpret_cast<pi_device>(2222);
+  }
   return PI_SUCCESS;
 }
 
@@ -188,15 +187,18 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
 
     auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
         Queue.get_context());
-
     Queue.submit([&](cl::sycl::handler &cgh) {
-      cgh.parallel_for<MultTestKernel>(cl::sycl::nd_range<1>(10, 10),
-                                       MultTestKernel{});
+      cgh.single_task<MultTestKernel>([](){});
     });
 
     auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
     auto KernelID = cl::sycl::get_kernel_id<MultTestKernel>();
     auto Kernel = BundleObject.get_kernel(KernelID);
+
+    // Because of emulating 2 devices program is retained for each one in build().
+    // It is also depends on number of device images. This test has one image,
+    // but other tests can create other images. Additional variable is added
+    // to control count of piProgramRetain calls
     auto BundleImpl = getSyclObjImpl(Bundle);
     int NumRetains = BundleImpl->size() * 2;
 
@@ -210,5 +212,7 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
     EXPECT_EQ(KernelCache.size(), (size_t)2) << "Expect 2 kernels in cache";
   }
   // Cache is cleared here, check kernel release
+  // 3 kernel releases is expected because kernel_bundle::get_kernel() calls piKernelRetain
+  // so one more kernel release is needed
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";
 }

From eb5abfba6b496138207c405143d47a25fdd30c58 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Thu, 16 Dec 2021 17:25:21 +0300
Subject: [PATCH 12/18] Clang-format fix

---
 .../kernel-and-program/MultipleDevsCache.cpp       | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 0f38ad95ea206..9463c5d7f34b9 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -188,17 +188,17 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
     auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
         Queue.get_context());
     Queue.submit([&](cl::sycl::handler &cgh) {
-      cgh.single_task<MultTestKernel>([](){});
+      cgh.single_task<MultTestKernel>([]() {});
     });
 
     auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
     auto KernelID = cl::sycl::get_kernel_id<MultTestKernel>();
     auto Kernel = BundleObject.get_kernel(KernelID);
 
-    // Because of emulating 2 devices program is retained for each one in build().
-    // It is also depends on number of device images. This test has one image,
-    // but other tests can create other images. Additional variable is added
-    // to control count of piProgramRetain calls
+    // Because of emulating 2 devices program is retained for each one in
+    // build(). It is also depends on number of device images. This test has one
+    // image, but other tests can create other images. Additional variable is
+    // added to control count of piProgramRetain calls
     auto BundleImpl = getSyclObjImpl(Bundle);
     int NumRetains = BundleImpl->size() * 2;
 
@@ -212,7 +212,7 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
     EXPECT_EQ(KernelCache.size(), (size_t)2) << "Expect 2 kernels in cache";
   }
   // Cache is cleared here, check kernel release
-  // 3 kernel releases is expected because kernel_bundle::get_kernel() calls piKernelRetain
-  // so one more kernel release is needed
+  // 3 kernel releases is expected because kernel_bundle::get_kernel() calls
+  // piKernelRetain so one more kernel release is needed
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";
 }

From bc4be30eef2479315110323d3942246087a3ac89 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Sun, 19 Dec 2021 22:13:28 +0300
Subject: [PATCH 13/18] Add more clarifying comment

---
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 9463c5d7f34b9..404bd6aca524f 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -211,8 +211,11 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
 
     EXPECT_EQ(KernelCache.size(), (size_t)2) << "Expect 2 kernels in cache";
   }
-  // Cache is cleared here, check kernel release
-  // 3 kernel releases is expected because kernel_bundle::get_kernel() calls
-  // piKernelRetain so one more kernel release is needed
+  // First kernel creating is called in handler::single_task().
+  // kernel_bundle::get_kernel() creates a kernel and shares it with created programs.
+  // Also the kernel is retained in kernel_bundle::get_kernel().
+  // A kernel is removed from cache if piKernelRelease was called for it,
+  // so it will not be removed twice for the other programs.
+  // As a result we must expect 3 piKernelRelease calls.
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";
 }

From a530172cf041d4d1103fa5abf2f01cfac2200fe4 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Mon, 20 Dec 2021 14:12:47 +0300
Subject: [PATCH 14/18] Clang-format fix

---
 .../unittests/kernel-and-program/MultipleDevsCache.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 404bd6aca524f..a4e26b582088b 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -212,10 +212,10 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
     EXPECT_EQ(KernelCache.size(), (size_t)2) << "Expect 2 kernels in cache";
   }
   // First kernel creating is called in handler::single_task().
-  // kernel_bundle::get_kernel() creates a kernel and shares it with created programs.
-  // Also the kernel is retained in kernel_bundle::get_kernel().
-  // A kernel is removed from cache if piKernelRelease was called for it,
-  // so it will not be removed twice for the other programs.
-  // As a result we must expect 3 piKernelRelease calls.
+  // kernel_bundle::get_kernel() creates a kernel and shares it with created
+  // programs. Also the kernel is retained in kernel_bundle::get_kernel(). A
+  // kernel is removed from cache if piKernelRelease was called for it, so it
+  // will not be removed twice for the other programs. As a result we must
+  // expect 3 piKernelRelease calls.
   EXPECT_EQ(KernelReleaseCounter, 3) << "Expect 3 piKernelRelease calls";
 }

From b3f60a12f027be6e3f4476ba5f6ebbae631e1185 Mon Sep 17 00:00:00 2001
From: Alexey Bader <alexey.bader@intel.com>
Date: Tue, 28 Dec 2021 09:10:02 +0300
Subject: [PATCH 15/18] [SYCL][NFC] Don't include sycl.hpp from headers

Apply LLVM's coding style rule - include as little as possible.
https://llvm.org/docs/CodingStandards.html#include-as-little-as-possible
---
 sycl/unittests/helpers/CommonRedefinitions.hpp  | 1 -
 sycl/unittests/helpers/PiImage.hpp              | 1 -
 sycl/unittests/helpers/PiMock.hpp               | 4 +++-
 sycl/unittests/kernel-and-program/Cache.cpp     | 1 +
 sycl/unittests/pi/PiMock.cpp                    | 4 +++-
 sycl/unittests/pi/TestGetPlatforms.hpp          | 3 ++-
 sycl/unittests/queue/USM.cpp                    | 5 ++++-
 sycl/unittests/scheduler/SchedulerTest.hpp      | 3 ++-
 sycl/unittests/scheduler/SchedulerTestUtils.hpp | 1 -
 9 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/sycl/unittests/helpers/CommonRedefinitions.hpp b/sycl/unittests/helpers/CommonRedefinitions.hpp
index 71c2712573c92..8bed27d438bf9 100644
--- a/sycl/unittests/helpers/CommonRedefinitions.hpp
+++ b/sycl/unittests/helpers/CommonRedefinitions.hpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <CL/sycl.hpp>
 #include <helpers/PiImage.hpp>
 #include <helpers/PiMock.hpp>
 
diff --git a/sycl/unittests/helpers/PiImage.hpp b/sycl/unittests/helpers/PiImage.hpp
index 761058fdef128..599e9ec7b6bae 100644
--- a/sycl/unittests/helpers/PiImage.hpp
+++ b/sycl/unittests/helpers/PiImage.hpp
@@ -8,7 +8,6 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/pi.hpp>
 #include <detail/platform_impl.hpp>
diff --git a/sycl/unittests/helpers/PiMock.hpp b/sycl/unittests/helpers/PiMock.hpp
index b8d27edd6fb09..50f076b0f8222 100644
--- a/sycl/unittests/helpers/PiMock.hpp
+++ b/sycl/unittests/helpers/PiMock.hpp
@@ -27,9 +27,11 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/pi.hpp>
+#include <CL/sycl/device_selector.hpp>
+#include <CL/sycl/platform.hpp>
+#include <CL/sycl/queue.hpp>
 #include <detail/platform_impl.hpp>
 
 #include <functional>
diff --git a/sycl/unittests/kernel-and-program/Cache.cpp b/sycl/unittests/kernel-and-program/Cache.cpp
index 116faff524d12..e25a55c2f9ff7 100644
--- a/sycl/unittests/kernel-and-program/Cache.cpp
+++ b/sycl/unittests/kernel-and-program/Cache.cpp
@@ -107,6 +107,7 @@ static pi_result redefinedKernelCreate(pi_program program,
                                        pi_kernel *ret_kernel) {
   return PI_SUCCESS;
 }
+
 static pi_result redefinedKernelRelease(pi_kernel kernel) { return PI_SUCCESS; }
 
 class KernelAndProgramCacheTest : public ::testing::Test {
diff --git a/sycl/unittests/pi/PiMock.cpp b/sycl/unittests/pi/PiMock.cpp
index 7f28104a97654..cfba6ee480334 100644
--- a/sycl/unittests/pi/PiMock.cpp
+++ b/sycl/unittests/pi/PiMock.cpp
@@ -6,10 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <gtest/gtest.h>
 #include <helpers/PiMock.hpp>
+
 #include <detail/queue_impl.hpp>
 
+#include <gtest/gtest.h>
+
 using namespace cl::sycl;
 
 pi_result piProgramBuildRedefine(pi_program, pi_uint32, const pi_device *,
diff --git a/sycl/unittests/pi/TestGetPlatforms.hpp b/sycl/unittests/pi/TestGetPlatforms.hpp
index 3696709217be9..ae2b6d0f25a7e 100644
--- a/sycl/unittests/pi/TestGetPlatforms.hpp
+++ b/sycl/unittests/pi/TestGetPlatforms.hpp
@@ -4,7 +4,8 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
+#include <CL/sycl/platform.hpp>
+
 #include <algorithm>
 #include <functional>
 #include <vector>
diff --git a/sycl/unittests/queue/USM.cpp b/sycl/unittests/queue/USM.cpp
index 3bb6d563e9176..a3b7eb7deaeb0 100644
--- a/sycl/unittests/queue/USM.cpp
+++ b/sycl/unittests/queue/USM.cpp
@@ -6,10 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <CL/sycl/usm.hpp>
 #include <detail/event_impl.hpp>
-#include <gtest/gtest.h>
+
 #include <helpers/PiMock.hpp>
 
+#include <gtest/gtest.h>
+
 namespace {
 using namespace sycl;
 
diff --git a/sycl/unittests/scheduler/SchedulerTest.hpp b/sycl/unittests/scheduler/SchedulerTest.hpp
index 6c4cd5f00adda..c127ca06d5120 100644
--- a/sycl/unittests/scheduler/SchedulerTest.hpp
+++ b/sycl/unittests/scheduler/SchedulerTest.hpp
@@ -8,7 +8,8 @@
 
 #pragma once
 
-#include <CL/sycl.hpp>
+#include <CL/sycl/queue.hpp>
+
 #include <gtest/gtest.h>
 
 class SchedulerTest : public ::testing::Test {
diff --git a/sycl/unittests/scheduler/SchedulerTestUtils.hpp b/sycl/unittests/scheduler/SchedulerTestUtils.hpp
index 60f5cbf05e04a..2f6073d1d3672 100644
--- a/sycl/unittests/scheduler/SchedulerTestUtils.hpp
+++ b/sycl/unittests/scheduler/SchedulerTestUtils.hpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #pragma once
-#include <CL/sycl.hpp>
 #include <CL/sycl/detail/cl.h>
 #include <detail/queue_impl.hpp>
 #include <detail/scheduler/commands.hpp>

From cb440dcb2f64370558fd9638a837b50c1eb3f190 Mon Sep 17 00:00:00 2001
From: Alexey Bader <alexey.bader@intel.com>
Date: Tue, 28 Dec 2021 10:53:28 +0300
Subject: [PATCH 16/18] [SYCL][NFC] Factor out empty kernel creation
 boilerplate

---
 sycl/unittests/helpers/TestKernel.hpp         | 55 +++++++++++++++++++
 .../program_manager/itt_annotations.cpp       | 46 +---------------
 .../scheduler/InOrderQueueHostTaskDeps.cpp    | 44 +--------------
 sycl/unittests/scheduler/RequiredWGSize.cpp   | 44 +--------------
 sycl/unittests/stream/stream.cpp              | 43 +--------------
 5 files changed, 60 insertions(+), 172 deletions(-)
 create mode 100644 sycl/unittests/helpers/TestKernel.hpp

diff --git a/sycl/unittests/helpers/TestKernel.hpp b/sycl/unittests/helpers/TestKernel.hpp
new file mode 100644
index 0000000000000..d8c2a8b87caec
--- /dev/null
+++ b/sycl/unittests/helpers/TestKernel.hpp
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "PiImage.hpp"
+
+class TestKernel;
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace detail {
+template <> struct KernelInfo<TestKernel> {
+  static constexpr unsigned getNumParams() { return 0; }
+  static const kernel_param_desc_t &getParamDesc(int) {
+    static kernel_param_desc_t Dummy;
+    return Dummy;
+  }
+  static constexpr const char *getName() { return "TestKernel"; }
+  static constexpr bool isESIMD() { return false; }
+  static constexpr bool callsThisItem() { return false; }
+  static constexpr bool callsAnyThisFreeFunction() { return false; }
+};
+
+} // namespace detail
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)
+
+static sycl::unittest::PiImage generateDefaultImage() {
+  using namespace sycl::unittest;
+
+  PiPropertySet PropSet;
+
+  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
+
+  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"TestKernel"});
+
+  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
+              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
+              "",                                     // Compile options
+              "",                                     // Link options
+              std::move(Bin),
+              std::move(Entries),
+              std::move(PropSet)};
+
+  return Img;
+}
+
+static sycl::unittest::PiImage Img = generateDefaultImage();
+static sycl::unittest::PiImageArray<1> ImgArray{&Img};
\ No newline at end of file
diff --git a/sycl/unittests/program_manager/itt_annotations.cpp b/sycl/unittests/program_manager/itt_annotations.cpp
index e0d117910d0c9..ad024bf43501d 100644
--- a/sycl/unittests/program_manager/itt_annotations.cpp
+++ b/sycl/unittests/program_manager/itt_annotations.cpp
@@ -18,6 +18,8 @@
 
 #include <stdlib.h>
 
+#include <helpers/TestKernel.hpp>
+
 // Same as defined in config.def
 static constexpr auto ITTProfileEnvVarName = "INTEL_ENABLE_OFFLOAD_ANNOTATIONS";
 
@@ -37,27 +39,6 @@ static void unset_env(const char *name) {
 #endif
 }
 
-class TestKernel;
-
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-namespace detail {
-template <> struct KernelInfo<TestKernel> {
-  static constexpr unsigned getNumParams() { return 0; }
-  static const kernel_param_desc_t &getParamDesc(int) {
-    static kernel_param_desc_t Dummy;
-    return Dummy;
-  }
-  static constexpr const char *getName() { return "TestKernel"; }
-  static constexpr bool isESIMD() { return false; }
-  static constexpr bool callsThisItem() { return false; }
-  static constexpr bool callsAnyThisFreeFunction() { return false; }
-};
-
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
-
 static pi_result redefinedProgramCreate(pi_context, const void *, size_t,
                                         pi_program *) {
   return PI_SUCCESS;
@@ -218,29 +199,6 @@ static void setupDefaultMockAPIs(sycl::unittest::PiMock &Mock) {
   Mock.redefine<PiApiKind::piKernelGetGroupInfo>(redefinedKernelGetGroupInfo);
 }
 
-static sycl::unittest::PiImage generateDefaultImage() {
-  using namespace sycl::unittest;
-
-  PiPropertySet PropSet;
-
-  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
-
-  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"TestKernel"});
-
-  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
-              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
-              "",                                     // Compile options
-              "",                                     // Link options
-              std::move(Bin),
-              std::move(Entries),
-              std::move(PropSet)};
-
-  return Img;
-}
-
-sycl::unittest::PiImage Img = generateDefaultImage();
-sycl::unittest::PiImageArray<1> ImgArray{&Img};
-
 TEST(ITTNotify, UseKernelBundle) {
   set_env(ITTProfileEnvVarName, "1");
 
diff --git a/sycl/unittests/scheduler/InOrderQueueHostTaskDeps.cpp b/sycl/unittests/scheduler/InOrderQueueHostTaskDeps.cpp
index b28ec3686ea46..99a12427c5e7a 100644
--- a/sycl/unittests/scheduler/InOrderQueueHostTaskDeps.cpp
+++ b/sycl/unittests/scheduler/InOrderQueueHostTaskDeps.cpp
@@ -18,49 +18,7 @@
 
 #include <gtest/gtest.h>
 
-class TestKernel;
-
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-namespace detail {
-template <> struct KernelInfo<TestKernel> {
-  static constexpr unsigned getNumParams() { return 0; }
-  static const kernel_param_desc_t &getParamDesc(int) {
-    static kernel_param_desc_t Dummy;
-    return Dummy;
-  }
-  static constexpr const char *getName() { return "TestKernel"; }
-  static constexpr bool isESIMD() { return false; }
-  static constexpr bool callsThisItem() { return false; }
-  static constexpr bool callsAnyThisFreeFunction() { return false; }
-};
-
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
-
-static sycl::unittest::PiImage generateDefaultImage() {
-  using namespace sycl::unittest;
-
-  PiPropertySet PropSet;
-
-  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
-
-  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"TestKernel"});
-
-  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
-              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
-              "",                                     // Compile options
-              "",                                     // Link options
-              std::move(Bin),
-              std::move(Entries),
-              std::move(PropSet)};
-
-  return Img;
-}
-
-static sycl::unittest::PiImage Img = generateDefaultImage();
-static sycl::unittest::PiImageArray<1> ImgArray{&Img};
+#include <helpers/TestKernel.hpp>
 
 using namespace sycl;
 
diff --git a/sycl/unittests/scheduler/RequiredWGSize.cpp b/sycl/unittests/scheduler/RequiredWGSize.cpp
index fe83d8a612dc2..449e3b8071cd9 100644
--- a/sycl/unittests/scheduler/RequiredWGSize.cpp
+++ b/sycl/unittests/scheduler/RequiredWGSize.cpp
@@ -18,31 +18,12 @@
 
 #include <stdlib.h>
 
-class TestKernel;
+#include <helpers/TestKernel.hpp>
 
 bool KernelGetGroupInfoCalled = false;
 std::array<size_t, 3> IncomingLocalSize = {0, 0, 0};
 std::array<size_t, 3> RequiredLocalSize = {0, 0, 0};
 
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-namespace detail {
-template <> struct KernelInfo<TestKernel> {
-  static constexpr unsigned getNumParams() { return 0; }
-  static const kernel_param_desc_t &getParamDesc(int) {
-    static kernel_param_desc_t Dummy;
-    return Dummy;
-  }
-  static constexpr const char *getName() { return "TestKernel"; }
-  static constexpr bool isESIMD() { return false; }
-  static constexpr bool callsThisItem() { return false; }
-  static constexpr bool callsAnyThisFreeFunction() { return false; }
-};
-
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
-
 static pi_result redefinedProgramCreate(pi_context, const void *, size_t,
                                         pi_program *) {
   return PI_SUCCESS;
@@ -205,29 +186,6 @@ static void setupDefaultMockAPIs(sycl::unittest::PiMock &Mock) {
   Mock.redefine<PiApiKind::piKernelGetGroupInfo>(redefinedKernelGetGroupInfo);
 }
 
-static sycl::unittest::PiImage generateDefaultImage() {
-  using namespace sycl::unittest;
-
-  PiPropertySet PropSet;
-
-  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
-
-  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"TestKernel"});
-
-  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
-              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
-              "",                                     // Compile options
-              "",                                     // Link options
-              std::move(Bin),
-              std::move(Entries),
-              std::move(PropSet)};
-
-  return Img;
-}
-
-sycl::unittest::PiImage Img = generateDefaultImage();
-sycl::unittest::PiImageArray<1> ImgArray{&Img};
-
 static void performChecks() {
   sycl::platform Plt{sycl::default_selector()};
   if (Plt.is_host()) {
diff --git a/sycl/unittests/stream/stream.cpp b/sycl/unittests/stream/stream.cpp
index 94522444aa657..c7d8fbf19bf91 100644
--- a/sycl/unittests/stream/stream.cpp
+++ b/sycl/unittests/stream/stream.cpp
@@ -16,48 +16,7 @@
 
 #include <limits>
 
-class TestKernel;
-
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-namespace detail {
-template <> struct KernelInfo<TestKernel> {
-  static constexpr unsigned getNumParams() { return 0; }
-  static const kernel_param_desc_t &getParamDesc(int) {
-    static kernel_param_desc_t Dummy;
-    return Dummy;
-  }
-  static constexpr const char *getName() { return "Stream_TestKernel"; }
-  static constexpr bool isESIMD() { return false; }
-  static constexpr bool callsThisItem() { return false; }
-  static constexpr bool callsAnyThisFreeFunction() { return false; }
-};
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
-
-static sycl::unittest::PiImage generateDefaultImage() {
-  using namespace sycl::unittest;
-
-  PiPropertySet PropSet;
-
-  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
-
-  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"Stream_TestKernel"});
-
-  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
-              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
-              "",                                     // Compile options
-              "",                                     // Link options
-              std::move(Bin),
-              std::move(Entries),
-              std::move(PropSet)};
-
-  return Img;
-}
-
-static sycl::unittest::PiImage Img = generateDefaultImage();
-static sycl::unittest::PiImageArray<1> ImgArray{&Img};
+#include <helpers/TestKernel.hpp>
 
 size_t GBufferCreateCounter = 0;
 

From ed4811e4f997d9ea12c8635b0ea968bb48742e52 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Mon, 10 Jan 2022 18:43:16 +0300
Subject: [PATCH 17/18] Refactor some tests

---
 sycl/unittests/kernel-and-program/Cache.cpp   | 15 +++++--
 .../kernel-and-program/HelperKernelInfo.hpp   | 29 -------------
 .../kernel-and-program/MultipleDevsCache.cpp  | 42 ++-----------------
 3 files changed, 15 insertions(+), 71 deletions(-)
 delete mode 100644 sycl/unittests/kernel-and-program/HelperKernelInfo.hpp

diff --git a/sycl/unittests/kernel-and-program/Cache.cpp b/sycl/unittests/kernel-and-program/Cache.cpp
index e25a55c2f9ff7..a5845f6ce0e1e 100644
--- a/sycl/unittests/kernel-and-program/Cache.cpp
+++ b/sycl/unittests/kernel-and-program/Cache.cpp
@@ -9,7 +9,6 @@
 #define SYCL2020_DISABLE_DEPRECATION_WARNINGS
 
 #include "CL/sycl/detail/pi.h"
-#include "HelperKernelInfo.hpp"
 #include "detail/context_impl.hpp"
 #include "detail/kernel_program_cache.hpp"
 #include "detail/program_impl.hpp"
@@ -38,14 +37,24 @@ class TestKernel2 {
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 namespace detail {
+struct MockKernelInfo {
+  static constexpr unsigned getNumParams() { return 0; }
+  static const kernel_param_desc_t &getParamDesc(int) {
+    static kernel_param_desc_t Dummy;
+    return Dummy;
+  }
+  static constexpr const char *getName() { return "TestKernel2"; }
+  static constexpr bool isESIMD() { return false; }
+  static constexpr bool callsThisItem() { return false; }
+  static constexpr bool callsAnyThisFreeFunction() { return false; }
+};
+
 template <> struct KernelInfo<TestKernel> : public MockKernelInfo {
   static constexpr const char *getName() { return "TestKernel"; }
 };
-
 template <> struct KernelInfo<TestKernel2> : public MockKernelInfo {
   static constexpr const char *getName() { return "TestKernel2"; }
 };
-
 } // namespace detail
 } // namespace sycl
 } // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/unittests/kernel-and-program/HelperKernelInfo.hpp b/sycl/unittests/kernel-and-program/HelperKernelInfo.hpp
deleted file mode 100644
index 34de647cf5cfd..0000000000000
--- a/sycl/unittests/kernel-and-program/HelperKernelInfo.hpp
+++ /dev/null
@@ -1,29 +0,0 @@
-//==----------------------- HelperKernelInfo.hpp ---------------------------==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#pragma once
-
-#include <CL/sycl.hpp>
-
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-namespace detail {
-struct MockKernelInfo {
-  static constexpr unsigned getNumParams() { return 0; }
-  static const kernel_param_desc_t &getParamDesc(int) {
-    static kernel_param_desc_t Dummy;
-    return Dummy;
-  }
-  static constexpr bool isESIMD() { return false; }
-  static constexpr bool callsThisItem() { return false; }
-  static constexpr bool callsAnyThisFreeFunction() { return false; }
-};
-
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index a4e26b582088b..475e9495261ee 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -8,14 +8,13 @@
 
 #define SYCL2020_DISABLE_DEPRECATION_WARNINGS
 
-#include "HelperKernelInfo.hpp"
 #include "detail/context_impl.hpp"
 #include "detail/kernel_bundle_impl.hpp"
 #include "detail/kernel_program_cache.hpp"
-#include <CL/sycl.hpp>
 #include <helpers/CommonRedefinitions.hpp>
 #include <helpers/PiImage.hpp>
 #include <helpers/PiMock.hpp>
+#include <helpers/TestKernel.hpp>
 
 #include <gtest/gtest.h>
 
@@ -23,41 +22,6 @@
 
 using namespace sycl;
 
-class MultTestKernel;
-
-__SYCL_INLINE_NAMESPACE(cl) {
-namespace sycl {
-namespace detail {
-template <> struct KernelInfo<MultTestKernel> : public MockKernelInfo {
-  static constexpr const char *getName() { return "MultTestKernel"; }
-};
-} // namespace detail
-} // namespace sycl
-} // __SYCL_INLINE_NAMESPACE(cl)
-
-static sycl::unittest::PiImage generateDefaultImage() {
-  using namespace sycl::unittest;
-
-  PiPropertySet PropSet;
-
-  std::vector<unsigned char> Bin{0, 1, 2, 3, 4, 5}; // Random data
-
-  PiArray<PiOffloadEntry> Entries = makeEmptyKernels({"MultTestKernel"});
-
-  PiImage Img{PI_DEVICE_BINARY_TYPE_SPIRV,            // Format
-              __SYCL_PI_DEVICE_BINARY_TARGET_SPIRV64, // DeviceTargetSpec
-              "",                                     // Compile options
-              "",                                     // Link options
-              std::move(Bin),
-              std::move(Entries),
-              std::move(PropSet)};
-
-  return Img;
-}
-
-static sycl::unittest::PiImage Img = generateDefaultImage();
-static sycl::unittest::PiImageArray<1> ImgArray{&Img};
-
 static pi_result redefinedContextCreate(
     const pi_context_properties *properties, pi_uint32 num_devices,
     const pi_device *devices,
@@ -188,11 +152,11 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
     auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
         Queue.get_context());
     Queue.submit([&](cl::sycl::handler &cgh) {
-      cgh.single_task<MultTestKernel>([]() {});
+      cgh.single_task<TestKernel>([]() {});
     });
 
     auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
-    auto KernelID = cl::sycl::get_kernel_id<MultTestKernel>();
+    auto KernelID = cl::sycl::get_kernel_id<TestKernel>();
     auto Kernel = BundleObject.get_kernel(KernelID);
 
     // Because of emulating 2 devices program is retained for each one in

From ad007d9cd7a6f5e0504ae50f2d2790881841af82 Mon Sep 17 00:00:00 2001
From: mdimakov <maxim.dimakov@intel.com>
Date: Mon, 10 Jan 2022 18:50:28 +0300
Subject: [PATCH 18/18] Clang-format fix

---
 sycl/unittests/kernel-and-program/Cache.cpp             | 1 -
 sycl/unittests/kernel-and-program/MultipleDevsCache.cpp | 5 ++---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/sycl/unittests/kernel-and-program/Cache.cpp b/sycl/unittests/kernel-and-program/Cache.cpp
index a5845f6ce0e1e..5a41da137ed28 100644
--- a/sycl/unittests/kernel-and-program/Cache.cpp
+++ b/sycl/unittests/kernel-and-program/Cache.cpp
@@ -43,7 +43,6 @@ struct MockKernelInfo {
     static kernel_param_desc_t Dummy;
     return Dummy;
   }
-  static constexpr const char *getName() { return "TestKernel2"; }
   static constexpr bool isESIMD() { return false; }
   static constexpr bool callsThisItem() { return false; }
   static constexpr bool callsAnyThisFreeFunction() { return false; }
diff --git a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
index 475e9495261ee..15ac42424d728 100644
--- a/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
+++ b/sycl/unittests/kernel-and-program/MultipleDevsCache.cpp
@@ -151,9 +151,8 @@ TEST_F(MultipleDeviceCacheTest, ProgramRetain) {
 
     auto Bundle = cl::sycl::get_kernel_bundle<sycl::bundle_state::input>(
         Queue.get_context());
-    Queue.submit([&](cl::sycl::handler &cgh) {
-      cgh.single_task<TestKernel>([]() {});
-    });
+    Queue.submit(
+        [&](cl::sycl::handler &cgh) { cgh.single_task<TestKernel>([]() {}); });
 
     auto BundleObject = cl::sycl::build(Bundle, Bundle.get_devices());
     auto KernelID = cl::sycl::get_kernel_id<TestKernel>();