diff --git a/projects/hip-tests/catch/performance/scenarios/stream/hipPerfIncreasingNumberOfStreams.cc b/projects/hip-tests/catch/performance/scenarios/stream/hipPerfIncreasingNumberOfStreams.cc index 55ecaa66fd97..1975b8b6a2cb 100644 --- a/projects/hip-tests/catch/performance/scenarios/stream/hipPerfIncreasingNumberOfStreams.cc +++ b/projects/hip-tests/catch/performance/scenarios/stream/hipPerfIncreasingNumberOfStreams.cc @@ -8,13 +8,15 @@ #include #include +#include "stream_create_helper.hh" + using namespace std; __global__ static void _noop_kernel() {} HIP_TEST_CASE(Performance_KernelLaunchLatency_IncreasingNumberOfStreams) { - vector streamsNumber{1, 10, 50, 100, 1000, 5000}; + vector streamsNumber{1, 10, 50, 100, 1000}; hipError_t err = hipSuccess; hipEvent_t start, stop; int32_t NITER = 100000; @@ -50,16 +52,14 @@ HIP_TEST_CASE(Performance_KernelLaunchLatency_IncreasingNumberOfStreams) { HIP_CHECK(hipEventCreate(&stop)); for (auto& numHipStreams : streamsNumber) { - vector streams(numHipStreams); - - if (isBlocking) { - for (int i = 0; i < numHipStreams; ++i) { - HIP_CHECK(hipStreamCreate(&streams[i])); - } - } else { - for (int i = 0; i < numHipStreams; ++i) { - HIP_CHECK(hipStreamCreateWithFlags(&streams[i], hipStreamNonBlocking)); - } + vector streams; + streams.reserve(numHipStreams); + + if (stream_test::CreateStreams(numHipStreams, !isBlocking, streams) == + stream_test::CreateResult::kUnavailable) { + cout << "Skipping " << numHipStreams << " streams: the device cannot provide that many" + << std::endl; + break; } auto kernelStart = std::chrono::steady_clock::now(); @@ -85,8 +85,8 @@ HIP_TEST_CASE(Performance_KernelLaunchLatency_IncreasingNumberOfStreams) { HIP_CHECK(hipDeviceSynchronize()); - for (int i = 0; i < numHipStreams; ++i) { - HIP_CHECK(hipStreamDestroy(streams[i])); + for (hipStream_t s : streams) { + HIP_CHECK(hipStreamDestroy(s)); } } HIP_CHECK(hipEventDestroy(start)); diff --git a/projects/hip-tests/catch/performance/scenarios/stream/hipPerfMultiStreamKernelLaunch.cc b/projects/hip-tests/catch/performance/scenarios/stream/hipPerfMultiStreamKernelLaunch.cc index e9f7a380b649..ee82247baf95 100644 --- a/projects/hip-tests/catch/performance/scenarios/stream/hipPerfMultiStreamKernelLaunch.cc +++ b/projects/hip-tests/catch/performance/scenarios/stream/hipPerfMultiStreamKernelLaunch.cc @@ -16,6 +16,8 @@ #include #include +#include "stream_create_helper.hh" + #if HT_AMD #define device_clock64() wall_clock64() #else @@ -38,16 +40,15 @@ class Experiment { double walltime_us; }; - Experiment(uint32_t num_streams): + Experiment(size_t num_streams): num_streams_{num_streams}, streams_{num_streams, nullptr} { } Experiment(const Experiment& other): num_streams_{other.num_streams_}, streams_{other.num_streams_, nullptr} { } - void init() { - for (hipStream_t& s: streams_){ - HIP_CHECK(hipStreamCreate(&s)); - } + stream_test::CreateResult init() { + streams_.clear(); + return stream_test::CreateStreams(num_streams_, /*nonBlocking=*/false, streams_); } void cleanup() { @@ -122,7 +123,11 @@ HIP_TEST_CASE(Performance_hipPerfMultiStreamKernelLaunch) { for (const auto& num_streams : STREAMS_PER_EXPERIMENT) { Experiment exp(num_streams); Experiment::Metrics metrics; - exp.init(); + if (exp.init() == stream_test::CreateResult::kUnavailable) { + std::cout << "Skipping " << num_streams << " streams: the device cannot provide that many" + << std::endl; + break; + } exp.do_warmup(WARMUP_ITERATIONS, WARMUP_KERNEL_DISPATCHES_PER_STREAM, TimingKernel, timing_count); HIP_CHECK(hipDeviceSynchronize()); if (KERNEL_SLEEP_US == 0) { diff --git a/projects/hip-tests/catch/performance/scenarios/stream/stream_create_helper.hh b/projects/hip-tests/catch/performance/scenarios/stream/stream_create_helper.hh new file mode 100644 index 000000000000..4232bf91a794 --- /dev/null +++ b/projects/hip-tests/catch/performance/scenarios/stream/stream_create_helper.hh @@ -0,0 +1,82 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include +#include + +#include "hip_test_common.hh" + +namespace stream_test { + +/** Which backend this process is running on. + * + * GPU_ENABLE_PAL | Windows | Linux + * unset or "" | PAL | ROCr + * "0" | ROCr | ROCr + * "1" | PAL | PAL + * "2" | auto | auto + * + * "2" hands the choice to the runtime, so it is reported as Unknown and the caller stays + * permissive rather than guessing. + */ +enum class Backend { kRocr, kPal, kUnknown }; + +inline Backend ActiveBackend() { + const std::string env = TestContext::getEnvVar("GPU_ENABLE_PAL"); + if (env.empty()) { + return TestContext::get().isWindows() ? Backend::kPal : Backend::kRocr; + } + if (env == "0") return Backend::kRocr; + if (env == "1") return Backend::kPal; + return Backend::kUnknown; +} + + +enum class CreateResult { + kCreated, ///< All `count` streams were created. + kUnavailable, ///< The device cannot provide this many; nothing was left allocated. +}; + +/** Creates `count` streams, appending them to `streams`. + * + * On kUnavailable the streams created so far are destroyed and `streams` is left empty. The + * counts in these sweeps only increase, so a caller that gets kUnavailable should stop rather + * than try the larger ones. Errors other than the exhaustion codes fail the test as usual. + */ +inline CreateResult CreateStreams(size_t count, bool nonBlocking, + std::vector& streams) { + for (size_t i = 0; i < count; ++i) { + hipStream_t s = nullptr; + const hipError_t err = nonBlocking ? hipStreamCreateWithFlags(&s, hipStreamNonBlocking) + : hipStreamCreate(&s); + + const bool exhausted = (err == hipErrorOutOfMemory) || + (err == hipErrorLaunchFailure && ActiveBackend() != Backend::kRocr); + if (!exhausted) { + HIP_CHECK(err); + streams.push_back(s); + continue; + } + + // Best effort: on a lost context these fail too, and HIP_CHECK here would turn a skip into + // a failure. + for (hipStream_t created : streams) { + static_cast(hipStreamDestroy(created)); + } + streams.clear(); + // The failed creation, and any failed destroy above, leaves a sticky thread-local error + // behind, and this path does not fail the test. Consume it, so that a later test case in + // this process does not fail on an error this one left behind. + static_cast(hipGetLastError()); + + return CreateResult::kUnavailable; + } + return CreateResult::kCreated; +} + +} // namespace stream_test