Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,13 @@ update_agent_runtime_visibility(rocprofiler_agent_t& agent_info)
auto rocr_visible =
parse_env_visible("ROCR_VISIBLE_DEVICES", agent_info.logical_node_type_id);

// HIP enumerates only ROCR-visible agents, so a ROCR-hidden agent is never HIP-visible
if(rocr_visible && !*rocr_visible)
{
set_hsa_visibility(false);
return;
}

auto rocr_index =
(rocr_visible && *rocr_visible) ? rocr_visible->index : agent_info.logical_node_type_id;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
#include <cstdlib>
#include <iostream>
#include <random>
#include <set>
#include <sstream>
#include <type_traits>
#include <typeinfo>
Expand Down Expand Up @@ -820,6 +821,75 @@ TEST(rocprofiler_lib, agent_visibility_multigpu)
}
}

// AIPROFSDK-1169: HIP_VISIBLE_DEVICES indexes the ROCR-visible list, so ROCR-hidden agents
// must not trigger the "not visible to HSA" warning
TEST(rocprofiler_lib, agent_visibility_rocr_hip_stacked)
{
constexpr auto noval = std::string_view{};

common::set_env("ROCR_VISIBLE_DEVICES", noval, 1);
common::set_env("HIP_VISIBLE_DEVICES", noval, 1);
common::set_env("GPU_DEVICE_ORDINAL", noval, 1);
common::set_env("CUDA_VISIBLE_DEVICES", noval, 1);

auto num_gpu_agents = get_gpu_agents().size();
if(num_gpu_agents < 3)
{
GTEST_SKIP() << "requires 3 or more gpu agents";
}

// on an 8-GPU system this reproduces the ticket's table exactly (target = GPU 4)
const uint32_t target = (num_gpu_agents >= 6) ? 4 : 1;

struct visibility_case
{
std::string rocr;
std::string hip;
std::set<uint32_t> hsa_visible;
std::set<uint32_t> hip_visible;
};

auto cases = std::vector<visibility_case>{
{"", std::to_string(target), {}, {target}},
{fmt::format("{},{}", target - 1, target), "1", {target - 1, target}, {target}},
{fmt::format("{},{}", target, target + 1), "0", {target, target + 1}, {target}},
{std::to_string(target), std::to_string(target), {target}, {}},
};

for(const auto& itr : cases)
{
common::set_env("ROCR_VISIBLE_DEVICES", itr.rocr, 1);
common::set_env("HIP_VISIBLE_DEVICES", itr.hip, 1);

testing::internal::CaptureStderr();
auto agents = get_gpu_agents();
auto err = testing::internal::GetCapturedStderr();

auto msg = fmt::format("ROCR_VISIBLE_DEVICES={} HIP_VISIBLE_DEVICES={}", itr.rocr, itr.hip);

// get_gpu_agents() always warns; its absence means stderr was not captured
ASSERT_NE(err.find("refreshing internal topology"), std::string::npos) << msg << "\n"
<< err;
EXPECT_EQ(err.find("not visible to HSA"), std::string::npos) << msg << "\n" << err;

ASSERT_EQ(agents.size(), num_gpu_agents) << msg;
for(const auto* agent : agents)
{
auto id = agent->logical_node_type_id;
auto hsa_exp = static_cast<uint32_t>(itr.rocr.empty() || itr.hsa_visible.count(id) > 0);
auto hip_exp = static_cast<uint32_t>(itr.hip_visible.count(id) > 0);

EXPECT_EQ(agent->runtime_visibility.hsa, hsa_exp) << msg << " :: GPU " << id;
EXPECT_EQ(agent->runtime_visibility.hip, hip_exp) << msg << " :: GPU " << id;
EXPECT_EQ(agent->runtime_visibility.rccl, hip_exp) << msg << " :: GPU " << id;
EXPECT_EQ(agent->runtime_visibility.rocdecode, hip_exp) << msg << " :: GPU " << id;
}
}

common::set_env("ROCR_VISIBLE_DEVICES", noval, 1);
common::set_env("HIP_VISIBLE_DEVICES", noval, 1);
}

TEST(rocprofiler_lib, agent_visibility_inverted_multigpu)
{
constexpr auto noval = std::string_view{};
Expand Down
Loading