diff --git a/projects/hipfile/include/hipfile-api-trace.h b/projects/hipfile/include/hipfile-api-trace.h index 8ea63433e419..c4e1b3081577 100644 --- a/projects/hipfile/include/hipfile-api-trace.h +++ b/projects/hipfile/include/hipfile-api-trace.h @@ -15,7 +15,7 @@ // Increment the step version when new runtime API functions are added. // If the corresponding table version increases, reset the step version // to zero. -#define HIPFILE_RUNTIME_API_TABLE_STEP_VERSION 0 +#define HIPFILE_RUNTIME_API_TABLE_STEP_VERSION 1 // hipFile API interface typedef const char *(*PfnHipFileGetOpErrorString)(hipFileOpError_t status); @@ -60,6 +60,9 @@ typedef hipFileError_t (*PfnHipFileSetParameterSizeT)(hipFileSizeTConfigParamete typedef hipFileError_t (*PfnHipFileSetParameterBool)(hipFileBoolConfigParameter_t param, bool value); typedef hipFileError_t (*PfnHipFileSetParameterString)(hipFileStringConfigParameter_t param, const char *desc_str); +typedef hipFileError_t (*PfnHipFileGetStatsL1)(hipFileStatsLevel1_t *stats); +typedef hipFileError_t (*PfnHipFileGetStatsL2)(hipFileStatsLevel2_t *stats); +typedef hipFileError_t (*PfnHipFileGetStatsL3)(hipFileStatsLevel3_t *stats); // hipFile API dispatch table struct hipFileDispatchTable { @@ -99,6 +102,12 @@ struct hipFileDispatchTable { // PLEASE DO NOT EDIT ABOVE! // HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 1 + PfnHipFileGetStatsL1 pfn_hipfile_get_stats_l1; + PfnHipFileGetStatsL2 pfn_hipfile_get_stats_l2; + PfnHipFileGetStatsL3 pfn_hipfile_get_stats_l3; + // PLEASE DO NOT EDIT ABOVE! + + // HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 2 // ******************************************************************************************* // READ BELOW diff --git a/projects/hipfile/include/hipfile.h b/projects/hipfile/include/hipfile.h index d17983ed2240..f20a60c5fe46 100644 --- a/projects/hipfile/include/hipfile.h +++ b/projects/hipfile/include/hipfile.h @@ -1069,6 +1069,195 @@ hipFileError_t hipFileSetParameterBool(hipFileBoolConfigParameter_t param, bool HIPFILE_API hipFileError_t hipFileSetParameterString(hipFileStringConfigParameter_t param, const char *desc_str); +// *********************************************************************** +// STATS API +// *********************************************************************** + +/*! + * @defgroup stats Stats API + * + * @brief Functions for querying hipFile I/O statistics + */ + +/*! + * @brief Maximum number of GPUs tracked by the stats API + * @ingroup stats + */ +#define HIPFILE_MAX_GPUS 16 + +/*! + * @brief Length of the GPU UUID field in hipFilePerGpuStats_t + * @ingroup stats + */ +#define HIPFILE_GPU_UUID_LEN 16 + +/*! + * @brief Counter tracking successful and failed occurrences of an operation + * @ingroup stats + */ +typedef struct hipFileOpCounter { + uint64_t ok; //!< Number of successful operations + uint64_t err; //!< Number of failed operations +} hipFileOpCounter_t; + +/*! + * @brief Level 1 statistics: basic I/O and operation counters + * @ingroup stats + * + * Fields that hipFile does not track are zero-filled. Calculated fields + * (bandwidth, average latency) are derived from the counters that are tracked. + */ +typedef struct hipFileStatsLevel1 { + hipFileOpCounter_t read_ops; //!< Read operation counters + hipFileOpCounter_t write_ops; //!< Write operation counters + hipFileOpCounter_t hdl_register_ops; //!< File handle register operation counters + hipFileOpCounter_t hdl_deregister_ops; //!< File handle deregister operation counters (zero-filled) + hipFileOpCounter_t buf_register_ops; //!< Buffer register operation counters + hipFileOpCounter_t buf_deregister_ops; //!< Buffer deregister operation counters (zero-filled) + + uint64_t read_bytes; //!< Total bytes read + uint64_t write_bytes; //!< Total bytes written + uint64_t read_bw_bytes_per_sec; //!< Read bandwidth (bytes/sec), derived from bytes and latency sum + uint64_t write_bw_bytes_per_sec; //!< Write bandwidth (bytes/sec), derived from bytes and latency sum + uint64_t read_lat_avg_us; //!< Average read latency (us), derived from latency sum and op count + uint64_t write_lat_avg_us; //!< Average write latency (us), derived from latency sum and op count + + uint64_t read_ops_per_sec; //!< Read operations per second (zero-filled) + uint64_t write_ops_per_sec; //!< Write operations per second (zero-filled) + + uint64_t read_lat_sum_us; //!< Sum of read latencies (us) + uint64_t write_lat_sum_us; //!< Sum of write latencies (us) + + hipFileOpCounter_t batch_submit_ops; //!< Batch submit operation counters (zero-filled) + hipFileOpCounter_t batch_complete_ops; //!< Batch complete operation counters (zero-filled) + hipFileOpCounter_t batch_setup_ops; //!< Batch setup operation counters (zero-filled) + hipFileOpCounter_t batch_cancel_ops; //!< Batch cancel operation counters (zero-filled) + hipFileOpCounter_t batch_destroy_ops; //!< Batch destroy operation counters (zero-filled) + hipFileOpCounter_t batch_enqueued_ops; //!< Batch enqueue operation counters (zero-filled) + hipFileOpCounter_t batch_posix_enqueued_ops; //!< POSIX batch enqueue operation counters (zero-filled) + hipFileOpCounter_t batch_processed_ops; //!< Batch process operation counters (zero-filled) + hipFileOpCounter_t batch_posix_processed_ops; //!< POSIX batch process operation counters (zero-filled) + hipFileOpCounter_t batch_nvfs_submit_ops; //!< NVFS batch submit operation counters (zero-filled) + hipFileOpCounter_t batch_p2p_submit_ops; //!< P2P batch submit operation counters (zero-filled) + hipFileOpCounter_t batch_aio_submit_ops; //!< AIO batch submit operation counters (zero-filled) + hipFileOpCounter_t batch_iouring_submit_ops; //!< io_uring batch submit operation counters (zero-filled) + hipFileOpCounter_t batch_mixed_io_submit_ops; //!< Mixed IO batch submit operation counters (zero-filled) + hipFileOpCounter_t batch_total_submit_ops; //!< Total batch submit operation counters (zero-filled) + + uint64_t batch_read_bytes; //!< Total batch read bytes (zero-filled) + uint64_t batch_write_bytes; //!< Total batch write bytes (zero-filled) + uint64_t batch_read_bw_bytes; //!< Batch read bandwidth (zero-filled) + uint64_t batch_write_bw_bytes; //!< Batch write bandwidth (zero-filled) + uint64_t batch_submit_lat_avg_us; //!< Average batch submit latency (zero-filled) + uint64_t batch_completion_lat_avg_us; //!< Average batch completion latency (zero-filled) + uint64_t batch_submit_ops_per_sec; //!< Batch submit operations per second (zero-filled) + uint64_t batch_complete_ops_per_sec; //!< Batch complete operations per second (zero-filled) + uint64_t batch_submit_lat_sum_us; //!< Sum of batch submit latencies (zero-filled) + uint64_t batch_completion_lat_sum_us; //!< Sum of batch completion latencies (zero-filled) + uint64_t last_batch_read_bytes; //!< Last batch read bytes (zero-filled) + uint64_t last_batch_write_bytes; //!< Last batch write bytes (zero-filled) +} hipFileStatsLevel1_t; + +/*! + * @brief Level 2 statistics: includes Level 1 plus I/O size histograms + * @ingroup stats + * + * The size histograms are zero-filled; hipFile tracks histograms with different + * bucket boundaries that are not directly mappable to this format. + */ +typedef struct hipFileStatsLevel2 { + hipFileStatsLevel1_t basic; //!< Level 1 statistics + uint64_t read_size_kb_hist[32]; //!< Histogram of read sizes in KiB (zero-filled) + uint64_t write_size_kb_hist[32]; //!< Histogram of write sizes in KiB (zero-filled) +} hipFileStatsLevel2_t; + +/*! + * @brief Per-GPU statistics used in Level 3 + * @ingroup stats + * + * Fields that hipFile does not track are zero-filled. The fastpath backend + * maps to @c n_nvfs_reads / @c n_nvfs_writes, and the fallback backend maps + * to @c n_posix_reads / @c n_posix_writes. + */ +typedef struct hipFilePerGpuStats { + char uuid[HIPFILE_GPU_UUID_LEN]; //!< GPU UUID (zero-filled) + + uint64_t read_bytes; //!< Total bytes read + uint64_t read_bw_bytes_per_sec; //!< Read bandwidth (bytes/sec), derived from bytes and duration + uint64_t read_utilization; //!< Read utilization percentage (zero-filled) + uint64_t read_duration_us; //!< Total read duration (us) + uint64_t n_total_reads; //!< Total number of reads across all backends + uint64_t n_p2p_reads; //!< Number of PCIe P2P DMA reads (zero-filled) + uint64_t n_nvfs_reads; //!< Number of fastpath reads + uint64_t n_posix_reads; //!< Number of fallback (POSIX) reads + uint64_t n_unaligned_reads; //!< Number of unaligned reads + uint64_t n_dr_reads; //!< Number of dynamic-routing reads (zero-filled) + uint64_t n_sparse_regions; //!< Number of sparse regions (zero-filled) + uint64_t n_inline_regions; //!< Number of inline regions (zero-filled) + uint64_t n_reads_err; //!< Number of read errors + + uint64_t writes_bytes; //!< Total bytes written + uint64_t write_bw_bytes_per_sec; //!< Write bandwidth (bytes/sec), derived from bytes and duration + uint64_t write_utilization; //!< Write utilization percentage (zero-filled) + uint64_t write_duration_us; //!< Total write duration (us) + uint64_t n_total_writes; //!< Total number of writes across all backends + uint64_t n_p2p_writes; //!< Number of PCIe P2P DMA writes (zero-filled) + uint64_t n_nvfs_writes; //!< Number of fastpath writes + uint64_t n_posix_writes; //!< Number of fallback (POSIX) writes + uint64_t n_unaligned_writes; //!< Number of unaligned writes + uint64_t n_dr_writes; //!< Number of dynamic-routing writes (zero-filled) + uint64_t n_writes_err; //!< Number of write errors + + uint64_t n_mmap; //!< Number of buffer registrations (global counter, not per-GPU) + uint64_t n_mmap_ok; //!< Successful buffer registrations (same as n_mmap) + uint64_t n_mmap_err; //!< Failed buffer registrations (zero-filled) + uint64_t n_mmap_free; //!< Number of buffer deregistrations (zero-filled) + uint64_t reg_bytes; //!< Total bytes registered (zero-filled) +} hipFilePerGpuStats_t; + +/*! + * @brief Level 3 statistics: includes Level 2 plus per-GPU statistics + * @ingroup stats + */ +typedef struct hipFileStatsLevel3 { + hipFileStatsLevel2_t detailed; //!< Level 2 statistics + uint32_t num_gpus; //!< Number of GPUs with recorded activity + hipFilePerGpuStats_t per_gpu_stats[HIPFILE_MAX_GPUS]; //!< Per-GPU statistics +} hipFileStatsLevel3_t; + +/*! + * @brief Get Level 1 hipFile statistics + * @ingroup stats + * + * @param [out] stats Pointer to a hipFileStatsLevel1_t structure to be filled + * + * @return \hipfile_error_return + */ +HIPFILE_API +hipFileError_t hipFileGetStatsL1(hipFileStatsLevel1_t *stats); + +/*! + * @brief Get Level 2 hipFile statistics + * @ingroup stats + * + * @param [out] stats Pointer to a hipFileStatsLevel2_t structure to be filled + * + * @return \hipfile_error_return + */ +HIPFILE_API +hipFileError_t hipFileGetStatsL2(hipFileStatsLevel2_t *stats); + +/*! + * @brief Get Level 3 hipFile statistics + * @ingroup stats + * + * @param [out] stats Pointer to a hipFileStatsLevel3_t structure to be filled + * + * @return \hipfile_error_return + */ +HIPFILE_API +hipFileError_t hipFileGetStatsL3(hipFileStatsLevel3_t *stats); + // Not a part of the public API #undef __HIPFILE_NODISCARD diff --git a/projects/hipfile/src/amd_detail/api_trace/api-dispatch-interface.cpp b/projects/hipfile/src/amd_detail/api_trace/api-dispatch-interface.cpp index dbaa0a49afc9..8c1a35578dd5 100644 --- a/projects/hipfile/src/amd_detail/api_trace/api-dispatch-interface.cpp +++ b/projects/hipfile/src/amd_detail/api_trace/api-dispatch-interface.cpp @@ -169,3 +169,18 @@ hipFileSetParameterString(hipFileStringConfigParameter_t param, const char *desc { return hipFile::GetHipFileDispatchTable()->pfn_hipfile_set_parameter_string(param, desc_str); } +hipFileError_t +hipFileGetStatsL1(hipFileStatsLevel1_t *stats) +{ + return hipFile::GetHipFileDispatchTable()->pfn_hipfile_get_stats_l1(stats); +} +hipFileError_t +hipFileGetStatsL2(hipFileStatsLevel2_t *stats) +{ + return hipFile::GetHipFileDispatchTable()->pfn_hipfile_get_stats_l2(stats); +} +hipFileError_t +hipFileGetStatsL3(hipFileStatsLevel3_t *stats) +{ + return hipFile::GetHipFileDispatchTable()->pfn_hipfile_get_stats_l3(stats); +} diff --git a/projects/hipfile/src/amd_detail/api_trace/api-trace-internal.h b/projects/hipfile/src/amd_detail/api_trace/api-trace-internal.h index d724de32a431..2a7bedfe5ce1 100644 --- a/projects/hipfile/src/amd_detail/api_trace/api-trace-internal.h +++ b/projects/hipfile/src/amd_detail/api_trace/api-trace-internal.h @@ -61,5 +61,8 @@ hipFileError_t hipFileGetParameterString(hipFileStringConfigParameter_t param, c hipFileError_t hipFileSetParameterSizeT(hipFileSizeTConfigParameter_t param, size_t value); hipFileError_t hipFileSetParameterBool(hipFileBoolConfigParameter_t param, bool value); hipFileError_t hipFileSetParameterString(hipFileStringConfigParameter_t param, const char *desc_str); +hipFileError_t hipFileGetStatsL1(hipFileStatsLevel1_t *stats); +hipFileError_t hipFileGetStatsL2(hipFileStatsLevel2_t *stats); +hipFileError_t hipFileGetStatsL3(hipFileStatsLevel3_t *stats); } // namespace hipFile diff --git a/projects/hipfile/src/amd_detail/api_trace/api-trace.cpp b/projects/hipfile/src/amd_detail/api_trace/api-trace.cpp index 8355c87c70de..3fae592bdd49 100644 --- a/projects/hipfile/src/amd_detail/api_trace/api-trace.cpp +++ b/projects/hipfile/src/amd_detail/api_trace/api-trace.cpp @@ -58,6 +58,9 @@ namespace { ptr_dispatch_table->pfn_hipfile_set_parameter_size_t = hipFile::hipFileSetParameterSizeT; ptr_dispatch_table->pfn_hipfile_set_parameter_bool = hipFile::hipFileSetParameterBool; ptr_dispatch_table->pfn_hipfile_set_parameter_string = hipFile::hipFileSetParameterString; + ptr_dispatch_table->pfn_hipfile_get_stats_l1 = hipFile::hipFileGetStatsL1; + ptr_dispatch_table->pfn_hipfile_get_stats_l2 = hipFile::hipFileGetStatsL2; + ptr_dispatch_table->pfn_hipfile_get_stats_l3 = hipFile::hipFileGetStatsL3; } #if HIPFILE_ROCPROFILER_REGISTER > 0 @@ -169,14 +172,18 @@ HIPFILE_ENFORCE_ABI(hipFileDispatchTable, pfn_hipfile_set_parameter_size_t, 28) HIPFILE_ENFORCE_ABI(hipFileDispatchTable, pfn_hipfile_set_parameter_bool, 29) HIPFILE_ENFORCE_ABI(hipFileDispatchTable, pfn_hipfile_set_parameter_string, 30) // HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 1 +HIPFILE_ENFORCE_ABI(hipFileDispatchTable, pfn_hipfile_get_stats_l1, 31) +HIPFILE_ENFORCE_ABI(hipFileDispatchTable, pfn_hipfile_get_stats_l2, 32) +HIPFILE_ENFORCE_ABI(hipFileDispatchTable, pfn_hipfile_get_stats_l3, 33) +// HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 2 // If HIPFILE_ENFORCE_ABI entries are added for each new function pointer in the table, // the number below will be one greater than the number in the last HIPFILE_ENFORCE_ABI line. For example: -// HIPFILE_ENFORCE_ABI(, , 30) -// HIPFILE_ENFORCE_ABI_VERSIONING(
, 31) <- 30 + 1 = 31 -HIPFILE_ENFORCE_ABI_VERSIONING(hipFileDispatchTable, 31) +// HIPFILE_ENFORCE_ABI(
, , 33) +// HIPFILE_ENFORCE_ABI_VERSIONING(
, 34) <- 33 + 1 = 34 +HIPFILE_ENFORCE_ABI_VERSIONING(hipFileDispatchTable, 34) -static_assert(HIPFILE_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 0, +static_assert(HIPFILE_RUNTIME_API_TABLE_MAJOR_VERSION == 0 && HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 1, "If you encounter this error, add the new HIPFILE_ENFORCE_ABI(...) code for the updated " "function pointers, " "and then modify this check to ensure it evaluates to true."); diff --git a/projects/hipfile/src/amd_detail/hipfile.cpp b/projects/hipfile/src/amd_detail/hipfile.cpp index 0509cb8546a4..f5fd284efb35 100644 --- a/projects/hipfile/src/amd_detail/hipfile.cpp +++ b/projects/hipfile/src/amd_detail/hipfile.cpp @@ -622,4 +622,189 @@ catch (...) { return handle_exception(); } +hipFileError_t +hipFileGetStatsL1(hipFileStatsLevel1_t *stats) +{ + try { + if (stats == nullptr) { + return {hipFileInvalidValue, hipSuccess}; + } + const Stats *s{Context::get()->getStats()}; + if (s == nullptr) { + return {hipFileInternalError, hipSuccess}; + } + + *stats = {}; + + static constexpr IoType ioTypes[]{IoType::Read, IoType::Write}; + static constexpr StatsBackend backends[]{StatsBackend::Fastpath, StatsBackend::Fallback}; + + for (size_t gpuId{}; gpuId < Stats::MaxGpus; ++gpuId) { + for (const auto &backend : backends) { + const PerGpuStatsV1 *g{s->getPerGpuStats(gpuId, backend)}; + if (g == nullptr) { + continue; + } + for (const auto &ioType : ioTypes) { + const auto [sizeHist, countHist, timeHist, errorHist] = g->getHistograms(ioType); + if (sizeHist == nullptr || countHist == nullptr || timeHist == nullptr || + errorHist == nullptr) { + continue; + } + uint64_t bytes{sizeHist->accumulate()}; + uint64_t count{countHist->accumulate()}; + uint64_t timeUs{timeHist->accumulate()}; + uint64_t errors{errorHist->accumulate()}; + if (ioType == IoType::Read) { + stats->read_ops.ok += count; + stats->read_ops.err += errors; + stats->read_bytes += bytes; + stats->read_lat_sum_us += timeUs; + } + else { + stats->write_ops.ok += count; + stats->write_ops.err += errors; + stats->write_bytes += bytes; + stats->write_lat_sum_us += timeUs; + } + } + } + } + + stats->hdl_register_ops.ok = s->getFileRegistrations().load(); + stats->buf_register_ops.ok = s->getBufferRegistrations().load(); + + if (stats->read_ops.ok > 0) { + stats->read_lat_avg_us = stats->read_lat_sum_us / stats->read_ops.ok; + } + if (stats->write_ops.ok > 0) { + stats->write_lat_avg_us = stats->write_lat_sum_us / stats->write_ops.ok; + } + if (stats->read_lat_sum_us > 0) { + stats->read_bw_bytes_per_sec = static_cast( + static_cast(stats->read_bytes) * 1e6 / static_cast(stats->read_lat_sum_us)); + } + if (stats->write_lat_sum_us > 0) { + stats->write_bw_bytes_per_sec = static_cast( + static_cast(stats->write_bytes) * 1e6 / static_cast(stats->write_lat_sum_us)); + } + + return {hipFileSuccess, hipSuccess}; + } + catch (...) { + return handle_exception(); + } +} + +hipFileError_t +hipFileGetStatsL2(hipFileStatsLevel2_t *stats) +{ + try { + if (stats == nullptr) { + return {hipFileInvalidValue, hipSuccess}; + } + *stats = {}; + return hipFile::hipFileGetStatsL1(&stats->basic); + } + catch (...) { + return handle_exception(); + } +} + +hipFileError_t +hipFileGetStatsL3(hipFileStatsLevel3_t *stats) +{ + try { + if (stats == nullptr) { + return {hipFileInvalidValue, hipSuccess}; + } + const Stats *s{Context::get()->getStats()}; + if (s == nullptr) { + return {hipFileInternalError, hipSuccess}; + } + + *stats = {}; + + hipFileError_t err{hipFile::hipFileGetStatsL2(&stats->detailed)}; + if (err.err != hipFileSuccess) { + return err; + } + + uint64_t globalBufRegs{s->getBufferRegistrations().load()}; + + uint32_t numGpus{}; + for (size_t gpuId{}; gpuId < Stats::MaxGpus; ++gpuId) { + if (!s->gpuInUse(gpuId)) { + continue; + } + ++numGpus; + + hipFilePerGpuStats_t &g{stats->per_gpu_stats[gpuId]}; + + // Fastpath maps to nvfs; fallback maps to posix + static constexpr StatsBackend backends[]{StatsBackend::Fastpath, StatsBackend::Fallback}; + for (const auto &backend : backends) { + const PerGpuStatsV1 *pg{s->getPerGpuStats(gpuId, backend)}; + if (pg == nullptr) { + continue; + } + + const auto [rSizeHist, rCountHist, rTimeHist, rErrorHist] = pg->getHistograms(IoType::Read); + const auto [wSizeHist, wCountHist, wTimeHist, wErrorHist] = pg->getHistograms(IoType::Write); + + uint64_t rBytes{rSizeHist ? rSizeHist->accumulate() : 0}; + uint64_t rCount{rCountHist ? rCountHist->accumulate() : 0}; + uint64_t rTimeUs{rTimeHist ? rTimeHist->accumulate() : 0}; + uint64_t rErrors{rErrorHist ? rErrorHist->accumulate() : 0}; + + uint64_t wBytes{wSizeHist ? wSizeHist->accumulate() : 0}; + uint64_t wCount{wCountHist ? wCountHist->accumulate() : 0}; + uint64_t wTimeUs{wTimeHist ? wTimeHist->accumulate() : 0}; + uint64_t wErrors{wErrorHist ? wErrorHist->accumulate() : 0}; + + g.read_bytes += rBytes; + g.read_duration_us += rTimeUs; + g.n_total_reads += rCount; + g.n_reads_err += rErrors; + g.n_unaligned_reads += pg->unalignedCount[static_cast(IoType::Read)].load(); + + g.writes_bytes += wBytes; + g.write_duration_us += wTimeUs; + g.n_total_writes += wCount; + g.n_writes_err += wErrors; + g.n_unaligned_writes += pg->unalignedCount[static_cast(IoType::Write)].load(); + + if (backend == StatsBackend::Fastpath) { + g.n_nvfs_reads = rCount; + g.n_nvfs_writes = wCount; + } + else { + g.n_posix_reads = rCount; + g.n_posix_writes = wCount; + } + } + + if (g.read_duration_us > 0) { + g.read_bw_bytes_per_sec = static_cast(static_cast(g.read_bytes) * 1e6 / + static_cast(g.read_duration_us)); + } + if (g.write_duration_us > 0) { + g.write_bw_bytes_per_sec = static_cast(static_cast(g.writes_bytes) * 1e6 / + static_cast(g.write_duration_us)); + } + + // Buffer registrations are a global counter; report it on each active GPU + // as the best available approximation. + g.n_mmap = globalBufRegs; + g.n_mmap_ok = globalBufRegs; + } + + stats->num_gpus = numGpus; + return {hipFileSuccess, hipSuccess}; + } + catch (...) { + return handle_exception(); + } +} + } // namespace diff --git a/projects/hipfile/test/amd_detail/api_trace.cpp b/projects/hipfile/test/amd_detail/api_trace.cpp index f73397f3fdc0..b06d32899a6e 100644 --- a/projects/hipfile/test/amd_detail/api_trace.cpp +++ b/projects/hipfile/test/amd_detail/api_trace.cpp @@ -86,6 +86,10 @@ TEST(ApiTrace, DispatchTable_all_function_pointers_populated) EXPECT_NE(t->pfn_hipfile_set_parameter_size_t, nullptr); EXPECT_NE(t->pfn_hipfile_set_parameter_bool, nullptr); EXPECT_NE(t->pfn_hipfile_set_parameter_string, nullptr); + // HIPFILE_RUNTIME_API_TABLE_STEP_VERSION == 1 + EXPECT_NE(t->pfn_hipfile_get_stats_l1, nullptr); + EXPECT_NE(t->pfn_hipfile_get_stats_l2, nullptr); + EXPECT_NE(t->pfn_hipfile_get_stats_l3, nullptr); } // Smoke test the call-through. With no profiler attached, going through the diff --git a/projects/hipfile/test/amd_detail/hipfile-api.cpp b/projects/hipfile/test/amd_detail/hipfile-api.cpp index beb67fe6979e..527814efe459 100644 --- a/projects/hipfile/test/amd_detail/hipfile-api.cpp +++ b/projects/hipfile/test/amd_detail/hipfile-api.cpp @@ -27,8 +27,10 @@ #include "mhip.h" #include "mmountinfo.h" #include "mstate.h" +#include "mstats.h" #include "msys.h" #include "state.h" +#include "stats.h" #include #include @@ -335,4 +337,359 @@ TEST_P(HipFileIoBackendSelectionParam, HipFileIoIssuesIoToHighestScoringBackend) INSTANTIATE_TEST_SUITE_P(HipFileIoBackendSelection, HipFileIoBackendSelectionParam, ::testing::Values(IoType::Read, IoType::Write)); +// *********************************************************************** +// hipFileGetStatsL1 tests +// *********************************************************************** + +struct HipFileGetStatsL1 : public HipFileUnopened { + Stats stats{}; + StrictMock mstats{}; + + void SetUp() override + { + EXPECT_CALL(mstats, getStats).WillRepeatedly(testing::Return(&stats)); + stats.setLevel(StatsLevel::Basic); + } +}; + +TEST_F(HipFileGetStatsL1, NullptrReturnsInvalidValue) +{ + EXPECT_EQ(hipFileGetStatsL1(nullptr), HIPFILE_INVALID_VALUE); +} + +TEST_F(HipFileGetStatsL1, NullStatsServerReturnsInternalError) +{ + EXPECT_CALL(mstats, getStats).WillRepeatedly(testing::Return(nullptr)); + hipFileStatsLevel1_t out{}; + EXPECT_EQ(hipFileGetStatsL1(&out), HipFileOpError(hipFileInternalError)); +} + +TEST_F(HipFileGetStatsL1, ZeroWhenNoActivity) +{ + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.read_bytes, 0u); + EXPECT_EQ(out.write_bytes, 0u); + EXPECT_EQ(out.read_ops.ok, 0u); + EXPECT_EQ(out.write_ops.ok, 0u); +} + +TEST_F(HipFileGetStatsL1, AggregatesReadBytesAcrossGpusAndBackends) +{ + // GPU 0 fastpath: 100 bytes, 1 op, 1000us + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 100; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Read)].buckets[0] = + 1; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioTimeUs[static_cast(IoType::Read)].buckets[0] = + 1000; + // GPU 1 fallback: 200 bytes, 2 ops, 2000us + stats.getPerGpuStats(1, StatsBackend::Fallback) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 200; + stats.getPerGpuStats(1, StatsBackend::Fallback)->ioCount[static_cast(IoType::Read)].buckets[0] = + 2; + stats.getPerGpuStats(1, StatsBackend::Fallback)->ioTimeUs[static_cast(IoType::Read)].buckets[0] = + 2000; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.read_bytes, 300u); + EXPECT_EQ(out.read_ops.ok, 3u); + EXPECT_EQ(out.read_lat_sum_us, 3000u); +} + +TEST_F(HipFileGetStatsL1, AggregatesWriteBytesAcrossGpusAndBackends) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Write)] + .buckets[0] = 512; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Write)].buckets[0] = + 4; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioTimeUs[static_cast(IoType::Write)].buckets[0] = + 4000; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.write_bytes, 512u); + EXPECT_EQ(out.write_ops.ok, 4u); + EXPECT_EQ(out.write_lat_sum_us, 4000u); +} + +TEST_F(HipFileGetStatsL1, AggregatesErrorCounts) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->errorCount[static_cast(IoType::Read)] + .buckets[0] = 3; + stats.getPerGpuStats(1, StatsBackend::Fallback) + ->errorCount[static_cast(IoType::Write)] + .buckets[0] = 5; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.read_ops.err, 3u); + EXPECT_EQ(out.write_ops.err, 5u); +} + +TEST_F(HipFileGetStatsL1, PopulatesRegistrationCounters) +{ + stats.getFileRegistrations() = 7; + stats.getBufferRegistrations() = 11; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.hdl_register_ops.ok, 7u); + EXPECT_EQ(out.buf_register_ops.ok, 11u); +} + +TEST_F(HipFileGetStatsL1, DerivesAverageLatency) +{ + // 2 reads totalling 200us -> avg 100us + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Read)].buckets[0] = + 2; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioTimeUs[static_cast(IoType::Read)].buckets[0] = + 200; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.read_lat_avg_us, 100u); + EXPECT_EQ(out.write_lat_avg_us, 0u); +} + +TEST_F(HipFileGetStatsL1, DerivesBandwidth) +{ + // 1e6 bytes over 1e6 us -> 1e6 bytes/sec + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 1000000; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Read)].buckets[0] = + 1; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioTimeUs[static_cast(IoType::Read)].buckets[0] = + 1000000; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.read_bw_bytes_per_sec, 1000000u); +} + +TEST_F(HipFileGetStatsL1, ZeroLatencySumLeavesAverageAndBandwidthZero) +{ + // bytes recorded but no time -> no divide-by-zero + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 1000; + + hipFileStatsLevel1_t out{}; + ASSERT_EQ(hipFileGetStatsL1(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.read_lat_avg_us, 0u); + EXPECT_EQ(out.read_bw_bytes_per_sec, 0u); +} + +// *********************************************************************** +// hipFileGetStatsL2 tests +// *********************************************************************** + +struct HipFileGetStatsL2 : public HipFileUnopened { + Stats stats{}; + StrictMock mstats{}; + + void SetUp() override + { + EXPECT_CALL(mstats, getStats).WillRepeatedly(testing::Return(&stats)); + stats.setLevel(StatsLevel::Basic); + } +}; + +TEST_F(HipFileGetStatsL2, NullptrReturnsInvalidValue) +{ + EXPECT_EQ(hipFileGetStatsL2(nullptr), HIPFILE_INVALID_VALUE); +} + +TEST_F(HipFileGetStatsL2, PopulatesL1BasicFields) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 256; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Read)].buckets[0] = + 1; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioTimeUs[static_cast(IoType::Read)].buckets[0] = + 500; + + hipFileStatsLevel2_t out{}; + ASSERT_EQ(hipFileGetStatsL2(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.basic.read_bytes, 256u); + EXPECT_EQ(out.basic.read_ops.ok, 1u); + EXPECT_EQ(out.basic.read_lat_sum_us, 500u); +} + +TEST_F(HipFileGetStatsL2, SizeHistogramsAreZeroFilled) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 1024; + + hipFileStatsLevel2_t out{}; + ASSERT_EQ(hipFileGetStatsL2(&out), HIPFILE_SUCCESS); + for (size_t i = 0; i < 32; ++i) { + EXPECT_EQ(out.read_size_kb_hist[i], 0u) << "read_size_kb_hist[" << i << "] should be zero"; + EXPECT_EQ(out.write_size_kb_hist[i], 0u) << "write_size_kb_hist[" << i << "] should be zero"; + } +} + +// *********************************************************************** +// hipFileGetStatsL3 tests +// *********************************************************************** + +struct HipFileGetStatsL3 : public HipFileUnopened { + Stats stats{}; + StrictMock mstats{}; + + void SetUp() override + { + EXPECT_CALL(mstats, getStats).WillRepeatedly(testing::Return(&stats)); + stats.setLevel(StatsLevel::Basic); + } +}; + +TEST_F(HipFileGetStatsL3, NullptrReturnsInvalidValue) +{ + EXPECT_EQ(hipFileGetStatsL3(nullptr), HIPFILE_INVALID_VALUE); +} + +TEST_F(HipFileGetStatsL3, NullStatsServerReturnsInternalError) +{ + EXPECT_CALL(mstats, getStats).WillRepeatedly(testing::Return(nullptr)); + hipFileStatsLevel3_t out{}; + EXPECT_EQ(hipFileGetStatsL3(&out), HipFileOpError(hipFileInternalError)); +} + +TEST_F(HipFileGetStatsL3, ZeroWhenNoActivity) +{ + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.num_gpus, 0u); + EXPECT_EQ(out.detailed.basic.read_bytes, 0u); +} + +TEST_F(HipFileGetStatsL3, PopulatesDetailedL2Fields) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Write)] + .buckets[0] = 128; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Write)].buckets[0] = + 1; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.detailed.basic.write_bytes, 128u); +} + +TEST_F(HipFileGetStatsL3, CountsActiveGpus) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + stats.getPerGpuStats(2, StatsBackend::Fallback)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.num_gpus, 2u); +} + +TEST_F(HipFileGetStatsL3, PerGpuReadBytesAggregatesBothBackends) +{ + // GPU 0: fastpath 100 bytes, fallback 50 bytes + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 100; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + stats.getPerGpuStats(0, StatsBackend::Fallback) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 50; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.per_gpu_stats[0].read_bytes, 150u); +} + +TEST_F(HipFileGetStatsL3, FastpathMapsToNvfsAndFallbackMapsToPosix) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioCount[static_cast(IoType::Read)].buckets[0] = + 3; + stats.getPerGpuStats(0, StatsBackend::Fallback)->ioCount[static_cast(IoType::Read)].buckets[0] = + 7; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.per_gpu_stats[0].n_nvfs_reads, 3u); + EXPECT_EQ(out.per_gpu_stats[0].n_posix_reads, 7u); + EXPECT_EQ(out.per_gpu_stats[0].n_total_reads, 10u); +} + +TEST_F(HipFileGetStatsL3, PerGpuUnalignedCountsPopulated) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath)->unalignedCount[static_cast(IoType::Read)] = 4; + stats.getPerGpuStats(0, StatsBackend::Fallback)->unalignedCount[static_cast(IoType::Write)] = 6; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.per_gpu_stats[0].n_unaligned_reads, 4u); + EXPECT_EQ(out.per_gpu_stats[0].n_unaligned_writes, 6u); +} + +TEST_F(HipFileGetStatsL3, PerGpuErrorCountsPopulated) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->errorCount[static_cast(IoType::Read)] + .buckets[0] = 2; + stats.getPerGpuStats(0, StatsBackend::Fallback) + ->errorCount[static_cast(IoType::Write)] + .buckets[0] = 9; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.per_gpu_stats[0].n_reads_err, 2u); + EXPECT_EQ(out.per_gpu_stats[0].n_writes_err, 9u); +} + +TEST_F(HipFileGetStatsL3, PerGpuBandwidthDerived) +{ + // 1e6 bytes over 1e6 us -> 1e6 bytes/sec + stats.getPerGpuStats(0, StatsBackend::Fastpath) + ->ioSizeBytes[static_cast(IoType::Read)] + .buckets[0] = 1000000; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->ioTimeUs[static_cast(IoType::Read)].buckets[0] = + 1000000; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.per_gpu_stats[0].read_bw_bytes_per_sec, 1000000u); +} + +TEST_F(HipFileGetStatsL3, InactiveGpuSlotIsZero) +{ + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + // GPU 1 was never used — its slot must be zeroed + EXPECT_EQ(out.per_gpu_stats[1].read_bytes, 0u); + EXPECT_EQ(out.per_gpu_stats[1].n_total_reads, 0u); +} + +TEST_F(HipFileGetStatsL3, BufferRegistrationsReportedOnActiveGpu) +{ + stats.getBufferRegistrations() = 5; + stats.getPerGpuStats(0, StatsBackend::Fastpath)->inUse = 1; + + hipFileStatsLevel3_t out{}; + ASSERT_EQ(hipFileGetStatsL3(&out), HIPFILE_SUCCESS); + EXPECT_EQ(out.per_gpu_stats[0].n_mmap, 5u); + EXPECT_EQ(out.per_gpu_stats[0].n_mmap_ok, 5u); +} + HIPFILE_WARN_NO_GLOBAL_CTOR_ON