diff --git a/projects/rdc/CHANGELOG.md b/projects/rdc/CHANGELOG.md index 204cb6c4def7..59a47491e936 100644 --- a/projects/rdc/CHANGELOG.md +++ b/projects/rdc/CHANGELOG.md @@ -27,6 +27,14 @@ Full documentation for RDC is available at [ROCm DataCenter Tool User Guide](htt - RDC no longer reports another GPU's XGMI link states, or garbage values that could crash `rdcd`, for a GPU whose link status AMD SMI cannot read. That GPU now reports no links. - `rdci link` now prints an error and exits with a nonzero status when it cannot get the link status, instead of printing nothing and exiting with 0. +- `rdcd` no longer crashes reading `RDC_FI_GPU_CLOCK` or `RDC_FI_MEM_CLOCK` when AMD SMI reports no current clock level, as it does for a power-gated clock domain on some APUs. The field is N/A then. + +- Fields that AMD SMI cannot read are N/A instead of AMD SMI's "not available" value, such as 4294967295 for `RDC_FI_GPU_GFX_BUSY_INST` or 65535 for `RDC_FI_GPU_VCN_BUSY_INST` on Radeon GPUs. This covers the utilization and clock fields, the `RDC_FI_GPU_*_BUSY_INST` fields, and the `RDC_HEALTH_*` throttle and residency counters. + +- `RDC_FI_ECC_CORRECT_TOTAL`, `RDC_FI_ECC_UNCORRECT_TOTAL`, and `RDC_FI_ECC_DEFERRED_TOTAL` are N/A on GPUs without ECC, like the per-block ECC fields, instead of 0. The memory health check still checks such a GPU's retired pages. + +- `RDC_FI_GPU_MEMORY_CUR_BANDWIDTH` is N/A instead of 0 when the GPU reports no memory activity. + - `RDC_FI_GPU_MEMORY_CUR_BANDWIDTH` no longer reports zero under DMA or copy-only memory traffic on GPUs whose instantaneous UMC activity does not reflect DMA transfers. It now derives memory activity from the `mem_activity_acc` accumulator over firmware time, and falls back to the instantaneous `umc_activity` reading when the accumulator or firmware timestamp is unavailable. - `rdci topo` and `rdc_device_topology_get()` now report the links of the requested GPU. They reported links between other GPU pairs, which showed the wrong link types on systems where GPU pairs are not all linked the same way, such as XGMI hives joined by PCIe. diff --git a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc index f6c505f3d14c..8a1276830390 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc @@ -61,6 +61,14 @@ static const std::unordered_map rdc_evnt_2_smi {RDC_EVNT_XGMI_5_THRPUT, AMDSMI_EVNT_XGMI_DATA_OUT_5}, }; +// amd-smi reports a value it cannot read as the maximum of the field's type. Some uint32_t +// fields hold the uint16_t maximum of the gpu_metrics field they are copied from, as older +// amd-smi did for power. Only for values that stay far below 65535: percentages, W, MHz, +// counts. +static bool smi_not_available(uint32_t v) { + return v == std::numeric_limits::max() || v == std::numeric_limits::max(); +} + RdcMetricFetcherImpl::RdcMetricFetcherImpl() : task_started_(true) { // kick off another thread for async fetch updater_ = std::async(std::launch::async, [this]() { @@ -301,6 +309,7 @@ void RdcMetricFetcherImpl::get_ecc_total(uint32_t gpu_index, rdc_field_t field_i amdsmi_status_t err = AMDSMI_STATUS_SUCCESS; uint64_t correctable_count = 0; uint64_t uncorrectable_count = 0; + bool counted = false; // a block reported its counts amdsmi_ras_err_state_t err_state = AMDSMI_RAS_ERR_STATE_INVALID; amdsmi_processor_handle processor_handle = nullptr; @@ -324,10 +333,12 @@ void RdcMetricFetcherImpl::get_ecc_total(uint32_t gpu_index, rdc_field_t field_i if (err == AMDSMI_STATUS_SUCCESS) { correctable_count += ec.correctable_count; uncorrectable_count += ec.uncorrectable_count; + counted = true; } } - value->status = AMDSMI_STATUS_SUCCESS; + // Without a count from any block, as on GPUs without ECC, report N/A rather than 0 errors. + value->status = counted ? AMDSMI_STATUS_SUCCESS : AMDSMI_STATUS_NOT_SUPPORTED; value->type = INTEGER; if (field_id == RDC_FI_ECC_CORRECT_TOTAL) { value->value.l_int = correctable_count; @@ -422,6 +433,7 @@ void RdcMetricFetcherImpl::get_ecc_deferred_total(uint32_t gpu_index, rdc_field_ amdsmi_status_t err = get_processor_handle_from_id(gpu_index, &processor_handle); uint64_t deferred_count = 0; + bool counted = false; // a block reported its counts // amdsmi_gpu_block_t is a 64-bit flag enum; a 32-bit counter wraps to 0 and never exits. for (uint64_t b = AMDSMI_GPU_BLOCK_FIRST; b <= AMDSMI_GPU_BLOCK_LAST; b = b * 2) { amdsmi_ras_err_state_t err_state = AMDSMI_RAS_ERR_STATE_INVALID; @@ -433,10 +445,11 @@ void RdcMetricFetcherImpl::get_ecc_deferred_total(uint32_t gpu_index, rdc_field_ err = amdsmi_get_gpu_ecc_count(processor_handle, static_cast(b), &ec); if (err == AMDSMI_STATUS_SUCCESS) { deferred_count += ec.deferred_count; + counted = true; } } - value->status = AMDSMI_STATUS_SUCCESS; + value->status = counted ? AMDSMI_STATUS_SUCCESS : AMDSMI_STATUS_NOT_SUPPORTED; value->type = INTEGER; value->value.l_int = deferred_count; } @@ -547,6 +560,10 @@ rdc_status_t RdcMetricFetcherImpl::bulk_fetch_smi_fields( RDC_FI_GPU_UTIL // average_gfx_activity }; + // gpu_metrics reports a value it does not have as UINT16_MAX. Such a field is left out + // here, as for a power of 0, and fetched one by one. + constexpr uint16_t kU16NotSupported = std::numeric_limits::max(); + // To prevent always call the bulk API even if it is not supported, // the static is used to cache last try. static amdsmi_status_t rs = AMDSMI_STATUS_SUCCESS; @@ -587,13 +604,16 @@ rdc_status_t RdcMetricFetcherImpl::bulk_fetch_smi_fields( switch (field_id) { case RDC_FI_GPU_CLOCK: // current_gfxclk * 1000000 + if (gpu_metrics.current_gfxclk == kU16NotSupported) continue; value.field_value.value.l_int = static_cast(gpu_metrics.current_gfxclk) * 1000000; break; case RDC_FI_MEMORY_TEMP: // temperature_mem * 1000 + if (gpu_metrics.temperature_mem == kU16NotSupported) continue; value.field_value.value.l_int = static_cast(gpu_metrics.temperature_mem) * 1000; break; case RDC_FI_GPU_TEMP: // temperature_edge * 1000 + if (gpu_metrics.temperature_edge == kU16NotSupported) continue; value.field_value.value.l_int = static_cast(gpu_metrics.temperature_edge) * 1000; break; case RDC_FI_POWER_USAGE: // average_socket_power @@ -617,6 +637,7 @@ rdc_status_t RdcMetricFetcherImpl::bulk_fetch_smi_fields( value.field_value.value.l_int *= 1000000; break; case RDC_FI_GPU_UTIL: // average_gfx_activity + if (gpu_metrics.average_gfx_activity == kU16NotSupported) continue; value.field_value.value.l_int = static_cast(gpu_metrics.average_gfx_activity); break; default: @@ -700,6 +721,11 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel return; } + // Map a narrower field's "not available" maximum to uint64_t's maximum. + auto widen = [](auto v) -> uint64_t { + using T = decltype(v); + return v == std::numeric_limits::max() ? std::numeric_limits::max() : v; + }; const std::unordered_map rdc_field_to_gpu_metrics = { {RDC_FI_XGMI_0_READ_KB, gpu_metrics.xgmi_read_data_acc[0]}, {RDC_FI_XGMI_1_READ_KB, gpu_metrics.xgmi_read_data_acc[1]}, @@ -726,10 +752,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel {RDC_FI_GFX_ACTIVITY_ACC, gpu_metrics.gfx_activity_acc}, {RDC_FI_MEM_ACTIVITY_ACC, gpu_metrics.mem_activity_acc}, {RDC_FI_ACCUMULATION_COUNTER, gpu_metrics.accumulation_counter}, - {RDC_FI_GPU_GFX_BUSY_INST, - static_cast(gpu_metrics.xcp_stats[0].gfx_busy_inst[0])}, - {RDC_FI_GPU_VCN_BUSY_INST, static_cast(gpu_metrics.xcp_stats[0].vcn_busy[0])}, - {RDC_FI_GPU_JPEG_BUSY_INST, static_cast(gpu_metrics.xcp_stats[0].jpeg_busy[0])}, + {RDC_FI_GPU_GFX_BUSY_INST, widen(gpu_metrics.xcp_stats[0].gfx_busy_inst[0])}, + {RDC_FI_GPU_VCN_BUSY_INST, widen(gpu_metrics.xcp_stats[0].vcn_busy[0])}, + {RDC_FI_GPU_JPEG_BUSY_INST, widen(gpu_metrics.xcp_stats[0].jpeg_busy[0])}, }; // In gpu_metrics,the max value means not supported @@ -772,7 +797,11 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel value->status = amdsmi_get_gpu_activity(processor_handle, &engine_usage); value->type = INTEGER; if (value->status == AMDSMI_STATUS_SUCCESS) { - value->value.l_int = static_cast(engine_usage.umc_activity); + if (smi_not_available(engine_usage.umc_activity)) { + value->status = AMDSMI_STATUS_NOT_SUPPORTED; + } else { + value->value.l_int = static_cast(engine_usage.umc_activity); + } } break; } @@ -804,9 +833,11 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel // Instantaneous UMC controller activity. On some ASICs this only reflects // compute-shader memory traffic and reads zero under DMA/copy traffic. double activity_pct = 0.0; + bool have_activity = false; if (metrics_status == AMDSMI_STATUS_SUCCESS && gpu_metrics.average_umc_activity != kU16NotSupported) { activity_pct = static_cast(gpu_metrics.average_umc_activity); + have_activity = true; } // Prefer the memory-activity accumulator when available: unlike the @@ -819,6 +850,7 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel std::lock_guard lock(mem_activity_mutex_); auto prev = mem_activity_cache_.find(gpu_index); const bool have_prev = prev != mem_activity_cache_.end(); + have_activity = have_activity || have_prev; // a first sample has no rate yet activity_pct = derive_mem_activity_percent( activity_pct, have_prev, have_prev ? prev->second.mem_activity_acc : 0, have_prev ? prev->second.firmware_timestamp : 0, gpu_metrics.mem_activity_acc, @@ -841,6 +873,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel : 0; // Don't mask a metrics-fetch failure with vram-info success. value->status = (metrics_status != AMDSMI_STATUS_SUCCESS) ? metrics_status : vram_status; + if (value->status == AMDSMI_STATUS_SUCCESS && !have_activity) { + value->status = AMDSMI_STATUS_NOT_SUPPORTED; + } break; } case RDC_FI_GPU_MEMORY_FREE: { @@ -1007,7 +1042,12 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel value->status = amdsmi_get_clk_freq(processor_handle, clk_type, &f); value->type = INTEGER; if (value->status == AMDSMI_STATUS_SUCCESS) { - value->value.l_int = f.frequency[f.current]; + // current is UINT32_MAX when no level is current, as for a power-gated clock domain. + if (f.current < f.num_supported && f.current < AMDSMI_MAX_NUM_FREQUENCIES) { + value->value.l_int = static_cast(f.frequency[f.current]); + } else { + value->status = AMDSMI_STATUS_NOT_SUPPORTED; + } } break; } @@ -1034,7 +1074,11 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel value->status = amdsmi_get_clock_info(processor_handle, clk_type, &clk_info); value->type = INTEGER; if (value->status == AMDSMI_STATUS_SUCCESS) { - value->value.l_int = static_cast(clk_info.clk) * 1000000; + if (smi_not_available(clk_info.clk)) { + value->status = AMDSMI_STATUS_NOT_SUPPORTED; + } else { + value->value.l_int = static_cast(clk_info.clk) * 1000000; + } } break; } @@ -1043,7 +1087,11 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel value->status = amdsmi_get_gpu_activity(processor_handle, &engine_usage); value->type = INTEGER; if (value->status == AMDSMI_STATUS_SUCCESS) { - value->value.l_int = static_cast(engine_usage.gfx_activity); + if (smi_not_available(engine_usage.gfx_activity)) { + value->status = AMDSMI_STATUS_NOT_SUPPORTED; + } else { + value->value.l_int = static_cast(engine_usage.gfx_activity); + } } break; } @@ -1385,40 +1433,43 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel value->status = Smi2RdcError(ret); value->type = INTEGER; if (value->status == AMDSMI_STATUS_SUCCESS) { + // amd-smi reports a counter the GPU does not provide as UINT64_MAX. + constexpr uint64_t kU64NotSupported = std::numeric_limits::max(); + uint64_t raw = kU64NotSupported; switch (field_id) { case RDC_HEALTH_POWER_THROTTLE_TIME: - value->value.l_int = static_cast(violation_status.acc_ppt_pwr); + raw = violation_status.acc_ppt_pwr; break; case RDC_HEALTH_THERMAL_THROTTLE_TIME: case RDC_HEALTH_SOCKET_THRM_ACC: // alias — same underlying acc_socket_thrm counter - value->value.l_int = static_cast(violation_status.acc_socket_thrm); + raw = violation_status.acc_socket_thrm; break; case RDC_HEALTH_VIOLATION_COUNTER: - value->value.l_int = static_cast(violation_status.acc_counter); + raw = violation_status.acc_counter; break; case RDC_HEALTH_PROCHOT_RESIDENCY_ACC: - value->value.l_int = static_cast(violation_status.acc_prochot_thrm); + raw = violation_status.acc_prochot_thrm; break; case RDC_HEALTH_PROCHOT_RESIDENCY_PCT: - value->value.l_int = static_cast(violation_status.per_prochot_thrm); + raw = violation_status.per_prochot_thrm; break; case RDC_HEALTH_PPT_RESIDENCY_PCT: - value->value.l_int = static_cast(violation_status.per_ppt_pwr); + raw = violation_status.per_ppt_pwr; break; case RDC_HEALTH_SOCKET_THRM_PCT: - value->value.l_int = static_cast(violation_status.per_socket_thrm); + raw = violation_status.per_socket_thrm; break; case RDC_HEALTH_VR_THRM_ACC: - value->value.l_int = static_cast(violation_status.acc_vr_thrm); + raw = violation_status.acc_vr_thrm; break; case RDC_HEALTH_VR_THRM_PCT: - value->value.l_int = static_cast(violation_status.per_vr_thrm); + raw = violation_status.per_vr_thrm; break; case RDC_HEALTH_HBM_THRM_ACC: - value->value.l_int = static_cast(violation_status.acc_hbm_thrm); + raw = violation_status.acc_hbm_thrm; break; case RDC_HEALTH_HBM_THRM_PCT: - value->value.l_int = static_cast(violation_status.per_hbm_thrm); + raw = violation_status.per_hbm_thrm; break; case RDC_HEALTH_GFX_CLK_LMT_PWR_ACC: case RDC_HEALTH_GFX_CLK_LMT_PWR_PCT: @@ -1428,9 +1479,7 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel case RDC_HEALTH_LOW_UTIL_PCT: case RDC_HEALTH_GFX_CLK_LMT_TOTAL_ACC: case RDC_HEALTH_GFX_CLK_LMT_TOTAL_PCT: { - // Driver 1.8 XCP/XCC fields — max uint64 means not supported - const auto not_sup = std::numeric_limits::max(); - uint64_t raw = not_sup; + // Driver 1.8 XCP/XCC fields switch (field_id) { case RDC_HEALTH_GFX_CLK_LMT_PWR_ACC: raw = violation_status.acc_gfx_clk_below_host_limit_pwr[0][0]; @@ -1459,16 +1508,16 @@ rdc_status_t RdcMetricFetcherImpl::fetch_gpu_field_(uint32_t gpu_index, rdc_fiel default: break; } - if (raw == not_sup) { - value->status = AMDSMI_STATUS_NOT_SUPPORTED; - } else { - value->value.l_int = static_cast(raw); - } break; } default: break; } + if (raw == kU64NotSupported) { + value->status = AMDSMI_STATUS_NOT_SUPPORTED; + } else { + value->value.l_int = static_cast(raw); + } } break; } diff --git a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc index 50af681bf9cd..cd3abada33c5 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc @@ -30,6 +30,7 @@ THE SOFTWARE. #include #include +#include "amd_smi/amdsmi.h" #include "common/rdc_utils.h" #include "rdc/rdc.h" #include "rdc_lib/RdcLogger.h" @@ -614,10 +615,12 @@ rdc_status_t RdcWatchTableImpl::memory_check(rdc_gpu_group_t group_id, uint32_t rdc_field_value start = {}, end = {}; rdc_status_t result = get_start_end_values(group_id, gpu_index, RDC_FI_ECC_UNCORRECT_TOTAL, 0, nullptr, &end); - if (result != RDC_ST_OK) return result; + // A GPU without ECC has no count to check, but can still have retired pages. The + // fetcher returns RDC_ST_SMI_ERROR for every failure and keeps amd-smi's status in the value. + if (result != RDC_ST_OK && end.status != AMDSMI_STATUS_NOT_SUPPORTED) return result; uint64_t ecc_uncorrectable_count = 0; - ecc_uncorrectable_count = end.value.l_int; + if (result == RDC_ST_OK) ecc_uncorrectable_count = end.value.l_int; if (ecc_uncorrectable_count > 0) { rdc_health_incidents_t* incident = &response->incidents[response->incidents_count]; diff --git a/projects/rdc/tests/ci/test_fake_gpu_not_available.py b/projects/rdc/tests/ci/test_fake_gpu_not_available.py new file mode 100644 index 000000000000..378afe5cd1ef --- /dev/null +++ b/projects/rdc/tests/ci/test_fake_gpu_not_available.py @@ -0,0 +1,140 @@ +"""Values amd-smi cannot read, against the fake amd-smi: RDC reports N/A, not a number. + +Runs like test_fake_gpu.py (same environment variables and helpers). RDC_FAKE_SENTINEL makes +a call succeed with what amd-smi reports for a value it cannot read; see +tests/fake_amdsmi/README.md. +""" + +import pytest + +# The helpers and the `tools` fixture come from test_fake_gpu.py, not conftest.py: +# tests/ci/conftest.py is not installed with the suite (share/rdc/rdctst_tests/fake_gpu). +from test_fake_gpu import Rdcd, _created_id, golden, tools # noqa: F401 (fixture) + +# amd-smi call that reports "not available" on GPU 1 -> RDC fields that must be N/A there. +NOT_AVAILABLE = { + # No current clock level (a power-gated clock domain): rdcd read past frequency[]. + "amdsmi_get_clk_freq": [100, 101], # GPU_CLOCK, MEM_CLOCK + "amdsmi_get_power_info": [300], # POWER_USAGE + "amdsmi_get_gpu_activity": [500, 505], # GPU_UTIL, GPU_MEMORY_ACTIVITY + "amdsmi_get_clock_info": [104, 105, 106], # SOC_CLOCK, VCLK0, DCLK0 + "amdsmi_get_gpu_metrics_info": [507], # GPU_MEMORY_CUR_BANDWIDTH + "amdsmi_get_violation_status": [ + 3006, # RDC_HEALTH_POWER_THROTTLE_TIME + 3007, # RDC_HEALTH_THERMAL_THROTTLE_TIME + 3008, # RDC_HEALTH_VIOLATION_COUNTER + 3009, # RDC_HEALTH_PROCHOT_RESIDENCY_ACC + 3010, # RDC_HEALTH_PROCHOT_RESIDENCY_PCT + 3011, # RDC_HEALTH_PPT_RESIDENCY_PCT + 3012, # RDC_HEALTH_SOCKET_THRM_ACC + 3013, # RDC_HEALTH_SOCKET_THRM_PCT + 3014, # RDC_HEALTH_VR_THRM_ACC + 3015, # RDC_HEALTH_VR_THRM_PCT + 3016, # RDC_HEALTH_HBM_THRM_ACC + 3017, # RDC_HEALTH_HBM_THRM_PCT + 3018, # RDC_HEALTH_GFX_CLK_LMT_PWR_ACC + 3019, # RDC_HEALTH_GFX_CLK_LMT_PWR_PCT + 3020, # RDC_HEALTH_GFX_CLK_LMT_THM_ACC + 3021, # RDC_HEALTH_GFX_CLK_LMT_THM_PCT + 3022, # RDC_HEALTH_LOW_UTIL_ACC + 3023, # RDC_HEALTH_LOW_UTIL_PCT + 3024, # RDC_HEALTH_GFX_CLK_LMT_TOTAL_ACC + 3025, # RDC_HEALTH_GFX_CLK_LMT_TOTAL_PCT + ], +} + + +@pytest.fixture +def start_rdcd(tools, tmp_path): + """Start rdcd with fake settings; faults apply from the start of the test.""" + started = [] + + def start(**fake_env): + d = Rdcd(tools, tmp_path, **fake_env).start() + started.append(d) + d.arm() + return d + + yield start + for d in started: + code = d.stop() + assert code == 0, f"rdcd exited with {code} after SIGTERM\n{d.output()}" + + +def _dmon(d, fields, gpus=(0, 1)): + """{gpu: {field: value}} from one dmon sample. + + Columns are matched by position: dmon runs header names of 20 or more characters together. + """ + out = d.ok( + "dmon", + "-e", + ",".join(map(str, fields)), + "-i", + ",".join(map(str, gpus)), + "-c", + "1", + "-d", + "500", + ) + rows = {} + for line in out.splitlines(): + cols = line.split() + if cols and cols[0].isdigit(): + assert len(cols) == len(fields) + 1, f"malformed dmon row {line!r} in:\n{out}" + rows[int(cols[0])] = dict(zip(fields, cols[1:])) + assert sorted(rows) == sorted(gpus), out + return rows + + +@pytest.mark.parametrize("api", sorted(NOT_AVAILABLE)) +def test_value_amd_smi_cannot_read_is_na(start_rdcd, api): + d = start_rdcd(RDC_FAKE_SENTINEL=f"{api}@1") + fields = NOT_AVAILABLE[api] + rows = _dmon(d, fields) + for field in fields: + assert rows[1][field] == "N/A", f"GPU 1 field {field}: {rows[1][field]}, not N/A" + assert rows[0][field] != "N/A", f"GPU 0 field {field} is N/A" + if field in golden.FIELDS: + assert rows[0][field] == str(golden.expected(field, 0)), rows[0] + + +def test_busy_values_the_gpu_does_not_report_are_na(start_rdcd): + # Like an RDNA GPU's, the fake's gpu_metrics has no per-XCP busy values (so no GPU + # shows a number here). + d = start_rdcd() + rows = _dmon(d, [519, 520, 521]) # GPU_GFX_BUSY_INST, GPU_VCN_BUSY_INST, GPU_JPEG_BUSY_INST + for gpu, values in rows.items(): + assert set(values.values()) == {"N/A"}, f"GPU {gpu}: {values}" + + +def test_ecc_totals_are_na_without_ecc(start_rdcd): + # amd-smi on a GPU without ECC fails every block's ECC status with NOT_SUPPORTED. + d = start_rdcd(RDC_FAKE_FAIL="amdsmi_get_gpu_ecc_status@1=AMDSMI_STATUS_NOT_SUPPORTED") + fields = [600, 601, 659] # ECC_CORRECT_TOTAL, ECC_UNCORRECT_TOTAL, ECC_DEFERRED_TOTAL + rows = _dmon(d, fields) + assert set(rows[1].values()) == {"N/A"}, f"GPU 1: {rows[1]}" + assert rows[0][600] == str(golden.reading("ECC_UMC_CORRECTABLE", 0)), rows[0] + assert "N/A" not in rows[0].values(), rows[0] + + +def test_health_checks_pages_of_a_gpu_without_ecc(start_rdcd): + d = start_rdcd(RDC_FAKE_FAIL="amdsmi_get_gpu_ecc_status@1=AMDSMI_STATUS_NOT_SUPPORTED") + gid = _created_id(d.ok("group", "-c", "no_ecc")) + d.ok("group", "-g", gid, "-a", "1") + d.ok("health", "-g", gid, "-s", "m") + out = d.ok("health", "-g", gid, "-c") + # Fake GPU 1 has one bad page, pending retirement. + assert "Detected 1 pending retired page(s)." in out, out + + +def test_bulk_fetch_leaves_metrics_the_gpu_does_not_report_to_other_calls(start_rdcd): + d = start_rdcd(RDC_BULK_FETCH_ENABLED="true", RDC_FAKE_SENTINEL="amdsmi_get_gpu_metrics_info@1") + fields = [100, 200, 201, 300, 500] # what RDC reads in bulk from gpu_metrics + rows = _dmon(d, fields) + for gpu in (0, 1): + for field in fields: + want = str(golden.expected(field, gpu)) + assert rows[gpu][field] == want, ( + f"GPU {gpu} field {field}: {rows[gpu][field]}, not {want}" + ) diff --git a/projects/rdc/tests/fake_amdsmi/README.md b/projects/rdc/tests/fake_amdsmi/README.md index 6ea1d04d5828..ec00a58e3a58 100644 --- a/projects/rdc/tests/fake_amdsmi/README.md +++ b/projects/rdc/tests/fake_amdsmi/README.md @@ -29,7 +29,7 @@ a value shown for the wrong GPU or the wrong field fails the comparison. | `RDC_FAKE_HIVES=H` | XGMI hives, 1 to 128; GPU `g` is in hive `g % H` (default 1) | | `RDC_FAKE_FAIL=rule,...` | `rule` is `api[@gpu][=status]` (`gpu` is the processor index, which counts partitions when `RDC_FAKE_PARTITIONS` is set): the call returns `status` (a number or an `AMDSMI_STATUS_*` name, default `AMDSMI_STATUS_UNKNOWN_ERROR`; anything else aborts). `api` `*` matches every call that takes a GPU. Calls without a GPU argument, such as `amdsmi_init`, match only rules without `@gpu`. | | `RDC_FAKE_POISON=1` | a failing call first fills its outputs with `0xA5` bytes, as a caller that ignores the status would read from a dirty stack | -| `RDC_FAKE_SENTINEL=api[@gpu],...` | the call succeeds but reports amd-smi's "not available" value (all bits set); honored by `amdsmi_get_power_info`, `amdsmi_get_gpu_activity`, `amdsmi_get_gpu_busy_percent`, `amdsmi_get_clock_info` and `amdsmi_get_gpu_metrics_info` | +| `RDC_FAKE_SENTINEL=api[@gpu],...` | the call succeeds but reports what amd-smi reports for a value it cannot read: all bits set in `amdsmi_get_power_info`, `amdsmi_get_gpu_metrics_info` and `amdsmi_get_violation_status`; activities of `0xFFFF` in `amdsmi_get_gpu_activity`; `clk` = `UINT32_MAX` in `amdsmi_get_clock_info`; `current` = `UINT32_MAX` (no current level) in `amdsmi_get_clk_freq` | | `RDC_FAKE_ARM=path` | `FAIL` and `SENTINEL` rules apply only once `path` exists, so a test can set RDC up first | | `RDC_FAKE_EVENT_WAIT_MS=ms` | upper bound for the event-notification wait (default 100), so `rdcd` stops promptly | | `RDC_FAKE_LOG=path` | append `api gpu` for every call (`-1` for calls without a GPU), plus ` !` when a rule failed the call | diff --git a/projects/rdc/tests/fake_amdsmi/fake_amdsmi.cc b/projects/rdc/tests/fake_amdsmi/fake_amdsmi.cc index 65d6f42bfede..e5a576f418b6 100644 --- a/projects/rdc/tests/fake_amdsmi/fake_amdsmi.cc +++ b/projects/rdc/tests/fake_amdsmi/fake_amdsmi.cc @@ -40,10 +40,13 @@ THE SOFTWARE. // RDC_FAKE_POISON=1 a failing call first fills its outputs with 0xA5 bytes, which // is what a caller that ignores the status reads from a dirty // stack -// RDC_FAKE_SENTINEL=api[@gpu],... the call succeeds but reports amd-smi's "not available" -// value (all bits set); honored by amdsmi_get_power_info, -// amdsmi_get_gpu_activity, amdsmi_get_gpu_busy_percent, -// amdsmi_get_clock_info and amdsmi_get_gpu_metrics_info +// RDC_FAKE_SENTINEL=api[@gpu],... the call succeeds but reports what amd-smi reports for a +// value it cannot read: all bits set in amdsmi_get_power_info, +// amdsmi_get_gpu_metrics_info and +// amdsmi_get_violation_status; 0xFFFF activities in +// amdsmi_get_gpu_activity; clk = UINT32_MAX in +// amdsmi_get_clock_info; current = UINT32_MAX in +// amdsmi_get_clk_freq // RDC_FAKE_ARM=path FAIL and SENTINEL rules apply only once path exists, so a // test can set RDC up first // RDC_FAKE_EVENT_WAIT_MS=ms upper bound for the event-notification wait (default 100) @@ -559,8 +562,12 @@ amdsmi_status_t amdsmi_get_gpu_activity(amdsmi_processor_handle processor_handle const FakeProc* p; if (auto st = enter(__func__, processor_handle, &p, info)) return st; if (info == nullptr) return AMDSMI_STATUS_INVAL; - if (sentinel(__func__, p)) return fill_sentinel(info); memset(info, 0, sizeof(*info)); + if (sentinel(__func__, p)) { + // amd-smi copies gpu_metrics' uint16_t marker into these uint32_t fields. + info->gfx_activity = info->umc_activity = info->mm_activity = UINT16_MAX; + return AMDSMI_STATUS_SUCCESS; + } info->gfx_activity = V(kGFX_ACTIVITY_PCT, p->gpu); info->umc_activity = V(kUMC_ACTIVITY_PCT, p->gpu); return AMDSMI_STATUS_SUCCESS; @@ -571,7 +578,6 @@ amdsmi_status_t amdsmi_get_gpu_busy_percent(amdsmi_processor_handle processor_ha const FakeProc* p; if (auto st = enter(__func__, processor_handle, &p, gpu_busy_percent)) return st; if (gpu_busy_percent == nullptr) return AMDSMI_STATUS_INVAL; - if (sentinel(__func__, p)) return fill_sentinel(gpu_busy_percent); *gpu_busy_percent = V(kGFX_ACTIVITY_PCT, p->gpu); return AMDSMI_STATUS_SUCCESS; } @@ -581,7 +587,6 @@ amdsmi_status_t amdsmi_get_clock_info(amdsmi_processor_handle processor_handle, const FakeProc* p; if (auto st = enter(__func__, processor_handle, &p, info)) return st; if (info == nullptr) return AMDSMI_STATUS_INVAL; - if (sentinel(__func__, p)) return fill_sentinel(info); memset(info, 0, sizeof(*info)); if (clk_type == AMDSMI_CLK_TYPE_GFX) { info->clk = V(kGFX_CLK_MHZ, p->gpu); @@ -596,6 +601,8 @@ amdsmi_status_t amdsmi_get_clock_info(amdsmi_processor_handle processor_handle, info->min_clk = 100; info->max_clk = 1000; } + // amd-smi: the range is known, the current clock is not. + if (sentinel(__func__, p)) info->clk = UINT32_MAX; return AMDSMI_STATUS_SUCCESS; } @@ -613,6 +620,8 @@ amdsmi_status_t amdsmi_get_clk_freq(amdsmi_processor_handle processor_handle, f->frequency[0] = (mem ? 900 : 500) * mhz; f->frequency[1] = V(mem ? kMEM_CLK_MHZ : kGFX_CLK_MHZ, p->gpu) * mhz; f->frequency[2] = (mem ? 1300 : 2100) * mhz; + // amd-smi: no level is current, as for a power-gated clock domain. + if (sentinel(__func__, p)) f->current = UINT32_MAX; return AMDSMI_STATUS_SUCCESS; } @@ -1096,6 +1105,8 @@ amdsmi_status_t amdsmi_get_violation_status(amdsmi_processor_handle processor_ha const FakeProc* p; if (auto st = enter(__func__, processor_handle, &p, info)) return st; if (info == nullptr) return AMDSMI_STATUS_INVAL; + // amd-smi starts every counter at UINT64_MAX and fills in what the GPU provides. + if (sentinel(__func__, p)) return fill_sentinel(info); memset(static_cast(info), 0, sizeof(*info)); info->reference_timestamp = now_ms() * 1000; info->acc_counter = now_ms();