From 15432e923f2da5224971d28be07c73267201e4b1 Mon Sep 17 00:00:00 2001 From: Bindhiya Kanangot Balakrishnan Date: Tue, 28 Jul 2026 10:20:23 -0500 Subject: [PATCH 1/5] fix(amdsmi): [ROCM-1553] Bump GPU metrics accumulator counters (#8210) Widen five amdsmi_gpu_metrics_t accumulator counter fields from uint32_t to uint64_t to match the amdgpu pmfw MetricsTable_t definition and prevent overflow: - gfx_activity_acc - mem_activity_acc - pcie_nak_sent_count_acc - pcie_nak_rcvd_count_acc - pcie_lc_perf_other_end_recovery This is a breaking ABI change: the struct layout and field offsets shift, so callers reading these fields must be recompiled. The change cascades through the ROCm SMI headers (rocm_smi.h, rocm_smi_dyn_gpu_metrics.h) and the C++ implementation, and the ctypes and Rust wrappers are regenerated to match. A 7.15.0 CHANGELOG entry documents the widening. --------- Signed-off-by: Bindhiya Kanangot Balakrishnan Signed-off-by: Maisam Arif (cherry picked from commit aeb9da985bd9083f65f9f44cf82e8d9ed6af4279) --- projects/amdsmi/CHANGELOG.md | 3 + projects/amdsmi/include/amd_smi/amdsmi.h | 10 +- .../amdsmi/py-interface/amdsmi_interface.py | 20 ++-- .../amdsmi/py-interface/amdsmi_wrapper.py | 11 +- .../rocm_smi/include/rocm_smi/rocm_smi.h | 10 +- .../rocm_smi/rocm_smi_dyn_gpu_metrics.h | 10 +- .../rust-interface/src/amdsmi_wrapper.rs | 113 ++++++++++-------- projects/amdsmi/src/amd_smi/amd_smi.cc | 19 +-- 8 files changed, 99 insertions(+), 97 deletions(-) diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md index a2a0cd6d933f..c465cda4eda1 100644 --- a/projects/amdsmi/CHANGELOG.md +++ b/projects/amdsmi/CHANGELOG.md @@ -17,6 +17,9 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - Consumers that pass explicit `--gtest_filter` values should update those filters to the new suite names. - See the [AMD SMI test design](docs/conceptual/test-design.md#naming-conventions) for the suite naming convention and `--gtest_filter` usage. +- **Widened five `amdsmi_gpu_metrics_t` accumulator counters from 32-bit to 64-bit** (breaking). + - `gfx_activity_acc`, `mem_activity_acc`, `pcie_nak_sent_count_acc`, `pcie_nak_rcvd_count_acc`, and `pcie_lc_perf_other_end_recovery` are now `uint64_t` to match the amdgpu pmfw metrics header. Recompile callers that read these fields; the struct layout and field offsets have changed. + ### Fixed - **Fixed `amd-smi ras --cper --json` emitting nothing when there are no CPER entries**. diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index 097d8bea9309..f5460400098d 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -2381,8 +2381,8 @@ typedef struct { /* * v1.1 additions */ - uint32_t gfx_activity_acc; //!< new in v1 - uint32_t mem_activity_acc; //!< new in v1 + uint64_t gfx_activity_acc; //!< new in v1 + uint64_t mem_activity_acc; //!< new in v1 uint16_t temperature_hbm[AMDSMI_NUM_HBM_INSTANCES]; //!< new in v1 /* @@ -2435,8 +2435,8 @@ typedef struct { * @brief v1.5 additions */ uint16_t jpeg_activity[AMDSMI_MAX_NUM_JPEG]; //!< JPEG activity percent (encode/decode) - uint32_t pcie_nak_sent_count_acc; //!< PCIE NAK sent accumulated count - uint32_t pcie_nak_rcvd_count_acc; //!< PCIE NAK received accumulated count + uint64_t pcie_nak_sent_count_acc; //!< PCIE NAK sent accumulated count + uint64_t pcie_nak_rcvd_count_acc; //!< PCIE NAK received accumulated count /** * @brief v1.6 additions @@ -2489,7 +2489,7 @@ typedef struct { amdsmi_gpu_xcp_metrics_t xcp_stats[AMDSMI_MAX_NUM_XCP]; /**< XCP (Graphic Cluster Partitions) metrics stats */ - uint32_t pcie_lc_perf_other_end_recovery; //!< PCIE other end recovery counter + uint64_t pcie_lc_perf_other_end_recovery; //!< PCIE other end recovery counter /** * @brief v1.7 additions diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index b82d8e8bb4b8..26b9d68032de 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -5717,10 +5717,10 @@ def amdsmi_get_gpu_metrics_info(processor_handle: processor_handle_t) -> Dict[st gpu_metrics.pcie_link_speed, MaxUIntegerTypes.UINT16_T ), "gfx_activity_acc": _validate_if_max_uint( - gpu_metrics.gfx_activity_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.gfx_activity_acc, MaxUIntegerTypes.UINT64_T ), "mem_activity_acc": _validate_if_max_uint( - gpu_metrics.mem_activity_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.mem_activity_acc, MaxUIntegerTypes.UINT64_T ), "temperature_hbm": _validate_if_max_uint( list(gpu_metrics.temperature_hbm), MaxUIntegerTypes.UINT16_T @@ -5786,10 +5786,10 @@ def amdsmi_get_gpu_metrics_info(processor_handle: processor_handle_t) -> Dict[st list(gpu_metrics.jpeg_activity), MaxUIntegerTypes.UINT16_T, isActivity=True ), "pcie_nak_sent_count_acc": _validate_if_max_uint( - gpu_metrics.pcie_nak_sent_count_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.pcie_nak_sent_count_acc, MaxUIntegerTypes.UINT64_T ), "pcie_nak_rcvd_count_acc": _validate_if_max_uint( - gpu_metrics.pcie_nak_rcvd_count_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.pcie_nak_rcvd_count_acc, MaxUIntegerTypes.UINT64_T ), "accumulation_counter": _validate_if_max_uint( gpu_metrics.accumulation_counter, MaxUIntegerTypes.UINT64_T @@ -5823,7 +5823,7 @@ def amdsmi_get_gpu_metrics_info(processor_handle: processor_handle_t) -> Dict[st "xcp_stats.gfx_below_host_limit_total_acc": list(gpu_metrics.xcp_stats), "xcp_stats.temperature_xcd": list(gpu_metrics.xcp_stats), "pcie_lc_perf_other_end_recovery": _validate_if_max_uint( - gpu_metrics.pcie_lc_perf_other_end_recovery, MaxUIntegerTypes.UINT32_T + gpu_metrics.pcie_lc_perf_other_end_recovery, MaxUIntegerTypes.UINT64_T ), "vram_max_bandwidth": _validate_if_max_uint( gpu_metrics.vram_max_bandwidth, MaxUIntegerTypes.UINT64_T @@ -6053,10 +6053,10 @@ def amdsmi_get_gpu_partition_metrics_info(processor_handle: processor_handle_t) gpu_metrics.pcie_link_speed, MaxUIntegerTypes.UINT16_T ), "gfx_activity_acc": _validate_if_max_uint( - gpu_metrics.gfx_activity_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.gfx_activity_acc, MaxUIntegerTypes.UINT64_T ), "mem_activity_acc": _validate_if_max_uint( - gpu_metrics.mem_activity_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.mem_activity_acc, MaxUIntegerTypes.UINT64_T ), "temperature_hbm": _validate_if_max_uint( list(gpu_metrics.temperature_hbm), MaxUIntegerTypes.UINT16_T @@ -6122,10 +6122,10 @@ def amdsmi_get_gpu_partition_metrics_info(processor_handle: processor_handle_t) list(gpu_metrics.jpeg_activity), MaxUIntegerTypes.UINT16_T, isActivity=True ), "pcie_nak_sent_count_acc": _validate_if_max_uint( - gpu_metrics.pcie_nak_sent_count_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.pcie_nak_sent_count_acc, MaxUIntegerTypes.UINT64_T ), "pcie_nak_rcvd_count_acc": _validate_if_max_uint( - gpu_metrics.pcie_nak_rcvd_count_acc, MaxUIntegerTypes.UINT32_T + gpu_metrics.pcie_nak_rcvd_count_acc, MaxUIntegerTypes.UINT64_T ), "accumulation_counter": _validate_if_max_uint( gpu_metrics.accumulation_counter, MaxUIntegerTypes.UINT64_T @@ -6159,7 +6159,7 @@ def amdsmi_get_gpu_partition_metrics_info(processor_handle: processor_handle_t) "xcp_stats.gfx_below_host_limit_total_acc": list(gpu_metrics.xcp_stats), "xcp_stats.temperature_xcd": list(gpu_metrics.xcp_stats), "pcie_lc_perf_other_end_recovery": _validate_if_max_uint( - gpu_metrics.pcie_lc_perf_other_end_recovery, MaxUIntegerTypes.UINT32_T + gpu_metrics.pcie_lc_perf_other_end_recovery, MaxUIntegerTypes.UINT64_T ), "vram_max_bandwidth": _validate_if_max_uint( gpu_metrics.vram_max_bandwidth, MaxUIntegerTypes.UINT64_T diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index b65dede4da8c..df41a1a7070c 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -2396,8 +2396,8 @@ class struct_amdsmi_gpu_metrics_t(Structure): ('pcie_link_width', ctypes.c_uint16), ('pcie_link_speed', ctypes.c_uint16), ('PADDING_0', ctypes.c_ubyte * 2), - ('gfx_activity_acc', ctypes.c_uint32), - ('mem_activity_acc', ctypes.c_uint32), + ('gfx_activity_acc', ctypes.c_uint64), + ('mem_activity_acc', ctypes.c_uint64), ('temperature_hbm', ctypes.c_uint16 * 4), ('firmware_timestamp', ctypes.c_uint64), ('voltage_soc', ctypes.c_uint16), @@ -2424,8 +2424,8 @@ class struct_amdsmi_gpu_metrics_t(Structure): ('current_vclk0s', ctypes.c_uint16 * 4), ('current_dclk0s', ctypes.c_uint16 * 4), ('jpeg_activity', ctypes.c_uint16 * 32), - ('pcie_nak_sent_count_acc', ctypes.c_uint32), - ('pcie_nak_rcvd_count_acc', ctypes.c_uint32), + ('pcie_nak_sent_count_acc', ctypes.c_uint64), + ('pcie_nak_rcvd_count_acc', ctypes.c_uint64), ('accumulation_counter', ctypes.c_uint64), ('prochot_residency_acc', ctypes.c_uint64), ('ppt_residency_acc', ctypes.c_uint64), @@ -2435,8 +2435,7 @@ class struct_amdsmi_gpu_metrics_t(Structure): ('num_partition', ctypes.c_uint16), ('PADDING_4', ctypes.c_ubyte * 6), ('xcp_stats', struct_amdsmi_gpu_xcp_metrics_t * 8), - ('pcie_lc_perf_other_end_recovery', ctypes.c_uint32), - ('PADDING_5', ctypes.c_ubyte * 4), + ('pcie_lc_perf_other_end_recovery', ctypes.c_uint64), ('vram_max_bandwidth', ctypes.c_uint64), ('xgmi_link_status', ctypes.c_uint16 * 8), ('temperature_hbm_stacks', ctypes.c_uint16 * 12), diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h index d91c079fec0a..474110270553 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h @@ -1520,8 +1520,8 @@ typedef struct { /* * v1.1 additions */ - uint32_t gfx_activity_acc; // new in v1 - uint32_t mem_activity_acc; // new in v1 + uint64_t gfx_activity_acc; // new in v1 + uint64_t mem_activity_acc; // new in v1 uint16_t temperature_hbm[RSMI_NUM_HBM_INSTANCES]; // new in v1 /* @@ -1589,10 +1589,10 @@ typedef struct { uint16_t jpeg_activity[RSMI_MAX_NUM_JPEG_ENGS]; // PCIE NAK sent accumulated count - uint32_t pcie_nak_sent_count_acc; + uint64_t pcie_nak_sent_count_acc; // PCIE NAK received accumulated count - uint32_t pcie_nak_rcvd_count_acc; + uint64_t pcie_nak_rcvd_count_acc; /* * v1.6 additions @@ -1646,7 +1646,7 @@ typedef struct { struct amdgpu_xcp_metrics_t xcp_stats[RSMI_MAX_NUM_XCP]; /* PCIE other end recovery counter */ - uint32_t pcie_lc_perf_other_end_recovery; + uint64_t pcie_lc_perf_other_end_recovery; /* * v1.7 additions diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_dyn_gpu_metrics.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_dyn_gpu_metrics.h index 194a18ba501d..fa2f123e4345 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_dyn_gpu_metrics.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_dyn_gpu_metrics.h @@ -894,7 +894,7 @@ static const auto AMDGpuMetricsBaseSchema = details::AMDGpuMetricSchemaMapType_t details::AMDGpuMetricAttributeInstance_t( "GFX Activity Accumulator", "Accumulator for GFX activity percentage", details::AMDGpuMetricAttributeId_t::GFX_ACTIVITY_ACC, - details::AMDGpuMetricAttributeType_t::TYPE_UINT32, + details::AMDGpuMetricAttributeType_t::TYPE_UINT64, details::AMDGpuMetricUnitType_t::PERCENT), static_cast(0)}}, @@ -903,7 +903,7 @@ static const auto AMDGpuMetricsBaseSchema = details::AMDGpuMetricSchemaMapType_t details::AMDGpuMetricAttributeInstance_t( "Memory Activity Accumulator", "Accumulator for memory activity percentage", details::AMDGpuMetricAttributeId_t::MEM_ACTIVITY_ACC, - details::AMDGpuMetricAttributeType_t::TYPE_UINT32, + details::AMDGpuMetricAttributeType_t::TYPE_UINT64, details::AMDGpuMetricUnitType_t::PERCENT), static_cast(0)}}, @@ -957,7 +957,7 @@ static const auto AMDGpuMetricsBaseSchema = details::AMDGpuMetricSchemaMapType_t details::AMDGpuMetricAttributeInstance_t( "PCIe NAK Sent Count Accumulator", "Accumulator for PCIe NAK sent count", details::AMDGpuMetricAttributeId_t::PCIE_NAK_SENT_COUNT_ACC, - details::AMDGpuMetricAttributeType_t::TYPE_UINT32, + details::AMDGpuMetricAttributeType_t::TYPE_UINT64, details::AMDGpuMetricUnitType_t::COUNT_ACCUMULATOR), static_cast(0)}}, @@ -966,7 +966,7 @@ static const auto AMDGpuMetricsBaseSchema = details::AMDGpuMetricSchemaMapType_t details::AMDGpuMetricAttributeInstance_t( "PCIe NAK Received Count Accumulator", "Accumulator for PCIe NAK received count", details::AMDGpuMetricAttributeId_t::PCIE_NAK_RCVD_COUNT_ACC, - details::AMDGpuMetricAttributeType_t::TYPE_UINT32, + details::AMDGpuMetricAttributeType_t::TYPE_UINT64, details::AMDGpuMetricUnitType_t::COUNT_ACCUMULATOR), static_cast(0)}}, @@ -1061,7 +1061,7 @@ static const auto AMDGpuMetricsBaseSchema = details::AMDGpuMetricSchemaMapType_t details::AMDGpuMetricAttributeInstance_t( "PCIe LC Perf Other End Recovery", "PCIe link control performance other end recovery", details::AMDGpuMetricAttributeId_t::PCIE_LC_PERF_OTHER_END_RECOVERY, - details::AMDGpuMetricAttributeType_t::TYPE_UINT32, + details::AMDGpuMetricAttributeType_t::TYPE_UINT64, details::AMDGpuMetricUnitType_t::COUNT_ACCUMULATOR), static_cast(0)}}, diff --git a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs index 473c9382b301..b617d4e68c0f 100644 --- a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs +++ b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs @@ -137,6 +137,15 @@ pub const AMDSMI_DATE_FORMAT: &[u8; 35] = b"%04d-%02d-%02d:%02d:%02d:%02d.%03d\0 pub const AMDSMI_LIB_VERSION_MAJOR: u32 = 27; pub const AMDSMI_LIB_VERSION_MINOR: u32 = 0; pub const AMDSMI_LIB_VERSION_RELEASE: u32 = 0; +pub const AMDSMI_MAX_VF_COUNT: u32 = 32; +pub const AMDSMI_MAX_DRIVER_NUM: u32 = 2; +pub const AMDSMI_DFC_FW_NUMBER_OF_ENTRIES: u32 = 9; +pub const AMDSMI_MAX_WHITE_LIST_ELEMENTS: u32 = 16; +pub const AMDSMI_MAX_BLACK_LIST_ELEMENTS: u32 = 64; +pub const AMDSMI_MAX_TA_WHITE_LIST_ELEMENTS: u32 = 8; +pub const AMDSMI_MAX_ERR_RECORDS: u32 = 10; +pub const AMDSMI_MAX_PROFILE_COUNT: u32 = 16; +pub const AMDSMI_PF_INDEX: u32 = 31; pub const AMDSMI_MAX_DRIVER_INFO_RSVD: u32 = 64; pub const AMDSMI_MAX_UUID_ELEMENTS: u32 = 16; pub const AMDSMI_MAX_NUM_FREQUENCIES: u32 = 33; @@ -2931,8 +2940,8 @@ pub struct AmdsmiGpuMetricsT { pub current_fan_speed: u16, pub pcie_link_width: u16, pub pcie_link_speed: u16, - pub gfx_activity_acc: u32, - pub mem_activity_acc: u32, + pub gfx_activity_acc: u64, + pub mem_activity_acc: u64, pub temperature_hbm: [u16; 4usize], pub firmware_timestamp: u64, pub voltage_soc: u16, @@ -2956,8 +2965,8 @@ pub struct AmdsmiGpuMetricsT { pub current_vclk0s: [u16; 4usize], pub current_dclk0s: [u16; 4usize], pub jpeg_activity: [u16; 32usize], - pub pcie_nak_sent_count_acc: u32, - pub pcie_nak_rcvd_count_acc: u32, + pub pcie_nak_sent_count_acc: u64, + pub pcie_nak_rcvd_count_acc: u64, pub accumulation_counter: u64, pub prochot_residency_acc: u64, pub ppt_residency_acc: u64, @@ -2966,7 +2975,7 @@ pub struct AmdsmiGpuMetricsT { pub hbm_thm_residency_acc: u64, pub num_partition: u16, pub xcp_stats: [AmdsmiGpuXcpMetricsT; 8usize], - pub pcie_lc_perf_other_end_recovery: u32, + pub pcie_lc_perf_other_end_recovery: u64, pub vram_max_bandwidth: u64, pub xgmi_link_status: [u16; 8usize], pub temperature_hbm_stacks: [u16; 12usize], @@ -2978,7 +2987,7 @@ pub struct AmdsmiGpuMetricsT { } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of AmdsmiGpuMetricsT"][::std::mem::size_of::() - 4720usize]; + ["Size of AmdsmiGpuMetricsT"][::std::mem::size_of::() - 4736usize]; ["Alignment of AmdsmiGpuMetricsT"][::std::mem::align_of::() - 8usize]; ["Offset of field: AmdsmiGpuMetricsT::common_header"] [::std::mem::offset_of!(AmdsmiGpuMetricsT, common_header) - 0usize]; @@ -3045,91 +3054,91 @@ const _: () = { ["Offset of field: AmdsmiGpuMetricsT::gfx_activity_acc"] [::std::mem::offset_of!(AmdsmiGpuMetricsT, gfx_activity_acc) - 80usize]; ["Offset of field: AmdsmiGpuMetricsT::mem_activity_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, mem_activity_acc) - 84usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, mem_activity_acc) - 88usize]; ["Offset of field: AmdsmiGpuMetricsT::temperature_hbm"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_hbm) - 88usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_hbm) - 96usize]; ["Offset of field: AmdsmiGpuMetricsT::firmware_timestamp"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, firmware_timestamp) - 96usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, firmware_timestamp) - 104usize]; ["Offset of field: AmdsmiGpuMetricsT::voltage_soc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, voltage_soc) - 104usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, voltage_soc) - 112usize]; ["Offset of field: AmdsmiGpuMetricsT::voltage_gfx"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, voltage_gfx) - 106usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, voltage_gfx) - 114usize]; ["Offset of field: AmdsmiGpuMetricsT::voltage_mem"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, voltage_mem) - 108usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, voltage_mem) - 116usize]; ["Offset of field: AmdsmiGpuMetricsT::indep_throttle_status"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, indep_throttle_status) - 112usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, indep_throttle_status) - 120usize]; ["Offset of field: AmdsmiGpuMetricsT::current_socket_power"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_socket_power) - 120usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_socket_power) - 128usize]; ["Offset of field: AmdsmiGpuMetricsT::vcn_activity"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, vcn_activity) - 122usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, vcn_activity) - 130usize]; ["Offset of field: AmdsmiGpuMetricsT::gfxclk_lock_status"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, gfxclk_lock_status) - 132usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, gfxclk_lock_status) - 140usize]; ["Offset of field: AmdsmiGpuMetricsT::xgmi_link_width"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_link_width) - 136usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_link_width) - 144usize]; ["Offset of field: AmdsmiGpuMetricsT::xgmi_link_speed"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_link_speed) - 138usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_link_speed) - 146usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_bandwidth_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_bandwidth_acc) - 144usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_bandwidth_acc) - 152usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_bandwidth_inst"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_bandwidth_inst) - 152usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_bandwidth_inst) - 160usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_l0_to_recov_count_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_l0_to_recov_count_acc) - 160usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_l0_to_recov_count_acc) - 168usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_replay_count_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_replay_count_acc) - 168usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_replay_count_acc) - 176usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_replay_rover_count_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_replay_rover_count_acc) - 176usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_replay_rover_count_acc) - 184usize]; ["Offset of field: AmdsmiGpuMetricsT::xgmi_read_data_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_read_data_acc) - 184usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_read_data_acc) - 192usize]; ["Offset of field: AmdsmiGpuMetricsT::xgmi_write_data_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_write_data_acc) - 248usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_write_data_acc) - 256usize]; ["Offset of field: AmdsmiGpuMetricsT::current_gfxclks"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_gfxclks) - 312usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_gfxclks) - 320usize]; ["Offset of field: AmdsmiGpuMetricsT::current_socclks"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_socclks) - 328usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_socclks) - 336usize]; ["Offset of field: AmdsmiGpuMetricsT::current_vclk0s"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_vclk0s) - 336usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_vclk0s) - 344usize]; ["Offset of field: AmdsmiGpuMetricsT::current_dclk0s"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_dclk0s) - 344usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_dclk0s) - 352usize]; ["Offset of field: AmdsmiGpuMetricsT::jpeg_activity"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, jpeg_activity) - 352usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, jpeg_activity) - 360usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_nak_sent_count_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_nak_sent_count_acc) - 416usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_nak_sent_count_acc) - 424usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_nak_rcvd_count_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_nak_rcvd_count_acc) - 420usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_nak_rcvd_count_acc) - 432usize]; ["Offset of field: AmdsmiGpuMetricsT::accumulation_counter"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, accumulation_counter) - 424usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, accumulation_counter) - 440usize]; ["Offset of field: AmdsmiGpuMetricsT::prochot_residency_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, prochot_residency_acc) - 432usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, prochot_residency_acc) - 448usize]; ["Offset of field: AmdsmiGpuMetricsT::ppt_residency_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, ppt_residency_acc) - 440usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, ppt_residency_acc) - 456usize]; ["Offset of field: AmdsmiGpuMetricsT::socket_thm_residency_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, socket_thm_residency_acc) - 448usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, socket_thm_residency_acc) - 464usize]; ["Offset of field: AmdsmiGpuMetricsT::vr_thm_residency_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, vr_thm_residency_acc) - 456usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, vr_thm_residency_acc) - 472usize]; ["Offset of field: AmdsmiGpuMetricsT::hbm_thm_residency_acc"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, hbm_thm_residency_acc) - 464usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, hbm_thm_residency_acc) - 480usize]; ["Offset of field: AmdsmiGpuMetricsT::num_partition"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, num_partition) - 472usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, num_partition) - 488usize]; ["Offset of field: AmdsmiGpuMetricsT::xcp_stats"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, xcp_stats) - 480usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, xcp_stats) - 496usize]; ["Offset of field: AmdsmiGpuMetricsT::pcie_lc_perf_other_end_recovery"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_lc_perf_other_end_recovery) - 4640usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, pcie_lc_perf_other_end_recovery) - 4656usize]; ["Offset of field: AmdsmiGpuMetricsT::vram_max_bandwidth"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, vram_max_bandwidth) - 4648usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, vram_max_bandwidth) - 4664usize]; ["Offset of field: AmdsmiGpuMetricsT::xgmi_link_status"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_link_status) - 4656usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, xgmi_link_status) - 4672usize]; ["Offset of field: AmdsmiGpuMetricsT::temperature_hbm_stacks"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_hbm_stacks) - 4672usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_hbm_stacks) - 4688usize]; ["Offset of field: AmdsmiGpuMetricsT::temperature_mid"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_mid) - 4696usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_mid) - 4712usize]; ["Offset of field: AmdsmiGpuMetricsT::temperature_aid"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_aid) - 4700usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, temperature_aid) - 4716usize]; ["Offset of field: AmdsmiGpuMetricsT::current_uclk_aid"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_uclk_aid) - 4704usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_uclk_aid) - 4720usize]; ["Offset of field: AmdsmiGpuMetricsT::current_socclks_mid"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_socclks_mid) - 4708usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, current_socclks_mid) - 4724usize]; ["Offset of field: AmdsmiGpuMetricsT::apu_metrics"] - [::std::mem::offset_of!(AmdsmiGpuMetricsT, apu_metrics) - 4712usize]; + [::std::mem::offset_of!(AmdsmiGpuMetricsT, apu_metrics) - 4728usize]; }; #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -4073,6 +4082,12 @@ extern "C" { gpu_busy_percent: *mut u32, ) -> AmdsmiStatusT; } +extern "C" { + pub fn amdsmi_get_vcn_busy_percent( + processor_handle: AmdsmiProcessorHandle, + vcn_busy_percent: *mut u32, + ) -> AmdsmiStatusT; +} extern "C" { pub fn amdsmi_get_utilization_count( processor_handle: AmdsmiProcessorHandle, diff --git a/projects/amdsmi/src/amd_smi/amd_smi.cc b/projects/amdsmi/src/amd_smi/amd_smi.cc index 389ae113a671..0e99da000949 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi.cc @@ -5682,23 +5682,8 @@ amdsmi_status_t amdsmi_get_pcie_info(amdsmi_processor_handle processor_handle, info->pcie_metric.pcie_replay_count = metric_info.pcie_replay_count_acc; info->pcie_metric.pcie_l0_to_recovery_count = metric_info.pcie_l0_to_recov_count_acc; info->pcie_metric.pcie_replay_roll_over_count = metric_info.pcie_replay_rover_count_acc; - /** - * pcie_metric.pcie_nak_received_count: (uint64_t) - * metric_info.pcie_nak_rcvd_count_acc: (uint32_t) - */ - info->pcie_metric.pcie_nak_received_count = - translate_umax_or_assign_valuepcie_metric.pcie_nak_received_count)>( - metric_info.pcie_nak_rcvd_count_acc, (metric_info.pcie_nak_rcvd_count_acc)); - /** - * pcie_metric.pcie_nak_sent_count: (uint64_t) - * metric_info.pcie_nak_sent_count_acc: (uint32_t) - */ - info->pcie_metric.pcie_nak_sent_count = - translate_umax_or_assign_valuepcie_metric.pcie_nak_sent_count)>( - metric_info.pcie_nak_sent_count_acc, (metric_info.pcie_nak_sent_count_acc)); - /** - * pcie_metric.pcie_lc_perf_other_end_recovery: (uint32_t) - */ + info->pcie_metric.pcie_nak_received_count = metric_info.pcie_nak_rcvd_count_acc; + info->pcie_metric.pcie_nak_sent_count = metric_info.pcie_nak_sent_count_acc; info->pcie_metric.pcie_lc_perf_other_end_recovery_count = translate_umax_or_assign_value< decltype(info->pcie_metric.pcie_lc_perf_other_end_recovery_count)>( metric_info.pcie_lc_perf_other_end_recovery, (metric_info.pcie_lc_perf_other_end_recovery)); From 7ff1686439dfca6ac972a0727d228c57f6cf500e Mon Sep 17 00:00:00 2001 From: gabrpham Date: Tue, 28 Jul 2026 17:46:53 -0500 Subject: [PATCH 2/5] refactor(amdsmi): remove driver reload API and related functions (#8575) A static code scan flagged an insecure pattern in the driver-reload path. The functionality was already removed from the amd-smi CLI, so remove the API entirely rather than leave a vulnerable, unused entry point. - include/amd_smi/amdsmi.h: drop the amdsmi_gpu_driver_reload() declaration and its tagDriverControl doxygen group. - src/amd_smi/amd_smi.cc: delete the implementation. - rocm_smi: remove the underlying driver-reload plumbing (rocm_smi.cc, rocm_smi_device.cc, rocm_smi_utils.cc and their headers). - py-interface: remove the Python binding and __init__ export; regenerate amdsmi_wrapper.py and the Rust wrapper to match. - Callers now reload the driver out of band with `sudo modprobe -r amdgpu && sudo modprobe amdgpu` to apply memory partition changes; examples, docs/conceptual/partition.md, and hw_compatibility_status.py are updated accordingly. - tests: rework memorypartition_read_write_test.cc and test_partition.py for the removed reload step and add test_memory_partition_lifecycle.py. - CHANGELOG: add the 7.15.0 Removed entry. Signed-off-by: gabrpham Signed-off-by: Maisam Arif (cherry picked from commit 706209f4f54ad7f0500187539a211da08dc258cd) --- projects/amdsmi/CHANGELOG.md | 5 + projects/amdsmi/docs/conceptual/partition.md | 11 +- .../amdsmi/example/amd_smi_drm_example.cc | 26 +- .../example/amd_smi_partition_example.cc | 19 +- .../example/amd_smi_partition_example.py | 27 +- projects/amdsmi/include/amd_smi/amdsmi.h | 62 +--- projects/amdsmi/py-interface/__init__.py | 1 - .../amdsmi/py-interface/amdsmi_interface.py | 4 - .../amdsmi/py-interface/amdsmi_wrapper.py | 23 +- .../rocm_smi/include/rocm_smi/rocm_smi.h | 50 --- .../include/rocm_smi/rocm_smi_device.h | 2 - .../include/rocm_smi/rocm_smi_utils.h | 1 - projects/amdsmi/rocm_smi/src/rocm_smi.cc | 39 --- .../amdsmi/rocm_smi/src/rocm_smi_device.cc | 141 -------- .../amdsmi/rocm_smi/src/rocm_smi_utils.cc | 33 -- .../rust-interface/src/amdsmi_wrapper.rs | 3 - projects/amdsmi/src/amd_smi/amd_smi.cc | 20 -- .../memorypartition_read_write_test.cc | 329 +++--------------- .../python/functional/gpu/test_partition.py | 145 +++++++- .../test_memory_partition_lifecycle.py | 314 +++++++++++++++++ .../amdsmi/tools/hw_compatibility_status.py | 6 - 21 files changed, 542 insertions(+), 719 deletions(-) create mode 100644 projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md index c465cda4eda1..a915af8d9ecc 100644 --- a/projects/amdsmi/CHANGELOG.md +++ b/projects/amdsmi/CHANGELOG.md @@ -31,6 +31,11 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - The per-process KFD lookup rebuilt the entire KFD node topology (an expensive sysfs walk) on every call just to translate the device BDF into its KFD GPU id. - The caller already knows this value, so it is now passed through to `gpuvsmi_get_pid_info()`, eliminating one full topology discovery per process per refresh. Falls back to the original discovery path when the id is unavailable. +### Removed + +- **Removed the `amdsmi_gpu_driver_reload()` API and its Python binding** (breaking). + - Reload the amdgpu driver out of band with `sudo modprobe -r amdgpu && sudo modprobe amdgpu` to apply memory partition changes instead. + ### Resolved Issues - **Fixed `amd-smi set --ptl-status` silently failing to change PTL state**. diff --git a/projects/amdsmi/docs/conceptual/partition.md b/projects/amdsmi/docs/conceptual/partition.md index e997fb456eb8..d242a5991204 100644 --- a/projects/amdsmi/docs/conceptual/partition.md +++ b/projects/amdsmi/docs/conceptual/partition.md @@ -460,8 +460,6 @@ Changing partition settings has strict requirements: sudo modprobe -r amdgpu && sudo modprobe amdgpu ``` - Alternatively, call `amdsmi_gpu_driver_reload()` from the library. - - A driver reload affects **all GPUs in the hive** -- every GPU in the system is reconfigured to the new memory partition configuration at once. @@ -475,7 +473,7 @@ set API automatically triggered an immediate driver reload on invocation; the CL reloaded after the user explicitly requested the partition change. The API-level reload was separated to give applications control over when the disruptive reload occurs. Additionally, as of [**ROCm 7.13.0**](https://github.com/ROCm/rocm-systems/blob/develop/projects/amdsmi/CHANGELOG.md#amd_smi_lib-for-rocm-7130), `amd-smi reset -r` is no longer available for driver reloading — use -`sudo modprobe -r amdgpu && sudo modprobe amdgpu` or `amdsmi_gpu_driver_reload()` instead. +`sudo modprobe -r amdgpu && sudo modprobe amdgpu` instead. ``` ## Workload isolation and assignment @@ -695,7 +693,8 @@ int main() { // Step 4: Reload the driver -- required to apply the memory partition change. // Stop all GPU workloads first. The reload may reset the accelerator partition. - amdsmi_gpu_driver_reload(); + // The reload needs to occur out of band via calls to `modprobe -r amdgpu` and + // then `modprobe amdgpu` // Step 5: Re-initialize to pull in the updated topology (new device count/handles) amdsmi_shut_down(); @@ -731,7 +730,6 @@ APIs, see index (obtained from {c:func}`amdsmi_get_gpu_accelerator_partition_profile_config`). - {c:func}`amdsmi_get_gpu_memory_partition_config` -- Query the current NPS mode and supported NPS modes. - {c:func}`amdsmi_set_gpu_memory_partition_mode` -- Set the NPS memory partition mode. -- {c:func}`amdsmi_gpu_driver_reload` -- Reload the amdgpu driver to apply memory partition changes. **Bare metal only:** - {c:func}`amdsmi_get_gpu_compute_partition` -- Query the current compute partition setting as a string. @@ -769,7 +767,8 @@ amdsmi.amdsmi_set_gpu_memory_partition_mode(gpu, amdsmi.AmdSmiMemoryPartitionTyp # Step 4: Reload the driver -- required to apply the memory partition change. # Stop all GPU workloads first. The reload may reset the accelerator partition. -amdsmi.amdsmi_gpu_driver_reload() +# The reload needs to occur out of band via calls to `modprobe -r amdgpu` and +# then `modprobe amdgpu` # Step 5: Re-initialize to pull in the updated topology (new device count/handles) amdsmi.amdsmi_shut_down() diff --git a/projects/amdsmi/example/amd_smi_drm_example.cc b/projects/amdsmi/example/amd_smi_drm_example.cc index 7c820fc6fb21..e762666bc64b 100644 --- a/projects/amdsmi/example/amd_smi_drm_example.cc +++ b/projects/amdsmi/example/amd_smi_drm_example.cc @@ -742,18 +742,9 @@ int main() { std::cout << "\tamdsmi_set_gpu_memory_partition(" << gpu_number << ", " << memoryPartitionString(updatePartition) << "): " << err_str << "\n\n"; - // Reload only if the memory partition was set successfully if (ret_set == AMDSMI_STATUS_SUCCESS) { - std::cout << "\t**Reloading GPU driver to apply memory " - << "partition change, this may take some time... **\n"; - amdsmi_status_t reload_status = amdsmi_gpu_driver_reload(); - amdsmi_status_code_to_string(reload_status, &err_str); - if (reload_status == AMDSMI_STATUS_SUCCESS) { - PRINT_AMDSMI_RET(reload_status) - std::cout << "\tamdsmi_gpu_driver_reload(): " << err_str << "\n\n"; - } else { - std::cout << "\tamdsmi_gpu_driver_reload(): " << err_str << "\n\n"; - } + std::cout << "\t** Memory partition staged. Run to apply:\n" + << "\t sudo modprobe -r amdgpu && sudo modprobe amdgpu\n\n"; } // Get the current memory partition @@ -841,18 +832,9 @@ int main() { } std::cout << "\tamdsmi_set_gpu_memory_partition(" << gpu_number << ", " << memoryPartitionString(orig_partition) << "): " << err_str << "\n\n"; - // Reload only if the memory partition was set successfully if (ret_set == AMDSMI_STATUS_SUCCESS) { - std::cout << "\t**Reloading GPU driver to apply memory " - << "partition change, this may take some time... **\n"; - amdsmi_status_t reload_status = amdsmi_gpu_driver_reload(); - amdsmi_status_code_to_string(reload_status, &err_str); - if (reload_status == AMDSMI_STATUS_SUCCESS) { - PRINT_AMDSMI_RET(reload_status) - std::cout << "\tamdsmi_gpu_driver_reload(): " << err_str << "\n\n"; - } else { - std::cout << "\tamdsmi_gpu_driver_reload(): " << err_str << "\n\n"; - } + std::cout << "\t** Memory partition staged. Run to apply:\n" + << "\t sudo modprobe -r amdgpu && sudo modprobe amdgpu\n\n"; } // Get the current memory partition char current_memory_partition[AMDSMI_MAX_STRING_LENGTH]; diff --git a/projects/amdsmi/example/amd_smi_partition_example.cc b/projects/amdsmi/example/amd_smi_partition_example.cc index e4fcc90c444d..c6df030745a8 100644 --- a/projects/amdsmi/example/amd_smi_partition_example.cc +++ b/projects/amdsmi/example/amd_smi_partition_example.cc @@ -41,7 +41,7 @@ // invalid and must not be used. // // Memory partition (NPS mode): -// Requires amdsmi_gpu_driver_reload() to take effect. The driver reload +// Requires driver reload to take effect. The driver reload // tears down and rebuilds all kernel device objects, which also destroys // any existing handles. amdsmi_shut_down() + amdsmi_init() is therefore // mandatory before querying or changing anything further. @@ -363,18 +363,6 @@ static void print_available_partition_modes(const std::vector& gpus) { @@ -514,9 +502,8 @@ int main() { } else { mem_changed = set_memory_partition(gpus.front(), target_memory_partition); if (mem_changed) { - if (!reload_driver()) - std::cout << "[warn] Driver reload failed; memory partition change may not have " - "taken effect.\n"; + std::cout << " Memory partition staged. Run to apply:\n" + << " sudo modprobe -r amdgpu && sudo modprobe amdgpu\n"; } else { std::cout << "\n[info] Memory partition unchanged; skipping driver reload.\n"; } diff --git a/projects/amdsmi/example/amd_smi_partition_example.py b/projects/amdsmi/example/amd_smi_partition_example.py index 57c5967dd5a1..872e231dc488 100755 --- a/projects/amdsmi/example/amd_smi_partition_example.py +++ b/projects/amdsmi/example/amd_smi_partition_example.py @@ -29,7 +29,7 @@ # invalid and must not be used. # # Memory partition (NPS mode): -# Requires amdsmi_gpu_driver_reload() to take effect. The driver reload +# Requires driver reload to take effect. The driver reload # tears down and rebuilds all kernel device objects, which also destroys # any existing handles. amdsmi_shut_down() + amdsmi_init() is therefore # mandatory before querying or changing anything further. @@ -244,22 +244,6 @@ def set_memory_partition(gpu0: Any, target: amdsmi.AmdSmiMemoryPartitionType) -> return False -# Returns True if the driver was reloaded successfully. -def reload_driver() -> bool: - print_separator("Reload driver") - # Mandatory to apply memory partition change. All GPU activity must be - # stopped first. The reload may reset the accelerator partition to default. - # Root privileges (sudo) are required for driver reload. - print(" Reloading driver, this may take some time...") - try: - amdsmi.amdsmi_gpu_driver_reload() - print(" amdsmi_gpu_driver_reload: success") - return True - except amdsmi.AmdSmiException as e: - print(f" amdsmi_gpu_driver_reload: {e}") - return False - - # For each GPU: build the NPS->profiles map, print every entry, then demonstrate # iterating over only the profiles compatible with the currently active NPS mode. def print_profiles_by_nps(gpus: List) -> None: @@ -388,11 +372,10 @@ def main() -> None: else: mem_changed = set_memory_partition(gpus[0], target_memory_partition) if mem_changed: - if not reload_driver(): - print( - "[warn] Driver reload failed; memory partition change " - "may not have taken effect." - ) + print( + " Memory partition staged. Run to apply:\n" + " sudo modprobe -r amdgpu && sudo modprobe amdgpu" + ) else: print("\n[info] Memory partition unchanged; skipping driver reload.") diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index f5460400098d..ce8b6d48cb8f 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -7112,7 +7112,7 @@ amdsmi_status_t amdsmi_get_gpu_memory_partition(amdsmi_processor_handle processo * Device must be idle and have no workloads when performing set partition operations. * * On @platform{gpu_bm_linux} AMDGPU driver restart is REQUIRED to complete updating to - * the new memory partition setting. Refer to `amdsmi_gpu_driver_reload()` for more details. + * the new memory partition setting. * * @param[in] processor_handle Device which to query * @@ -7163,10 +7163,10 @@ amdsmi_status_t amdsmi_get_gpu_memory_partition_config(amdsmi_processor_handle p * Device must be idle and have no workloads when performing set partition operations. * * @details On @platform{gpu_bm_linux} AMDGPU driver restart is REQUIRED to complete updating - * to the new memory partition setting. Refer to `amdsmi_gpu_driver_reload()` for more details. + * to the new memory partition setting. * * On @platform{gpu_bm_linux} AMDGPU driver restart is REQUIRED to complete updating to - * the new memory partition setting. Refer to `amdsmi_gpu_driver_reload()` for more details. + * the new memory partition setting. * * @param[in] processor_handle A processor handle * @@ -7859,62 +7859,6 @@ amdsmi_status_t amdsmi_get_gpu_process_list_by_pid(amdsmi_processor_handle* proc /** @} End tagProcessInfo */ -/*****************************************************************************/ -/** @defgroup tagDriverControl Driver control mechanisms - * These functions provide control over the driver. Users should use with - * caution as they may cause the driver to become unstable. - * @{ - */ -/** - * @brief Restart the device driver (kmod module) for all AMD GPUs on the - * system. - * - * @ingroup tagDriverControl - * - * @platform{gpu_bm_linux} @platform{guest_1vf} @platform{guest_mvf} - * - * @details This function will reload the AMD GPU driver as described in - * the Linux kernel documentation - - * https://docs.kernel.org/admin-guide/sysctl/kernel.html#modprobe - * with no extra parameters as specified in - * https://docs.kernel.org/gpu/amdgpu/module-parameters.html. - * - * Use this function with caution, as it will unload and reload the AMD GPU - * driver: `modprobe -r amdgpu && modprobe amdgpu`. - * - * Any process or workload using the AMD GPU driver is REQUIRED to be - * stopped before calling this function. Otherwise, function will return - * ::AMDSMI_STATUS_AMDGPU_RESTART_ERR could not successfully restart - * the amdgpu driver. - * - * User is REQUIRED to have root/admin privileges to call this function. - * Otherwise, this function will return ::AMDSMI_STATUS_NO_PERM. - * - * This API will take time to complete, as we are checking the driver's - * loading status to confirm it reloaded properly. If - * ::AMDSMI_STATUS_AMDGPU_RESTART_ERR is returned, it means the driver - * did not reload properly and the user should check dmesg logs. - * - * This function has been created in order to conveniently reload the - * AMD GPU driver once `amdsmi_set_gpu_memory_partition()` or - * `amdsmi_set_gpu_memory_partition_mode()` successfully has been changed - * on Baremetal systems. Now users can control the reload once all GPU - * processes/workloads have been stopped on the AMD GPU driver. - * A (AMD GPU) driver reload is REQUIRED to complete changing - * to the new memory partition configuration - * (`amdsmi_set_gpu_memory_partition()`/`amdsmi_set_gpu_memory_partition_mode()`) - * operation MUST be successful. This function WILL EFFECT all GPUs in the - * hive to be reconfigured with the specified memory partition configuration. - * - * @return ::amdsmi_status_t | ::AMDSMI_STATUS_SUCCESS on success - * @return | ::AMDSMI_STATUS_NO_PERM function requires admin/sudo privileges - * @return | ::AMDSMI_STATUS_AMDGPU_RESTART_ERR could not successfully restart - * the amdgpu driver. - */ -amdsmi_status_t amdsmi_gpu_driver_reload(void); - -/** @} End tagDriverControl */ - /*****************************************************************************/ /** @defgroup tagPTL Peak Tops Limiter * @{ diff --git a/projects/amdsmi/py-interface/__init__.py b/projects/amdsmi/py-interface/__init__.py index 513ba9734360..bf49a6bc4218 100644 --- a/projects/amdsmi/py-interface/__init__.py +++ b/projects/amdsmi/py-interface/__init__.py @@ -188,7 +188,6 @@ from .amdsmi_interface import amdsmi_set_gpu_perf_level from .amdsmi_interface import amdsmi_get_gpu_power_profile_presets from .amdsmi_interface import amdsmi_reset_gpu -from .amdsmi_interface import amdsmi_gpu_driver_reload from .amdsmi_interface import amdsmi_set_gpu_perf_determinism_mode from .amdsmi_interface import amdsmi_set_gpu_fan_speed from .amdsmi_interface import amdsmi_reset_gpu_fan diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index 26b9d68032de..93ba440d3d40 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -4774,10 +4774,6 @@ def amdsmi_reset_gpu(processor_handle: processor_handle_t): _check_res(amdsmi_wrapper.amdsmi_reset_gpu(processor_handle)) -def amdsmi_gpu_driver_reload(): - _check_res(amdsmi_wrapper.amdsmi_gpu_driver_reload()) - - def amdsmi_set_gpu_fan_speed(processor_handle: processor_handle_t, sensor_idx: int, fan_speed: int): if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index df41a1a7070c..01bbebfae99b 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -4252,12 +4252,6 @@ class struct_amdsmi_cper_hdr_t(Structure): amdsmi_get_gpu_process_list_by_pid.argtypes = [ctypes.POINTER(ctypes.POINTER(None)), uint32_t, ctypes.POINTER(struct_amdsmi_proc_info_by_pid_t), ctypes.POINTER(ctypes.c_uint32)] except AttributeError: pass -try: - amdsmi_gpu_driver_reload = _libraries['libamd_smi.so'].amdsmi_gpu_driver_reload - amdsmi_gpu_driver_reload.restype = amdsmi_status_t - amdsmi_gpu_driver_reload.argtypes = [] -except AttributeError: - pass try: amdsmi_get_gpu_ptl_state = _libraries['libamd_smi.so'].amdsmi_get_gpu_ptl_state amdsmi_get_gpu_ptl_state.restype = amdsmi_status_t @@ -5309,15 +5303,14 @@ class struct_amdsmi_ttm_info_t(Structure): 'amdsmi_gpu_block_t', 'amdsmi_gpu_cache_info_t', 'amdsmi_gpu_control_counter', 'amdsmi_gpu_counter_group_supported', 'amdsmi_gpu_create_counter', - 'amdsmi_gpu_destroy_counter', 'amdsmi_gpu_driver_reload', - 'amdsmi_gpu_metrics_t', 'amdsmi_gpu_ras_policy_info_t', - 'amdsmi_gpu_ras_policy_v4_0_t', 'amdsmi_gpu_read_counter', - 'amdsmi_gpu_validate_ras_eeprom', 'amdsmi_gpu_xcp_metrics_t', - 'amdsmi_gpu_xgmi_error_status', 'amdsmi_hsmp_driver_version_t', - 'amdsmi_hsmp_freqlimit_src_names', 'amdsmi_hsmp_metrics_table_t', - 'amdsmi_init', 'amdsmi_init_flags_t', - 'amdsmi_init_gpu_event_notification', 'amdsmi_io_bw_encoding_t', - 'amdsmi_is_P2P_accessible', + 'amdsmi_gpu_destroy_counter', 'amdsmi_gpu_metrics_t', + 'amdsmi_gpu_ras_policy_info_t', 'amdsmi_gpu_ras_policy_v4_0_t', + 'amdsmi_gpu_read_counter', 'amdsmi_gpu_validate_ras_eeprom', + 'amdsmi_gpu_xcp_metrics_t', 'amdsmi_gpu_xgmi_error_status', + 'amdsmi_hsmp_driver_version_t', 'amdsmi_hsmp_freqlimit_src_names', + 'amdsmi_hsmp_metrics_table_t', 'amdsmi_init', + 'amdsmi_init_flags_t', 'amdsmi_init_gpu_event_notification', + 'amdsmi_io_bw_encoding_t', 'amdsmi_is_P2P_accessible', 'amdsmi_is_gpu_power_management_enabled', 'amdsmi_kfd_info_t', 'amdsmi_link_id_bw_type_t', 'amdsmi_link_metrics_t', 'amdsmi_link_status_t', 'amdsmi_link_type_t', diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h index 474110270553..a1b0da0bb566 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi.h @@ -5929,56 +5929,6 @@ rsmi_status_t rsmi_dev_metrics_log_get(uint32_t dv_ind); /** @} */ // end of DevMetricsHeaderInfoGet -/*****************************************************************************/ -/** @defgroup DriverControl Driver control mechanisms - * These functions provide control over the driver. Users should use with - * caution as they may cause the driver to become unstable. - * @{ - */ -/** - * @brief Restart the device driver (kmod module) for all AMD GPUs on the - * system. - * - * @details This function will reload the AMD GPU driver as described in - * the Linux kernel documentation - - * https://docs.kernel.org/admin-guide/sysctl/kernel.html#modprobe - * with no extra parameters as specified in - * https://docs.kernel.org/gpu/amdgpu/module-parameters.html. - * - * Use this function with caution, as it will unload and reload the AMD GPU - * driver: `modprobe -r amdgpu && modprobe amdgpu`. - * - * Any process or workload using the AMD GPU driver is REQUIRED to be - * stopped before calling this function. Otherwise, function will return - * ::RSMI_STATUS_AMDGPU_RESTART_ERR could not successfully restart - * the amdgpu driver. - * - * User is REQUIRED to have root/admin privileges to call this function. - * Otherwise, this function will return ::RSMI_STATUS_PERMISSION. - * - * This API will take time to complete, as we are checking the driver's - * loading status to confirm it reloaded properly. If - * ::RSMI_STATUS_AMDGPU_RESTART_ERR is returned, it means the driver - * did not reload properly and the user should check dmesg logs. - * - * This function has been created in order to conveniently reload the - * AMD GPU driver once `rsmi_dev_memory_partition_set()` - * successfully has been changed on Baremetal systems. - * Now users can control the reload once all GPU processes/workloads - * have been stopped on the AMD GPU driver. A (AMD GPU) driver reload - * is REQUIRED to complete changing to the new memory partition - * configuration (`rsmi_dev_memory_partition_set()`) operation MUST - * be successful. This function WILL EFFECT all GPUs in the hive to - * be reconfigured with the specified memory partition configuration. - * - * @retval ::RSMI_STATUS_SUCCESS call was successful - * @retval ::RSMI_STATUS_PERMISSION function requires root access - * @retval ::RSMI_STATUS_AMDGPU_RESTART_ERR could not successfully restart - * the amdgpu driver. - */ -rsmi_status_t rsmi_dev_amdgpu_driver_reload(void); -/** @} */ // end of DriverControl - #ifdef __cplusplus } #endif // __cplusplus diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_device.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_device.h index 4a5b06720c04..38d17d96b18b 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_device.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_device.h @@ -253,8 +253,6 @@ class Device { void fillSupportedFuncs(void); void DumpSupportedFunctions(void); bool DeviceAPISupported(std::string name, uint64_t variant, uint64_t sub_variant); - rsmi_status_t restartAMDGpuDriver(void); - rsmi_status_t isRestartInProgress(bool* isRestartInProgress, bool* isAMDGPUModuleLive); rsmi_status_t storeDevicePartitions(uint32_t dv_ind); template std::string readBootPartitionState(uint32_t dv_ind); diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_utils.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_utils.h index 950789e7e535..e7c407c682f8 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_utils.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_utils.h @@ -66,7 +66,6 @@ int ReadSysfsStr(std::string path, std::string* retStr); int WriteSysfsStr(std::string path, std::string val); bool IsInteger(const std::string& n_str); bool stringToInteger(const std::string& n_str, int& value); -std::pair executeCommand(std::string command, bool stdOut = true); rsmi_status_t storeTmpFile(uint32_t dv_ind, std::string parameterName, std::string stateName, std::string storageData); std::vector getListOfAppTmpFiles(); diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi.cc b/projects/amdsmi/rocm_smi/src/rocm_smi.cc index f2bd56919dfa..b203c093deb3 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi.cc @@ -3798,45 +3798,6 @@ rsmi_status_t rsmi_dev_gpu_reset(uint32_t dv_ind) { CATCH } -rsmi_status_t rsmi_dev_amdgpu_driver_reload(void) { - TRY std::ostringstream ss; - ss << __PRETTY_FUNCTION__ << "| ======= start ======="; - LOG_TRACE(ss); - // TODO(amdsmi_team): technically, we should block for all devices - // As this is a global operation, we can use a mutex to ensure - // that only one thread is trying to restart the driver at a time. - uint32_t dv_ind = 0; // Default to first device - DEVICE_MUTEX - GET_DEV_FROM_INDX - - rsmi_status_t restartRet = dev->restartAMDGpuDriver(); - - // Attempting to speed up processing time - bool is_logger_enabled = ROCmLogging::Logger::getInstance()->isLoggerEnabled(); - if (restartRet != RSMI_STATUS_SUCCESS) { - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | ======= end ======= " - << " | Fail - restart AMD GPU detected" - << " | Device #: " << dv_ind << " | Type: AMDGPU Driver Reload" - << " | Cause: AMDGPU Driver Reload failed " - << " | Returning = " << getRSMIStatusString(restartRet, false); - LOG_ERROR(ss); - } - return restartRet; - } - - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | ======= end ======= " - << " | Success - if restart completed successfully" - << " | Device #: " << dv_ind << " | Type: AMDGPU Driver Reload" - << " | Returning = " << getRSMIStatusString(restartRet, false); - LOG_INFO(ss); - } - return restartRet; - - CATCH -} - rsmi_status_t rsmi_dev_od_volt_curve_regions_get(uint32_t dv_ind, uint32_t* num_regions, rsmi_freq_volt_region_t* buffer) { TRY std::ostringstream ss; diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi_device.cc b/projects/amdsmi/rocm_smi/src/rocm_smi_device.cc index 91b9a17c6edc..89857d6b293d 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi_device.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi_device.cc @@ -1664,147 +1664,6 @@ bool Device::DeviceAPISupported(std::string name, uint64_t variant, uint64_t sub return subvariant_match(&(var_it->second), sub_variant); } -rsmi_status_t Device::restartAMDGpuDriver(void) { - REQUIRE_ROOT_ACCESS - std::ostringstream ss; - bool restartSuccessful = true; - bool success = false; - std::string out; - bool wasGdmServiceActive = false; - bool isRestartInProgress = true; - bool isAMDGPUModuleLive = false; - bool restartGDM = false; - std::string captureRestartErr; - // 1 sec = 1000 ms = 1000000 us - const int kTimeToWaitForDriverMSec = 1000; - // Attempting to speed up processing time - bool is_logger_enabled = ROCmLogging::Logger::getInstance()->isLoggerEnabled(); - - // sudo systemctl is-active gdm - // we do not care about the success of checking if gdm is active - std::tie(success, out) = executeCommand("systemctl is-active gdm 2>/dev/null", true); - (out == "active") ? (restartGDM = true) : (restartGDM = false); - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | systemctl is-active gdm: out = " << out - << "; success = " << (success ? "True" : "False") - << "; restartGDM = " << (restartGDM ? "True" : "False"); - LOG_INFO(ss); - } - - // if gdm is active -> sudo systemctl stop gdm - // TODO(AMD_SMI_team): are are there other display manager's we need to take into account? - // see https://help.gnome.org/admin/gdm/stable/overview.html.en_GB - if (success && (out == "active") && (restartGDM)) { - wasGdmServiceActive = true; - std::tie(success, out) = executeCommand("systemctl stop gdm& 2>/dev/null", true); - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | systemctl stop gdm&: out = " << out - << "; success = " << (success ? "True" : "False"); - LOG_INFO(ss); - } - } else { - success = true; // ignore failures to restart gdm - } - - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | B4 modprobing anything!!! out = " << out - << "; success = " << (success ? "True" : "False") - << "; restartSuccessful = " << (restartSuccessful ? "True" : "False") - << "; captureRestartErr = " << captureRestartErr; - LOG_INFO(ss); - } - - // sudo modprobe -r amdgpu - // sudo modprobe amdgpu - std::tie(success, out) = executeCommand( - "modprobe -r -v amdgpu >/dev/null 2>&1 && modprobe -v amdgpu >/dev/null 2>&1", true); - restartSuccessful &= success; - captureRestartErr = out; - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | modprobe -r -v amdgpu && modprobe -v amdgpu: out = " << out - << "; success = " << (success ? "True" : "False") - << "; restartSuccessful = " << (restartSuccessful ? "True" : "False") - << "; captureRestartErr = " << captureRestartErr; - LOG_INFO(ss); - } - - // if gdm was active -> sudo systemctl start gdm - // We don't care if successful or not, just try to restart as a courtesy - if (wasGdmServiceActive && restartGDM) { - std::tie(success, out) = executeCommand("systemctl start gdm& 2>/dev/null", true); - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | systemctl start gdm&: out = " << out - << "; success = " << (success ? "True" : "False"); - LOG_INFO(ss); - } - } - - // Return early if there was an issue restarting amdgpu - if (!restartSuccessful) { - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ - << " | [ERROR] Issue found during amdgpu restart: " << captureRestartErr - << "; retartSuccessful: " << (restartSuccessful ? "True" : "False"); - LOG_ERROR(ss); - } - return RSMI_STATUS_AMDGPU_RESTART_ERR; - } - - // wait for amdgpu module to come back up - rsmi_status_t status = Device::isRestartInProgress(&isRestartInProgress, &isAMDGPUModuleLive); - int maxLoops = 10; // wait a max of 10 sec - while (status != RSMI_STATUS_SUCCESS) { - maxLoops -= 1; - if (maxLoops == 0) { - break; - } - amd::smi::system_wait(kTimeToWaitForDriverMSec); - status = Device::isRestartInProgress(&isRestartInProgress, &isAMDGPUModuleLive); - } - - return ((restartSuccessful && (!isRestartInProgress && isAMDGPUModuleLive)) - ? RSMI_STATUS_SUCCESS - : RSMI_STATUS_AMDGPU_RESTART_ERR); -} - -rsmi_status_t Device::isRestartInProgress(bool* isRestartInProgress, bool* isAMDGPUModuleLive) { - REQUIRE_ROOT_ACCESS - std::ostringstream ss; - bool success = false; - std::string out; - bool deviceRestartInProgress = true; // Assume in progress, we intend to disprove - bool isSystemAMDGPUModuleLive = false; // Assume AMD GPU module is not live, - // we intend to disprove - // Attempting to speed up processing time - bool is_logger_enabled = ROCmLogging::Logger::getInstance()->isLoggerEnabled(); - - // wait for amdgpu module to come back up - std::tie(success, out) = executeCommand("cat /sys/module/amdgpu/initstate", true); - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | success = " << (success ? "True" : "False") - << " | out = " << out; - LOG_DEBUG(ss); - } - if ((success == true) && (!out.empty())) { - isSystemAMDGPUModuleLive = containsString(out, "live"); - } - if (isSystemAMDGPUModuleLive) { - deviceRestartInProgress = false; - } - *isRestartInProgress = deviceRestartInProgress; - *isAMDGPUModuleLive = isSystemAMDGPUModuleLive; - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ - << " | *isRestartInProgress = " << (*isRestartInProgress ? "True" : "False") - << " | *isAMDGPUModuleLive = " << (*isAMDGPUModuleLive ? "True" : "False") - << " | out = " << out; - LOG_DEBUG(ss); - } - - return ((*isAMDGPUModuleLive && !*isRestartInProgress) ? RSMI_STATUS_SUCCESS - : RSMI_STATUS_AMDGPU_RESTART_ERR); -} - template rsmi_status_t storeParameter(uint32_t dv_ind); diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi_utils.cc b/projects/amdsmi/rocm_smi/src/rocm_smi_utils.cc index ccb40b93485f..8d521e91c143 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi_utils.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi_utils.cc @@ -592,39 +592,6 @@ std::string removeString(const std::string origStr, const std::string& removeMe) return modifiedStr; } -// defaults to trim stdOut -std::pair executeCommand(std::string command, bool stdOut) { - char buffer[128]; - std::string stdoutAndErr; - bool successfulRun = true; - command = "stdbuf -i0 -o0 -e0 " + command; // remove stdOut and err buffering - - FILE* pipe = popen(command.c_str(), "r"); - if (!pipe) { - stdoutAndErr = "[ERROR] popen failed to call " + command; - successfulRun = false; - } else { - // read until end of process - while (!feof(pipe)) { - // use buffer to read and add to stdoutAndErr - if (fgets(buffer, sizeof(buffer), pipe) != nullptr) { - stdoutAndErr += buffer; - } - } - } - - // any return code other than 0, is a failed execution - if (pipe && pclose(pipe) != 0) { - successfulRun = false; - } - - if (stdOut) { - // remove leading and trailing spaces of output and new lines - stdoutAndErr = trim(stdoutAndErr); - } - return std::make_pair(successfulRun, stdoutAndErr); -} - // originalString - string to search for substring // substring - string looking to find // displayComparisons = defaults to false, set to true to see debug prints diff --git a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs index b617d4e68c0f..642a9e21e0e7 100644 --- a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs +++ b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs @@ -5156,9 +5156,6 @@ extern "C" { max_processes: *mut u32, ) -> AmdsmiStatusT; } -extern "C" { - pub fn amdsmi_gpu_driver_reload() -> AmdsmiStatusT; -} extern "C" { pub fn amdsmi_get_gpu_ptl_state( processor_handle: AmdsmiProcessorHandle, diff --git a/projects/amdsmi/src/amd_smi/amd_smi.cc b/projects/amdsmi/src/amd_smi/amd_smi.cc index 0e99da000949..7454998f066c 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi.cc @@ -4464,26 +4464,6 @@ amdsmi_status_t amdsmi_reset_gpu(amdsmi_processor_handle processor_handle) { return ret; } -amdsmi_status_t amdsmi_gpu_driver_reload(void) { - std::ostringstream ss; - AMDSMI_CHECK_INIT(); - - // Attempting to speed up processing time - bool is_logger_enabled = ROCmLogging::Logger::getInstance()->isLoggerEnabled(); - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ << " | ======= start ======="; - LOG_INFO(ss); - } - rsmi_status_t ret = rsmi_dev_amdgpu_driver_reload(); - amdsmi_status_t amdsmi_status = amd::smi::rsmi_to_amdsmi_status(ret); - if (is_logger_enabled) { - ss << __PRETTY_FUNCTION__ - << " | Returning: " << smi_amdgpu_get_status_string(amdsmi_status, false); - LOG_INFO(ss); - } - return amdsmi_status; -} - amdsmi_status_t amdsmi_get_gpu_busy_percent(amdsmi_processor_handle processor_handle, uint32_t* gpu_busy_percent) { auto status = rsmi_wrapper(rsmi_dev_busy_percent_get, processor_handle, 0, gpu_busy_percent); diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc index f91507941b4f..3233f9e60684 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc @@ -22,7 +22,6 @@ #include "memorypartition_read_write.h" #include -#include #include #include @@ -42,79 +41,6 @@ const uint32_t MAX_DPX_PARTITIONS = 2; const uint32_t MAX_TPX_PARTITIONS = 3; const uint32_t MAX_QPX_PARTITIONS = 4; -void ReloadDriverWithMessages(bool isVerbose, const std::string& preReloadMessage, - const std::string& successMessage, const std::string& errorMessage, - const std::string& restartErrorMessage, - amdsmi_status_t* reload_status) { - if (isVerbose) { - std::cout << "\t**" << preReloadMessage << std::endl; - } - - DISPLAY_AMDSMI_API("amdsmi_gpu_driver_reload", "", isVerbose); - auto start_time = std::chrono::steady_clock::now(); - auto driver_reload_status = amdsmi_gpu_driver_reload(); - auto end_time = std::chrono::steady_clock::now(); - auto elapsed_time = std::chrono::duration_cast(end_time - start_time); - auto elapsed_seconds = std::chrono::duration_cast(end_time - start_time); - *reload_status = driver_reload_status; - DISPLAY_AMDSMI_STATUS(isVerbose, __FILE__, __LINE__, driver_reload_status, AMDSMI_STATUS_SUCCESS); - - if (isVerbose) { - std::cout << "\t**" - << "amdsmi_gpu_driver_reload() took " << elapsed_time.count() << " milliseconds (" - << elapsed_seconds.count() << " seconds)" << std::endl; - } - - if (driver_reload_status == AMDSMI_STATUS_SUCCESS) { - if (isVerbose) { - std::cout << "\t**" << successMessage << std::endl; - } - ASSERT_EQ(driver_reload_status, AMDSMI_STATUS_SUCCESS); - } else if (driver_reload_status == AMDSMI_STATUS_AMDGPU_RESTART_ERR) { - if (isVerbose) { - std::cout << "\t**" << restartErrorMessage << std::endl; - } - ASSERT_TRUE(driver_reload_status == AMDSMI_STATUS_AMDGPU_RESTART_ERR); - } else { - if (isVerbose) { - std::cout << "\t**" << errorMessage << ": " - << smi_amdgpu_get_status_string(driver_reload_status, false) << std::endl; - } - } - - // Tests should fail if the driver reload fails - // TODO(amdsmi_team): This is a temporary solution until CQE can update - // how their containers are ran. - // This is because the driver reload requires: - // 1) Containers must run serially - // (i.e. no parallel containers running at the same time) - // 2) Containers must run with extra parameters: - // --cap-add=SYS_ADMIN -v /lib/modules:/lib/modules - // See: - // https://rocm.docs.amd.com/projects/amdsmi/en/latest/how-to/setup-docker-container.html - // 3) Require kmod to be installed within the docker container - // (if ASIC supports memory partitions) -} - -// Helper function to check if kmod is available -bool IsKmodInstalled() { - // One time check for modprobe existence - static bool installed = [] { - // Check common paths for modprobe - constexpr std::array paths = {"/usr/sbin/modprobe", "/sbin/modprobe", - "/usr/bin/modprobe", "/bin/modprobe"}; - - struct stat st; - for (const auto& path : paths) { - if (stat(path, &st) == 0 && (st.st_mode & S_IXUSR)) { - return true; - } - } - return false; - }(); - return installed; -} - TestMemoryPartitionReadWrite::TestMemoryPartitionReadWrite() : TestBase() { set_title("AMDSMI Memory Partition Read Test"); set_description( @@ -427,8 +353,6 @@ void TestMemoryPartitionReadWrite::Run(void) { // uint32_t num_devices_to_test = 1; uint32_t num_devices_to_test = current_num_devices; for (uint32_t dv_ind = 0; dv_ind < num_devices_to_test; ++dv_ind) { - bool wasSetSuccess = false; - bool isNewNPSMode = false; if (dv_ind != 0) { IF_VERB(STANDARD) { std::cout << std::endl; } } @@ -596,7 +520,6 @@ void TestMemoryPartitionReadWrite::Run(void) { for (int partition = static_cast(AMDSMI_MEMORY_PARTITION_NPS1); partition <= static_cast(AMDSMI_MEMORY_PARTITION_NPS8); partition++) { ret_set = AMDSMI_STATUS_NOT_SUPPORTED; - wasSetSuccess = false; new_memory_partition = static_cast(partition); if (new_memory_partition != AMDSMI_MEMORY_PARTITION_NPS1 && new_memory_partition != AMDSMI_MEMORY_PARTITION_NPS2 && @@ -610,60 +533,53 @@ void TestMemoryPartitionReadWrite::Run(void) { << "======== TEST AMDSMI_MEMORY_PARTITION_" << memoryPartitionString(new_memory_partition) << " ===============" << std::endl; } - IF_VERB(STANDARD) { - std::cout << "\t**" - << "Attempting to set memory partition to: " - << memoryPartitionString(new_memory_partition) << std::endl; - } + // Read capabilities before attempting set; used to validate set return code. + current_memory_config = {}; DISPLAY_AMDSMI_API("amdsmi_get_gpu_memory_partition_config", "gpu=" + std::to_string(dv_ind), VERB(STANDARD)); auto ret_caps = amdsmi_get_gpu_memory_partition_config(processor_handles_[dv_ind], ¤t_memory_config); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret_caps, AMDSMI_STATUS_SUCCESS); + ASSERT_TRUE((ret_caps == AMDSMI_STATUS_NOT_SUPPORTED) || (ret_caps == AMDSMI_STATUS_SUCCESS)); + std::string memory_caps_str = "N/A"; + bool partition_is_supported = false; if (ret_caps == AMDSMI_STATUS_SUCCESS) { memory_caps_str.clear(); if (current_memory_config.partition_caps.nps_flags.nps1_cap) { memory_caps_str += (memory_caps_str.empty() ? "NPS1" : ", NPS1"); + if (new_memory_partition == AMDSMI_MEMORY_PARTITION_NPS1) partition_is_supported = true; } if (current_memory_config.partition_caps.nps_flags.nps2_cap) { memory_caps_str += (memory_caps_str.empty() ? "NPS2" : ", NPS2"); + if (new_memory_partition == AMDSMI_MEMORY_PARTITION_NPS2) partition_is_supported = true; } if (current_memory_config.partition_caps.nps_flags.nps4_cap) { memory_caps_str += (memory_caps_str.empty() ? "NPS4" : ", NPS4"); + if (new_memory_partition == AMDSMI_MEMORY_PARTITION_NPS4) partition_is_supported = true; } if (current_memory_config.partition_caps.nps_flags.nps8_cap) { memory_caps_str += (memory_caps_str.empty() ? "NPS8" : ", NPS8"); + if (new_memory_partition == AMDSMI_MEMORY_PARTITION_NPS8) partition_is_supported = true; } } IF_VERB(STANDARD) { - std::cout << "\t**" - << "amdsmi_get_gpu_memory_partition_config(processor_handles_[" << dv_ind - << "], current_memory_config): " << smi_amdgpu_get_status_string(ret_caps, false) - << std::endl; std::cout << "\t**" << "Available Memory Partition Capabilities: " << memory_caps_str << "\n" << "\t**" << "current_memory_partition_mode: " - << memoryPartitionString(current_memory_config.mp_mode) << "\n" + << (ret_caps == AMDSMI_STATUS_SUCCESS + ? memoryPartitionString(current_memory_config.mp_mode) + : "N/A") + << "\n" << "\t**" - << "num_numa_ranges: " << current_memory_config.num_numa_ranges << std::endl; - } - ASSERT_TRUE((ret_caps == AMDSMI_STATUS_NOT_SUPPORTED) || (ret_caps == AMDSMI_STATUS_SUCCESS)); - // Save original memory partition - amdsmi_memory_partition_type_t saved_orig_memory_partition = current_memory_config.mp_mode; - // Detect if we're changing to a different NPS mode - if (ret_caps == AMDSMI_STATUS_SUCCESS) { - isNewNPSMode = (current_memory_config.mp_mode != new_memory_partition); - IF_VERB(STANDARD) { - std::cout << "\t**" - << "NPS mode change detected: " << (isNewNPSMode ? "YES" : "NO") - << " (current (Saved): |" - << memoryPartitionString(current_memory_config.mp_mode) << "| -> Requested: |" - << memoryPartitionString(new_memory_partition) << "|)" << std::endl; - } + << "Requested partition supported by hardware: " + << (partition_is_supported ? "YES" : "NO") << "\n" + << "\t**" + << "Attempting to set memory partition to: " + << memoryPartitionString(new_memory_partition) << std::endl; } DISPLAY_AMDSMI_API("amdsmi_set_gpu_memory_partition_mode", "gpu=" + std::to_string(dv_ind), @@ -680,136 +596,39 @@ void TestMemoryPartitionReadWrite::Run(void) { } if (ret_set == AMDSMI_STATUS_NOT_SUPPORTED) { break; - } else { - ASSERT_TRUE((ret_set == AMDSMI_STATUS_SUCCESS) || (ret_set == AMDSMI_STATUS_BUSY) || - (ret_set == AMDSMI_STATUS_AMDGPU_RESTART_ERR) || - (ret_set == AMDSMI_STATUS_INVAL) || (ret_set == AMDSMI_STATUS_NOT_SUPPORTED)); - } - - amdsmi_status_t driver_reload_status = AMDSMI_STATUS_NOT_SUPPORTED; - if (ret_set == AMDSMI_STATUS_SUCCESS) { // do not continue trying to reset - // Now we require a separate call to reload the driver, since this operation - // has been removed from the amdsmi_set_gpu_memory_partition_mode and - // amdsmi_set_gpu_memory_partition(). - // This is to allow the user to select the appropriate time to reload the driver - // since there can be errors if any device has a workload/process running on it. - std::string reload_message = - "\t Reloading the AMD GPU driver after setting memory partition to " + - memoryPartitionString(new_memory_partition) + - ". This may take some time, please wait..."; - std::string driver_reload_success_message = - "amdsmi_gpu_driver_reload() successful after setting memory partition to " + - memoryPartitionString(new_memory_partition); - std::string failure_message = - "amdsmi_gpu_driver_reload() failed after setting memory partition to " + - memoryPartitionString(new_memory_partition); - std::string restart_error_message = - "amdsmi_gpu_driver_reload() failed with AMDGPU_RESTART_ERR after " - "setting memory partition to " + - memoryPartitionString(new_memory_partition); - ReloadDriverWithMessages(isVerbose, reload_message, driver_reload_success_message, - failure_message, restart_error_message, &driver_reload_status); - if (driver_reload_status == AMDSMI_STATUS_SUCCESS) { - wasSetSuccess = true; - } - if (driver_reload_status == AMDSMI_STATUS_AMDGPU_RESTART_ERR) { - // Check kmod availability for driver reload operations - // This is required in order to fully test changing memory partitions works - - bool kmod_available = IsKmodInstalled(); - - IF_VERB(STANDARD) { - std::cout << "\t** kmod (modprobe) installed: " << (kmod_available ? "YES" : "NO") - << std::endl; - } - - if (!kmod_available) { - IF_VERB(STANDARD) { - std::cout << "** ERROR: kmod is not installed. " - << "This device has been detected as supporting memory partitions. " - << "\n** Memory partition tests require kmod for " - << "driver reload operations to fully validate functionality. " - << "\n** Install with: apt-get install kmod (Debian/Ubuntu) " - << "or dnf install kmod (RHEL) **" << std::endl; - } - ASSERT_TRUE(IsKmodInstalled()); - } - } } + ASSERT_TRUE((ret_set == AMDSMI_STATUS_SUCCESS) || (ret_set == AMDSMI_STATUS_BUSY) || + (ret_set == AMDSMI_STATUS_AMDGPU_RESTART_ERR) || + (ret_set == AMDSMI_STATUS_INVAL) || (ret_set == AMDSMI_STATUS_NOT_SUPPORTED)); - DISPLAY_AMDSMI_API("amdsmi_get_gpu_memory_partition_config", "gpu=" + std::to_string(dv_ind), - VERB(STANDARD)); - ret = amdsmi_get_gpu_memory_partition_config(processor_handles_[dv_ind], - ¤t_memory_config); - DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); - if (ret == AMDSMI_STATUS_NOT_SUPPORTED) { - continue; - } - CHK_ERR_ASRT(ret) - IF_VERB(STANDARD) { - std::cout << "\t**" - << "Current memory partition: " - << memoryPartitionString(current_memory_config.mp_mode) << std::endl; - } - IF_VERB(STANDARD) { - std::cout << "\t**WasSetSuccess (Set Memory Partition AND Driver reload was successful): " - << (wasSetSuccess ? "true" : "false") - << ", isNewNPSMode: " << (isNewNPSMode ? "true" : "false") - << "\n\t**Saved Memory Partition: " - << memoryPartitionString(saved_orig_memory_partition) - << "\n\t**Current Memory Partition: " - << memoryPartitionString(current_memory_config.mp_mode) - << "\n\t**Requested Memory Partition: " - << memoryPartitionString(new_memory_partition) << std::endl; + // If the hardware advertises support for this mode, the set should succeed unless the + // device is busy (e.g., workloads running). + if (partition_is_supported) { + EXPECT_TRUE(ret_set == AMDSMI_STATUS_SUCCESS || ret_set == AMDSMI_STATUS_BUSY) + << "Set did not succeed for a partition mode advertised as supported by hardware: " + << memoryPartitionString(new_memory_partition); } - if (wasSetSuccess) { // driver reload was successful - ASSERT_EQ(AMDSMI_STATUS_SUCCESS, ret_set); - CHK_ERR_ASRT(ret_set) - if (isNewNPSMode) { - IF_VERB(STANDARD) { - std::cout << "\t**Since driver reload (and set) was successful and a new NPS mode " - << "was requested; current memory partition (" - << memoryPartitionString(current_memory_config.mp_mode) - << ") is expected to be different than original (" - << memoryPartitionString(saved_orig_memory_partition) - << ") and equal to requested (" << memoryPartitionString(new_memory_partition) - << ")" << std::endl; - } - ASSERT_STRNE(memoryPartitionString(current_memory_config.mp_mode).c_str(), - memoryPartitionString(saved_orig_memory_partition).c_str()); - ASSERT_STREQ(memoryPartitionString(current_memory_config.mp_mode).c_str(), - memoryPartitionString(new_memory_partition).c_str()); - } else { - // if driver reload (and set) was successful, but not a new NPS mode - IF_VERB(STANDARD) { - std::cout << "\t**" - << "Since driver reload (and set) was successful, but no new NPS mode " - << "was requested; current memory partition (" - << memoryPartitionString(current_memory_config.mp_mode) - << ") is expected to be equal to original (" - << memoryPartitionString(saved_orig_memory_partition) - << ") and equal to requested (" << memoryPartitionString(new_memory_partition) - << ")" << std::endl; - } - ASSERT_STREQ(memoryPartitionString(current_memory_config.mp_mode).c_str(), - memoryPartitionString(saved_orig_memory_partition).c_str()); - ASSERT_STREQ(memoryPartitionString(current_memory_config.mp_mode).c_str(), - memoryPartitionString(new_memory_partition).c_str()); - } - } else { - ASSERT_TRUE(ret_set == AMDSMI_STATUS_SUCCESS || ret_set == AMDSMI_STATUS_INVAL || - ret_set == AMDSMI_STATUS_NOT_SUPPORTED); - // Since driver reload or set memory partition was not successful - // we don't care about comparison - // There are times when these can be equal or not - IF_VERB(STANDARD) { - std::cout << "\t**Since driver reload or set memory partition was NOT successful, " - << "we cannot guarantee current memory partition (" - << memoryPartitionString(current_memory_config.mp_mode) - << ") will or will not match requested (" - << memoryPartitionString(new_memory_partition) << ")" << std::endl; - } + if (ret_set == AMDSMI_STATUS_SUCCESS) { + std::cout << "\t** Memory partition staged to " + << memoryPartitionString(new_memory_partition) + << ". A driver reload is required to apply: " + << "sudo modprobe -r amdgpu && sudo modprobe amdgpu\n"; + } + + // Verify capabilities are unchanged after the set attempt. + amdsmi_memory_partition_config_t post_set_config = {}; + auto ret_caps_post = + amdsmi_get_gpu_memory_partition_config(processor_handles_[dv_ind], &post_set_config); + if (ret_caps_post == AMDSMI_STATUS_SUCCESS && ret_caps == AMDSMI_STATUS_SUCCESS) { + EXPECT_EQ(post_set_config.partition_caps.nps_flags.nps1_cap, + current_memory_config.partition_caps.nps_flags.nps1_cap); + EXPECT_EQ(post_set_config.partition_caps.nps_flags.nps2_cap, + current_memory_config.partition_caps.nps_flags.nps2_cap); + EXPECT_EQ(post_set_config.partition_caps.nps_flags.nps4_cap, + current_memory_config.partition_caps.nps_flags.nps4_cap); + EXPECT_EQ(post_set_config.partition_caps.nps_flags.nps8_cap, + current_memory_config.partition_caps.nps_flags.nps8_cap); } } // END MEMORY PARTITION FOR LOOP @@ -856,51 +675,15 @@ void TestMemoryPartitionReadWrite::Run(void) { << orig_memory_partition << "): " << smi_amdgpu_get_status_string(ret, false) << std::endl; } - CHK_ERR_ASRT(ret) - if (ret == AMDSMI_STATUS_SUCCESS) { - // Now we require a separate call to reload the driver, since this operation - // has been removed from the amdsmi_set_gpu_memory_partition_mode and - // amdsmi_set_gpu_memory_partition(). - // This is to allow the user to select the appropriate time to reload the driver - // since there can be errors if any device has a workload/process running on it. - amdsmi_status_t driver_reload_status = AMDSMI_STATUS_NOT_SUPPORTED; - std::string reload_message = - "\t Reloading the AMD GPU driver after resetting memory partition to " + - std::string(orig_memory_partition) + ". This may take some time, please wait..."; - std::string driver_reload_success_message = - "amdsmi_gpu_driver_reload() successful after resetting memory partition to " + - std::string(orig_memory_partition); - std::string failure_message = - "amdsmi_gpu_driver_reload() failed after resetting memory partition to " + - std::string(orig_memory_partition); - std::string restart_error_message = - "amdsmi_gpu_driver_reload() failed with AMDGPU_RESTART_ERR after " - "resetting memory partition to " + - std::string(orig_memory_partition); - ReloadDriverWithMessages(isVerbose, reload_message, driver_reload_success_message, - failure_message, restart_error_message, &driver_reload_status); - } - DISPLAY_AMDSMI_API("amdsmi_get_gpu_memory_partition", "gpu=" + std::to_string(dv_ind), - VERB(STANDARD)); - ret = amdsmi_get_gpu_memory_partition(processor_handles_[dv_ind], current_memory_partition, - k255Len); - DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); - CHK_ERR_ASRT(ret) - IF_VERB(STANDARD) { - std::cout << "\t**" - << "Attempted to set memory partition: " - << memoryPartitionString(new_memory_partition) << std::endl - << "\t**" - << "Current memory partition: " << current_memory_partition << std::endl; - } - ASSERT_EQ(AMDSMI_STATUS_SUCCESS, ret); - ASSERT_STREQ(orig_memory_partition, current_memory_partition); - IF_VERB(STANDARD) { - std::cout << "\t**" - << "Confirmed prior memory partition (" << orig_memory_partition - << ") is equal to current memory partition (" << current_memory_partition << ")" - << std::endl; - } + // The set stages the restore; a driver reload is required before the sysfs + // value flips. Assert only that the API accepted the request. + ASSERT_TRUE(ret == AMDSMI_STATUS_SUCCESS || ret == AMDSMI_STATUS_INVAL || + ret == AMDSMI_STATUS_NOT_SUPPORTED) + << "amdsmi_set_gpu_memory_partition returned unexpected status: " + << smi_amdgpu_get_status_string(ret, false); + std::cout << "\t** Memory partition restore staged to " << orig_memory_partition + << ". A driver reload is required to apply: " + << "sudo modprobe -r amdgpu && sudo modprobe amdgpu\n"; } // END DEVICE FOR LOOP // Restore original compute partition settings (see orig_dev_config ^) diff --git a/projects/amdsmi/tests/python/functional/gpu/test_partition.py b/projects/amdsmi/tests/python/functional/gpu/test_partition.py index 557db91fcf11..09620e74fbf1 100755 --- a/projects/amdsmi/tests/python/functional/gpu/test_partition.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_partition.py @@ -324,14 +324,147 @@ def test_set_gpu_accelerator_partition_profile(self): # Uses clk_type_name instead of clk_type # Uses clk_limit_type_name instead of clk_limit_type - def test_set_gpu_memory_partition(self): + def test_set_gpu_memory_partition_return_codes(self): + """Verify that set accepts supported modes and rejects unsupported ones.""" self.common.print_func_name("") - self.common.Test_Per_GPU_With_One_Enum( - amdsmi_set_gpu_memory_partition=amdsmi.amdsmi_set_gpu_memory_partition, - memory_partition_type=common.MEMORY_PARTITION_TYPES, - ) - return + NPS_NAME_TO_TYPE = { + "NPS1": amdsmi.AmdSmiMemoryPartitionType.NPS1, + "NPS2": amdsmi.AmdSmiMemoryPartitionType.NPS2, + "NPS4": amdsmi.AmdSmiMemoryPartitionType.NPS4, + "NPS8": amdsmi.AmdSmiMemoryPartitionType.NPS8, + } + + raise_exception = None + for i, gpu in enumerate(self.common.processors): + self.common.print_device_header(i) + + # Read current partition for restore. + original_partition = None + msg = f"\t### amdsmi_get_gpu_memory_partition(gpu={i}):" + try: + original_partition = amdsmi.amdsmi_get_gpu_memory_partition(gpu) + self.common.print(msg, original_partition) + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + if self.common.check_ret(msg, e, self.common.PASS): + raise_exception = e + continue + + # Read hardware capabilities to build per-GPU condition map. + supported_modes = set() + msg = f"\t### amdsmi_get_gpu_memory_partition_config(gpu={i}):" + try: + config = amdsmi.amdsmi_get_gpu_memory_partition_config(gpu) + caps = config.get("partition_caps", []) + self.common.print(msg, caps) + supported_modes = {c for c in caps if c in NPS_NAME_TO_TYPE} + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + # Older kernels may not support config query; fall through with empty set. + self.common.print(msg, e) + + # Attempt each NPS mode; condition is driven by hardware capabilities. + for mode_name, mode_type in NPS_NAME_TO_TYPE.items(): + cond = self.common.PASS if mode_name in supported_modes else self.common.FAIL + msg = f"\t### amdsmi_set_gpu_memory_partition(gpu={i}, mode={mode_name}):" + try: + amdsmi.amdsmi_set_gpu_memory_partition(gpu, mode_type) + self.common.print(msg, "SUCCESS") + if cond == self.common.FAIL: + self.common.print( + f"\t TEST FAILURE: expected failure for {mode_name} but call succeeded", + "", + ) + raise_exception = amdsmi.AmdSmiLibraryException(0) + else: + self.common.check_ret("", "", self.common.PASS) + self.common.print( + f"\t [info] Staged {mode_name}. Apply with: " + "sudo modprobe -r amdgpu && sudo modprobe amdgpu", + "", + ) + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + if self.common.check_ret(msg, e, cond): + raise_exception = e + + # Restore original partition (stages the write; driver reload applies it). + if original_partition and original_partition in NPS_NAME_TO_TYPE: + msg = f"\t### amdsmi_set_gpu_memory_partition restore(gpu={i}, mode={original_partition}):" + try: + amdsmi.amdsmi_set_gpu_memory_partition( + gpu, NPS_NAME_TO_TYPE[original_partition] + ) + self.common.print(msg, "SUCCESS") + self.common.print( + f"\t [info] Restore staged to {original_partition}. Apply with: " + "sudo modprobe -r amdgpu && sudo modprobe amdgpu", + "", + ) + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + if self.common.check_ret(msg, e, self.common.PASS): + raise_exception = e + + if raise_exception: + raise raise_exception + + def test_set_gpu_memory_partition_idempotent(self): + """Setting the current partition mode must succeed and leave the read-back unchanged.""" + self.common.print_func_name("") + + NPS_NAME_TO_TYPE = { + "NPS1": amdsmi.AmdSmiMemoryPartitionType.NPS1, + "NPS2": amdsmi.AmdSmiMemoryPartitionType.NPS2, + "NPS4": amdsmi.AmdSmiMemoryPartitionType.NPS4, + "NPS8": amdsmi.AmdSmiMemoryPartitionType.NPS8, + } + + raise_exception = None + for i, gpu in enumerate(self.common.processors): + self.common.print_device_header(i) + + # Read current partition. + current_partition = None + msg = f"\t### amdsmi_get_gpu_memory_partition(gpu={i}):" + try: + current_partition = amdsmi.amdsmi_get_gpu_memory_partition(gpu) + self.common.print(msg, current_partition) + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + if self.common.check_ret(msg, e, self.common.PASS): + raise_exception = e + continue + + if current_partition not in NPS_NAME_TO_TYPE: + self.common.print(f"\t [skip] Unknown partition mode: {current_partition}", "") + continue + + # Set the same mode again — must succeed. + msg = f"\t### amdsmi_set_gpu_memory_partition idempotent(gpu={i}, mode={current_partition}):" + try: + amdsmi.amdsmi_set_gpu_memory_partition(gpu, NPS_NAME_TO_TYPE[current_partition]) + self.common.print(msg, "SUCCESS") + self.common.check_ret("", "", self.common.PASS) + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + if self.common.check_ret(msg, e, self.common.PASS): + raise_exception = e + continue + + # Read back — value must be unchanged (no reload occurred). + msg = f"\t### amdsmi_get_gpu_memory_partition post-idempotent-set(gpu={i}):" + try: + post_partition = amdsmi.amdsmi_get_gpu_memory_partition(gpu) + self.common.print(msg, post_partition) + if post_partition != current_partition: + self.common.print( + f"\t TEST FAILURE: partition changed from {current_partition} " + f"to {post_partition} without a driver reload", + "", + ) + raise_exception = amdsmi.AmdSmiLibraryException(0) + except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: + if self.common.check_ret(msg, e, self.common.PASS): + raise_exception = e + + if raise_exception: + raise raise_exception def test_set_gpu_memory_partition_mode(self): self.common.print_func_name("") diff --git a/projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py b/projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py new file mode 100644 index 000000000000..d537ddfa541e --- /dev/null +++ b/projects/amdsmi/tests/python/integration/test_memory_partition_lifecycle.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# +# Copyright (C) Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +"""Memory partition driver lifecycle integration test. + +Exercises the full set -> driver reload -> verify round-trip that cannot be +covered by the standard functional test suite. + +Requirements +------------ +- Root privileges (modprobe requires root) +- kmod installed (modprobe must be on PATH) +- No GPU workloads running during the test +- Run in isolation -- NOT via the standard unittest/pytest runner + +Usage +----- + sudo python3 tests/python/integration/test_memory_partition_lifecycle.py [--target NPS2] + +Exit codes +---------- + 0 all phases passed + 1 a phase failed (reason printed to stdout) + 2 precondition not met (no root, no kmod, bad --target) + 3 unexpected exception +""" + +import argparse +import os +import shutil +import subprocess +import sys +import time + +import amdsmi + +_MODPROBE_TIMEOUT_SEC = 120 +_DRIVER_SETTLE_SEC = 5 + +_NPS_NAME_TO_TYPE = { + "NPS1": amdsmi.AmdSmiMemoryPartitionType.NPS1, + "NPS2": amdsmi.AmdSmiMemoryPartitionType.NPS2, + "NPS4": amdsmi.AmdSmiMemoryPartitionType.NPS4, + "NPS8": amdsmi.AmdSmiMemoryPartitionType.NPS8, +} + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _print(msg: str) -> None: + print(msg, flush=True) + + +def _fail(msg: str) -> int: + _print(f" [FAIL] {msg}") + return 1 + + +def _check_preconditions(target_mode: str) -> int: + """Return 0 if all preconditions are satisfied, 2 otherwise.""" + if target_mode not in _NPS_NAME_TO_TYPE: + _print( + f"[ERROR] Unknown target mode '{target_mode}'. Choose from: {list(_NPS_NAME_TO_TYPE)}" + ) + return 2 + + if os.geteuid() != 0: + _print("[ERROR] This test requires root privileges (run with sudo).") + return 2 + + if not shutil.which("modprobe"): + _print("[ERROR] modprobe not found. Install kmod and retry.") + return 2 + + return 0 + + +def _reload_driver() -> bool: + """ + Unload then reload the amdgpu kernel module. + + Uses two separate subprocess.run() calls with shell=False so the arguments + are passed directly to execvp -- no shell is involved and there is no + injection surface. Returns True on success. + """ + _print(" Running: modprobe -r amdgpu") + try: + subprocess.run(["modprobe", "-r", "amdgpu"], check=True, timeout=_MODPROBE_TIMEOUT_SEC) + except subprocess.CalledProcessError as e: + _print(f" [FAIL] modprobe -r amdgpu exited with code {e.returncode}; check dmesg") + return False + except subprocess.TimeoutExpired: + _print(f" [FAIL] modprobe -r amdgpu timed out after {_MODPROBE_TIMEOUT_SEC}s") + return False + + _print(" Running: modprobe amdgpu") + try: + subprocess.run(["modprobe", "amdgpu"], check=True, timeout=_MODPROBE_TIMEOUT_SEC) + except subprocess.CalledProcessError as e: + _print(f" [FAIL] modprobe amdgpu exited with code {e.returncode}; check dmesg") + return False + except subprocess.TimeoutExpired: + _print(f" [FAIL] modprobe amdgpu timed out after {_MODPROBE_TIMEOUT_SEC}s") + return False + + _print(f" Driver reloaded. Waiting {_DRIVER_SETTLE_SEC}s for devices to settle...") + time.sleep(_DRIVER_SETTLE_SEC) + return True + + +def _init_and_get_primary() -> tuple: + """ + Init amdsmi and return (gpu_handle, partition_string) for the first GPU. + Caller is responsible for calling amdsmi.amdsmi_shut_down() when done. + Raises RuntimeError if no GPUs are found. + """ + amdsmi.amdsmi_init() + gpus = amdsmi.amdsmi_get_processor_handles() + if not gpus: + amdsmi.amdsmi_shut_down() + raise RuntimeError("No GPU handles returned after amdsmi_init()") + gpu = gpus[0] + partition = amdsmi.amdsmi_get_gpu_memory_partition(gpu) + return gpu, partition + + +# --------------------------------------------------------------------------- +# Test phases +# --------------------------------------------------------------------------- + + +def _phase1_read_original() -> tuple: + """Return (gpu, original_partition). Shuts down amdsmi before returning.""" + _print("\n=== Phase 1: Read original partition ===") + gpu, original_partition = _init_and_get_primary() + _print(f" Original partition: {original_partition}") + amdsmi.amdsmi_shut_down() + return gpu, original_partition + + +def _phase2_check_capabilities(target_mode: str) -> bool: + """ + Return True if target_mode is advertised by hardware capabilities. + Returns True unconditionally if the config API is unavailable (older kernel). + Shuts down amdsmi before returning. + """ + _print("\n=== Phase 2: Check hardware capabilities ===") + gpu, _ = _init_and_get_primary() + supported = True + try: + config = amdsmi.amdsmi_get_gpu_memory_partition_config(gpu) + caps = config.get("partition_caps", []) + _print(f" Hardware capabilities: {caps}") + if target_mode not in caps: + _print(f" [SKIP] {target_mode} not advertised in partition_caps") + supported = False + except amdsmi.AmdSmiLibraryException as e: + _print(f" [WARN] amdsmi_get_gpu_memory_partition_config unavailable: {e}") + _print(" Proceeding without capability pre-check.") + amdsmi.amdsmi_shut_down() + return supported + + +def _phase3_stage_change(target_mode: str) -> int: + """Stage the partition change. Returns 0 on success, 1 on failure.""" + _print(f"\n=== Phase 3: Stage partition change to {target_mode} ===") + gpu, _ = _init_and_get_primary() + try: + amdsmi.amdsmi_set_gpu_memory_partition(gpu, _NPS_NAME_TO_TYPE[target_mode]) + _print(f" amdsmi_set_gpu_memory_partition({target_mode}): SUCCESS") + except amdsmi.AmdSmiLibraryException as e: + amdsmi.amdsmi_shut_down() + return _fail(f"amdsmi_set_gpu_memory_partition failed: {e}") + amdsmi.amdsmi_shut_down() + return 0 + + +def _phase4_reload() -> int: + """Reload the driver. Returns 0 on success, 1 on failure.""" + _print("\n=== Phase 4: Reload driver to apply change ===") + return 0 if _reload_driver() else _fail("Driver reload failed") + + +def _phase5_verify(expected_mode: str) -> int: + """Re-init and verify the active partition matches expected_mode. Returns 0 or 1.""" + _print(f"\n=== Phase 5: Re-init and verify partition is {expected_mode} ===") + _, post_partition = _init_and_get_primary() + _print(f" Post-reload partition: {post_partition}") + amdsmi.amdsmi_shut_down() + if post_partition != expected_mode: + return _fail(f"Expected {expected_mode}, got {post_partition}") + _print(f" [PASS] Partition is {post_partition}") + return 0 + + +def _phase6_restore(original_partition: str) -> int: + """Stage and apply restore to original_partition. Returns 0 or 1.""" + _print(f"\n=== Phase 6: Restore to {original_partition} ===") + gpu, _ = _init_and_get_primary() + try: + amdsmi.amdsmi_set_gpu_memory_partition(gpu, _NPS_NAME_TO_TYPE[original_partition]) + _print(f" amdsmi_set_gpu_memory_partition({original_partition}): SUCCESS") + except amdsmi.AmdSmiLibraryException as e: + amdsmi.amdsmi_shut_down() + return _fail(f"Restore set failed: {e}") + amdsmi.amdsmi_shut_down() + + if not _reload_driver(): + return _fail("Driver reload for restore failed") + + _, restored = _init_and_get_primary() + amdsmi.amdsmi_shut_down() + _print(f" Post-restore partition: {restored}") + if restored != original_partition: + return _fail(f"Restore failed: expected {original_partition}, got {restored}") + _print(f" [PASS] Partition restored to {restored}") + return 0 + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def run(target_mode: str) -> int: + rc = _check_preconditions(target_mode) + if rc != 0: + return rc + + try: + _, original_partition = _phase1_read_original() + + if original_partition == target_mode: + _print( + f"\n[SKIP] Device is already in {target_mode}. " + "Choose a different --target to exercise a partition change." + ) + return 0 + + if original_partition not in _NPS_NAME_TO_TYPE: + _print(f"\n[ERROR] Original partition '{original_partition}' is not a known NPS mode.") + return 1 + + if not _phase2_check_capabilities(target_mode): + return 0 # hardware does not support this mode -- skip, not fail + + rc = _phase3_stage_change(target_mode) + if rc != 0: + return rc + + rc = _phase4_reload() + if rc != 0: + # Driver did not reload -- attempt restore so the system is not left + # with a staged-but-unapplied partition change. + _print(" Attempting restore despite reload failure...") + _phase6_restore(original_partition) + return rc + + rc = _phase5_verify(target_mode) + # Always attempt restore, even if verify failed. + restore_rc = _phase6_restore(original_partition) + + if rc != 0: + return rc + if restore_rc != 0: + return restore_rc + + except Exception as e: + _print(f"\n[ERROR] Unexpected exception: {e}") + try: + amdsmi.amdsmi_shut_down() + except Exception: + pass + return 3 + + _print( + f"\n[PASS] Full lifecycle test passed: " + f"{original_partition} -> {target_mode} -> {original_partition}" + ) + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Memory partition driver lifecycle integration test." + ) + parser.add_argument( + "--target", + default="NPS2", + choices=list(_NPS_NAME_TO_TYPE), + help="Partition mode to change to (default: NPS2)", + ) + args = parser.parse_args() + sys.exit(run(args.target)) diff --git a/projects/amdsmi/tools/hw_compatibility_status.py b/projects/amdsmi/tools/hw_compatibility_status.py index 21a520822cd6..0560b41b9358 100644 --- a/projects/amdsmi/tools/hw_compatibility_status.py +++ b/projects/amdsmi/tools/hw_compatibility_status.py @@ -598,12 +598,6 @@ def run_tests(): test_api("amdsmi_get_violation_status", lambda: amdsmi.amdsmi_get_violation_status(gpu_handle)) - test_api( - "amdsmi_gpu_driver_reload", - None, - skip_reason="Risky - reloads GPU driver, could crash system", - ) - # ======================================================================== # SECTION 7: PCIe INFO (12 APIs) # ======================================================================== From 23ec5edc37232b9181269496cb59522e60b01611 Mon Sep 17 00:00:00 2001 From: Joseph Narlo Date: Wed, 29 Jul 2026 19:49:30 -0500 Subject: [PATCH 3/5] refactor(amdsmi): Sync with unified with known abi changes (#7844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fabric (UALoE/IFoE) telemetry, fabric-info, and NIC firmware APIs were introduced in the in-progress ROCm 7.14.0 release but their declarations had drifted from the "unified" ABI that downstream consumers build against. The public surface carried inconsistencies that would force source churn on every caller and leave dead symbols in the header: a string-returning telemetry helper that couldn't report failure, a misspelled fabric-type enumerator (UALLINK), an intermediate versioning struct that added an extra access level, a NIC firmware typedef whose name didn't reflect that it describes a single entry, a duplicate telemetry-category enumerator, and a cluster of unreferenced max-count macros. Because the feature is still unreleased, this is the last clean opportunity to reconcile the headers with the agreed ABI before the symbols are frozen, avoiding a future breaking change once external code depends on them. - `amdsmi_fabric_telem_id_to_string()` signature changed from `const char* f(uint64_t)` to `amdsmi_status_t f(uint64_t, const char**)`, so the function can surface lookup failures via a status code and return the name through an out-parameter. Callers (the GTest, the example, and the Python interface) were updated to pass a `const char**` and check the status. - Flattened `amdsmi_fabric_info_t` by removing the intermediate `amdsmi_fabric_info_ver_t` type. Its payload union is now the `fabric_info` member of `amdsmi_fabric_info_t`, and its version field is exposed directly as the top-level `fabric_version`. Field access simplifies from `fabric_info.fabric_version.v1.` to `fabric_info.v1.`, and `fabric_info.version` becomes `fabric_version`. The change is ABI-preserving: field offsets and overall struct size are unchanged, and the Python `amdsmi_get_gpu_fabric_info()` dictionary keys are unchanged. - Typedef rename: `amdsmi_nic_fw_t` → `amdsmi_nic_fw_entry_t` (it models one firmware entry within `amdsmi_nic_fw_info_t::fw[]`), with the `AMDSmiAINICDevice` member updated to match. - Enumerator rename: `AMDSMI_FABRIC_TYPE_UALLINK` → `AMDSMI_FABRIC_TYPE_UALINK`, including the `UALoeLinkTypeMap` entry. - Removed symbols: the duplicate `AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN` enumerator (identical `0xFFFFFFFF` value to `..._INVALID`), and eight unreferenced public macros (`AMDSMI_MAX_VF_COUNT`, `AMDSMI_MAX_DRIVER_NUM`, `AMDSMI_DFC_FW_NUMBER_OF_ENTRIES`, `AMDSMI_MAX_WHITE_LIST_ELEMENTS`, `AMDSMI_MAX_BLACK_LIST_ELEMENTS`, `AMDSMI_MAX_TA_WHITE_LIST_ELEMENTS`, `AMDSMI_MAX_ERR_RECORDS`, `AMDSMI_MAX_PROFILE_COUNT`). - Cascade: the ctypes wrapper (`amdsmi_wrapper.py`) and Rust bindings (`amdsmi_wrapper.rs`) were regenerated; `amdsmi_interface.py` was updated by hand; the CHANGELOG records the Changed/Removed entries. JIRA ID : AILITOOLS-270 C++ functional tests Python CLI test All tests passed as expected --------- Signed-off-by: josnarlo Signed-off-by: Maisam Arif Signed-off-by: Galantsev, Dmitrii Co-authored-by: Maisam Arif Co-authored-by: Galantsev, Dmitrii Co-authored-by: Mika Laitio (cherry picked from commit 923767a953371a3c9a01c60162ef6da5e3720ec5) --- projects/amdsmi/CHANGELOG.md | 59 ++++- .../amdsmi_cli/subcommands/set_value.py | 22 +- .../amdsmi/amdsmi_cli/subcommands/static.py | 6 +- .../amdsmi/docs/reference/amdsmi-py-api.md | 143 +--------- .../amdsmi/example/amd_smi_drm_example.cc | 224 ++++++++-------- .../amdsmi/example/amd_smi_fabric_example.cc | 45 ++-- projects/amdsmi/include/amd_smi/amdsmi.h | 232 ++++++++++------- .../amd_smi/impl/nic/amd_smi_ainic_device.h | 2 +- projects/amdsmi/py-interface/__init__.py | 7 +- .../amdsmi/py-interface/amdsmi_interface.py | 245 +++++++++--------- .../amdsmi/py-interface/amdsmi_wrapper.py | 120 +++++---- .../include/rocm_smi/rocm_smi_properties.h | 1 - projects/amdsmi/rocm_smi/src/rocm_smi.cc | 3 +- projects/amdsmi/rust-interface/README.md | 8 +- .../examples/amdsmi_get_gpu_info.rs | 6 - projects/amdsmi/rust-interface/src/amdsmi.rs | 112 -------- .../rust-interface/src/amdsmi_wrapper.rs | 121 ++++----- projects/amdsmi/src/amd_smi/amd_smi.cc | 183 +++++++------ .../amdsmi/src/amd_smi/amd_smi_gpu_device.cc | 147 +++++------ projects/amdsmi/src/amd_smi/amd_smi_ualoe.cc | 11 +- projects/amdsmi/src/rocm_smi_properties.cc | 6 - .../gpu/clock/clock_limit_read_write.h | 51 ++++ .../gpu/clock/clock_limit_read_write_test.cc | 182 +++++++++++++ .../gpu/identity/id_info_read_test.cc | 13 - ...epartition_memallocmode_read_write_test.cc | 54 ++-- .../computepartition_read_write_test.cc | 85 +++--- .../memorypartition_read_write_test.cc | 10 +- .../ifoe/fabric/fabric_read_test.cc | 12 +- .../ifoe/identity/ifoe_info_read_test.cc | 15 -- .../system/mutual_exclusion_test.cc | 7 +- projects/amdsmi/tests/amd_smi_test/main.cc | 7 + projects/amdsmi/tests/python/cli/base.py | 33 ++- projects/amdsmi/tests/python/cli/test_set.py | 9 +- projects/amdsmi/tests/python/common/common.py | 14 +- .../python/functional/gpu/test_benchmark.py | 67 ----- .../python/functional/gpu/test_identity.py | 5 - .../python/functional/gpu/test_overdrive.py | 25 -- .../python/functional/gpu/test_partition.py | 92 +------ .../amdsmi/tools/hw_compatibility_status.py | 13 +- projects/rccl/src/include/amdsmi_wrap.h | 35 ++- projects/rccl/src/misc/amdsmi_wrap.cc | 15 +- 41 files changed, 1198 insertions(+), 1249 deletions(-) create mode 100644 projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write.h create mode 100644 projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write_test.cc diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md index a915af8d9ecc..534f55a1648e 100644 --- a/projects/amdsmi/CHANGELOG.md +++ b/projects/amdsmi/CHANGELOG.md @@ -6,20 +6,68 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr ## amd_smi_lib for ROCm 7.15.0 +### Added + +- **Added accelerator partition memory allocation mode API**. + - New APIs: `amdsmi_get_gpu_accelerator_partition_mem_alloc_mode()`, `amdsmi_set_gpu_accelerator_partition_mem_alloc_mode()`. + - New enum: `amdsmi_accelerator_partition_mem_alloc_mode_t` (`AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING`, `AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL`). + - Supersedes the equivalent `compute_partition` memory allocation mode APIs, which are now deprecated. + ### Changed - **Bumped the library major version to 27.0.0** (breaking). - The shared library SONAME is now `libamd_smi.so.27`. Consumers linked against `libamd_smi.so.26` must relink; no source changes are required beyond the API changes listed elsewhere in this release. +- **Widened five `amdsmi_gpu_metrics_t` accumulator counters from 32-bit to 64-bit** (breaking). + - `gfx_activity_acc`, `mem_activity_acc`, `pcie_nak_sent_count_acc`, `pcie_nak_rcvd_count_acc`, and `pcie_lc_perf_other_end_recovery` are now `uint64_t` to match the amdgpu pmfw metrics header. Recompile callers that read these fields; the struct layout and field offsets have changed. + +- **Aligned the fabric telemetry and NIC firmware API surface with the unified ABI** (breaking). + - `amdsmi_fabric_telem_id_to_string()` now returns `amdsmi_status_t` and writes the name through a `const char**` out-parameter, instead of returning a `const char*` directly. + - Renamed `amdsmi_nic_fw_t` to `amdsmi_nic_fw_entry_t`. + - Added the `amdsmi_fabric_type_t` enumerator `AMDSMI_FABRIC_TYPE_UALINK`; the previous spelling `AMDSMI_FABRIC_TYPE_UALLINK` is retained as a deprecated alias with the same value and is slated for removal in a future ROCm release. + +- **Flattened the `amdsmi_fabric_info_t` structure and removed `amdsmi_fabric_info_ver_t`**. + - The intermediate `amdsmi_fabric_info_ver_t` type was removed from the public header. Its payload union is now the `fabric_info` member of `amdsmi_fabric_info_t`, and its version field is exposed directly as the top-level `fabric_version` field. + - Field access simplifies from `fabric_info.fabric_version.v1.` to `fabric_info.v1.`, and `fabric_info.version` becomes `fabric_version`. + - The change is ABI-preserving: field offsets and the overall structure size are unchanged. The Python `amdsmi_get_gpu_fabric_info()` dictionary keys are also unchanged. + +- **Prefixed public preprocessor macros with `AMDSMI_` in `amdsmi.h`** (breaking). + - `MAX_SVI3_RAIL_INDEX`, `MAX_SVI3_RAIL_SELECTION`, `POWER_EFFICIENCY_MODE_4`, `POWER_EFFICIENCY_MODE_5`, and `MAX_NUMBER_OF_AFIDS_PER_RECORD` are now `AMDSMI_MAX_SVI3_RAIL_INDEX`, `AMDSMI_MAX_SVI3_RAIL_SELECTION`, `AMDSMI_POWER_EFFICIENCY_MODE_4`, `AMDSMI_POWER_EFFICIENCY_MODE_5`, and `AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD`. The unused `CENTRIGRADE_TO_MILLI_CENTIGRADE` macro was removed. Update references to the new names. + +- **Deprecated the `compute_partition` APIs in favor of `accelerator_partition` equivalents**. + - `amdsmi_get_gpu_compute_partition()`, `amdsmi_set_gpu_compute_partition()`, `amdsmi_get_gpu_compute_partition_mem_alloc_mode()`, and `amdsmi_set_gpu_compute_partition_mem_alloc_mode()` are slated for removal in a future ROCm release. They now emit a `DeprecationWarning` from the Python interface and delegate to their `accelerator_partition` counterparts. + - The `amdsmi_compute_partition_type_t` and `amdsmi_compute_partition_mem_alloc_mode_t` enums are deprecated in favor of `amdsmi_accelerator_partition_type_t` and `amdsmi_accelerator_partition_mem_alloc_mode_t`. + +- **Deprecated `amdsmi_set_gpu_memory_partition()` in favor of `amdsmi_set_gpu_memory_partition_mode()`**. + - `amdsmi_set_gpu_memory_partition` is slated for removal in a future ROCm release. It now emits a `DeprecationWarning` from the Python interface and functions as a wrapper of `amdsmi_set_gpu_memory_partition_mode()`. + +- **Namespaced `amdsmi_clk_limit_type_t` and `amdsmi_io_bw_encoding_t` enumerators with an `AMDSMI_` prefix**. + - Added `AMDSMI_CLK_LIMIT_MIN`/`AMDSMI_CLK_LIMIT_MAX` and `AMDSMI_AGG_BW0`/`AMDSMI_RD_BW0`/`AMDSMI_WR_BW0`. + - The unprefixed names (`CLK_LIMIT_MIN`, `CLK_LIMIT_MAX`, `AGG_BW0`, `RD_BW0`, `WR_BW0`) are retained as deprecated aliases with unchanged values and are slated for removal in a future ROCm release. + +- **Removed the `amdsmi_gpu_driver_reload()` API and its Python binding** (breaking). + - Reload the amdgpu driver out of band with `sudo modprobe -r amdgpu && sudo modprobe amdgpu` to apply memory partition changes instead. + +- **Removed unused public macros to match the unified ABI** (breaking). + - Dropped the `AMDSMI_MAX_VF_COUNT`, `AMDSMI_MAX_DRIVER_NUM`, `AMDSMI_DFC_FW_NUMBER_OF_ENTRIES`, `AMDSMI_MAX_WHITE_LIST_ELEMENTS`, `AMDSMI_MAX_BLACK_LIST_ELEMENTS`, `AMDSMI_MAX_TA_WHITE_LIST_ELEMENTS`, `AMDSMI_MAX_ERR_RECORDS`, `AMDSMI_MAX_PROFILE_COUNT`, and `AMDSMI_PF_INDEX` defines, which were unreferenced by any API or struct. + +- **Deduplicated the `amdsmi_fabric_telemetry_category_t` enumerators**. + - `AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID` is now the canonical `0xFFFFFFFF` value; the previous `AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN` name (same value) is retained as a deprecated alias and is slated for removal in a future ROCm release. + +- **Deprecated `amdsmi_get_gpu_vram_vendor()`**. Use `amdsmi_get_gpu_vram_info()` and read the `vram_vendor` field instead. The function is retained (it now delegates to `amdsmi_get_gpu_vram_info()`, and the Python binding emits a `DeprecationWarning`) and is slated for removal in a future ROCm release. + +- **Removed `amdsmi_get_cpusocket_handles()` from the Python interface** (breaking). Use `amdsmi_get_cpu_handles()` instead. + +- **Removed the `plpds` key from `amdsmi_get_xgmi_plpd()` Python output** (breaking). Use the `policies` key instead. + +- **Removed `amdsmi_set_gpu_clk_range()`** (breaking). Use `amdsmi_set_gpu_clk_limit()` instead. + - **Restructured AMD SMI C++ tests into unit and functional suites**. - The `amdsmitst` source tree now separates unit tests from hardware-backed functional tests under `tests/amd_smi_test/unit/` and `tests/amd_smi_test/functional/`. - GTest suite names now follow a `[]` scheme: functional tests are `FunctionalReadOnly`/`FunctionalReadWrite` (e.g. `GpuFunctionalReadOnly`) and unit tests are `Unit` (e.g. `GpuUnit`). This replaces the old `amdsmitstReadOnly`/`amdsmitstReadWrite` and `AmdSmiDynamicMetricTest` names. - Consumers that pass explicit `--gtest_filter` values should update those filters to the new suite names. - See the [AMD SMI test design](docs/conceptual/test-design.md#naming-conventions) for the suite naming convention and `--gtest_filter` usage. -- **Widened five `amdsmi_gpu_metrics_t` accumulator counters from 32-bit to 64-bit** (breaking). - - `gfx_activity_acc`, `mem_activity_acc`, `pcie_nak_sent_count_acc`, `pcie_nak_rcvd_count_acc`, and `pcie_lc_perf_other_end_recovery` are now `uint64_t` to match the amdgpu pmfw metrics header. Recompile callers that read these fields; the struct layout and field offsets have changed. - ### Fixed - **Fixed `amd-smi ras --cper --json` emitting nothing when there are no CPER entries**. @@ -31,11 +79,6 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr - The per-process KFD lookup rebuilt the entire KFD node topology (an expensive sysfs walk) on every call just to translate the device BDF into its KFD GPU id. - The caller already knows this value, so it is now passed through to `gpuvsmi_get_pid_info()`, eliminating one full topology discovery per process per refresh. Falls back to the original discovery path when the id is unavailable. -### Removed - -- **Removed the `amdsmi_gpu_driver_reload()` API and its Python binding** (breaking). - - Reload the amdgpu driver out of band with `sudo modprobe -r amdgpu && sudo modprobe amdgpu` to apply memory partition changes instead. - ### Resolved Issues - **Fixed `amd-smi set --ptl-status` silently failing to change PTL state**. diff --git a/projects/amdsmi/amdsmi_cli/subcommands/set_value.py b/projects/amdsmi/amdsmi_cli/subcommands/set_value.py index 76b7a8a3b5d3..f5ecb31206d2 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/set_value.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/set_value.py @@ -1254,7 +1254,9 @@ def format_fan_error(message, include_driver_note=False): memory_partition = amdsmi_interface.AmdSmiMemoryPartitionType[ args.memory_partition ] - amdsmi_interface.amdsmi_set_gpu_memory_partition(args.gpu, memory_partition) + amdsmi_interface.amdsmi_set_gpu_memory_partition_mode( + args.gpu, memory_partition + ) out = f"Successfully set memory partition to {args.memory_partition}, use `sudo modprobe -r amdgpu && sudo modprobe amdgpu` to reload driver" except amdsmi_exception.AmdSmiLibraryException as e: out = f"[{e.get_error_info(detailed=False)}] Unable to set memory partition to {args.memory_partition}" @@ -1805,34 +1807,34 @@ def format_fan_error(message, include_driver_note=False): if getattr(args, "compute_partition_mem_alloc_mode", None): try: - mode = amdsmi_interface.AmdSmiComputePartitionMemAllocModeType[ + mode = amdsmi_interface.AmdSmiAcceleratorPartitionMemAllocModeType[ args.compute_partition_mem_alloc_mode ] - amdsmi_interface.amdsmi_set_gpu_compute_partition_mem_alloc_mode(args.gpu, mode) - out = f"Successfully set compute partition memory allocation mode to {args.compute_partition_mem_alloc_mode}" + amdsmi_interface.amdsmi_set_gpu_accelerator_partition_mem_alloc_mode(args.gpu, mode) + out = f"Successfully set accelerator partition memory allocation mode to {args.compute_partition_mem_alloc_mode}" except KeyError: out = ( - "Invalid compute partition memory allocation mode " + "Invalid accelerator partition memory allocation mode " f"{args.compute_partition_mem_alloc_mode}" ) - self.logger.store_output(args.gpu, "compute_partition_mem_alloc_mode", out) + self.logger.store_output(args.gpu, "accelerator_partition_mem_alloc_mode", out) self.logger.print_output() self.logger.clear_multiple_devices_output() return except amdsmi_exception.AmdSmiLibraryException as e: - out = f"[{e.get_error_info(detailed=False)}] Unable to set compute partition memory allocation mode to {args.compute_partition_mem_alloc_mode}" + out = f"[{e.get_error_info(detailed=False)}] Unable to set accelerator partition memory allocation mode to {args.compute_partition_mem_alloc_mode}" if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: out = "[AMDSMI_STATUS_NO_PERM] Command requires elevation" - self.logger.store_output(args.gpu, "compute_partition_mem_alloc_mode", out) + self.logger.store_output(args.gpu, "accelerator_partition_mem_alloc_mode", out) self.logger.print_output() self.logger.clear_multiple_devices_output() raise PermissionError("Command requires elevation") from e else: - self.logger.store_output(args.gpu, "compute_partition_mem_alloc_mode", out) + self.logger.store_output(args.gpu, "accelerator_partition_mem_alloc_mode", out) self.logger.print_output() self.logger.clear_multiple_devices_output() return - self.logger.store_output(args.gpu, "compute_partition_mem_alloc_mode", out) + self.logger.store_output(args.gpu, "accelerator_partition_mem_alloc_mode", out) self.logger.print_output() self.logger.clear_multiple_devices_output() return diff --git a/projects/amdsmi/amdsmi_cli/subcommands/static.py b/projects/amdsmi/amdsmi_cli/subcommands/static.py index e797208d08cb..e80047d3017a 100644 --- a/projects/amdsmi/amdsmi_cli/subcommands/static.py +++ b/projects/amdsmi/amdsmi_cli/subcommands/static.py @@ -836,13 +836,13 @@ def static_gpu( e.get_error_info(), ) try: - mem_alloc_mode = amdsmi_interface.amdsmi_get_gpu_compute_partition_mem_alloc_mode( - args.gpu + mem_alloc_mode = ( + amdsmi_interface.amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(args.gpu) ) except amdsmi_exception.AmdSmiLibraryException as e: mem_alloc_mode = "N/A" logging.debug( - "Failed to get compute partition mem alloc mode for gpu %s | %s", + "Failed to get accelerator partition mem alloc mode for gpu %s | %s", gpu_id, e.get_error_info(), ) diff --git a/projects/amdsmi/docs/reference/amdsmi-py-api.md b/projects/amdsmi/docs/reference/amdsmi-py-api.md index 8ad4cb5d5019..2f1848d80c29 100644 --- a/projects/amdsmi/docs/reference/amdsmi-py-api.md +++ b/projects/amdsmi/docs/reference/amdsmi-py-api.md @@ -2317,51 +2317,6 @@ finally: amdsmi.amdsmi_shut_down() ``` -### amdsmi_set_gpu_clk_range - -Description: This function sets the clock range information. -It is not supported on virtual machine guest - -Input parameters: - -* `processor_handle` handle for the given device -* `min_clk_value` minimum clock value for desired clock range -* `max_clk_value` maximum clock value for desired clock range -* `clk_type` SYS | MEM range type - -Output: None - -Exceptions that can be thrown by `amdsmi_set_gpu_clk_range` function: - -* `AmdSmiLibraryException` -* `AmdSmiParameterException` - -#### Possible Library Exceptions - -- `AMDSMI_STATUS_NO_HSMP_MSG_SUP` - HSMP message/feature not supported -- `AMDSMI_STATUS_NOT_SUPPORTED` - Feature not supported -- `AMDSMI_STATUS_NOT_YET_IMPLEMENTED` - Feature not yet implemented -- `AMDSMI_STATUS_TIMEOUT` - Timeout in API call -- `AMDSMI_STATUS_INVAL` - Invalid parameters - -Example: - -```python -import amdsmi -try: - amdsmi.amdsmi_init() - devices = amdsmi.amdsmi_get_processor_handles() - if len(devices) == 0: - print("No GPUs on machine") - else: - for device in devices: - amdsmi.amdsmi_set_gpu_clk_range(device, 0, 1000, amdsmi.AmdSmiClkType.SYS) -except amdsmi.AmdSmiException as e: - print(e) -finally: - amdsmi.amdsmi_shut_down() -``` - ### amdsmi_get_gpu_bdf_id Description: Get the unique PCI device identifier associated for a device @@ -4575,7 +4530,7 @@ Field | Description ---|--- `num_supported` | The number of supported policies `current_id` | The current policy index -`policies` | List of policies. (`plpds` marked for deprecation in next major release) +`policies` | List of policies. Exceptions that can be thrown by `amdsmi_get_xgmi_plpd` function: @@ -5151,48 +5106,6 @@ finally: amdsmi.amdsmi_shut_down() ``` -### amdsmi_get_gpu_vram_vendor - -Description: **Deprecated** (slated for removal in a future ROCm release; use `amdsmi_get_gpu_vram_info()` instead). Get the vram vendor string of a gpu device. - -Input parameters: - -* `processor_handle` device which to query - -Output: vram vendor - -Exceptions that can be thrown by `amdsmi_get_gpu_vram_vendor` function: - -* `AmdSmiLibraryException` -* `AmdSmiParameterException` - -#### Possible Library Exceptions - -- `AMDSMI_STATUS_NOT_SUPPORTED` - Feature not supported -- `AMDSMI_STATUS_NOT_YET_IMPLEMENTED` - Feature not yet implemented -- `AMDSMI_STATUS_NO_HSMP_MSG_SUP` - HSMP message/feature not supported -- `AMDSMI_STATUS_INVAL` - Invalid parameters -- `AMDSMI_STATUS_TIMEOUT` - Timeout in API call - -Example: - -```python -import amdsmi -try: - amdsmi.amdsmi_init() - devices = amdsmi.amdsmi_get_processor_handles() - if len(devices) == 0: - print("No GPUs on machine") - else: - for device in devices: - vram_vendor = amdsmi.amdsmi_get_gpu_vram_vendor(device) - print(vram_vendor) -except amdsmi.AmdSmiException as e: - print(e) -finally: - amdsmi.amdsmi_shut_down() -``` - ### amdsmi_get_gpu_subsystem_id Description: Get the subsystem device id associated with the device with provided device handle. @@ -5690,7 +5603,7 @@ finally: amdsmi.amdsmi_shut_down() ``` -### amdsmi_get_gpu_compute_partition_mem_alloc_mode +### amdsmi_get_gpu_accelerator_partition_mem_alloc_mode Description: Get the compute partition memory allocation mode from the given GPU. Controls how HBM capacity is distributed across XCPs within each memory partition. @@ -5700,7 +5613,7 @@ Input parameters: Output: String of the memory allocation mode (`"CAPPING"` or `"ALL"`) -Exceptions that can be thrown by `amdsmi_get_gpu_compute_partition_mem_alloc_mode` function: +Exceptions that can be thrown by `amdsmi_get_gpu_accelerator_partition_mem_alloc_mode` function: * `AmdSmiLibraryException` * `AmdSmiParameterException` @@ -5724,7 +5637,7 @@ try: print("No GPUs on machine") else: for device in devices: - mode = amdsmi.amdsmi_get_gpu_compute_partition_mem_alloc_mode(device) + mode = amdsmi.amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(device) print(mode) except amdsmi.AmdSmiException as e: print(e) @@ -5732,7 +5645,7 @@ finally: amdsmi.amdsmi_shut_down() ``` -### amdsmi_set_gpu_compute_partition_mem_alloc_mode +### amdsmi_set_gpu_accelerator_partition_mem_alloc_mode Description: Set the compute partition memory allocation mode for the given GPU. Requires elevated privileges (sudo). Controls whether each XCP is capped to an even share of memory (`CAPPING`) or may use the full partition memory (`ALL`). @@ -5743,7 +5656,7 @@ Input parameters: Output: None -Exceptions that can be thrown by `amdsmi_set_gpu_compute_partition_mem_alloc_mode` function: +Exceptions that can be thrown by `amdsmi_set_gpu_accelerator_partition_mem_alloc_mode` function: * `AmdSmiLibraryException` * `AmdSmiParameterException` @@ -5767,7 +5680,7 @@ try: print("No GPUs on machine") else: for device in devices: - amdsmi.amdsmi_set_gpu_compute_partition_mem_alloc_mode( + amdsmi.amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( device, amdsmi.AmdSmiComputePartitionMemAllocModeType.CAPPING ) except amdsmi.AmdSmiException as e: @@ -5860,48 +5773,6 @@ finally: amdsmi.amdsmi_shut_down() ``` -### amdsmi_set_gpu_memory_partition - -Description: Set the memory partition to the given GPU. This function does not allow any concurrent operations. Devices must be idle and have no workloads when performing set partition operations. - -Input parameters: - -* `processor_handle` the device handle -* `memory_partition` the type of memory_partition to set - -Output: `None` - -Exceptions that can be thrown by `amdsmi_set_gpu_memory_partition` function: - -* `AmdSmiLibraryException` -* `AmdSmiParameterException` - -#### Possible Library Exceptions - -- `AMDSMI_STATUS_NOT_SUPPORTED` - Feature not supported -- `AMDSMI_STATUS_INVAL` - Invalid parameters -- `AMDSMI_STATUS_NO_PERM` - Permission Denied -- `AMDSMI_STATUS_BUSY` - Device is busy, could not acquire resource or mutex - -Example: - -```python -import amdsmi -try: - amdsmi.amdsmi_init() - memory_partition = amdsmi.AmdSmiMemoryPartitionType.NPS1 - devices = amdsmi.amdsmi_get_processor_handles() - if len(devices) == 0: - print("No GPUs on machine") - else: - for device in devices: - amdsmi.amdsmi_set_gpu_memory_partition(device, memory_partition) -except amdsmi.AmdSmiException as e: - print(e) -finally: - amdsmi.amdsmi_shut_down() -``` - ### amdsmi_get_gpu_memory_partition_config Description: Get the current memory partition mode and supported NPS modes for the given GPU. diff --git a/projects/amdsmi/example/amd_smi_drm_example.cc b/projects/amdsmi/example/amd_smi_drm_example.cc index e762666bc64b..2db0d87d769a 100644 --- a/projects/amdsmi/example/amd_smi_drm_example.cc +++ b/projects/amdsmi/example/amd_smi_drm_example.cc @@ -369,30 +369,24 @@ static void print_apu_metrics_info(const amdsmi_gpu_metrics_t& smu) { } } -static const std::string computePartitionString( - amdsmi_compute_partition_type_t computeParitionType) { - switch (computeParitionType) { - case AMDSMI_COMPUTE_PARTITION_SPX: +static const std::string acceleratorPartitionString( + amdsmi_accelerator_partition_type_t acceleratorParitionType) { + switch (acceleratorParitionType) { + case AMDSMI_ACCELERATOR_PARTITION_SPX: return "SPX"; - case AMDSMI_COMPUTE_PARTITION_DPX: + case AMDSMI_ACCELERATOR_PARTITION_DPX: return "DPX"; - case AMDSMI_COMPUTE_PARTITION_TPX: + case AMDSMI_ACCELERATOR_PARTITION_TPX: return "TPX"; - case AMDSMI_COMPUTE_PARTITION_QPX: + case AMDSMI_ACCELERATOR_PARTITION_QPX: return "QPX"; - case AMDSMI_COMPUTE_PARTITION_CPX: + case AMDSMI_ACCELERATOR_PARTITION_CPX: return "CPX"; default: return "N/A"; } } -static const std::map - mapStringToSMIComputePartitionTypes{ - {"SPX", AMDSMI_COMPUTE_PARTITION_SPX}, {"DPX", AMDSMI_COMPUTE_PARTITION_DPX}, - {"TPX", AMDSMI_COMPUTE_PARTITION_TPX}, {"QPX", AMDSMI_COMPUTE_PARTITION_QPX}, - {"CPX", AMDSMI_COMPUTE_PARTITION_CPX}, {"N/A", AMDSMI_COMPUTE_PARTITION_INVALID}}; - static const std::string memoryPartitionString(amdsmi_memory_partition_type_t memoryParitionType) { switch (memoryParitionType) { case AMDSMI_MEMORY_PARTITION_NPS1: @@ -442,7 +436,7 @@ static const std::map link_type_map = { int main() { amdsmi_status_t ret; - std::vector orig_accelerator_partitions; + std::vector orig_accelerator_partitions; std::vector orig_memory_partitions; uint32_t gpu_number = 0; @@ -467,19 +461,19 @@ int main() { std::cout << "Total Socket: " << socket_count << std::endl; // WARNING: Do not put any other settings before/inside/or between these lambda functions - // Required to save/change/reset the compute/accelerator & memory partition settings + // Required to save/change/reset the accelerator & memory partition settings // Reason: Modifies total number of gpu count, which will affect other API calls. // Requires amdsmi_shut_down()/amdsmi_init(AMDSMI_INIT_AMD_GPUS) to re-enumerate // total number of GPUs (AKA "processors per socket"). - // Changing back to original settings (compute/accelerator & memory partition) + // Changing back to original settings (accelerator & memory partition) // will not modify the GPU count. // Save all original partition settings for later auto save_original_partitions = [socket_count, &ret, sockets]( - std::vector& orig_partitions, + std::vector& orig_partitions, std::vector& orig_memory_partitions, uint32_t& gpu_number) -> void { - std::cout << " **Saving Original Compute/Accelerator & Memory Partition Settings**\n"; + std::cout << " **Saving Original Accelerator & Memory Partition Settings**\n"; // For each socket, get identifier and devices for (uint32_t i = 0; i < socket_count; i++) { @@ -508,33 +502,34 @@ int main() { std::cout << "\t**Device Handle: " << processor_handles[device_index] << std::endl; std::cout << "\t**GPU Number: " << gpu_number << std::endl; - // Get the original compute partition - char original_compute_partition[AMDSMI_MAX_STRING_LENGTH]; - ret = amdsmi_get_gpu_compute_partition(processor_handles[device_index], - original_compute_partition, - static_cast(AMDSMI_MAX_STRING_LENGTH)); + // Get the original accelerator partition + amdsmi_accelerator_partition_profile_t profile; + uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PROFILE]; + ret = amdsmi_get_gpu_accelerator_partition_profile(processor_handles[device_index], + &profile, partition_id); + std::string original_accelerator_partition = + acceleratorPartitionString(profile.profile_type); const char* err_str; amdsmi_status_code_to_string(ret, &err_str); if (ret == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret) - std::cout << " Output of amdsmi_get_gpu_compute_partition:\n"; - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << mapStringToSMIComputePartitionTypes.at(original_compute_partition) - << "): " << err_str << "\n\n"; - std::cout << "\tCompute Partition (original): " << original_compute_partition << "\n\n"; - } else { - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(AMDSMI_COMPUTE_PARTITION_INVALID) << "): " << err_str + std::cout << " Output of amdsmi_get_gpu_accelerator_partition:\n"; + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << profile.profile_type << "): " << err_str << "\n\n"; + std::cout << "\tAccelerator Partition (original): " << original_accelerator_partition << "\n\n"; + } else { + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << acceleratorPartitionString(AMDSMI_ACCELERATOR_PARTITION_INVALID) + << "): " << err_str << "\n\n"; } - // Save the original compute/accelerator partition + // Save the original accelerator partition if (ret == AMDSMI_STATUS_SUCCESS) { - orig_partitions.push_back( - mapStringToSMIComputePartitionTypes.at(original_compute_partition)); + orig_partitions.push_back(profile.profile_index); } else { - orig_partitions.push_back(AMDSMI_COMPUTE_PARTITION_INVALID); + orig_partitions.push_back(0); } // Get the original memory partition @@ -569,13 +564,13 @@ int main() { // Reset GPU number for the next loop gpu_number = 0; }; - // Save the original compute/accelerator & memory partition settings + // Save the original accelerator & memory partition settings save_original_partitions(orig_accelerator_partitions, orig_memory_partitions, gpu_number); - std::cout << " **Version 1: Accelerator/Compute Partition & memory API Examples**\n"; + std::cout << " **Version 1: Accelerator Partition & memory API Examples**\n"; auto process_accelerator_partitions = [socket_count, &ret, sockets](uint32_t& gpu_number) -> void { - std::cout << " **Process Compute/Accelerator & Memory Partition Settings**\n"; + std::cout << " **Process Accelerator & Memory Partition Settings**\n"; // For each socket, get identifier and devices for (uint32_t i = 0; i < socket_count; i++) { @@ -604,56 +599,63 @@ int main() { std::cout << "\t**Device Handle: " << processor_handles[device_index] << std::endl; std::cout << "\t**GPU Number: " << gpu_number << std::endl; - // Get the original compute partition - char original_compute_partition[AMDSMI_MAX_STRING_LENGTH]; - ret = amdsmi_get_gpu_compute_partition(processor_handles[device_index], - original_compute_partition, - static_cast(AMDSMI_MAX_STRING_LENGTH)); + // Get the original accelerator partition + amdsmi_accelerator_partition_profile_t profile; + uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PROFILE]; + ret = amdsmi_get_gpu_accelerator_partition_profile(processor_handles[device_index], + &profile, partition_id); + std::string original_accelerator_partition = + acceleratorPartitionString(profile.profile_type); const char* err_str; amdsmi_status_code_to_string(ret, &err_str); if (ret == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret) - std::cout << " Output of amdsmi_get_gpu_compute_partition:\n"; - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << mapStringToSMIComputePartitionTypes.at(original_compute_partition) - << "): " << err_str << "\n\n"; - std::cout << "\tCompute Partition (original): " << original_compute_partition << "\n\n"; - } else { - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(AMDSMI_COMPUTE_PARTITION_INVALID) << "): " << err_str + std::cout << " Output of amdsmi_get_gpu_accelerator_partition:\n"; + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << profile.profile_type << "): " << err_str << "\n\n"; + std::cout << "\tAccelerator Partition (original): " << original_accelerator_partition << "\n\n"; + } else { + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << acceleratorPartitionString(AMDSMI_ACCELERATOR_PARTITION_INVALID) + << "): " << err_str << "\n\n"; } - // Iterate through all compute partitions - for (int partition = static_cast(AMDSMI_COMPUTE_PARTITION_SPX); - partition <= static_cast(AMDSMI_COMPUTE_PARTITION_CPX); partition++) { - amdsmi_compute_partition_type_t updatePartition = - static_cast(partition); - amdsmi_status_t ret_set = - amdsmi_set_gpu_compute_partition(processor_handles[device_index], updatePartition); + // Iterate through all available accelerator partition profiles + amdsmi_accelerator_partition_profile_config_t profile_config; + ret = amdsmi_get_gpu_accelerator_partition_profile_config(processor_handles[device_index], + &profile_config); + for (uint32_t profile_idx = 0; profile_idx < profile_config.num_profiles; profile_idx++) { + amdsmi_accelerator_partition_type_t updatePartition = + profile_config.profiles[profile_idx].profile_type; + amdsmi_status_t ret_set = amdsmi_set_gpu_accelerator_partition_profile( + processor_handles[device_index], profile_idx); amdsmi_status_code_to_string(ret_set, &err_str); if (ret_set == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret_set) } - std::cout << "\tamdsmi_set_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(updatePartition) << "): " << err_str << "\n\n"; - - // Get the current compute partition - char current_compute_partition[AMDSMI_MAX_STRING_LENGTH]; - ret = amdsmi_get_gpu_compute_partition(processor_handles[device_index], - current_compute_partition, - static_cast(AMDSMI_MAX_STRING_LENGTH)); + std::cout << "\tamdsmi_set_gpu_accelerator_partition_profile(" << gpu_number << ", " + << acceleratorPartitionString(updatePartition) << "): " << err_str << "\n\n"; + + // Get the current accelerator partition + amdsmi_accelerator_partition_profile_t profile; + uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PROFILE]; + ret = amdsmi_get_gpu_accelerator_partition_profile(processor_handles[device_index], + &profile, partition_id); + std::string current_accelerator_partition = + acceleratorPartitionString(profile.profile_type); amdsmi_status_code_to_string(ret, &err_str); if (ret == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret) - std::cout << " Output of amdsmi_get_gpu_compute_partition:\n"; - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(updatePartition) << "): " << err_str << "\n\n"; - std::cout << "\tCompute Partition (current): " << current_compute_partition << "\n\n"; + std::cout << " Output of amdsmi_get_gpu_accelerator_partition:\n"; + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << profile.profile_type << "): " << err_str << "\n\n"; + std::cout << "\tAccelerator Partition (current): " << current_accelerator_partition + << "\n\n"; } else { - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(AMDSMI_COMPUTE_PARTITION_INVALID) + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << acceleratorPartitionString(AMDSMI_ACCELERATOR_PARTITION_INVALID) << "): " << err_str << "\n\n"; } } @@ -716,7 +718,7 @@ int main() { } // Since memory partition effects entire GPU hive (and modifies current - // compute/accelerator partition), we'll default to only changing the + // accelerator partition), we'll default to only changing the // first device for the first socket (GPU #0) // Note: Any device can be requested to change memory partition, // but for simplicity, we will only change GPU #0. @@ -732,14 +734,14 @@ int main() { } amdsmi_memory_partition_type_t updatePartition = static_cast(partition); - auto ret_set = - amdsmi_set_gpu_memory_partition(processor_handles[device_index], updatePartition); + auto ret_set = amdsmi_set_gpu_memory_partition_mode(processor_handles[device_index], + updatePartition); amdsmi_status_code_to_string(ret_set, &err_str); if (ret_set == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret_set) - std::cout << " Output of amdsmi_set_gpu_memory_partition:\n"; + std::cout << " Output of amdsmi_set_gpu_memory_partition_mode:\n"; } - std::cout << "\tamdsmi_set_gpu_memory_partition(" << gpu_number << ", " + std::cout << "\tamdsmi_set_gpu_memory_partition_mode(" << gpu_number << ", " << memoryPartitionString(updatePartition) << "): " << err_str << "\n\n"; if (ret_set == AMDSMI_STATUS_SUCCESS) { @@ -810,7 +812,7 @@ int main() { std::cout << "\t**Device Handle: " << processor_handles[device_index] << std::endl; std::cout << "\t**GPU Number: " << gpu_number << std::endl; // Since memory partition effects entire GPU hive (and modifies current - // compute/accelerator partition), we'll default to only changing the + // accelerator partition), we'll default to only changing the // first device for the first socket (GPU #0) // Note: Any device can be requested to change memory partition, // but for simplicity, we will only change GPU #0. @@ -823,14 +825,14 @@ int main() { // Reset to original memory partition settings amdsmi_memory_partition_type_t orig_partition = orig_partitions[gpu_number]; amdsmi_status_t ret_set = - amdsmi_set_gpu_memory_partition(processor_handles[device_index], orig_partition); + amdsmi_set_gpu_memory_partition_mode(processor_handles[device_index], orig_partition); const char* err_str; amdsmi_status_code_to_string(ret_set, &err_str); if (ret_set == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret_set) - std::cout << " Output of amdsmi_set_gpu_memory_partition:\n"; + std::cout << " Output of amdsmi_set_gpu_memory_partition_mode:\n"; } - std::cout << "\tamdsmi_set_gpu_memory_partition(" << gpu_number << ", " + std::cout << "\tamdsmi_set_gpu_memory_partition_mode(" << gpu_number << ", " << memoryPartitionString(orig_partition) << "): " << err_str << "\n\n"; if (ret_set == AMDSMI_STATUS_SUCCESS) { std::cout << "\t** Memory partition staged. Run to apply:\n" @@ -862,12 +864,11 @@ int main() { // Reset to original memory partition settings reset_memory_partitions(orig_memory_partitions, gpu_number); - auto reset_accelerator_partitions = - [socket_count, &ret, sockets]( - const std::vector& orig_partitions, - uint32_t& gpu_number) -> void { + auto reset_accelerator_partitions = [socket_count, &ret, sockets]( + const std::vector& orig_partitions, + uint32_t& gpu_number) -> void { std::cout << " **Version 1: Memory Partition API Examples**\n"; - std::cout << " **Resetting Compute/Accelerator Partition Settings**\n"; + std::cout << " **Resetting Accelerator Partition Settings**\n"; // For each socket, get identifier and devices for (uint32_t i = 0; i < socket_count; i++) { @@ -896,35 +897,38 @@ int main() { std::cout << "\t**Device Handle: " << processor_handles[device_index] << std::endl; std::cout << "\t**GPU Number: " << gpu_number << std::endl; - // Reset to original compute/accelerator partition settings - amdsmi_compute_partition_type_t orig_partition = orig_partitions[gpu_number]; - amdsmi_status_t ret_set = - amdsmi_set_gpu_compute_partition(processor_handles[device_index], orig_partition); + // Reset to original accelerator partition settings + uint32_t orig_partition = orig_partitions[gpu_number]; + amdsmi_status_t ret_set = amdsmi_set_gpu_accelerator_partition_profile( + processor_handles[device_index], orig_partition); const char* err_str; amdsmi_status_code_to_string(ret_set, &err_str); if (ret_set == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret_set) - std::cout << " Output of amdsmi_set_gpu_compute_partition:\n"; + std::cout << " Output of amdsmi_set_gpu_accelerator_partition_profile:\n"; } - std::cout << "\tamdsmi_set_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(orig_partition) << "): " << err_str << "\n\n"; - - // Get the current compute/accelerator partition - char current_compute_partition[AMDSMI_MAX_STRING_LENGTH]; - ret = amdsmi_get_gpu_compute_partition(processor_handles[device_index], - current_compute_partition, - static_cast(AMDSMI_MAX_STRING_LENGTH)); + std::cout << "\tamdsmi_set_gpu_accelerator_partition_profile(" << gpu_number << ", " + << orig_partition << "): " << err_str << "\n\n"; + + // Get the current accelerator partition + amdsmi_accelerator_partition_profile_t profile; + uint32_t partition_id[AMDSMI_MAX_ACCELERATOR_PROFILE]; + ret = amdsmi_get_gpu_accelerator_partition_profile(processor_handles[device_index], + &profile, partition_id); + std::string current_accelerator_partition = + acceleratorPartitionString(profile.profile_type); amdsmi_status_code_to_string(ret, &err_str); if (ret == AMDSMI_STATUS_SUCCESS) { PRINT_AMDSMI_RET(ret) - std::cout << " Output of amdsmi_get_gpu_compute_partition:\n"; - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(orig_partition) << "): " << err_str << "\n\n"; - std::cout << "\tCompute Partition (current): " << current_compute_partition << "\n\n"; - } else { - std::cout << "\tamdsmi_get_gpu_compute_partition(" << gpu_number << ", " - << computePartitionString(AMDSMI_COMPUTE_PARTITION_INVALID) << "): " << err_str + std::cout << " Output of amdsmi_get_gpu_accelerator_partition:\n"; + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << profile.profile_type << "): " << err_str << "\n\n"; + std::cout << "\tAccelerator Partition (current): " << current_accelerator_partition << "\n\n"; + } else { + std::cout << "\tamdsmi_get_gpu_accelerator_partition_profile(" << gpu_number << ", " + << acceleratorPartitionString(AMDSMI_ACCELERATOR_PARTITION_INVALID) + << "): " << err_str << "\n\n"; } gpu_number++; } @@ -932,15 +936,15 @@ int main() { // Reset GPU number for the next loop gpu_number = 0; }; - // Reset to original compute/accelerator partition settings + // Reset to original accelerator partition settings reset_accelerator_partitions(orig_accelerator_partitions, gpu_number); // WARNING: Do not put any other settings before/inside/or between these lambda functions - // Required to save/change/reset the compute/accelerator & memory partition settings + // Required to save/change/reset the accelerator & memory partition settings // Reason: Modifies total number of gpu count, which will affect other API calls. // Requires amdsmi_shut_down()/amdsmi_init(AMDSMI_INIT_AMD_GPUS) to re-enumerate // total number of GPUs (AKA "processors per socket"). - // Changing back to original settings (compute/accelerator & memory partition) + // Changing back to original settings (accelerator & memory partition) // will not modify the GPU count. // Add new functionality below this line! diff --git a/projects/amdsmi/example/amd_smi_fabric_example.cc b/projects/amdsmi/example/amd_smi_fabric_example.cc index 5f052fa4aada..65ceeff9cc56 100644 --- a/projects/amdsmi/example/amd_smi_fabric_example.cc +++ b/projects/amdsmi/example/amd_smi_fabric_example.cc @@ -182,6 +182,7 @@ int main() { std::cout << "\t\t\tSuccessfully retrieved Fabric telemetry data" << std::endl; // Display telemetry information + const char* telem_name; for (int cat = AMDSMI_FABRIC_TELEMETRY_CATEGORY_UALOE; cat < static_cast(AMDSMI_FABRIC_TELEMETRY_CATEGORY_MAX); cat++) { auto category = static_cast(cat); @@ -201,9 +202,14 @@ int main() { // Display items for this instance for (uint32_t item_idx = 0; item_idx < instance->item_count; item_idx++) { auto& item = instance->items[item_idx]; + ret = amdsmi_fabric_telem_id_to_string(item.id, &telem_name); + if (ret != AMDSMI_STATUS_SUCCESS) { + std::cout << "\t\t\t\t\tWarning: Failed to retrieve Fabric name: " << ret + << std::endl; + continue; + } std::cout << "\t\t\t\t\t\tItem " << item_idx << ": " - << "ID=0x" << std::hex << item.id << std::dec << "(" - << amdsmi_fabric_telem_id_to_string(item.id) << ")" + << "ID=0x" << std::hex << item.id << std::dec << "(" << telem_name << ")" << ", Value=" << item.value << std::endl; } } @@ -245,33 +251,26 @@ int main() { } std::cout << "\t\t\tFabric info data: " << "\n"; std::cout << "\t\t\t\t** BDF: " << bdf_to_str(fabric_info.bdf) << "\n"; - std::cout << "\t\t\t\t** Fabric Version: " << fabric_info.fabric_info.version << "\n"; - std::cout << "\t\t\t\t** Fabric Type: " - << fabric_info.fabric_info.fabric_version.v1.fabric_type << "\n"; - std::cout << "\t\t\t\t** Accelerator ID: " - << fabric_info.fabric_info.fabric_version.v1.accelerator_id << "\n"; - std::cout << "\t\t\t\t** Bandwidth: " << fabric_info.fabric_info.fabric_version.v1.bandwidth + std::cout << "\t\t\t\t** Fabric Version: " << fabric_info.fabric_version << "\n"; + std::cout << "\t\t\t\t** Fabric Type: " << fabric_info.fabric_info.v1.fabric_type << "\n"; + std::cout << "\t\t\t\t** Accelerator ID: " << fabric_info.fabric_info.v1.accelerator_id << "\n"; - std::cout << "\t\t\t\t** Latency: " << fabric_info.fabric_info.fabric_version.v1.latency - << "\n"; - std::cout << "\t\t\t\t** PPOD ID: " - << ppod_id_to_str(fabric_info.fabric_info.fabric_version.v1.ppod_id) << "\n"; - std::cout << "\t\t\t\t** PPOD Size: " << fabric_info.fabric_info.fabric_version.v1.ppod_size - << "\n"; - std::cout << "\t\t\t\t** VPOD ID: " << fabric_info.fabric_info.fabric_version.v1.vpod_id - << "\n"; - std::cout << "\t\t\t\t** VPOD Size: " << fabric_info.fabric_info.fabric_version.v1.vpod_size + std::cout << "\t\t\t\t** Bandwidth: " << fabric_info.fabric_info.v1.bandwidth << "\n"; + std::cout << "\t\t\t\t** Latency: " << fabric_info.fabric_info.v1.latency << "\n"; + std::cout << "\t\t\t\t** PPOD ID: " << ppod_id_to_str(fabric_info.fabric_info.v1.ppod_id) << "\n"; + std::cout << "\t\t\t\t** PPOD Size: " << fabric_info.fabric_info.v1.ppod_size << "\n"; + std::cout << "\t\t\t\t** VPOD ID: " << fabric_info.fabric_info.v1.vpod_id << "\n"; + std::cout << "\t\t\t\t** VPOD Size: " << fabric_info.fabric_info.v1.vpod_size << "\n"; print_array_grid(std::cout, "\t\t\t\t", "VPOD Active Accelerators", - fabric_info.fabric_info.fabric_version.v1.vpod_active_accelerators, + fabric_info.fabric_info.v1.vpod_active_accelerators, AMDSMI_FABRIC_ACTIVE_ACCELERATORS_BITMAP_SIZE, MAX_COLUMNS_PER_GRID); print_array_grid(std::cout, "\t\t\t\t", "Local Accelerators", - fabric_info.fabric_info.fabric_version.v1.local_accelerators, + fabric_info.fabric_info.v1.local_accelerators, AMDSMI_FABRIC_MAX_LOCAL_GPUS, MAX_COLUMNS_PER_GRID); - std::cout << "\t\t\t\t** Address Mode: " - << fabric_info.fabric_info.fabric_version.v1.addr_mode << "\n"; - std::cout << "\t\t\t\t** Accelerator State: " - << fabric_info.fabric_info.fabric_version.v1.accel_state << "\n"; + std::cout << "\t\t\t\t** Address Mode: " << fabric_info.fabric_info.v1.addr_mode << "\n"; + std::cout << "\t\t\t\t** Accelerator State: " << fabric_info.fabric_info.v1.accel_state + << "\n"; std::cout << "\n"; } } diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index ce8b6d48cb8f..333ad50a7493 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -83,13 +83,6 @@ typedef enum { * @brief The following structure holds the gpu metrics values for a device. */ -/** - * @brief Unit conversion factor for HBM temperatures - * - * @cond @tag{gpu_bm_linux} @endcond - */ -#define CENTRIGRADE_TO_MILLI_CENTIGRADE 1000 - /** * @brief This should match NUM_HBM_INSTANCES * @@ -194,9 +187,9 @@ typedef enum { /** * @brief Max Number of AFIDs that will be inside one cper entry * - * @cond @tag{gpu_bm_linux} @tag{host} @tag{guest_windows} @endcond + * @cond @tag{gpu_bm_linux} @tag{host} @endcond */ -#define MAX_NUMBER_OF_AFIDS_PER_RECORD 12 //!< Maximum AFIDs per CPER record +#define AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD 12 //!< Maximum AFIDs per CPER record /** * @brief Introduced in gpu metrics v1.9+ @@ -246,16 +239,6 @@ typedef enum { * * @cond @tag{gpu_bm_linux} @endcond */ -// Deprecation targets: kept for source/ABI compatibility, slated for removal. -#define AMDSMI_MAX_VF_COUNT 32 //!< Maximum virtual functions supported -#define AMDSMI_MAX_DRIVER_NUM 2 //!< Maximum drivers supported -#define AMDSMI_DFC_FW_NUMBER_OF_ENTRIES 9 //!< DFC firmware entries supported -#define AMDSMI_MAX_WHITE_LIST_ELEMENTS 16 //!< Max white list elements for device access control -#define AMDSMI_MAX_BLACK_LIST_ELEMENTS 64 //!< Max black list elements for device access control -#define AMDSMI_MAX_TA_WHITE_LIST_ELEMENTS 8 //!< Max Trusted Application white list elements -#define AMDSMI_MAX_ERR_RECORDS 10 //!< Maximum error records that can be stored -#define AMDSMI_MAX_PROFILE_COUNT 16 //!< Maximum profiles supported -#define AMDSMI_PF_INDEX (AMDSMI_MAX_VF_COUNT - 1) #define AMDSMI_MAX_DRIVER_INFO_RSVD 64 // Deprecation target: kept for ABI compatibility; prefer AMDSMI_FABRIC_PPOD_ID_SIZE. #define AMDSMI_MAX_UUID_ELEMENTS 16 //!< Max UUID elements supported @@ -326,10 +309,10 @@ typedef struct { #define AMDSMI_MAX_SPD_REG_OFFSET 0x7FF //!< Maximum SPD register offset [22:12] #define AMDSMI_MAX_SPD_REG_SPACE 0x1 //!< Maximum SPD register space [23] #define AMDSMI_MAX_SPD_WRITE_DATA 0xFF //!< Maximum SPD write data [31:24] -#define MAX_SVI3_RAIL_INDEX 4 //!< Maximum SVI3 rail index -#define MAX_SVI3_RAIL_SELECTION 1 //!< Maximum SVI3 rail selection -#define POWER_EFFICIENCY_MODE_4 0x4 //!< Power Efficiency mode selection -#define POWER_EFFICIENCY_MODE_5 0x5 //!< Power Efficiency mode selection +#define AMDSMI_MAX_SVI3_RAIL_INDEX 4 //!< Maximum SVI3 rail index +#define AMDSMI_MAX_SVI3_RAIL_SELECTION 1 //!< Maximum SVI3 rail selection +#define AMDSMI_POWER_EFFICIENCY_MODE_4 0x4 //!< Power Efficiency mode selection +#define AMDSMI_POWER_EFFICIENCY_MODE_5 0x5 //!< Power Efficiency mode selection #define AMDSMI_MAX_POWER_EFFICIENCY_UTIL 0x7F //!< [9:3]=Balanced core mode utilization point(%) #define AMDSMI_MAX_POWER_EFFICIENCY_PPTLIMIT 0x1FFFFF //!< [30:10]=Balanced core mode PPT limit(mW) #define AMDSMI_RAIL_INDEX_NONE 0xFFFFFFFF //!< Rail Index value defined as maximum when not passed @@ -490,6 +473,9 @@ typedef enum { * @brief Compute Partition. This enum is used to identify * various compute partitioning settings. * + * @deprecated This enum is slated for removal in a future ROCm release; + * use amdsmi_accelerator_partition_type_t instead + * * @cond @tag{gpu_bm_linux} @tag{guest_windows} @endcond */ typedef enum { @@ -510,6 +496,9 @@ typedef enum { * @brief Compute Partition Memory Allocation Mode. Controls how GPU memory * is allocated across XCPs within a memory partition. * + * @deprecated This enum is slated for removal in a future ROCm release; + * use amdsmi_accelerator_partition_mem_alloc_mode_t instead + * * @cond @tag{gpu_bm_linux} @endcond */ typedef enum { @@ -519,6 +508,19 @@ typedef enum { //!< use the full partition memory } amdsmi_compute_partition_mem_alloc_mode_t; +/** + * @brief Accelerator Partition Memory Allocation Mode. Controls how GPU memory + * is allocated across XCPs within a memory partition. + * + * @cond @tag{gpu_bm_linux} @endcond + */ +typedef enum { + AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_INVALID = 0, //!< Invalid mode + AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING, //!< Memory is evenly capped per XCP + AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL //!< Each XCP in the partition may + //!< use the full partition memory +} amdsmi_accelerator_partition_mem_alloc_mode_t; + /** * @brief Memory Partitions * @@ -1790,8 +1792,10 @@ typedef enum { * @cond @tag{gpu_bm_linux} @endcond */ typedef enum { - CLK_LIMIT_MIN, //!< Min Clock value in MHz - CLK_LIMIT_MAX //!< Max Clock value in MHz + AMDSMI_CLK_LIMIT_MIN, //!< Min Clock value in MHz + AMDSMI_CLK_LIMIT_MAX, //!< Max Clock value in MHz + CLK_LIMIT_MIN = AMDSMI_CLK_LIMIT_MIN, //!< Deprecated, use AMDSMI_CLK_LIMIT_MIN instead + CLK_LIMIT_MAX = AMDSMI_CLK_LIMIT_MAX //!< Deprecated, use AMDSMI_CLK_LIMIT_MAX instead } amdsmi_clk_limit_type_t; /** @@ -2776,9 +2780,12 @@ typedef struct { * @cond @tag{cpu_bm} @endcond */ typedef enum { - AGG_BW0 = 1, //!< Aggregate Bandwidth - RD_BW0 = 2, //!< Read Bandwidth - WR_BW0 = 4 //!< Write Bandwidth + AMDSMI_AGG_BW0 = 1, //!< Aggregate Bandwidth + AMDSMI_RD_BW0 = 2, //!< Read Bandwidth + AMDSMI_WR_BW0 = 4, //!< Write Bandwidth + AGG_BW0 = AMDSMI_AGG_BW0, //!< Deprecated, use AMDSMI_AGG_BW0 instead + RD_BW0 = AMDSMI_RD_BW0, //!< Deprecated, use AMDSMI_RD_BW0 instead + WR_BW0 = AMDSMI_WR_BW0 //!< Deprecated, use AMDSMI_WR_BW0 instead } amdsmi_io_bw_encoding_t; /** @@ -3017,7 +3024,7 @@ typedef struct { typedef struct { char name[AMDSMI_MAX_STRING_LENGTH]; char version[AMDSMI_MAX_STRING_LENGTH]; -} amdsmi_nic_fw_t; +} amdsmi_nic_fw_entry_t; /** * @brief NIC firmware information collection @@ -3026,7 +3033,7 @@ typedef struct { */ typedef struct { uint32_t num_fw; - amdsmi_nic_fw_t fw[AMDSMI_MAX_NIC_FW]; + amdsmi_nic_fw_entry_t fw[AMDSMI_MAX_NIC_FW]; } amdsmi_nic_fw_info_t; /** @@ -5113,37 +5120,6 @@ amdsmi_status_t amdsmi_get_gpu_reg_table_info(amdsmi_processor_handle processor_ amdsmi_name_value_t** reg_metrics, uint32_t* num_of_metrics); -/** - * @brief This function sets the clock range information. It is not supported on virtual - * machine guest - * - * @deprecated ::amdsmi_set_gpu_clk_limit() should be used, with an - * interface that set the min_value and then max_value. - * - * @ingroup tagClkPowerPerfQuery - * - * @platform{gpu_bm_linux} - * - * @details Given a processor handle @p processor_handle, a minimum clock value @p minclkvalue, - * a maximum clock value @p maxclkvalue and a clock type @p clkType this function - * will set the sclk|mclk range - * - * @param[in] processor_handle a processor handle - * - * @param[in] minclkvalue value to apply to the clock range. Frequency values - * are in MHz. - * - * @param[in] maxclkvalue value to apply to the clock range. Frequency values - * are in MHz. - * - * @param[in] clkType AMDSMI_CLK_TYPE_SYS | AMDSMI_CLK_TYPE_MEM range type - * - * @return ::amdsmi_status_t | ::AMDSMI_STATUS_SUCCESS on success, non-zero on fail - */ -amdsmi_status_t amdsmi_set_gpu_clk_range(amdsmi_processor_handle processor_handle, - uint64_t minclkvalue, uint64_t maxclkvalue, - amdsmi_clk_type_t clkType); - /** * @brief This function sets the clock sets the clock min/max level * @@ -5587,7 +5563,6 @@ amdsmi_status_t amdsmi_clean_gpu_local_data(amdsmi_processor_handle processor_ha * @cond @tag{gpu_bm_linux} @tag{host} @endcond */ typedef enum { - AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN = 0xFFFFFFFF, //!< Unknown telemetry AMDSMI_FABRIC_TELEMETRY_CATEGORY_UALOE = 0, //!< UALOE telemetry AMDSMI_FABRIC_TELEMETRY_CATEGORY_SWITCH = 1, //!< Switch telemetry AMDSMI_FABRIC_TELEMETRY_CATEGORY_CRYPTO = 2, //!< Crypto telemetry @@ -5596,7 +5571,9 @@ typedef enum { AMDSMI_FABRIC_TELEMETRY_CATEGORY_DERIVED_UALOE = 5, //!< Derived UALOE telemetry AMDSMI_FABRIC_TELEMETRY_CATEGORY_DERIVED_NETPORT = 6, //!< Derived Network Port telemetry AMDSMI_FABRIC_TELEMETRY_CATEGORY_MAX = 7, //!< Maximum number of categories - AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID = 0xFFFFFFFF //!< Unknown telemetry + AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID = 0xFFFFFFFF, //!< Unknown telemetry + //!< Deprecated, use AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID instead + AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN = AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID } amdsmi_fabric_telemetry_category_t; /** @@ -5748,10 +5725,11 @@ amdsmi_status_t amdsmi_get_fabric_telemetry_data(amdsmi_processor_handle process * * @param[in] telem_id The telemetry item ID for which the name is requested * - * @return const char* | Pointer to string containing the telemetry item name, - * or UNKNOWN if the category or telemetry ID is not recognized + * @param[out] telem_name The telemetry item name + * + * @return ::amdsmi_status_t | ::AMDSMI_STATUS_SUCCESS on success, non-zero on fail */ -const char* amdsmi_fabric_telem_id_to_string(uint64_t telem_id); +amdsmi_status_t amdsmi_fabric_telem_id_to_string(uint64_t telem_id, const char** telem_name); /** * @brief Free Fabric telemetry storage @@ -5789,8 +5767,10 @@ typedef enum { */ typedef enum { AMDSMI_FABRIC_TYPE_UALOE, - AMDSMI_FABRIC_TYPE_UALLINK, - AMDSMI_FABRIC_TYPE_UNKNOWN + AMDSMI_FABRIC_TYPE_UALINK, + AMDSMI_FABRIC_TYPE_UNKNOWN, + AMDSMI_FABRIC_TYPE_UALLINK = + AMDSMI_FABRIC_TYPE_UALINK //!< Deprecated, use AMDSMI_FABRIC_TYPE_UALINK instead } amdsmi_fabric_type_t; /** @@ -5825,7 +5805,7 @@ typedef enum { */ typedef struct { uint32_t accelerator_id; //!< Accelerator identifier (range 0 to 1023) - amdsmi_fabric_type_t fabric_type; //!< UALOE or UALLINK + amdsmi_fabric_type_t fabric_type; //!< UALOE or UALINK uint32_t bandwidth; //!< Station bandwidth share in Mb/s uint32_t latency; //!< Latency in nanoseconds (depends on switch presence and type) uint8_t ppod_id[AMDSMI_MAX_UUID_ELEMENTS]; //!< Physical PoD Identifier (16 bytes) @@ -5841,22 +5821,18 @@ typedef struct { amdsmi_fabric_accelerator_vpod_state_t accel_state; //!< Accelerator vPoD State } amdsmi_fabric_info_v1_t; -typedef struct { - uint32_t version; - union fabric_info_ { - amdsmi_fabric_info_v1_t v1; - } fabric_version; -} amdsmi_fabric_info_ver_t; - /** * @brief Fabric device information structure * * @cond @tag{gpu_bm_linux} @tag{host} @endcond */ typedef struct { - amdsmi_bdf_t bdf; //!< BDF (Bus, Device, Function) of the Fabric device - amdsmi_fabric_info_ver_t fabric_info; //!< Fabric information structure (version 1) - uint32_t reserved[15]; //!< Reserved for future use + amdsmi_bdf_t bdf; //!< BDF (Bus, Device, Function) of the Fabric device + uint32_t fabric_version; + union fabric_info_ { + amdsmi_fabric_info_v1_t v1; + } fabric_info; + uint32_t reserved[15]; //!< Reserved for future use } amdsmi_fabric_info_t; /** @@ -6096,8 +6072,8 @@ typedef struct { * uint64_t that may be safely written to the memory pointed to by @p afids. This is the limit * on how many AF IDs will be written to @p afids. On return, @p num_afids will contain the * number of AF IDs written to @p afids, or the number of AF IDs that could have been written - * if enough memory had been provided. It is suggest to pass MAX_NUMBER_OF_AFIDS_PER_RECORD for all - * AF Ids. + * if enough memory had been provided. It is suggest to pass AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD + * for all AF Ids. * * @return ::amdsmi_status_t | ::AMDSMI_STATUS_SUCCESS on success, non-zero on fail */ @@ -6928,13 +6904,16 @@ amdsmi_status_t amdsmi_topo_get_p2p_status(amdsmi_processor_handle processor_han /** * @brief Retrieves the current compute partitioning for a desired device * + * @deprecated This API is slated for removal in a future ROCm release; + * ::amdsmi_get_gpu_accelerator_partition_profile() should be used instead + * * @ingroup tagComputePartition * * @platform{gpu_bm_linux} * * @details - * Given a processor handle @p processor_handle and a string @p compute_partition , - * and uint32 @p len , this function will attempt to obtain the device's + * Given a processor handle @p processor_handle and a string @p compute_partition, + * and uint32 @p len, this function will attempt to obtain the device's * current compute partition setting string. Upon successful retrieval, * the obtained device's compute partition settings string shall be stored in * the passed @p compute_partition char string variable. @@ -6963,6 +6942,9 @@ amdsmi_status_t amdsmi_get_gpu_compute_partition(amdsmi_processor_handle process /** * @brief Modifies a selected device's compute partition setting. * + * @deprecated This API is slated for removal in a future ROCm release; + * ::amdsmi_set_gpu_accelerator_partition_profile() should be used instead + * * @ingroup tagComputePartition * * @platform{gpu_bm_linux} @@ -6994,6 +6976,9 @@ amdsmi_status_t amdsmi_set_gpu_compute_partition(amdsmi_processor_handle process * @brief Retrieves the current compute partition memory allocation mode * for a desired device. * + * @deprecated This API is slated for removal in a future ROCm release; + * ::amdsmi_get_gpu_accelerator_partition_mem_alloc_mode() should be used instead + * * @ingroup tagComputePartition * * @platform{gpu_bm_linux} @@ -7002,9 +6987,9 @@ amdsmi_status_t amdsmi_set_gpu_compute_partition(amdsmi_processor_handle process * @p mode, this function will attempt to obtain the device's current * compute partition memory allocation mode. The mode controls how HBM * capacity is distributed across XCPs within each memory partition: - * - ::AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_CAPPING — each XCP is capped + * - ::AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING — each XCP is capped * to an even share. - * - ::AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_ALL — each XCP may use the + * - ::AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL — each XCP may use the * full memory partition size (useful when only one XCP is active). * * @param[in] processor_handle Device which to query @@ -7023,9 +7008,45 @@ amdsmi_status_t amdsmi_set_gpu_compute_partition(amdsmi_processor_handle process amdsmi_status_t amdsmi_get_gpu_compute_partition_mem_alloc_mode( amdsmi_processor_handle processor_handle, amdsmi_compute_partition_mem_alloc_mode_t* mode); +/** + * @brief Retrieves the current accelerator partition memory allocation mode + * for a desired device. + * + * @ingroup tagComputePartition + * + * @platform{gpu_bm_linux} + * + * @details Given a processor handle @p processor_handle and a pointer + * @p mode, this function will attempt to obtain the device's current + * accelerator partition memory allocation mode. The mode controls how HBM + * capacity is distributed across XCPs within each memory partition: + * - ::AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING — each XCP is capped + * to an even share. + * - ::AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL — each XCP may use the + * full memory partition size (useful when only one XCP is active). + * + * @param[in] processor_handle Device which to query + * + * @param[out] mode a pointer to an ::amdsmi_accelerator_partition_mem_alloc_mode_t + * variable, into which the device's current memory allocation mode will + * be written. + * + * @retval ::AMDSMI_STATUS_SUCCESS call was successful + * @retval ::AMDSMI_STATUS_INVAL the provided arguments are not valid + * @retval ::AMDSMI_STATUS_UNEXPECTED_DATA data provided to function is not valid + * @retval ::AMDSMI_STATUS_FILE_ERROR problem accessing the sysfs file + * @retval ::AMDSMI_STATUS_NOT_SUPPORTED installed software or hardware does not + * support this function + */ +amdsmi_status_t amdsmi_get_gpu_accelerator_partition_mem_alloc_mode( + amdsmi_processor_handle processor_handle, amdsmi_accelerator_partition_mem_alloc_mode_t* mode); + /** * @brief Modifies a selected device's compute partition memory allocation mode. * + * @deprecated This API is slated for removal in a future ROCm release; + * ::amdsmi_set_gpu_accelerator_partition_mem_alloc_mode() should be used instead + * * @ingroup tagComputePartition * * @platform{gpu_bm_linux} @@ -7055,6 +7076,38 @@ amdsmi_status_t amdsmi_get_gpu_compute_partition_mem_alloc_mode( amdsmi_status_t amdsmi_set_gpu_compute_partition_mem_alloc_mode( amdsmi_processor_handle processor_handle, amdsmi_compute_partition_mem_alloc_mode_t mode); +/** + * @brief Modifies a selected device's compute partition memory allocation mode. + * + * @ingroup tagComputePartition + * + * @platform{gpu_bm_linux} + * + * @details Given a processor handle @p processor_handle and a mode + * @p mode, this function will attempt to update the selected device's + * compute partition memory allocation mode. The mode controls how HBM + * capacity is distributed across XCPs within each memory partition: + * - ::AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING — each XCP is capped + * to an even share. This is the default. + * - ::AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL — each XCP may use the + * full memory partition size. + * + * @param[in] processor_handle Device which to modify + * + * @param[in] mode using enum ::amdsmi_accelerator_partition_mem_alloc_mode_t, + * define what the selected device's memory allocation mode should be + * updated to. + * + * @retval ::AMDSMI_STATUS_SUCCESS call was successful + * @retval ::AMDSMI_STATUS_NO_PERM function requires admin/sudo privileges + * @retval ::AMDSMI_STATUS_INVAL the provided arguments are not valid + * @retval ::AMDSMI_STATUS_FILE_ERROR problem accessing the sysfs file + * @retval ::AMDSMI_STATUS_NOT_SUPPORTED installed software or hardware does not + * support this function + */ +amdsmi_status_t amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( + amdsmi_processor_handle processor_handle, amdsmi_accelerator_partition_mem_alloc_mode_t mode); + /** @} End tagComputePartition */ /*****************************************************************************/ @@ -7072,8 +7125,8 @@ amdsmi_status_t amdsmi_set_gpu_compute_partition_mem_alloc_mode( * @platform{gpu_bm_linux} * * @details - * Given a processor handle @p processor_handle and a string @p memory_partition , - * and uint32 @p len , this function will attempt to obtain the device's + * Given a processor handle @p processor_handle and a string @p memory_partition, + * and uint32 @p len, this function will attempt to obtain the device's * memory partition string. Upon successful retrieval, the obtained device's * memory partition string shall be stored in the passed @p memory_partition * char string variable. @@ -7083,7 +7136,7 @@ amdsmi_status_t amdsmi_set_gpu_compute_partition_mem_alloc_mode( * @param[inout] memory_partition a pointer to a char string variable, * which the device's memory partition will be written to. * - * @param[in] len the length of the caller provided buffer @p memory_partition , + * @param[in] len the length of the caller provided buffer @p memory_partition, * suggested length is 5 or greater. * * @retval ::AMDSMI_STATUS_SUCCESS call was successful @@ -7102,6 +7155,9 @@ amdsmi_status_t amdsmi_get_gpu_memory_partition(amdsmi_processor_handle processo /** * @brief Modifies a selected device's current memory partition setting. * + * @deprecated This API is slated for removal in a future ROCm release; + * ::amdsmi_set_gpu_memory_partition_mode() should be used instead + * * @ingroup tagMemoryPartition * * @platform{gpu_bm_linux} diff --git a/projects/amdsmi/include/amd_smi/impl/nic/amd_smi_ainic_device.h b/projects/amdsmi/include/amd_smi/impl/nic/amd_smi_ainic_device.h index abd8f69bd667..fc41837fbdb0 100644 --- a/projects/amdsmi/include/amd_smi/impl/nic/amd_smi_ainic_device.h +++ b/projects/amdsmi/include/amd_smi/impl/nic/amd_smi_ainic_device.h @@ -45,7 +45,7 @@ class AMDSmiAINICDevice : public AMDSmiProcessor { amdsmi_nic_bus_info_t bus; amdsmi_nic_driver_info_t driver; amdsmi_nic_numa_info_t numa; - amdsmi_nic_fw_t versions; + amdsmi_nic_fw_entry_t versions; amdsmi_nic_port_info_t port; amdsmi_nic_rdma_devices_info_t rdma_dev; }; diff --git a/projects/amdsmi/py-interface/__init__.py b/projects/amdsmi/py-interface/__init__.py index bf49a6bc4218..681e75f7ac42 100644 --- a/projects/amdsmi/py-interface/__init__.py +++ b/projects/amdsmi/py-interface/__init__.py @@ -38,7 +38,6 @@ # ESMI Dependent Functions try: from .amdsmi_interface import amdsmi_get_cpu_handles - from .amdsmi_interface import amdsmi_get_cpusocket_handles # Deprecate in 8.0 from .amdsmi_interface import amdsmi_get_cpucore_handles from .amdsmi_interface import amdsmi_get_cpu_hsmp_proto_ver from .amdsmi_interface import amdsmi_get_cpu_smu_fw_version @@ -181,7 +180,6 @@ from .amdsmi_interface import amdsmi_set_gpu_pci_bandwidth from .amdsmi_interface import amdsmi_set_power_cap from .amdsmi_interface import amdsmi_set_gpu_power_profile -from .amdsmi_interface import amdsmi_set_gpu_clk_range from .amdsmi_interface import amdsmi_set_gpu_clk_limit from .amdsmi_interface import amdsmi_set_gpu_od_clk_info from .amdsmi_interface import amdsmi_set_gpu_od_volt_info @@ -284,13 +282,15 @@ from .amdsmi_interface import amdsmi_set_gpu_compute_partition from .amdsmi_interface import amdsmi_get_gpu_compute_partition_mem_alloc_mode from .amdsmi_interface import amdsmi_set_gpu_compute_partition_mem_alloc_mode +from .amdsmi_interface import amdsmi_get_gpu_accelerator_partition_mem_alloc_mode +from .amdsmi_interface import amdsmi_set_gpu_accelerator_partition_mem_alloc_mode from .amdsmi_interface import amdsmi_get_gpu_memory_partition from .amdsmi_interface import amdsmi_set_gpu_memory_partition +from .amdsmi_interface import amdsmi_set_gpu_memory_partition_mode from .amdsmi_interface import amdsmi_get_gpu_accelerator_partition_profile from .amdsmi_interface import amdsmi_get_gpu_accelerator_partition_profile_config from .amdsmi_interface import amdsmi_get_gpu_memory_partition_config from .amdsmi_interface import amdsmi_set_gpu_accelerator_partition_profile -from .amdsmi_interface import amdsmi_set_gpu_memory_partition_mode # # Individual GPU Metrics Functions from .amdsmi_interface import amdsmi_get_gpu_metrics_header_info @@ -333,6 +333,7 @@ from .amdsmi_interface import AmdSmiVoltageType from .amdsmi_interface import AmdSmiComputePartitionType from .amdsmi_interface import AmdSmiComputePartitionMemAllocModeType +from .amdsmi_interface import AmdSmiAcceleratorPartitionMemAllocModeType from .amdsmi_interface import AmdSmiMemoryPartitionType from .amdsmi_interface import AmdSmiPowerProfilePresetMasks from .amdsmi_interface import AmdSmiGpuBlock diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index 93ba440d3d40..5ba044f4a70f 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -21,6 +21,7 @@ import math import os import re +import warnings from collections.abc import Iterable from ctypes import POINTER, c_void_p from enum import IntEnum, Enum @@ -61,7 +62,7 @@ AMDSMI_MAX_NUM_XCP = 8 # max num afids per cper record -MAX_NUMBER_OF_AFIDS_PER_RECORD = 12 +AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD = 12 # Max number of DPM policies AMDSMI_MAX_NUM_PM_POLICIES = 32 @@ -85,6 +86,7 @@ AMDSMI_GPU_UUID_SIZE = 38 _AMDSMI_STRING_LENGTH = 80 _AMDSMI_MAX_STRING_LENGTH = 256 +AMDSMI_GPU_CUID_SIZE = AMDSMI_GPU_UUID_SIZE class AmdSmiStatus(IntEnum): @@ -266,8 +268,8 @@ class AmdSmiClkType(IntEnum): class AmdSmiClkLimitType(IntEnum): - MIN = amdsmi_wrapper.CLK_LIMIT_MIN - MAX = amdsmi_wrapper.CLK_LIMIT_MAX + MIN = amdsmi_wrapper.AMDSMI_CLK_LIMIT_MIN + MAX = amdsmi_wrapper.AMDSMI_CLK_LIMIT_MAX class AmdSmiTemperatureType(IntEnum): @@ -565,6 +567,7 @@ class AmdSmiAcceleratorPartitionType(IntEnum): INVALID = amdsmi_wrapper.AMDSMI_ACCELERATOR_PARTITION_INVALID +# This class is deprecated, use AmdSmiAcceleratorPartitionType instead class AmdSmiComputePartitionType(IntEnum): SPX = amdsmi_wrapper.AMDSMI_COMPUTE_PARTITION_SPX DPX = amdsmi_wrapper.AMDSMI_COMPUTE_PARTITION_DPX @@ -574,12 +577,19 @@ class AmdSmiComputePartitionType(IntEnum): INVALID = amdsmi_wrapper.AMDSMI_COMPUTE_PARTITION_INVALID +# This class is deprecated, use AmdSmiAcceleratorPartitionMemAllocModeType instead class AmdSmiComputePartitionMemAllocModeType(IntEnum): INVALID = amdsmi_wrapper.AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_INVALID CAPPING = amdsmi_wrapper.AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_CAPPING ALL = amdsmi_wrapper.AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_ALL +class AmdSmiAcceleratorPartitionMemAllocModeType(IntEnum): + INVALID = amdsmi_wrapper.AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_INVALID + CAPPING = amdsmi_wrapper.AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING + ALL = amdsmi_wrapper.AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL + + class AmdSmiMemoryPartitionType(IntEnum): NPS1 = amdsmi_wrapper.AMDSMI_MEMORY_PARTITION_NPS1 NPS2 = amdsmi_wrapper.AMDSMI_MEMORY_PARTITION_NPS2 @@ -1173,24 +1183,6 @@ def amdsmi_get_cpu_handles() -> Dict[str, Any]: return {"cpu_count": len(cpu_handles), "processor_handles": cpu_handles} -def amdsmi_get_cpusocket_handles() -> List[c_void_p]: - """Deprecated: Use amdsmi_get_cpu_handles() instead.\ - Will be deprecated in Rocm 8.0. - - Returns: - `List[c_void_p]`: List of CPU socket handles (legacy format). - """ - import warnings - - warnings.warn( - "amdsmi_get_cpusocket_handles() is deprecated, use amdsmi_get_cpu_handles() instead", - DeprecationWarning, - stacklevel=2, - ) - result = amdsmi_get_cpu_handles() - return result["processor_handles"] - - def amdsmi_get_socket_info(socket_handle): if not isinstance(socket_handle, amdsmi_wrapper.amdsmi_socket_handle): raise AmdSmiParameterException(socket_handle, amdsmi_wrapper.amdsmi_socket_handle) @@ -2478,7 +2470,6 @@ def amdsmi_get_gpu_device_bdf_bdf( Returns the raw amdsmi_bdf_t struct for a GPU. The same data is available as a formatted string from amdsmi_get_gpu_device_bdf(). """ - import warnings warnings.warn( "amdsmi_get_gpu_device_bdf_bdf() is deprecated, use amdsmi_get_gpu_device_bdf() " @@ -3578,8 +3569,8 @@ def amdsmi_get_afids_from_cper(cper_afid_data: bytes) -> Tuple[List[int], int]: buf = ctypes.create_string_buffer(raw_bytes, record_size) buf_ptr = ctypes.cast(buf, POINTER(ctypes.c_char)) - afid_array = (ctypes.c_uint64 * MAX_NUMBER_OF_AFIDS_PER_RECORD)() - num_afids_ct = ctypes.c_uint32(MAX_NUMBER_OF_AFIDS_PER_RECORD) + afid_array = (ctypes.c_uint64 * AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD)() + num_afids_ct = ctypes.c_uint32(AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD) # Call the wrapper function status = amdsmi_wrapper.amdsmi_get_afids_from_cper( @@ -3798,10 +3789,10 @@ def amdsmi_get_nic_fw_version(processor_handle: amdsmi_wrapper.amdsmi_processor_ if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - uuid = ctypes.create_string_buffer(_AMDSMI_MAX_STRING_LENGTH) + uuid = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) uuid_length = ctypes.c_uint32() - uuid_length.value = _AMDSMI_MAX_STRING_LENGTH + uuid_length.value = AMDSMI_MAX_STRING_LENGTH _check_res( amdsmi_wrapper.amdsmi_get_nic_fw_version(processor_handle, ctypes.byref(uuid_length), uuid) @@ -4076,32 +4067,20 @@ def amdsmi_get_gpu_vendor_name(processor_handle: processor_handle_t) -> str: raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) length = ctypes.c_uint64() - length.value = _AMDSMI_STRING_LENGTH + length.value = AMDSMI_MAX_STRING_LENGTH - vendor_name = ctypes.create_string_buffer(_AMDSMI_STRING_LENGTH) + vendor_name = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) _check_res(amdsmi_wrapper.amdsmi_get_gpu_vendor_name(processor_handle, vendor_name, length)) return vendor_name.value.decode("utf-8") -def amdsmi_get_gpu_id(processor_handle: processor_handle_t): - if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): - raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - gpu_id_16 = ctypes.c_uint16() - - _check_res(amdsmi_wrapper.amdsmi_get_gpu_id(processor_handle, ctypes.byref(gpu_id_16))) - - return gpu_id_16.value - - def amdsmi_get_gpu_vram_vendor(processor_handle: processor_handle_t): """Deprecated: use amdsmi_get_gpu_vram_info() instead. This API is slated for removal in a future ROCm release. """ - import warnings - warnings.warn( "amdsmi_get_gpu_vram_vendor() is deprecated, use amdsmi_get_gpu_vram_info() instead", DeprecationWarning, @@ -4111,15 +4090,25 @@ def amdsmi_get_gpu_vram_vendor(processor_handle: processor_handle_t): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) length = ctypes.c_uint32() - length.value = _AMDSMI_STRING_LENGTH + length.value = AMDSMI_MAX_STRING_LENGTH - vram_vendor = ctypes.create_string_buffer(_AMDSMI_STRING_LENGTH) + vram_vendor = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) _check_res(amdsmi_wrapper.amdsmi_get_gpu_vram_vendor(processor_handle, vram_vendor, length)) return vram_vendor.value.decode("utf-8") +def amdsmi_get_gpu_id(processor_handle: processor_handle_t): + if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): + raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) + gpu_id_16 = ctypes.c_uint16() + + _check_res(amdsmi_wrapper.amdsmi_get_gpu_id(processor_handle, ctypes.byref(gpu_id_16))) + + return gpu_id_16.value + + def amdsmi_get_gpu_subsystem_id(processor_handle: processor_handle_t): if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) @@ -4137,9 +4126,9 @@ def amdsmi_get_gpu_subsystem_name(processor_handle: processor_handle_t): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) length = ctypes.c_uint64() - length.value = _AMDSMI_STRING_LENGTH + length.value = AMDSMI_MAX_STRING_LENGTH - name = ctypes.create_string_buffer(_AMDSMI_STRING_LENGTH) + name = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) _check_res(amdsmi_wrapper.amdsmi_get_gpu_subsystem_name(processor_handle, name, length)) @@ -4351,13 +4340,21 @@ def amdsmi_is_P2P_accessible( def amdsmi_get_gpu_compute_partition(processor_handle: processor_handle_t): + """Deprecated: use amdsmi_get_gpu_accelerator_partition_profile() instead.""" if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) + warnings.warn( + "amdsmi_get_gpu_compute_partition() is deprecated, " + "use amdsmi_get_gpu_accelerator_partition_profile() instead", + DeprecationWarning, + stacklevel=2, + ) + length = ctypes.c_uint32() - length.value = _AMDSMI_STRING_LENGTH + length.value = AMDSMI_MAX_STRING_LENGTH - compute_partition = ctypes.create_string_buffer(_AMDSMI_STRING_LENGTH) + compute_partition = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) _check_res( amdsmi_wrapper.amdsmi_get_gpu_compute_partition(processor_handle, compute_partition, length) @@ -4369,6 +4366,14 @@ def amdsmi_get_gpu_compute_partition(processor_handle: processor_handle_t): def amdsmi_set_gpu_compute_partition( processor_handle: processor_handle_t, compute_partition: AmdSmiComputePartitionType ): + """Deprecated: use amdsmi_set_gpu_accelerator_partition_profile() instead.""" + + warnings.warn( + "amdsmi_set_gpu_compute_partition() is deprecated, " + "use amdsmi_set_gpu_accelerator_partition_profile() instead", + DeprecationWarning, + stacklevel=2, + ) if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) @@ -4380,29 +4385,57 @@ def amdsmi_set_gpu_compute_partition( def amdsmi_get_gpu_compute_partition_mem_alloc_mode(processor_handle: processor_handle_t): + """Deprecated: use amdsmi_get_gpu_accelerator_partition_mem_alloc_mode() instead.""" + + warnings.warn( + "amdsmi_get_gpu_compute_partition_mem_alloc_mode() is deprecated, " + "use amdsmi_get_gpu_accelerator_partition_mem_alloc_mode() instead", + DeprecationWarning, + stacklevel=2, + ) + name = amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(processor_handle) + return name + + +def amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(processor_handle: processor_handle_t): if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - mode = amdsmi_wrapper.amdsmi_compute_partition_mem_alloc_mode_t() + mode = amdsmi_wrapper.amdsmi_accelerator_partition_mem_alloc_mode_t() _check_res( - amdsmi_wrapper.amdsmi_get_gpu_compute_partition_mem_alloc_mode( + amdsmi_wrapper.amdsmi_get_gpu_accelerator_partition_mem_alloc_mode( processor_handle, ctypes.byref(mode) ) ) - return AmdSmiComputePartitionMemAllocModeType(mode.value).name + return AmdSmiAcceleratorPartitionMemAllocModeType(mode.value).name def amdsmi_set_gpu_compute_partition_mem_alloc_mode( processor_handle: processor_handle_t, mode: AmdSmiComputePartitionMemAllocModeType +): + """Deprecated: use amdsmi_set_gpu_accelerator_partition_mem_alloc_mode() instead.""" + + warnings.warn( + "amdsmi_set_gpu_compute_partition_mem_alloc_mode() is deprecated, " + "use amdsmi_set_gpu_accelerator_partition_mem_alloc_mode() instead", + DeprecationWarning, + stacklevel=2, + ) + acc_mode = AmdSmiAcceleratorPartitionMemAllocModeType(mode) + amdsmi_set_gpu_accelerator_partition_mem_alloc_mode(processor_handle, acc_mode) + + +def amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( + processor_handle: processor_handle_t, mode: AmdSmiAcceleratorPartitionMemAllocModeType ): if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - if not isinstance(mode, AmdSmiComputePartitionMemAllocModeType): - raise AmdSmiParameterException(mode, AmdSmiComputePartitionMemAllocModeType) + if not isinstance(mode, AmdSmiAcceleratorPartitionMemAllocModeType): + raise AmdSmiParameterException(mode, AmdSmiAcceleratorPartitionMemAllocModeType) _check_res( - amdsmi_wrapper.amdsmi_set_gpu_compute_partition_mem_alloc_mode(processor_handle, mode) + amdsmi_wrapper.amdsmi_set_gpu_accelerator_partition_mem_alloc_mode(processor_handle, mode) ) @@ -4425,9 +4458,9 @@ def amdsmi_get_gpu_memory_partition(processor_handle: processor_handle_t): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) length = ctypes.c_uint32() - length.value = _AMDSMI_STRING_LENGTH + length.value = AMDSMI_MAX_STRING_LENGTH - memory_partition = ctypes.create_string_buffer(_AMDSMI_STRING_LENGTH) + memory_partition = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) _check_res( amdsmi_wrapper.amdsmi_get_gpu_memory_partition(processor_handle, memory_partition, length) @@ -4474,13 +4507,14 @@ def amdsmi_get_gpu_memory_partition_config(processor_handle: processor_handle_t) def amdsmi_set_gpu_memory_partition( processor_handle: processor_handle_t, memory_partition: AmdSmiMemoryPartitionType ): - if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): - raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) + """Deprecated: Use amdsmi_set_gpu_memory_partition_mode() instead. Will be deprecated in future ROCM release""" - if not isinstance(memory_partition, AmdSmiMemoryPartitionType): - raise AmdSmiParameterException(memory_partition, AmdSmiMemoryPartitionType) - - _check_res(amdsmi_wrapper.amdsmi_set_gpu_memory_partition(processor_handle, memory_partition)) + warnings.warn( + "amdsmi_set_gpu_memory_partition() is deprecated, use amdsmi_set_gpu_memory_partition_mode() instead", + DeprecationWarning, + stacklevel=2, + ) + amdsmi_set_gpu_memory_partition_mode(processor_handle, memory_partition) def amdsmi_set_gpu_memory_partition_mode( @@ -4492,7 +4526,9 @@ def amdsmi_set_gpu_memory_partition_mode( if not isinstance(memory_partition, AmdSmiMemoryPartitionType): raise AmdSmiParameterException(memory_partition, AmdSmiMemoryPartitionType) - _check_res(amdsmi_wrapper.amdsmi_set_gpu_memory_partition(processor_handle, memory_partition)) + _check_res( + amdsmi_wrapper.amdsmi_set_gpu_memory_partition_mode(processor_handle, memory_partition) + ) def amdsmi_get_gpu_accelerator_partition_profile( @@ -4979,10 +5015,10 @@ def amdsmi_get_gpu_topo_cpu_affinity(processor_handle: amdsmi_wrapper.amdsmi_pro if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - gpucpuaffid = ctypes.create_string_buffer(_AMDSMI_MAX_STRING_LENGTH) + gpucpuaffid = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) gpucpuaffid_length = ctypes.c_uint32() - gpucpuaffid_length.value = _AMDSMI_MAX_STRING_LENGTH + gpucpuaffid_length.value = AMDSMI_MAX_STRING_LENGTH _check_res( amdsmi_wrapper.amdsmi_get_gpu_topo_cpu_affinity( @@ -4996,10 +5032,10 @@ def amdsmi_get_nic_topo_cpu_affinity(processor_handle: amdsmi_wrapper.amdsmi_pro if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - niccpuaffid = ctypes.create_string_buffer(_AMDSMI_MAX_STRING_LENGTH) + niccpuaffid = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) niccpuaffid_length = ctypes.c_uint32() - niccpuaffid_length.value = _AMDSMI_MAX_STRING_LENGTH + niccpuaffid_length.value = AMDSMI_MAX_STRING_LENGTH _check_res( amdsmi_wrapper.amdsmi_get_nic_topo_cpu_affinity( @@ -5013,10 +5049,10 @@ def amdsmi_get_switch_topo_cpu_affinity(processor_handle: amdsmi_wrapper.amdsmi_ if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - switchcpuaffid = ctypes.create_string_buffer(_AMDSMI_MAX_STRING_LENGTH) + switchcpuaffid = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) switchcpuaffid_length = ctypes.c_uint32() - switchcpuaffid_length.value = _AMDSMI_MAX_STRING_LENGTH + switchcpuaffid_length.value = AMDSMI_MAX_STRING_LENGTH _check_res( amdsmi_wrapper.amdsmi_get_switch_topo_cpu_affinity( @@ -5036,10 +5072,10 @@ def amdsmi_get_nic_gpu_topo_info( if not isinstance(processor_handle_dst, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle_dst, amdsmi_wrapper.amdsmi_processor_handle) - niccgpuinfo = ctypes.create_string_buffer(_AMDSMI_MAX_STRING_LENGTH) + niccgpuinfo = ctypes.create_string_buffer(AMDSMI_MAX_STRING_LENGTH) niccgpuinfo_length = ctypes.c_uint32() - niccgpuinfo_length.value = _AMDSMI_MAX_STRING_LENGTH + niccgpuinfo_length.value = AMDSMI_MAX_STRING_LENGTH _check_res( amdsmi_wrapper.amdsmi_get_nic_gpu_topo_info( @@ -5112,34 +5148,6 @@ def amdsmi_get_energy_count(processor_handle: processor_handle_t): } -def amdsmi_set_gpu_clk_range( - processor_handle: processor_handle_t, - min_clk_value: int, - max_clk_value: int, - clk_type: AmdSmiClkType, -) -> None: - if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): - raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - - if not isinstance(min_clk_value, int): - raise AmdSmiParameterException(min_clk_value, int) - - if not isinstance(max_clk_value, int): - raise AmdSmiParameterException(min_clk_value, int) - - if not isinstance(clk_type, AmdSmiClkType): - raise AmdSmiParameterException(clk_type, AmdSmiClkType) - - _check_res( - amdsmi_wrapper.amdsmi_set_gpu_clk_range( - processor_handle, - ctypes.c_uint64(min_clk_value), - ctypes.c_uint64(max_clk_value), - clk_type, - ) - ) - - def amdsmi_set_gpu_clk_limit( processor_handle: processor_handle_t, clk_type: str, limit_type: str, value: int ) -> None: @@ -5161,9 +5169,9 @@ def amdsmi_set_gpu_clk_limit( raise AmdSmiParameterException(f"Unsupported clock type: {clk_type}", str) if limit_type.lower() == "min": - limit_type_conversion = amdsmi_wrapper.CLK_LIMIT_MIN + limit_type_conversion = amdsmi_wrapper.AMDSMI_CLK_LIMIT_MIN elif limit_type.lower() == "max": - limit_type_conversion = amdsmi_wrapper.CLK_LIMIT_MAX + limit_type_conversion = amdsmi_wrapper.AMDSMI_CLK_LIMIT_MAX else: raise AmdSmiParameterException(f"Unsupported limit type: {limit_type}", str) _check_res( @@ -5536,7 +5544,6 @@ def amdsmi_get_xgmi_plpd(processor_handle: processor_handle_t) -> Dict[str, Any] return { "num_supported": policy.num_supported, "current_id": current_id, - "plpds": policies, # Marked for deprecation "policies": policies, # Correct field name } @@ -6983,7 +6990,7 @@ def amdsmi_stop_gpu_event_notification(processor_handle: processor_handle_t): "DERIVED_NETPORT", ] -_FABRIC_TYPE_NAMES = {0: "UALOE", 1: "UALLINK", 2: "UNKNOWN"} +_FABRIC_TYPE_NAMES = {0: "UALOE", 1: "UALINK", 2: "UNKNOWN"} _FABRIC_ADDR_MODE_NAMES = {0: "SOURCE_ALIASING", 1: "SOURCE_IDENTIFICATION", 2: "UNKNOWN"} _FABRIC_ACCEL_STATE_NAMES = { 0: "UNCONFIGURED", @@ -7027,21 +7034,19 @@ def amdsmi_get_fabric_telemetry_data( for item_idx in range(inst.item_count): item = inst.items[item_idx] telem_id = item.id - name_ptr = amdsmi_wrapper.amdsmi_fabric_telem_id_to_string(telem_id) - # Handle both c_char_p (string) and POINTER(c_char) (pointer) return types - if name_ptr: - if isinstance(name_ptr, bytes): - name_str = name_ptr.decode("utf-8") - elif hasattr(name_ptr, "value"): - # c_char_p has a .value attribute - name_str = ( - name_ptr.value.decode("utf-8") if name_ptr.value else "UNKNOWN" - ) - else: - # POINTER(c_char) - dereference and convert to string - name_str = ctypes.string_at(name_ptr).decode("utf-8") - else: + name_ptr = ctypes.POINTER(ctypes.c_char)() + status = amdsmi_wrapper.amdsmi_fabric_telem_id_to_string( + telem_id, ctypes.byref(name_ptr) + ) + # Unmapped telemetry ids (newer firmware) return NOT_FOUND; + # tolerate them so one unknown id does not abort the whole + # telemetry read. + if status == amdsmi_wrapper.AMDSMI_STATUS_SUCCESS and name_ptr: + name_str = ctypes.string_at(name_ptr).decode("utf-8") + elif status == amdsmi_wrapper.AMDSMI_STATUS_NOT_FOUND: name_str = "UNKNOWN" + else: + _check_res(status) items.append({"id": telem_id, "name": name_str, "value": item.value}) instances.append( { @@ -7077,8 +7082,8 @@ def amdsmi_get_gpu_fabric_info(processor_handle: processor_handle_t) -> Dict[str if not isinstance(processor_handle, amdsmi_wrapper.amdsmi_processor_handle): raise AmdSmiParameterException(processor_handle, amdsmi_wrapper.amdsmi_processor_handle) - info = amdsmi_wrapper.amdsmi_fabric_info_t() - ret = amdsmi_wrapper.amdsmi_get_gpu_fabric_info(processor_handle, ctypes.byref(info)) + fabric_info = amdsmi_wrapper.amdsmi_fabric_info_t() + ret = amdsmi_wrapper.amdsmi_get_gpu_fabric_info(processor_handle, ctypes.byref(fabric_info)) if ret == amdsmi_wrapper.AMDSMI_STATUS_RETRY: raise AmdSmiRetryException() if ret == amdsmi_wrapper.AMDSMI_STATUS_TIMEOUT: @@ -7086,10 +7091,10 @@ def amdsmi_get_gpu_fabric_info(processor_handle: processor_handle_t) -> Dict[str if ret not in (amdsmi_wrapper.AMDSMI_STATUS_SUCCESS, amdsmi_wrapper.AMDSMI_STATUS_NO_DATA): raise AmdSmiLibraryException(ret) - v1 = info.fabric_info.fabric_version.v1 + v1 = fabric_info.fabric_info.v1 return { - "bdf": _format_bdf(info.bdf), - "version": info.fabric_info.version, + "bdf": _format_bdf(fabric_info.bdf), + "version": fabric_info.fabric_version, "accelerator_id": v1.accelerator_id, "fabric_type": _FABRIC_TYPE_NAMES.get(v1.fabric_type, "UNKNOWN"), "bandwidth": v1.bandwidth, diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index 01bbebfae99b..ea4131dbde99 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -498,6 +498,17 @@ class struct_amdsmi_hsmp_driver_version_t(Structure): AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_ALL = 2 amdsmi_compute_partition_mem_alloc_mode_t = ctypes.c_uint32 # enum +# values for enumeration 'amdsmi_accelerator_partition_mem_alloc_mode_t' +amdsmi_accelerator_partition_mem_alloc_mode_t__enumvalues = { + 0: 'AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_INVALID', + 1: 'AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING', + 2: 'AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL', +} +AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_INVALID = 0 +AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING = 1 +AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL = 2 +amdsmi_accelerator_partition_mem_alloc_mode_t = ctypes.c_uint32 # enum + # values for enumeration 'amdsmi_memory_partition_type_t' amdsmi_memory_partition_type_t__enumvalues = { 0: 'AMDSMI_MEMORY_PARTITION_UNKNOWN', @@ -1921,9 +1932,13 @@ class struct_amdsmi_evt_notification_data_t(Structure): # values for enumeration 'amdsmi_clk_limit_type_t' amdsmi_clk_limit_type_t__enumvalues = { + 0: 'AMDSMI_CLK_LIMIT_MIN', + 1: 'AMDSMI_CLK_LIMIT_MAX', 0: 'CLK_LIMIT_MIN', 1: 'CLK_LIMIT_MAX', } +AMDSMI_CLK_LIMIT_MIN = 0 +AMDSMI_CLK_LIMIT_MAX = 1 CLK_LIMIT_MIN = 0 CLK_LIMIT_MAX = 1 amdsmi_clk_limit_type_t = ctypes.c_uint32 # enum @@ -2704,10 +2719,16 @@ class struct_amdsmi_dimm_thermal_t(Structure): # values for enumeration 'amdsmi_io_bw_encoding_t' amdsmi_io_bw_encoding_t__enumvalues = { + 1: 'AMDSMI_AGG_BW0', + 2: 'AMDSMI_RD_BW0', + 4: 'AMDSMI_WR_BW0', 1: 'AGG_BW0', 2: 'RD_BW0', 4: 'WR_BW0', } +AMDSMI_AGG_BW0 = 1 +AMDSMI_RD_BW0 = 2 +AMDSMI_WR_BW0 = 4 AGG_BW0 = 1 RD_BW0 = 2 WR_BW0 = 4 @@ -2901,17 +2922,17 @@ class struct_amdsmi_nic_numa_info_t(Structure): ] amdsmi_nic_numa_info_t = struct_amdsmi_nic_numa_info_t -class struct_amdsmi_nic_fw_t(Structure): +class struct_amdsmi_nic_fw_entry_t(Structure): pass -struct_amdsmi_nic_fw_t._pack_ = 1 # source:False -struct_amdsmi_nic_fw_t._layout_ = 'ms' -struct_amdsmi_nic_fw_t._fields_ = [ +struct_amdsmi_nic_fw_entry_t._pack_ = 1 # source:False +struct_amdsmi_nic_fw_entry_t._layout_ = 'ms' +struct_amdsmi_nic_fw_entry_t._fields_ = [ ('name', ctypes.c_char * 256), ('version', ctypes.c_char * 256), ] -amdsmi_nic_fw_t = struct_amdsmi_nic_fw_t +amdsmi_nic_fw_entry_t = struct_amdsmi_nic_fw_entry_t class struct_amdsmi_nic_fw_info_t(Structure): pass @@ -2919,7 +2940,7 @@ class struct_amdsmi_nic_fw_info_t(Structure): struct_amdsmi_nic_fw_info_t._layout_ = 'ms' struct_amdsmi_nic_fw_info_t._fields_ = [ ('num_fw', ctypes.c_uint32), - ('fw', struct_amdsmi_nic_fw_t * 16), + ('fw', struct_amdsmi_nic_fw_entry_t * 16), ] amdsmi_nic_fw_info_t = struct_amdsmi_nic_fw_info_t @@ -3445,12 +3466,6 @@ class struct_amdsmi_nic_rdma_devices_info_t(Structure): amdsmi_get_gpu_reg_table_info.argtypes = [amdsmi_processor_handle, amdsmi_reg_type_t, ctypes.POINTER(ctypes.POINTER(struct_amdsmi_name_value_t)), ctypes.POINTER(ctypes.c_uint32)] except AttributeError: pass -try: - amdsmi_set_gpu_clk_range = _libraries['libamd_smi.so'].amdsmi_set_gpu_clk_range - amdsmi_set_gpu_clk_range.restype = amdsmi_status_t - amdsmi_set_gpu_clk_range.argtypes = [amdsmi_processor_handle, uint64_t, uint64_t, amdsmi_clk_type_t] -except AttributeError: - pass try: amdsmi_set_gpu_clk_limit = _libraries['libamd_smi.so'].amdsmi_set_gpu_clk_limit amdsmi_set_gpu_clk_limit.restype = amdsmi_status_t @@ -3544,7 +3559,6 @@ class struct_amdsmi_nic_rdma_devices_info_t(Structure): # values for enumeration 'amdsmi_fabric_telemetry_category_t' amdsmi_fabric_telemetry_category_t__enumvalues = { - 4294967295: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN', 0: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_UALOE', 1: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_SWITCH', 2: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_CRYPTO', @@ -3554,8 +3568,8 @@ class struct_amdsmi_nic_rdma_devices_info_t(Structure): 6: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_DERIVED_NETPORT', 7: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_MAX', 4294967295: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID', + 4294967295: 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN', } -AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN = 4294967295 AMDSMI_FABRIC_TELEMETRY_CATEGORY_UALOE = 0 AMDSMI_FABRIC_TELEMETRY_CATEGORY_SWITCH = 1 AMDSMI_FABRIC_TELEMETRY_CATEGORY_CRYPTO = 2 @@ -3565,6 +3579,7 @@ class struct_amdsmi_nic_rdma_devices_info_t(Structure): AMDSMI_FABRIC_TELEMETRY_CATEGORY_DERIVED_NETPORT = 6 AMDSMI_FABRIC_TELEMETRY_CATEGORY_MAX = 7 AMDSMI_FABRIC_TELEMETRY_CATEGORY_INVALID = 4294967295 +AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN = 4294967295 amdsmi_fabric_telemetry_category_t = ctypes.c_uint32 # enum # values for enumeration 'amdsmi_fabric_telemetry_category_mask_t' @@ -3671,8 +3686,8 @@ class struct_amdsmi_fabric_telemetry_t(Structure): pass try: amdsmi_fabric_telem_id_to_string = _libraries['libamd_smi.so'].amdsmi_fabric_telem_id_to_string - amdsmi_fabric_telem_id_to_string.restype = ctypes.POINTER(ctypes.c_char) - amdsmi_fabric_telem_id_to_string.argtypes = [uint64_t] + amdsmi_fabric_telem_id_to_string.restype = amdsmi_status_t + amdsmi_fabric_telem_id_to_string.argtypes = [uint64_t, ctypes.POINTER(ctypes.POINTER(ctypes.c_char))] except AttributeError: pass try: @@ -3694,12 +3709,14 @@ class struct_amdsmi_fabric_telemetry_t(Structure): # values for enumeration 'amdsmi_fabric_type_t' amdsmi_fabric_type_t__enumvalues = { 0: 'AMDSMI_FABRIC_TYPE_UALOE', - 1: 'AMDSMI_FABRIC_TYPE_UALLINK', + 1: 'AMDSMI_FABRIC_TYPE_UALINK', 2: 'AMDSMI_FABRIC_TYPE_UNKNOWN', + 1: 'AMDSMI_FABRIC_TYPE_UALLINK', } AMDSMI_FABRIC_TYPE_UALOE = 0 -AMDSMI_FABRIC_TYPE_UALLINK = 1 +AMDSMI_FABRIC_TYPE_UALINK = 1 AMDSMI_FABRIC_TYPE_UNKNOWN = 2 +AMDSMI_FABRIC_TYPE_UALLINK = 1 amdsmi_fabric_type_t = ctypes.c_uint32 # enum # values for enumeration 'amdsmi_fabric_npa_address_mode_t' @@ -3750,7 +3767,7 @@ class struct_amdsmi_fabric_info_v1_t(Structure): ] amdsmi_fabric_info_v1_t = struct_amdsmi_fabric_info_v1_t -class struct_amdsmi_fabric_info_ver_t(Structure): +class struct_amdsmi_fabric_info_t(Structure): pass class union_fabric_info_(Union): @@ -3760,22 +3777,12 @@ class union_fabric_info_(Union): ('v1', amdsmi_fabric_info_v1_t), ] -struct_amdsmi_fabric_info_ver_t._pack_ = 1 # source:False -struct_amdsmi_fabric_info_ver_t._layout_ = 'ms' -struct_amdsmi_fabric_info_ver_t._fields_ = [ - ('version', ctypes.c_uint32), - ('fabric_version', union_fabric_info_), -] - -amdsmi_fabric_info_ver_t = struct_amdsmi_fabric_info_ver_t -class struct_amdsmi_fabric_info_t(Structure): - pass - struct_amdsmi_fabric_info_t._pack_ = 1 # source:False struct_amdsmi_fabric_info_t._layout_ = 'ms' struct_amdsmi_fabric_info_t._fields_ = [ ('bdf', amdsmi_bdf_t), - ('fabric_info', amdsmi_fabric_info_ver_t), + ('fabric_version', ctypes.c_uint32), + ('fabric_info', union_fabric_info_), ('reserved', ctypes.c_uint32 * 15), ('PADDING_0', ctypes.c_ubyte * 4), ] @@ -4060,12 +4067,24 @@ class struct_amdsmi_cper_hdr_t(Structure): amdsmi_get_gpu_compute_partition_mem_alloc_mode.argtypes = [amdsmi_processor_handle, ctypes.POINTER(amdsmi_compute_partition_mem_alloc_mode_t)] except AttributeError: pass +try: + amdsmi_get_gpu_accelerator_partition_mem_alloc_mode = _libraries['libamd_smi.so'].amdsmi_get_gpu_accelerator_partition_mem_alloc_mode + amdsmi_get_gpu_accelerator_partition_mem_alloc_mode.restype = amdsmi_status_t + amdsmi_get_gpu_accelerator_partition_mem_alloc_mode.argtypes = [amdsmi_processor_handle, ctypes.POINTER(amdsmi_accelerator_partition_mem_alloc_mode_t)] +except AttributeError: + pass try: amdsmi_set_gpu_compute_partition_mem_alloc_mode = _libraries['libamd_smi.so'].amdsmi_set_gpu_compute_partition_mem_alloc_mode amdsmi_set_gpu_compute_partition_mem_alloc_mode.restype = amdsmi_status_t amdsmi_set_gpu_compute_partition_mem_alloc_mode.argtypes = [amdsmi_processor_handle, amdsmi_compute_partition_mem_alloc_mode_t] except AttributeError: pass +try: + amdsmi_set_gpu_accelerator_partition_mem_alloc_mode = _libraries['libamd_smi.so'].amdsmi_set_gpu_accelerator_partition_mem_alloc_mode + amdsmi_set_gpu_accelerator_partition_mem_alloc_mode.restype = amdsmi_status_t + amdsmi_set_gpu_accelerator_partition_mem_alloc_mode.argtypes = [amdsmi_processor_handle, amdsmi_accelerator_partition_mem_alloc_mode_t] +except AttributeError: + pass try: amdsmi_get_gpu_memory_partition = _libraries['libamd_smi.so'].amdsmi_get_gpu_memory_partition amdsmi_get_gpu_memory_partition.restype = amdsmi_status_t @@ -4815,17 +4834,21 @@ class struct_amdsmi_ttm_info_t(Structure): 'AMDSMI_ACCELERATOR_PARTITION_DPX', 'AMDSMI_ACCELERATOR_PARTITION_INVALID', 'AMDSMI_ACCELERATOR_PARTITION_MAX', + 'AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL', + 'AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING', + 'AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_INVALID', 'AMDSMI_ACCELERATOR_PARTITION_QPX', 'AMDSMI_ACCELERATOR_PARTITION_SPX', 'AMDSMI_ACCELERATOR_PARTITION_TPX', 'AMDSMI_ACCELERATOR_XCC', 'AMDSMI_AFFINITY_SCOPE_NODE', 'AMDSMI_AFFINITY_SCOPE_SOCKET', - 'AMDSMI_CACHE_PROPERTY_CPU_CACHE', + 'AMDSMI_AGG_BW0', 'AMDSMI_CACHE_PROPERTY_CPU_CACHE', 'AMDSMI_CACHE_PROPERTY_DATA_CACHE', 'AMDSMI_CACHE_PROPERTY_ENABLED', 'AMDSMI_CACHE_PROPERTY_INST_CACHE', 'AMDSMI_CACHE_PROPERTY_SIMD_CACHE', 'AMDSMI_CARD_FORM_FACTOR_CEM', 'AMDSMI_CARD_FORM_FACTOR_OAM', 'AMDSMI_CARD_FORM_FACTOR_PCIE', - 'AMDSMI_CARD_FORM_FACTOR_UNKNOWN', 'AMDSMI_CLK_TYPE_DCEF', + 'AMDSMI_CARD_FORM_FACTOR_UNKNOWN', 'AMDSMI_CLK_LIMIT_MAX', + 'AMDSMI_CLK_LIMIT_MIN', 'AMDSMI_CLK_TYPE_DCEF', 'AMDSMI_CLK_TYPE_DCLK0', 'AMDSMI_CLK_TYPE_DCLK1', 'AMDSMI_CLK_TYPE_DF', 'AMDSMI_CLK_TYPE_FIRST', 'AMDSMI_CLK_TYPE_GFX', 'AMDSMI_CLK_TYPE_MEM', @@ -4913,9 +4936,9 @@ class struct_amdsmi_ttm_info_t(Structure): 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_SWITCH', 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_UALOE', 'AMDSMI_FABRIC_TELEMETRY_CATEGORY_UNKNOWN', - 'AMDSMI_FABRIC_TYPE_UALLINK', 'AMDSMI_FABRIC_TYPE_UALOE', - 'AMDSMI_FABRIC_TYPE_UNKNOWN', 'AMDSMI_FINE_DECODER_ACTIVITY', - 'AMDSMI_FINE_GRAIN_GFX_ACTIVITY', + 'AMDSMI_FABRIC_TYPE_UALINK', 'AMDSMI_FABRIC_TYPE_UALLINK', + 'AMDSMI_FABRIC_TYPE_UALOE', 'AMDSMI_FABRIC_TYPE_UNKNOWN', + 'AMDSMI_FINE_DECODER_ACTIVITY', 'AMDSMI_FINE_GRAIN_GFX_ACTIVITY', 'AMDSMI_FINE_GRAIN_MEM_ACTIVITY', 'AMDSMI_FREQ_IND_INVALID', 'AMDSMI_FREQ_IND_MAX', 'AMDSMI_FREQ_IND_MIN', 'AMDSMI_FW_ID_ASD', 'AMDSMI_FW_ID_CP_CE', 'AMDSMI_FW_ID_CP_ME', @@ -5012,7 +5035,7 @@ class struct_amdsmi_ttm_info_t(Structure): 'AMDSMI_RAS_ERR_STATE_INVALID', 'AMDSMI_RAS_ERR_STATE_LAST', 'AMDSMI_RAS_ERR_STATE_MULT_UC', 'AMDSMI_RAS_ERR_STATE_NONE', 'AMDSMI_RAS_ERR_STATE_PARITY', 'AMDSMI_RAS_ERR_STATE_POISON', - 'AMDSMI_RAS_ERR_STATE_SING_C', 'AMDSMI_REG_PCIE', + 'AMDSMI_RAS_ERR_STATE_SING_C', 'AMDSMI_RD_BW0', 'AMDSMI_REG_PCIE', 'AMDSMI_REG_USR', 'AMDSMI_REG_USR1', 'AMDSMI_REG_WAFL', 'AMDSMI_REG_XGMI', 'AMDSMI_STATUS_ADDRESS_FAULT', 'AMDSMI_STATUS_AMDGPU_RESTART_ERR', 'AMDSMI_STATUS_API_FAILED', @@ -5138,11 +5161,13 @@ class struct_amdsmi_ttm_info_t(Structure): 'AMDSMI_VRAM_TYPE_HBM2E', 'AMDSMI_VRAM_TYPE_HBM3', 'AMDSMI_VRAM_TYPE_HBM3E', 'AMDSMI_VRAM_TYPE_LPDDR4', 'AMDSMI_VRAM_TYPE_LPDDR5', 'AMDSMI_VRAM_TYPE_UNKNOWN', - 'AMDSMI_VRAM_TYPE__MAX', 'AMDSMI_XGMI_LINK_DISABLE', - 'AMDSMI_XGMI_LINK_DOWN', 'AMDSMI_XGMI_LINK_UP', - 'AMDSMI_XGMI_STATUS_ERROR', 'AMDSMI_XGMI_STATUS_MULTIPLE_ERRORS', + 'AMDSMI_VRAM_TYPE__MAX', 'AMDSMI_WR_BW0', + 'AMDSMI_XGMI_LINK_DISABLE', 'AMDSMI_XGMI_LINK_DOWN', + 'AMDSMI_XGMI_LINK_UP', 'AMDSMI_XGMI_STATUS_ERROR', + 'AMDSMI_XGMI_STATUS_MULTIPLE_ERRORS', 'AMDSMI_XGMI_STATUS_NO_ERRORS', 'CLK_LIMIT_MAX', 'CLK_LIMIT_MIN', 'RD_BW0', 'WR_BW0', 'amd_metrics_table_header_t', + 'amdsmi_accelerator_partition_mem_alloc_mode_t', 'amdsmi_accelerator_partition_profile_config_t', 'amdsmi_accelerator_partition_profile_t', 'amdsmi_accelerator_partition_resource_profile_t', @@ -5171,8 +5196,8 @@ class struct_amdsmi_ttm_info_t(Structure): 'amdsmi_event_type_t', 'amdsmi_evt_notification_data_t', 'amdsmi_evt_notification_type_t', 'amdsmi_fabric_accelerator_vpod_state_t', 'amdsmi_fabric_info_t', - 'amdsmi_fabric_info_v1_t', 'amdsmi_fabric_info_ver_t', - 'amdsmi_fabric_label_t', 'amdsmi_fabric_npa_address_mode_t', + 'amdsmi_fabric_info_v1_t', 'amdsmi_fabric_label_t', + 'amdsmi_fabric_npa_address_mode_t', 'amdsmi_fabric_size_constants_t', 'amdsmi_fabric_telem_id_to_string', 'amdsmi_fabric_telemetry_category_mask_t', @@ -5226,6 +5251,7 @@ class struct_amdsmi_ttm_info_t(Structure): 'amdsmi_get_cpu_xgmi_pstate_range', 'amdsmi_get_cpucore_handles', 'amdsmi_get_energy_count', 'amdsmi_get_esmi_err_msg', 'amdsmi_get_fabric_telemetry_data', 'amdsmi_get_fw_info', + 'amdsmi_get_gpu_accelerator_partition_mem_alloc_mode', 'amdsmi_get_gpu_accelerator_partition_profile', 'amdsmi_get_gpu_accelerator_partition_profile_config', 'amdsmi_get_gpu_activity', 'amdsmi_get_gpu_asic_info', @@ -5318,7 +5344,7 @@ class struct_amdsmi_ttm_info_t(Structure): 'amdsmi_memory_partition_type_t', 'amdsmi_memory_type_t', 'amdsmi_mm_ip_t', 'amdsmi_name_value_t', 'amdsmi_nic_asic_info_t', 'amdsmi_nic_bus_info_t', 'amdsmi_nic_driver_info_t', - 'amdsmi_nic_fw_info_t', 'amdsmi_nic_fw_t', + 'amdsmi_nic_fw_entry_t', 'amdsmi_nic_fw_info_t', 'amdsmi_nic_numa_info_t', 'amdsmi_nic_port_info_t', 'amdsmi_nic_port_t', 'amdsmi_nic_rdma_dev_info_t', 'amdsmi_nic_rdma_devices_info_t', 'amdsmi_nic_rdma_port_info_t', @@ -5352,9 +5378,9 @@ class struct_amdsmi_ttm_info_t(Structure): 'amdsmi_set_cpu_socket_lclk_dpm_level', 'amdsmi_set_cpu_socket_power_cap', 'amdsmi_set_cpu_xgmi_pstate_range', 'amdsmi_set_cpu_xgmi_width', + 'amdsmi_set_gpu_accelerator_partition_mem_alloc_mode', 'amdsmi_set_gpu_accelerator_partition_profile', - 'amdsmi_set_gpu_clk_limit', 'amdsmi_set_gpu_clk_range', - 'amdsmi_set_gpu_compute_partition', + 'amdsmi_set_gpu_clk_limit', 'amdsmi_set_gpu_compute_partition', 'amdsmi_set_gpu_compute_partition_mem_alloc_mode', 'amdsmi_set_gpu_event_notification_mask', 'amdsmi_set_gpu_fan_speed', 'amdsmi_set_gpu_memory_partition', @@ -5402,7 +5428,7 @@ class struct_amdsmi_ttm_info_t(Structure): 'struct_amdsmi_enumeration_info_t', 'struct_amdsmi_error_count_t', 'struct_amdsmi_evt_notification_data_t', 'struct_amdsmi_fabric_info_t', 'struct_amdsmi_fabric_info_v1_t', - 'struct_amdsmi_fabric_info_ver_t', 'struct_amdsmi_fabric_label_t', + 'struct_amdsmi_fabric_label_t', 'struct_amdsmi_fabric_telemetry_dataset_t', 'struct_amdsmi_fabric_telemetry_instance_t', 'struct_amdsmi_fabric_telemetry_item_t', @@ -5419,7 +5445,7 @@ class struct_amdsmi_ttm_info_t(Structure): 'struct_amdsmi_memory_partition_config_t', 'struct_amdsmi_name_value_t', 'struct_amdsmi_nic_asic_info_t', 'struct_amdsmi_nic_bus_info_t', 'struct_amdsmi_nic_driver_info_t', - 'struct_amdsmi_nic_fw_info_t', 'struct_amdsmi_nic_fw_t', + 'struct_amdsmi_nic_fw_entry_t', 'struct_amdsmi_nic_fw_info_t', 'struct_amdsmi_nic_numa_info_t', 'struct_amdsmi_nic_port_info_t', 'struct_amdsmi_nic_port_t', 'struct_amdsmi_nic_rdma_dev_info_t', 'struct_amdsmi_nic_rdma_devices_info_t', diff --git a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_properties.h b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_properties.h index a5c70f991649..74830bad8240 100644 --- a/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_properties.h +++ b/projects/amdsmi/rocm_smi/include/rocm_smi/rocm_smi_properties.h @@ -45,7 +45,6 @@ enum class AMDGpuVerbTypes_t : AMDGpuVerbId_t { kSetGpuPciBandwidth, kSetPowerCap, kSetGpuPowerProfile, - kSetGpuClkRange, kSetGpuOdClkInfo, kSetGpuOdVoltInfo, kSetGpuPerfLevelV1, diff --git a/projects/amdsmi/rocm_smi/src/rocm_smi.cc b/projects/amdsmi/rocm_smi/src/rocm_smi.cc index b203c093deb3..dab9c7320816 100644 --- a/projects/amdsmi/rocm_smi/src/rocm_smi.cc +++ b/projects/amdsmi/rocm_smi/src/rocm_smi.cc @@ -3456,7 +3456,8 @@ rsmi_status_t rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, return RSMI_STATUS_NOT_SUPPORTED; } - *temperature = static_cast(val_ui16) * CENTRIGRADE_TO_MILLI_CENTIGRADE; + // Multiple by 1000 to convert from Centigrade to mCentigrade + *temperature = static_cast(val_ui16) * 1000; ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " diff --git a/projects/amdsmi/rust-interface/README.md b/projects/amdsmi/rust-interface/README.md index 9c04b140450b..b04c262d4016 100644 --- a/projects/amdsmi/rust-interface/README.md +++ b/projects/amdsmi/rust-interface/README.md @@ -78,10 +78,10 @@ fn main() { Err(e) => eprintln!("Failed to get GPU vendor name: {}", e), } - // Get GPU VRAM vendor using the processor handle - match amdsmi_get_gpu_vram_vendor(processor_handle) { - Ok(gpu_vram_vendor) => println!("GPU VRAM Vendor: {}", gpu_vram_vendor), - Err(e) => eprintln!("Failed to get GPU VRAM vendor: {}", e), + // Get GPU VRAM info using the processor handle + match amdsmi_get_gpu_vram_info(processor_handle) { + Ok(gpu_vram_info) => println!("GPU VRAM Info: {:?}", gpu_vram_info), + Err(e) => eprintln!("Failed to get GPU VRAM info: {}", e), } // Get GPU BDF using the processor handle diff --git a/projects/amdsmi/rust-interface/examples/amdsmi_get_gpu_info.rs b/projects/amdsmi/rust-interface/examples/amdsmi_get_gpu_info.rs index 9a8cf9552ca0..4e84d4119ea4 100644 --- a/projects/amdsmi/rust-interface/examples/amdsmi_get_gpu_info.rs +++ b/projects/amdsmi/rust-interface/examples/amdsmi_get_gpu_info.rs @@ -69,12 +69,6 @@ fn main() { Err(e) => eprintln!("Failed to get GPU vendor name: {}", e), } - // Get GPU VRAM vendor using the processor handle - match amdsmi_get_gpu_vram_vendor(processor_handle) { - Ok(gpu_vram_vendor) => println!("GPU VRAM Vendor: {}", gpu_vram_vendor), - Err(e) => eprintln!("Failed to get GPU VRAM vendor: {}", e), - } - // Get GPU subsystem ID using the processor handle match amdsmi_get_gpu_subsystem_id(processor_handle) { Ok(gpu_subsystem_id) => println!("GPU Subsystem ID: {}", gpu_subsystem_id), diff --git a/projects/amdsmi/rust-interface/src/amdsmi.rs b/projects/amdsmi/rust-interface/src/amdsmi.rs index 6dbab97f6d0a..d3dca1bccf83 100644 --- a/projects/amdsmi/rust-interface/src/amdsmi.rs +++ b/projects/amdsmi/rust-interface/src/amdsmi.rs @@ -511,55 +511,6 @@ pub fn amdsmi_get_gpu_vendor_name(processor_handle: AmdsmiProcessorHandle) -> Am Ok(cstr_to_string!(name)) } -/// Retrieves the GPU VRAM vendor name for a given processor handle. -/// -/// This function returns the GPU VRAM vendor name associated with the specified processor handle. -/// The VRAM vendor name provides information about the manufacturer of the GPU's VRAM. -/// -/// # Arguments -/// -/// * `processor_handle` - A handle to the processor for which the GPU VRAM vendor name is being queried. -/// -/// # Returns -/// -/// * `AmdsmiResult` - Returns `Ok(String)` containing the GPU VRAM vendor name if successful, or an error if it fails. -/// -/// # Example -/// -/// ```rust -/// # use amdsmi::*; -/// # -/// # fn main() { -/// # // Initialize the AMD SMI library -/// # amdsmi_init(AmdsmiInitFlagsT::AmdsmiInitAmdGpus).expect("Failed to initialize AMD SMI"); -/// # -/// // Example processor_handle, assuming the number of processors is greater than zero -/// let processor_handle = amdsmi_get_processor_handles!()[0]; -/// -/// // Retrieve the GPU VRAM vendor name -/// match amdsmi_get_gpu_vram_vendor(processor_handle) { -/// Ok(vram_vendor_name) => println!("GPU VRAM Vendor Name: {}", vram_vendor_name), -/// Err(e) => panic!("Failed to get GPU VRAM vendor name: {}", e), -/// } -/// # -/// # // Shut down the AMD SMI library -/// # amdsmi_shut_down().expect("Failed to shut down AMD SMI"); -/// # } -/// ``` -/// -/// # Errors -/// -/// This function will return the error in [`AmdsmiStatusT`] if the underlying `amdsmi_wrapper::amdsmi_get_gpu_vram_vendor` call fails. -pub fn amdsmi_get_gpu_vram_vendor(processor_handle: AmdsmiProcessorHandle) -> AmdsmiResult { - let (mut brand, len) = define_cstr!(amdsmi_wrapper::AMDSMI_MAX_STRING_LENGTH); - call_unsafe!(amdsmi_wrapper::amdsmi_get_gpu_vram_vendor( - processor_handle, - brand.as_mut_ptr(), - len as u32, - )); - Ok(cstr_to_string!(brand)) -} - /// Retrieves the GPU subsystem ID for a given processor handle. /// /// This function returns the GPU subsystem ID associated with the specified processor handle. @@ -2685,69 +2636,6 @@ pub fn amdsmi_get_gpu_reg_table_info( Ok(reg_metrics) } -/// Set the GPU clock range of the device with the specified processor handle, minimum clock value, maximum clock value, and clock type. -/// -/// Given a processor handle `processor_handle`, a minimum clock value `minclkvalue`, a maximum clock value `maxclkvalue`, and a clock type `clk_type`, -/// this function sets the GPU clock range for the specified processor. -/// -/// # Arguments -/// -/// * `processor_handle` - A handle to the processor for which the GPU clock range is being set. -/// * `minclkvalue` - The minimum clock value to set. -/// * `maxclkvalue` - The maximum clock value to set. -/// * `clk_type` - The type of the clock to set. -/// -/// # Returns -/// -/// * `AmdsmiResult<()>` - Returns `Ok(())` if successful, or an error if it fails. -/// -/// # Example -/// -/// ```rust -/// # use amdsmi::*; -/// # -/// # fn main() { -/// # // Initialize the AMD SMI library -/// # amdsmi_init(AmdsmiInitFlagsT::AmdsmiInitAmdGpus).expect("Failed to initialize AMD SMI"); -/// # -/// // Example processor_handle, assuming the number of processors is greater than zero -/// let processor_handle = amdsmi_get_processor_handles!()[0]; -/// -/// // Example clock values and clock type -/// let minclkvalue = 500; // Set minimum clock value to 500 MHz -/// let maxclkvalue = 1500; // Set maximum clock value to 1500 MHz -/// let clk_type = AmdsmiClkTypeT::AmdsmiClkTypeGfx; -/// -/// // Set the GPU clock range -/// match amdsmi_set_gpu_clk_range(processor_handle, minclkvalue, maxclkvalue, clk_type) { -/// Ok(()) => println!("GPU clock range set successfully"), -/// Err(AmdsmiStatusT::AmdsmiStatusNotSupported) => println!("amdsmi_set_gpu_clk_range() not supported on this device"), -/// Err(e) => panic!("Failed to set GPU clock range: {}", e), -/// } -/// # -/// # // Shut down the AMD SMI library -/// # amdsmi_shut_down().expect("Failed to shut down AMD SMI"); -/// # } -/// ``` -/// -/// # Errors -/// -/// This function will return the error in [`AmdsmiStatusT`] if the underlying `amdsmi_wrapper::amdsmi_set_gpu_clk_range` call fails. -pub fn amdsmi_set_gpu_clk_range( - processor_handle: AmdsmiProcessorHandle, - minclkvalue: u64, - maxclkvalue: u64, - clk_type: AmdsmiClkTypeT, -) -> AmdsmiResult<()> { - call_unsafe!(amdsmi_wrapper::amdsmi_set_gpu_clk_range( - processor_handle, - minclkvalue, - maxclkvalue, - clk_type - )); - Ok(()) -} - /// Set the GPU clock limit of the device with the specified processor handle, clock type, limit type, and clock value. /// /// Given a processor handle `processor_handle`, a clock type `clk_type`, a limit type `limit_type`, and a clock value `clk_value`, diff --git a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs index 642a9e21e0e7..0de457d9f73a 100644 --- a/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs +++ b/projects/amdsmi/rust-interface/src/amdsmi_wrapper.rs @@ -127,6 +127,7 @@ pub const AMDSMI_APU_MAX_CORES: u32 = 16; pub const AMDSMI_APU_V24_CORES: u32 = 8; pub const AMDSMI_APU_MAX_L3: u32 = 2; pub const AMDSMI_APU_MAX_IPU: u32 = 8; +pub const AMDSMI_MAX_NUMBER_OF_AFIDS_PER_RECORD: u32 = 12; pub const AMDSMI_MAX_NUM_HBM_STACKS: u32 = 12; pub const AMDSMI_MAX_NUM_AID: u32 = 2; pub const AMDSMI_MAX_NUM_MID: u32 = 2; @@ -137,15 +138,6 @@ pub const AMDSMI_DATE_FORMAT: &[u8; 35] = b"%04d-%02d-%02d:%02d:%02d:%02d.%03d\0 pub const AMDSMI_LIB_VERSION_MAJOR: u32 = 27; pub const AMDSMI_LIB_VERSION_MINOR: u32 = 0; pub const AMDSMI_LIB_VERSION_RELEASE: u32 = 0; -pub const AMDSMI_MAX_VF_COUNT: u32 = 32; -pub const AMDSMI_MAX_DRIVER_NUM: u32 = 2; -pub const AMDSMI_DFC_FW_NUMBER_OF_ENTRIES: u32 = 9; -pub const AMDSMI_MAX_WHITE_LIST_ELEMENTS: u32 = 16; -pub const AMDSMI_MAX_BLACK_LIST_ELEMENTS: u32 = 64; -pub const AMDSMI_MAX_TA_WHITE_LIST_ELEMENTS: u32 = 8; -pub const AMDSMI_MAX_ERR_RECORDS: u32 = 10; -pub const AMDSMI_MAX_PROFILE_COUNT: u32 = 16; -pub const AMDSMI_PF_INDEX: u32 = 31; pub const AMDSMI_MAX_DRIVER_INFO_RSVD: u32 = 64; pub const AMDSMI_MAX_UUID_ELEMENTS: u32 = 16; pub const AMDSMI_MAX_NUM_FREQUENCIES: u32 = 33; @@ -330,6 +322,13 @@ pub enum AmdsmiComputePartitionMemAllocModeT { } #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum AmdsmiAcceleratorPartitionMemAllocModeT { + AmdsmiAcceleratorPartitionMemAllocInvalid = 0, + AmdsmiAcceleratorPartitionMemAllocCapping = 1, + AmdsmiAcceleratorPartitionMemAllocAll = 2, +} +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub enum AmdsmiMemoryPartitionTypeT { AmdsmiMemoryPartitionUnknown = 0, AmdsmiMemoryPartitionNps1 = 1, @@ -2256,11 +2255,17 @@ pub enum AmdsmiGpuBlockT { AmdsmiGpuBlockMpio = 262144, AmdsmiGpuBlockReserved = 9223372036854775808, } +impl AmdsmiClkLimitTypeT { + pub const ClkLimitMin: AmdsmiClkLimitTypeT = AmdsmiClkLimitTypeT::AmdsmiClkLimitMin; +} +impl AmdsmiClkLimitTypeT { + pub const ClkLimitMax: AmdsmiClkLimitTypeT = AmdsmiClkLimitTypeT::AmdsmiClkLimitMax; +} #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub enum AmdsmiClkLimitTypeT { - ClkLimitMin = 0, - ClkLimitMax = 1, + AmdsmiClkLimitMin = 0, + AmdsmiClkLimitMax = 1, } #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -3524,23 +3529,24 @@ const _: () = { }; #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct AmdsmiNicFwT { +pub struct AmdsmiNicFwEntryT { pub name: [::std::os::raw::c_char; 256usize], pub version: [::std::os::raw::c_char; 256usize], } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of AmdsmiNicFwT"][::std::mem::size_of::() - 512usize]; - ["Alignment of AmdsmiNicFwT"][::std::mem::align_of::() - 1usize]; - ["Offset of field: AmdsmiNicFwT::name"][::std::mem::offset_of!(AmdsmiNicFwT, name) - 0usize]; - ["Offset of field: AmdsmiNicFwT::version"] - [::std::mem::offset_of!(AmdsmiNicFwT, version) - 256usize]; + ["Size of AmdsmiNicFwEntryT"][::std::mem::size_of::() - 512usize]; + ["Alignment of AmdsmiNicFwEntryT"][::std::mem::align_of::() - 1usize]; + ["Offset of field: AmdsmiNicFwEntryT::name"] + [::std::mem::offset_of!(AmdsmiNicFwEntryT, name) - 0usize]; + ["Offset of field: AmdsmiNicFwEntryT::version"] + [::std::mem::offset_of!(AmdsmiNicFwEntryT, version) - 256usize]; }; #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct AmdsmiNicFwInfoT { pub num_fw: u32, - pub fw: [AmdsmiNicFwT; 16usize], + pub fw: [AmdsmiNicFwEntryT; 16usize], } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { @@ -4169,14 +4175,6 @@ extern "C" { num_of_metrics: *mut u32, ) -> AmdsmiStatusT; } -extern "C" { - pub fn amdsmi_set_gpu_clk_range( - processor_handle: AmdsmiProcessorHandle, - minclkvalue: u64, - maxclkvalue: u64, - clkType: AmdsmiClkTypeT, - ) -> AmdsmiStatusT; -} extern "C" { pub fn amdsmi_set_gpu_clk_limit( processor_handle: AmdsmiProcessorHandle, @@ -4274,13 +4272,12 @@ extern "C" { pub fn amdsmi_clean_gpu_local_data(processor_handle: AmdsmiProcessorHandle) -> AmdsmiStatusT; } impl AmdsmiFabricTelemetryCategoryT { - pub const AmdsmiFabricTelemetryCategoryInvalid: AmdsmiFabricTelemetryCategoryT = - AmdsmiFabricTelemetryCategoryT::AmdsmiFabricTelemetryCategoryUnknown; + pub const AmdsmiFabricTelemetryCategoryUnknown: AmdsmiFabricTelemetryCategoryT = + AmdsmiFabricTelemetryCategoryT::AmdsmiFabricTelemetryCategoryInvalid; } #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub enum AmdsmiFabricTelemetryCategoryT { - AmdsmiFabricTelemetryCategoryUnknown = 4294967295, AmdsmiFabricTelemetryCategoryUaloe = 0, AmdsmiFabricTelemetryCategorySwitch = 1, AmdsmiFabricTelemetryCategoryCrypto = 2, @@ -4289,6 +4286,7 @@ pub enum AmdsmiFabricTelemetryCategoryT { AmdsmiFabricTelemetryCategoryDerivedUaloe = 5, AmdsmiFabricTelemetryCategoryDerivedNetport = 6, AmdsmiFabricTelemetryCategoryMax = 7, + AmdsmiFabricTelemetryCategoryInvalid = 4294967295, } #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -4407,7 +4405,10 @@ extern "C" { ) -> AmdsmiStatusT; } extern "C" { - pub fn amdsmi_fabric_telem_id_to_string(telem_id: u64) -> *const ::std::os::raw::c_char; + pub fn amdsmi_fabric_telem_id_to_string( + telem_id: u64, + telem_name: *mut *const ::std::os::raw::c_char, + ) -> AmdsmiStatusT; } extern "C" { pub fn amdsmi_free_fabric_telemetry( @@ -4421,11 +4422,15 @@ pub enum AmdsmiFabricSizeConstantsT { AmdsmiFabricActiveAcceleratorsBitmapSize = 32, AmdsmiFabricMaxLocalGpus = 16, } +impl AmdsmiFabricTypeT { + pub const AmdsmiFabricTypeUallink: AmdsmiFabricTypeT = + AmdsmiFabricTypeT::AmdsmiFabricTypeUalink; +} #[repr(u32)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub enum AmdsmiFabricTypeT { AmdsmiFabricTypeUaloe = 0, - AmdsmiFabricTypeUallink = 1, + AmdsmiFabricTypeUalink = 1, AmdsmiFabricTypeUnknown = 2, } #[repr(u32)] @@ -4492,48 +4497,36 @@ const _: () = { }; #[repr(C)] #[derive(Copy, Clone)] -pub struct AmdsmiFabricInfoVerT { - pub version: u32, - pub fabric_version: AmdsmiFabricInfoVerTFabricInfo, +pub struct AmdsmiFabricInfoT { + pub bdf: AmdsmiBdfT, + pub fabric_version: u32, + pub fabric_info: AmdsmiFabricInfoTFabricInfo, + pub reserved: [u32; 15usize], } #[repr(C)] #[derive(Copy, Clone)] -pub union AmdsmiFabricInfoVerTFabricInfo { +pub union AmdsmiFabricInfoTFabricInfo { pub v1: AmdsmiFabricInfoV1T, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of AmdsmiFabricInfoVerTFabricInfo"] - [::std::mem::size_of::() - 244usize]; - ["Alignment of AmdsmiFabricInfoVerTFabricInfo"] - [::std::mem::align_of::() - 4usize]; - ["Offset of field: AmdsmiFabricInfoVerTFabricInfo::v1"] - [::std::mem::offset_of!(AmdsmiFabricInfoVerTFabricInfo, v1) - 0usize]; -}; -#[allow(clippy::unnecessary_operation, clippy::identity_op)] -const _: () = { - ["Size of AmdsmiFabricInfoVerT"][::std::mem::size_of::() - 248usize]; - ["Alignment of AmdsmiFabricInfoVerT"][::std::mem::align_of::() - 4usize]; - ["Offset of field: AmdsmiFabricInfoVerT::version"] - [::std::mem::offset_of!(AmdsmiFabricInfoVerT, version) - 0usize]; - ["Offset of field: AmdsmiFabricInfoVerT::fabric_version"] - [::std::mem::offset_of!(AmdsmiFabricInfoVerT, fabric_version) - 4usize]; + ["Size of AmdsmiFabricInfoTFabricInfo"] + [::std::mem::size_of::() - 244usize]; + ["Alignment of AmdsmiFabricInfoTFabricInfo"] + [::std::mem::align_of::() - 4usize]; + ["Offset of field: AmdsmiFabricInfoTFabricInfo::v1"] + [::std::mem::offset_of!(AmdsmiFabricInfoTFabricInfo, v1) - 0usize]; }; -#[repr(C)] -#[derive(Copy, Clone)] -pub struct AmdsmiFabricInfoT { - pub bdf: AmdsmiBdfT, - pub fabric_info: AmdsmiFabricInfoVerT, - pub reserved: [u32; 15usize], -} #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { ["Size of AmdsmiFabricInfoT"][::std::mem::size_of::() - 320usize]; ["Alignment of AmdsmiFabricInfoT"][::std::mem::align_of::() - 8usize]; ["Offset of field: AmdsmiFabricInfoT::bdf"] [::std::mem::offset_of!(AmdsmiFabricInfoT, bdf) - 0usize]; + ["Offset of field: AmdsmiFabricInfoT::fabric_version"] + [::std::mem::offset_of!(AmdsmiFabricInfoT, fabric_version) - 8usize]; ["Offset of field: AmdsmiFabricInfoT::fabric_info"] - [::std::mem::offset_of!(AmdsmiFabricInfoT, fabric_info) - 8usize]; + [::std::mem::offset_of!(AmdsmiFabricInfoT, fabric_info) - 12usize]; ["Offset of field: AmdsmiFabricInfoT::reserved"] [::std::mem::offset_of!(AmdsmiFabricInfoT, reserved) - 256usize]; }; @@ -4956,12 +4949,24 @@ extern "C" { mode: *mut AmdsmiComputePartitionMemAllocModeT, ) -> AmdsmiStatusT; } +extern "C" { + pub fn amdsmi_get_gpu_accelerator_partition_mem_alloc_mode( + processor_handle: AmdsmiProcessorHandle, + mode: *mut AmdsmiAcceleratorPartitionMemAllocModeT, + ) -> AmdsmiStatusT; +} extern "C" { pub fn amdsmi_set_gpu_compute_partition_mem_alloc_mode( processor_handle: AmdsmiProcessorHandle, mode: AmdsmiComputePartitionMemAllocModeT, ) -> AmdsmiStatusT; } +extern "C" { + pub fn amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( + processor_handle: AmdsmiProcessorHandle, + mode: AmdsmiAcceleratorPartitionMemAllocModeT, + ) -> AmdsmiStatusT; +} extern "C" { pub fn amdsmi_get_gpu_memory_partition( processor_handle: AmdsmiProcessorHandle, diff --git a/projects/amdsmi/src/amd_smi/amd_smi.cc b/projects/amdsmi/src/amd_smi/amd_smi.cc index 7454998f066c..63bdb132e0bb 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi.cc @@ -2653,6 +2653,8 @@ amdsmi_status_t amdsmi_get_gpu_vendor_name(amdsmi_processor_handle processor_han amdsmi_status_t amdsmi_get_gpu_vram_vendor(amdsmi_processor_handle processor_handle, char* brand, uint32_t len) { + // Deprecated: delegates to amdsmi_get_gpu_vram_info(); slated for removal in a + // future ROCm release. if (brand == nullptr) { return AMDSMI_STATUS_INVAL; } @@ -2692,7 +2694,7 @@ amdsmi_status_t amdsmi_get_gpu_vram_info(amdsmi_processor_handle processor_handl // --- sysfs first: vendor + total size (no DRM dependency) --- // The vendor string is only exposed via sysfs (mem_info_vram_vendor); the DRM // ioctl carries no vendor field. Read it before attempting the ioctl so callers - // (e.g. amdsmi_get_gpu_vram_vendor) still get the vendor when DRM is unavailable. + // (e.g. amdsmi_get_gpu_vram_info) still get the vendor when DRM is unavailable. char brand[256] = {'\0'}; amdsmi_status_t vendor_status = rsmi_wrapper(rsmi_dev_vram_vendor_get, processor_handle, 0, brand, 255); @@ -3052,6 +3054,7 @@ amdsmi_status_t amdsmi_topo_get_p2p_status(amdsmi_processor_handle processor_han } // Compute Partition functions +// This API is deprecated, use amdsmi_get_gpu_accelerator_partition_profile() instead amdsmi_status_t amdsmi_get_gpu_compute_partition(amdsmi_processor_handle processor_handle, char* compute_partition, uint32_t len) { AMDSMI_CHECK_INIT(); @@ -3065,6 +3068,7 @@ amdsmi_status_t amdsmi_get_gpu_compute_partition(amdsmi_processor_handle process return status; } +// This API is deprecated, use amdsmi_set_gpu_accelerator_partition_profile() instead amdsmi_status_t amdsmi_set_gpu_compute_partition( amdsmi_processor_handle processor_handle, amdsmi_compute_partition_type_t compute_partition) { AMDSMI_CHECK_INIT(); @@ -3073,8 +3077,18 @@ amdsmi_status_t amdsmi_set_gpu_compute_partition( return ret_resp; } +// This API is deprecated, use amdsmi_get_gpu_accelerator_partition_mem_alloc_mode() instead amdsmi_status_t amdsmi_get_gpu_compute_partition_mem_alloc_mode( amdsmi_processor_handle processor_handle, amdsmi_compute_partition_mem_alloc_mode_t* mode) { + amdsmi_accelerator_partition_mem_alloc_mode_t* acc_mode; + acc_mode = reinterpret_cast(mode); + amdsmi_status_t status = + amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(processor_handle, acc_mode); + return status; +} + +amdsmi_status_t amdsmi_get_gpu_accelerator_partition_mem_alloc_mode( + amdsmi_processor_handle processor_handle, amdsmi_accelerator_partition_mem_alloc_mode_t* mode) { AMDSMI_CHECK_INIT(); if (mode == nullptr) { return AMDSMI_STATUS_INVAL; @@ -3088,11 +3102,21 @@ amdsmi_status_t amdsmi_get_gpu_compute_partition_mem_alloc_mode( return status; } +// This API is deprecated, use amdsmi_set_gpu_accelerator_partition_mem_alloc_mode() instead amdsmi_status_t amdsmi_set_gpu_compute_partition_mem_alloc_mode( amdsmi_processor_handle processor_handle, amdsmi_compute_partition_mem_alloc_mode_t mode) { + amdsmi_accelerator_partition_mem_alloc_mode_t acc_mode; + acc_mode = static_cast(mode); + amdsmi_status_t status = + amdsmi_set_gpu_accelerator_partition_mem_alloc_mode(processor_handle, acc_mode); + return status; +} + +amdsmi_status_t amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( + amdsmi_processor_handle processor_handle, amdsmi_accelerator_partition_mem_alloc_mode_t mode) { AMDSMI_CHECK_INIT(); - if (mode != AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_CAPPING && - mode != AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_ALL) { + if (mode != AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING && + mode != AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL) { return AMDSMI_STATUS_INVAL; } return rsmi_wrapper(rsmi_dev_compute_partition_mem_alloc_mode_set, processor_handle, 0, @@ -3108,67 +3132,11 @@ amdsmi_status_t amdsmi_get_gpu_memory_partition(amdsmi_processor_handle processo return ret; } +// This API is deprecated, use amdsmi_set_gpu_memory_partition_mode() instead amdsmi_status_t amdsmi_set_gpu_memory_partition(amdsmi_processor_handle processor_handle, - amdsmi_memory_partition_type_t memory_partition) { + amdsmi_memory_partition_type_t mode) { AMDSMI_CHECK_INIT(); - if (memory_partition != AMDSMI_MEMORY_PARTITION_UNKNOWN && - memory_partition != AMDSMI_MEMORY_PARTITION_NPS1 && - memory_partition != AMDSMI_MEMORY_PARTITION_NPS2 && - memory_partition != AMDSMI_MEMORY_PARTITION_NPS4 && - memory_partition != AMDSMI_MEMORY_PARTITION_NPS8) { - return AMDSMI_STATUS_INVAL; - } - std::ostringstream ss; - std::lock_guard g(myMutex); - - const uint32_t k256 = 256; - char current_partition[k256]; - std::string current_partition_str = "UNKNOWN"; - std::string req_user_partition = "UNKNOWN"; - - req_user_partition.clear(); - switch (memory_partition) { - case AMDSMI_MEMORY_PARTITION_NPS1: - req_user_partition = "NPS1"; - break; - case AMDSMI_MEMORY_PARTITION_NPS2: - req_user_partition = "NPS2"; - break; - case AMDSMI_MEMORY_PARTITION_NPS4: - req_user_partition = "NPS4"; - break; - case AMDSMI_MEMORY_PARTITION_NPS8: - req_user_partition = "NPS8"; - break; - default: - req_user_partition = "UNKNOWN"; - break; - } - rsmi_memory_partition_type_t rsmi_type; - auto it = nps_amdsmi_to_RSMI.find(memory_partition); - if (it != nps_amdsmi_to_RSMI.end()) { - rsmi_type = it->second; - } else if (it == nps_amdsmi_to_RSMI.end()) { - return AMDSMI_STATUS_INVAL; - } - - amdsmi_status_t ret = rsmi_wrapper(rsmi_dev_memory_partition_set, processor_handle, 0, rsmi_type); - - amdsmi_status_t ret_get = - rsmi_wrapper(rsmi_dev_memory_partition_get, processor_handle, 0, current_partition, k256); - - if (ret_get == AMDSMI_STATUS_SUCCESS) { - current_partition_str.clear(); - current_partition_str = current_partition; - } - - ss << __PRETTY_FUNCTION__ << " | After attempting to set memory partition to " - << req_user_partition << "\n" - << " | Current memory partition is " << current_partition_str << "\n" - << " | Returning: " << smi_amdgpu_get_status_string(ret, false) - << " | User will need to reload driver in order to see a NPS mode change"; - LOG_INFO(ss); - return ret; + return amdsmi_set_gpu_memory_partition_mode(processor_handle, mode); } amdsmi_status_t amdsmi_get_gpu_memory_partition_config(amdsmi_processor_handle processor_handle, @@ -3240,11 +3208,67 @@ amdsmi_status_t amdsmi_get_gpu_memory_partition_config(amdsmi_processor_handle p config->partition_caps = flags; return status; } - -amdsmi_status_t amdsmi_set_gpu_memory_partition_mode(amdsmi_processor_handle processor_handle, - amdsmi_memory_partition_type_t mode) { +amdsmi_status_t amdsmi_set_gpu_memory_partition_mode( + amdsmi_processor_handle processor_handle, amdsmi_memory_partition_type_t memory_partition) { AMDSMI_CHECK_INIT(); - return amdsmi_set_gpu_memory_partition(processor_handle, mode); + if (memory_partition != AMDSMI_MEMORY_PARTITION_UNKNOWN && + memory_partition != AMDSMI_MEMORY_PARTITION_NPS1 && + memory_partition != AMDSMI_MEMORY_PARTITION_NPS2 && + memory_partition != AMDSMI_MEMORY_PARTITION_NPS4 && + memory_partition != AMDSMI_MEMORY_PARTITION_NPS8) { + return AMDSMI_STATUS_INVAL; + } + std::ostringstream ss; + std::lock_guard g(myMutex); + + const uint32_t k256 = 256; + char current_partition[k256]; + std::string current_partition_str = "UNKNOWN"; + std::string req_user_partition = "UNKNOWN"; + + req_user_partition.clear(); + switch (memory_partition) { + case AMDSMI_MEMORY_PARTITION_NPS1: + req_user_partition = "NPS1"; + break; + case AMDSMI_MEMORY_PARTITION_NPS2: + req_user_partition = "NPS2"; + break; + case AMDSMI_MEMORY_PARTITION_NPS4: + req_user_partition = "NPS4"; + break; + case AMDSMI_MEMORY_PARTITION_NPS8: + req_user_partition = "NPS8"; + break; + default: + req_user_partition = "UNKNOWN"; + break; + } + rsmi_memory_partition_type_t rsmi_type; + auto it = nps_amdsmi_to_RSMI.find(memory_partition); + if (it != nps_amdsmi_to_RSMI.end()) { + rsmi_type = it->second; + } else if (it == nps_amdsmi_to_RSMI.end()) { + return AMDSMI_STATUS_INVAL; + } + + amdsmi_status_t ret = rsmi_wrapper(rsmi_dev_memory_partition_set, processor_handle, 0, rsmi_type); + + amdsmi_status_t ret_get = + rsmi_wrapper(rsmi_dev_memory_partition_get, processor_handle, 0, current_partition, k256); + + if (ret_get == AMDSMI_STATUS_SUCCESS) { + current_partition_str.clear(); + current_partition_str = current_partition; + } + + ss << __PRETTY_FUNCTION__ << " | After attempting to set memory partition to " + << req_user_partition << "\n" + << " | Current memory partition is " << current_partition_str << "\n" + << " | Returning: " << smi_amdgpu_get_status_string(ret, false) + << " | User will need to reload driver in order to see a NPS mode change"; + LOG_INFO(ss); + return ret; } // Accelerator Partition functions @@ -3974,6 +3998,7 @@ amdsmi_status_t amdsmi_get_gpu_ecc_count(amdsmi_processor_handle processor_handl static_cast(block), reinterpret_cast(ec)); } + amdsmi_status_t amdsmi_get_gpu_ecc_enabled(amdsmi_processor_handle processor_handle, uint64_t* enabled_blocks) { AMDSMI_CHECK_INIT(); @@ -3981,6 +4006,7 @@ amdsmi_status_t amdsmi_get_gpu_ecc_enabled(amdsmi_processor_handle processor_han return rsmi_wrapper(rsmi_dev_ecc_enabled_get, processor_handle, 0, enabled_blocks); } + amdsmi_status_t amdsmi_get_gpu_ecc_status(amdsmi_processor_handle processor_handle, amdsmi_gpu_block_t block, amdsmi_ras_err_state_t* state) { AMDSMI_CHECK_INIT(); @@ -4433,21 +4459,6 @@ amdsmi_status_t amdsmi_set_gpu_od_volt_info(amdsmi_processor_handle processor_ha return rsmi_wrapper(rsmi_dev_od_volt_info_set, processor_handle, 0, vpoint, clkvalue, voltvalue); } -amdsmi_status_t amdsmi_set_gpu_clk_range(amdsmi_processor_handle processor_handle, - uint64_t minclkvalue, uint64_t maxclkvalue, - amdsmi_clk_type_t clkType) { - // Bare Metal and passthrough only feature - amdsmi_virtualization_mode_t virt_mode; - if (amdsmi_get_gpu_virtualization_mode(processor_handle, &virt_mode) == AMDSMI_STATUS_SUCCESS) { - if (virt_mode == AMDSMI_VIRTUALIZATION_MODE_GUEST) { - return AMDSMI_STATUS_NOT_SUPPORTED; - } - } - - return rsmi_wrapper(rsmi_dev_clk_range_set, processor_handle, 0, minclkvalue, maxclkvalue, - static_cast(clkType)); -} - amdsmi_status_t amdsmi_set_gpu_clk_limit(amdsmi_processor_handle processor_handle, amdsmi_clk_type_t clk_type, amdsmi_clk_limit_type_t limit_type, uint64_t clk_value) { @@ -6702,8 +6713,8 @@ amdsmi_status_t amdsmi_set_cpu_pwr_efficiency_mode(amdsmi_processor_handle proce pwreffmode_util = *utilization; pwreffmode_pptlimit = *ppt_limit; - if ((power_efficiency_mode == POWER_EFFICIENCY_MODE_4) || - (power_efficiency_mode == POWER_EFFICIENCY_MODE_5)) { + if ((power_efficiency_mode == AMDSMI_POWER_EFFICIENCY_MODE_4) || + (power_efficiency_mode == AMDSMI_POWER_EFFICIENCY_MODE_5)) { uint32_t cpu_family; uint32_t cpu_model; // cpu_family and cpu_model are only needed for mode 4/5 validation @@ -7834,14 +7845,14 @@ amdsmi_status_t amdsmi_get_cpu_svi3_vr_controller_temp(amdsmi_processor_handle p sock_ind = static_cast(std::stoi(proc_id, NULL, 0)); // Validate input parameters - if ((*rail_selection) > MAX_SVI3_RAIL_SELECTION) { + if ((*rail_selection) > AMDSMI_MAX_SVI3_RAIL_SELECTION) { return AMDSMI_STATUS_INVAL; } // Prepare ESMI SVI3 structure with input parameters svi3_info_.m_svi3_info_inarg.info.svi3_rail_selection = ((*rail_selection) & 0x1); - if ((*rail_selection) == MAX_SVI3_RAIL_SELECTION) { - if ((*rail_index) > MAX_SVI3_RAIL_INDEX) { + if ((*rail_selection) == AMDSMI_MAX_SVI3_RAIL_SELECTION) { + if ((*rail_index) > AMDSMI_MAX_SVI3_RAIL_INDEX) { return AMDSMI_STATUS_INVAL; } svi3_info_.m_svi3_info_inarg.info.svi3_rail_index = ((*rail_index) & 0x7); diff --git a/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc b/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc index 797b59b0dd1c..2f02841399ff 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi_gpu_device.cc @@ -557,7 +557,7 @@ namespace gpu_device::details { using UALoeLinkTypeMap_t = std::map; inline const auto UALoeLinkTypeMap = UALoeLinkTypeMap_t{{"ualoe", amdsmi_fabric_type_t::AMDSMI_FABRIC_TYPE_UALOE}, - {"ualink", amdsmi_fabric_type_t::AMDSMI_FABRIC_TYPE_UALLINK}}; + {"ualink", amdsmi_fabric_type_t::AMDSMI_FABRIC_TYPE_UALINK}}; using UALoeAddressModeTypeMap_t = std::map; inline const auto UALoeAddressModeTypeMap = UALoeAddressModeTypeMap_t{ @@ -599,65 +599,57 @@ auto get_fabric_type(const std::string& link_value, amdsmi_fabric_info_t& local_ const std::string_view fabric_type_sv{fabric_type_str}; if (auto fabric_type_itr = UALoeLinkTypeMap.find(fabric_type_sv); fabric_type_itr != UALoeLinkTypeMap.end()) { - local_fabric_info.fabric_info.fabric_version.v1.fabric_type = fabric_type_itr->second; + local_fabric_info.fabric_info.v1.fabric_type = fabric_type_itr->second; } else { - local_fabric_info.fabric_info.fabric_version.v1.fabric_type = - amdsmi_fabric_type_t::AMDSMI_FABRIC_TYPE_UNKNOWN; + local_fabric_info.fabric_info.v1.fabric_type = amdsmi_fabric_type_t::AMDSMI_FABRIC_TYPE_UNKNOWN; } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value << " -> " << fabric_type_str - << " | fabric_type: " << local_fabric_info.fabric_info.fabric_version.v1.fabric_type - << " |"; + << " | fabric_type: " << local_fabric_info.fabric_info.v1.fabric_type << " |"; log_ualoe_file_info(std::move(outstream)); } auto get_accelerator_id(const std::string& link_value, amdsmi_fabric_info_t& local_fabric_info) -> void { - auto result = parse_number_from_string< - decltype(local_fabric_info.fabric_info.fabric_version.v1.accelerator_id)>(link_value); + auto result = parse_number_from_string( + link_value); if (result.has_value()) { - local_fabric_info.fabric_info.fabric_version.v1.accelerator_id = - static_cast( - result.value()); + local_fabric_info.fabric_info.v1.accelerator_id = + static_cast(result.value()); } std::ostringstream outstream; - outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value << " | accelerator_id: " - << local_fabric_info.fabric_info.fabric_version.v1.accelerator_id << " |"; + outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value + << " | accelerator_id: " << local_fabric_info.fabric_info.v1.accelerator_id << " |"; log_ualoe_file_info(std::move(outstream)); } auto get_bandwidth(const std::string& link_value, amdsmi_fabric_info_t& local_fabric_info) -> void { auto result = - parse_number_from_string( - link_value); + parse_number_from_string(link_value); if (result.has_value()) { - local_fabric_info.fabric_info.fabric_version.v1.bandwidth = - static_cast( - result.value()); + local_fabric_info.fabric_info.v1.bandwidth = + static_cast(result.value()); } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value - << " | bandwidth: " << local_fabric_info.fabric_info.fabric_version.v1.bandwidth - << " |"; + << " | bandwidth: " << local_fabric_info.fabric_info.v1.bandwidth << " |"; log_ualoe_file_info(std::move(outstream)); } auto get_latency(const std::string& link_value, amdsmi_fabric_info_t& local_fabric_info) -> void { auto result = - parse_number_from_string( - link_value); + parse_number_from_string(link_value); if (result.has_value()) { - local_fabric_info.fabric_info.fabric_version.v1.latency = - static_cast( - result.value()); + local_fabric_info.fabric_info.v1.latency = + static_cast(result.value()); } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value - << " | latency: " << local_fabric_info.fabric_info.fabric_version.v1.latency << " |"; + << " | latency: " << local_fabric_info.fabric_info.v1.latency << " |"; log_ualoe_file_info(std::move(outstream)); } @@ -737,51 +729,43 @@ auto get_ppod_id(const std::string& link_value, std::array void { auto result = - parse_number_from_string( - link_value); + parse_number_from_string(link_value); if (result.has_value()) { - local_fabric_info.fabric_info.fabric_version.v1.ppod_size = - static_cast( - result.value()); + local_fabric_info.fabric_info.v1.ppod_size = + static_cast(result.value()); } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value - << " | ppod_size: " << local_fabric_info.fabric_info.fabric_version.v1.ppod_size - << " |"; + << " | ppod_size: " << local_fabric_info.fabric_info.v1.ppod_size << " |"; log_ualoe_file_info(std::move(outstream)); } auto get_vpod_id(const std::string& link_value, amdsmi_fabric_info_t& local_fabric_info) -> void { auto result = - parse_number_from_string( - link_value); + parse_number_from_string(link_value); if (result.has_value()) { - local_fabric_info.fabric_info.fabric_version.v1.vpod_id = - static_cast( - result.value()); + local_fabric_info.fabric_info.v1.vpod_id = + static_cast(result.value()); } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value - << " | vpod_id: " << local_fabric_info.fabric_info.fabric_version.v1.vpod_id << " |"; + << " | vpod_id: " << local_fabric_info.fabric_info.v1.vpod_id << " |"; log_ualoe_file_info(std::move(outstream)); } auto get_vpod_size(const std::string& link_value, amdsmi_fabric_info_t& local_fabric_info) -> void { auto result = - parse_number_from_string( - link_value); + parse_number_from_string(link_value); if (result.has_value()) { - local_fabric_info.fabric_info.fabric_version.v1.vpod_size = - static_cast( - result.value()); + local_fabric_info.fabric_info.v1.vpod_size = + static_cast(result.value()); } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value - << " | vpod_size: " << local_fabric_info.fabric_info.fabric_version.v1.vpod_size - << " |"; + << " | vpod_size: " << local_fabric_info.fabric_info.v1.vpod_size << " |"; log_ualoe_file_info(std::move(outstream)); } @@ -876,16 +860,15 @@ auto get_addr_mode(const std::string& link_value, amdsmi_fabric_info_t& local_fa const std::string_view addr_mode_sv{addr_mode_str}; if (auto addr_mode_itr = UALoeAddressModeTypeMap.find(addr_mode_sv); addr_mode_itr != UALoeAddressModeTypeMap.end()) { - local_fabric_info.fabric_info.fabric_version.v1.addr_mode = addr_mode_itr->second; + local_fabric_info.fabric_info.v1.addr_mode = addr_mode_itr->second; } else { - local_fabric_info.fabric_info.fabric_version.v1.addr_mode = + local_fabric_info.fabric_info.v1.addr_mode = amdsmi_fabric_npa_address_mode_t::AMDSMI_FABRIC_NPA_ADDRESS_MODE_UNKNOWN; } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value << " -> " << addr_mode_str - << " | addr_mode: " << local_fabric_info.fabric_info.fabric_version.v1.addr_mode - << " |"; + << " | addr_mode: " << local_fabric_info.fabric_info.v1.addr_mode << " |"; log_ualoe_file_info(std::move(outstream)); } @@ -899,16 +882,15 @@ auto get_accel_state(const std::string& link_value, amdsmi_fabric_info_t& local_ const std::string_view accel_state_sv{accel_state_str}; if (auto accel_state_itr = UALoeAcceleratorStateTypeMap.find(accel_state_sv); accel_state_itr != UALoeAcceleratorStateTypeMap.end()) { - local_fabric_info.fabric_info.fabric_version.v1.accel_state = accel_state_itr->second; + local_fabric_info.fabric_info.v1.accel_state = accel_state_itr->second; } else { - local_fabric_info.fabric_info.fabric_version.v1.accel_state = + local_fabric_info.fabric_info.v1.accel_state = amdsmi_fabric_accelerator_vpod_state_t::AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_UNKNOWN; } std::ostringstream outstream; outstream << __PRETTY_FUNCTION__ << " | Link value: " << link_value << " -> " << accel_state_str - << " | accel_state: " << local_fabric_info.fabric_info.fabric_version.v1.accel_state - << " |"; + << " | accel_state: " << local_fabric_info.fabric_info.v1.accel_state << " |"; log_ualoe_file_info(std::move(outstream)); } @@ -1066,7 +1048,7 @@ auto AMDSmiGPUDevice::get_fabric_info_from_ualoe(amdsmi_fabric_info_t& fabric_in LOG_TRACE(outstream); auto local_fabric_info = amdsmi_fabric_info_t{}; - auto local_ppod_id = std::array{}; + auto local_ppod_id = std::array{}; auto local_active_accelerators = std::array{}; auto local_accelerators = @@ -1092,45 +1074,40 @@ auto AMDSmiGPUDevice::get_fabric_info_from_ualoe(amdsmi_fabric_info_t& fabric_in * - 00000111 (last 3 bits are 1, all others 0) */ // local_fabric_info.bdf.function_number = ((bdf_.function_number + kUALOE_BDF_OFFSET) & 0x07); - local_fabric_info.fabric_info.version = - std::numeric_limits::max(); + local_fabric_info.fabric_version = + std::numeric_limits::max(); - local_fabric_info.fabric_info.fabric_version.v1.fabric_type = static_cast( - std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.fabric_type)>::max()); + local_fabric_info.fabric_info.v1.fabric_type = static_cast( + std::numeric_limits::max()); - local_fabric_info.fabric_info.fabric_version.v1.accelerator_id = std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.accelerator_id)>::max(); + local_fabric_info.fabric_info.v1.accelerator_id = + std::numeric_limits::max(); - local_fabric_info.fabric_info.fabric_version.v1.bandwidth = std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.bandwidth)>::max(); + local_fabric_info.fabric_info.v1.bandwidth = + std::numeric_limits::max(); - local_fabric_info.fabric_info.fabric_version.v1.latency = - std::numeric_limits::max(); + local_fabric_info.fabric_info.v1.latency = + std::numeric_limits::max(); // Sentinel when sysfs provides no ppod_id: UUID 99999999-9999-9999-9999-999999999999 (16 × 0x99) - std::fill(std::begin(local_fabric_info.fabric_info.fabric_version.v1.ppod_id), - std::end(local_fabric_info.fabric_info.fabric_version.v1.ppod_id), - static_cast(0x99)); + std::fill(std::begin(local_fabric_info.fabric_info.v1.ppod_id), + std::end(local_fabric_info.fabric_info.v1.ppod_id), static_cast(0x99)); - local_fabric_info.fabric_info.fabric_version.v1.ppod_size = std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.ppod_size)>::max(); + local_fabric_info.fabric_info.v1.ppod_size = + std::numeric_limits::max(); - local_fabric_info.fabric_info.fabric_version.v1.vpod_id = - std::numeric_limits::max(); + local_fabric_info.fabric_info.v1.vpod_id = + std::numeric_limits::max(); - local_fabric_info.fabric_info.fabric_version.v1.vpod_size = std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.vpod_size)>::max(); + local_fabric_info.fabric_info.v1.vpod_size = + std::numeric_limits::max(); - local_fabric_info.fabric_info.fabric_version.v1.addr_mode = - static_cast( - std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.addr_mode)>::max()); + local_fabric_info.fabric_info.v1.addr_mode = static_cast( + std::numeric_limits::max()); - local_fabric_info.fabric_info.fabric_version.v1.accel_state = + local_fabric_info.fabric_info.v1.accel_state = static_cast( - std::numeric_limits< - decltype(local_fabric_info.fabric_info.fabric_version.v1.accel_state)>::max()); + std::numeric_limits::max()); std::fill(std::begin(local_active_accelerators), std::end(local_active_accelerators), std::numeric_limits::max()); @@ -1205,13 +1182,13 @@ auto AMDSmiGPUDevice::get_fabric_info_from_ualoe(amdsmi_fabric_info_t& fabric_in local_active_accelerators); std::copy(local_active_accelerators.data(), (local_active_accelerators.data() + local_active_accelerators.size()), - local_fabric_info.fabric_info.fabric_version.v1.vpod_active_accelerators); + local_fabric_info.fabric_info.v1.vpod_active_accelerators); continue; } if (link_info_type_key == UALoeLinkInfo_t::LOCAL_ACCELS) { gpu_device::details::get_local_accelerators(ualoe_file_lines, local_accelerators); std::copy(local_accelerators.data(), (local_accelerators.data() + local_accelerators.size()), - local_fabric_info.fabric_info.fabric_version.v1.local_accelerators); + local_fabric_info.fabric_info.v1.local_accelerators); continue; } @@ -1236,7 +1213,7 @@ auto AMDSmiGPUDevice::get_fabric_info_from_ualoe(amdsmi_fabric_info_t& fabric_in case UALoeLinkInfo_t::PPOD_ID: gpu_device::details::get_ppod_id(link_value, local_ppod_id); std::copy(local_ppod_id.data(), (local_ppod_id.data() + local_ppod_id.size()), - local_fabric_info.fabric_info.fabric_version.v1.ppod_id); + local_fabric_info.fabric_info.v1.ppod_id); break; case UALoeLinkInfo_t::PPOD_SIZE: diff --git a/projects/amdsmi/src/amd_smi/amd_smi_ualoe.cc b/projects/amdsmi/src/amd_smi/amd_smi_ualoe.cc index df547c45f3a8..4e4586f695c9 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi_ualoe.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi_ualoe.cc @@ -561,12 +561,17 @@ static const std::unordered_map telemetry_id_map = { "NETPORT_FEC_CW_SYMBOL_ERRS_UNCORRECTABLE"}, }; -const char* amdsmi_fabric_telem_id_to_string(uint64_t telem_id) { +amdsmi_status_t amdsmi_fabric_telem_id_to_string(uint64_t telem_id, const char** telem_name) { + if (telem_name == nullptr) { + return AMDSMI_STATUS_INVAL; + } auto it = telemetry_id_map.find(telem_id); if (it != telemetry_id_map.end()) { - return it->second; + *telem_name = it->second; + return AMDSMI_STATUS_SUCCESS; } - return "UNKNOWN"; + *telem_name = "UNKNOWN"; + return AMDSMI_STATUS_NOT_FOUND; } // Note: AMDSMI and UALoE telemetry structures are now designed to be binary compatible diff --git a/projects/amdsmi/src/rocm_smi_properties.cc b/projects/amdsmi/src/rocm_smi_properties.cc index 244eb573e701..4e0ab417b0bd 100644 --- a/projects/amdsmi/src/rocm_smi_properties.cc +++ b/projects/amdsmi/src/rocm_smi_properties.cc @@ -125,7 +125,6 @@ const AMDGpuVerbList_t amdgpu_verb_check_list{ {AMDGpuVerbTypes_t::kSetGpuPciBandwidth, "amdsmi_set_gpu_pci_bandwidth"}, {AMDGpuVerbTypes_t::kSetPowerCap, "amdsmi_set_power_cap"}, {AMDGpuVerbTypes_t::kSetGpuPowerProfile, "amdsmi_set_gpu_power_profile"}, - {AMDGpuVerbTypes_t::kSetGpuClkRange, "amdsmi_set_gpu_clk_range"}, {AMDGpuVerbTypes_t::kSetGpuOdClkInfo, "amdsmi_set_gpu_od_clk_info"}, {AMDGpuVerbTypes_t::kSetGpuOdVoltInfo, "amdsmi_set_gpu_od_volt_info"}, {AMDGpuVerbTypes_t::kSetGpuPerfLevelV1, "amdsmi_set_gpu_perf_level_v1"}, @@ -187,11 +186,6 @@ const AMDGpuPropertyList_t amdgpu_property_reinforcement_list{ make_unique_property_id(AMDGpuPropertyTypesOffset_t::kDevInfoTypes, DevInfoTypes::kDevPowerProfileMode), AMDGpuVerbTypes_t::kSetGpuPowerProfile, AMDGpuPropertyOpModeTypes_t::kBoth, false}}, - {0x74A1, - {kDevRevIDAll, - make_unique_property_id(AMDGpuPropertyTypesOffset_t::kPerfTypes, - rsmi_dev_perf_level::RSMI_DEV_PERF_LEVEL_MANUAL), - AMDGpuVerbTypes_t::kSetGpuClkRange, AMDGpuPropertyOpModeTypes_t::kSrIov, false}}, {0x74A1, {kDevRevIDAll, make_unique_property_id(AMDGpuPropertyTypesOffset_t::kPerfTypes, diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write.h b/projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write.h new file mode 100644 index 000000000000..2a004c3f7f17 --- /dev/null +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef TESTS_AMD_SMI_TEST_FUNCTIONAL_CLOCK_LIMIT_READ_WRITE_H_ +#define TESTS_AMD_SMI_TEST_FUNCTIONAL_CLOCK_LIMIT_READ_WRITE_H_ + +#include "test_base.h" + +class TestClockLimitReadWrite : public TestBase { + public: + TestClockLimitReadWrite(); + + // @Brief: Destructor for test case of TestClockLimitReadWrite + virtual ~TestClockLimitReadWrite(); + + // @Brief: Setup the environment for measurement + virtual void SetUp(); + + // @Brief: Core measurement execution + virtual void Run(); + + // @Brief: Clean up and retrieve the resource + virtual void Close(); + + // @Brief: Display results + virtual void DisplayResults() const; + + // @Brief: Display information about what this test does + virtual void DisplayTestInfo(void); +}; + +#endif // TESTS_AMD_SMI_TEST_FUNCTIONAL_CLOCK_LIMIT_READ_WRITE_H_ diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write_test.cc new file mode 100644 index 000000000000..da3eeb3606cc --- /dev/null +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/clock/clock_limit_read_write_test.cc @@ -0,0 +1,182 @@ +/* + * Copyright (c) Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "clock_limit_read_write.h" + +#include +#include + +#include +#include +#include + +#include "amd_smi/amdsmi.h" +#include "test_common.h" + +TestClockLimitReadWrite::TestClockLimitReadWrite() : TestBase() { + set_title("AMDSMI Clock Limit Read/Write Test"); + set_description( + "The Clock Limit Read/Write test verifies that the sclk/mclk minimum " + "and maximum frequency limits can be set through " + "amdsmi_set_gpu_clk_limit() and restored to their original values."); +} + +TestClockLimitReadWrite::~TestClockLimitReadWrite(void) {} + +void TestClockLimitReadWrite::SetUp(void) { + TestBase::SetUp(); + + return; +} + +void TestClockLimitReadWrite::DisplayTestInfo(void) { TestBase::DisplayTestInfo(); } + +void TestClockLimitReadWrite::DisplayResults(void) const { + TestBase::DisplayResults(); + return; +} + +void TestClockLimitReadWrite::Close() { + // This will close handles opened within rsmitst utility calls and call + // amdsmi_shut_down(), so it should be done after other hsa cleanup + TestBase::Close(); +} + +// amdsmi_get_clk_freq() reports frequencies in Hz; amdsmi_set_gpu_clk_limit() +// expects MHz. +static uint64_t hz_to_mhz(uint64_t hz) { return hz / 1000000ULL; } + +void TestClockLimitReadWrite::Run(void) { + amdsmi_status_t ret; + amdsmi_frequencies_t f; + + TestBase::Run(); + PRINT_VERBOSITY(); + if (setup_failed_) { + std::cout << "** SetUp Failed for this test. Skipping.**" << std::endl; + return; + } + + // amdsmi_set_gpu_clk_limit() only operates on the SYS (gfx) and MEM clocks. + const amdsmi_clk_type_t clk_types[] = {AMDSMI_CLK_TYPE_SYS, AMDSMI_CLK_TYPE_MEM}; + + for (uint32_t dv_ind = 0; dv_ind < num_monitor_devs(); ++dv_ind) { + PrintDeviceHeader(processor_handles_[dv_ind]); + + for (auto clk : clk_types) { + const char* name = FreqEnumToStr(clk); + + // Read the supported frequency table to derive safe min/max values. + DISPLAY_AMDSMI_API("amdsmi_get_clk_freq", "gpu=" + std::to_string(dv_ind), VERB(STANDARD)); + ret = amdsmi_get_clk_freq(processor_handles_[dv_ind], clk, &f); + DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); + + if (ret == AMDSMI_STATUS_NOT_SUPPORTED || ret == AMDSMI_STATUS_NOT_YET_IMPLEMENTED) { + IF_VERB(STANDARD) { + std::cout << "\t**" << name << ": Not supported on this machine" << std::endl; + } + continue; + } + + if (ret == AMDSMI_STATUS_UNEXPECTED_DATA) { + std::cerr << "WARN: Clock [" << name << "] on device [" << dv_ind + << "] returned unexpected data. Likely a driver issue!" << std::endl; + continue; + } + + CHK_ERR_ASRT(ret) + + // Deep sleep (index 0 when present) is not a real DPM level; the lowest + // selectable level follows it. + uint32_t lowest_idx = (f.has_deep_sleep && f.num_supported > 0) ? 1 : 0; + if (f.num_supported <= lowest_idx) { + IF_VERB(STANDARD) { + std::cout << "\t**" << name << ": No selectable DPM levels, skipping." << std::endl; + } + continue; + } + + uint64_t orig_min_mhz = hz_to_mhz(f.frequency[lowest_idx]); + uint64_t orig_max_mhz = hz_to_mhz(f.frequency[f.num_supported - 1]); + + IF_VERB(STANDARD) { + std::cout << "\t**" << name << ": original min=" << orig_min_mhz + << " MHz, max=" << orig_max_mhz << " MHz" << std::endl; + } + + // ASSERT_*/CHK_ERR_ASRT expand to a bare "return;", so this helper must + // return void; it reports "not testable" through the captured flag. + bool limit_supported = true; + auto set_limit = [&](amdsmi_clk_limit_type_t limit_type, uint64_t mhz, const char* label) { + limit_supported = true; + DISPLAY_AMDSMI_API("amdsmi_set_gpu_clk_limit", + "gpu=" + std::to_string(dv_ind) + ", clk_type=" + std::string(name) + + ", clk_limit=" + std::string(label) + + ", value=" + std::to_string(mhz), + VERB(STANDARD)); + ret = amdsmi_set_gpu_clk_limit(processor_handles_[dv_ind], clk, limit_type, mhz); + DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS, + AMDSMI_STATUS_NOT_SUPPORTED, AMDSMI_STATUS_NO_PERM); + // Some ASICs reject clock-limit changes even with root. Treat those + // as "not testable here" rather than a defect. + if (ret == AMDSMI_STATUS_NOT_SUPPORTED || + (ret == AMDSMI_STATUS_NO_PERM && geteuid() == 0)) { + IF_VERB(STANDARD) { + std::cout << "\t**" << name << " set " << label + << ": Not supported on this machine. Skipping..." << std::endl; + } + limit_supported = false; + return; + } + CHK_ERR_ASRT(ret) + IF_VERB(STANDARD) { + std::cout << "\t**" << name << " set " << label << " -> " << mhz << " MHz: OK" + << std::endl; + } + }; + + // Exercise both limit types with their original values, then confirm the + // range is restored. Using the original min/max keeps the device in a + // safe, unchanged state after the test. + set_limit(AMDSMI_CLK_LIMIT_MAX, orig_max_mhz, "max"); + if (!limit_supported) { + continue; + } + set_limit(AMDSMI_CLK_LIMIT_MIN, orig_min_mhz, "min"); + + // Setting a clock limit switches the perf level to MANUAL; return the + // device to automatic control so later tests start from a clean state. + DISPLAY_AMDSMI_API("amdsmi_set_gpu_perf_level", "gpu=" + std::to_string(dv_ind), + VERB(STANDARD)); + ret = amdsmi_set_gpu_perf_level(processor_handles_[dv_ind], AMDSMI_DEV_PERF_LEVEL_AUTO); + DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS, + AMDSMI_STATUS_NOT_SUPPORTED); + + // Verify the restored range via a fresh read. + DISPLAY_AMDSMI_API("amdsmi_get_clk_freq", "gpu=" + std::to_string(dv_ind), VERB(STANDARD)); + ret = amdsmi_get_clk_freq(processor_handles_[dv_ind], clk, &f); + DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); + if (ret == AMDSMI_STATUS_SUCCESS) { + ASSERT_GT(f.num_supported, 0u); + } + } + } +} diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/identity/id_info_read_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/gpu/identity/id_info_read_test.cc index 23f34477d04a..fe9a5995318a 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/gpu/identity/id_info_read_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/identity/id_info_read_test.cc @@ -125,19 +125,6 @@ void TestIdInfoRead::Run(void) { DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_SUCCESS); CHK_ERR_ASRT(err) - DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_vendor", "gpu=" + std::to_string(i), VERB(STANDARD)); - err = amdsmi_get_gpu_vram_vendor(processor_handles_[i], buffer, kBufferLen); - DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_SUCCESS); - if (err == AMDSMI_STATUS_NOT_SUPPORTED) { - DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_vendor", "gpu=" + std::to_string(i), VERB(STANDARD)); - err = amdsmi_get_gpu_vram_vendor(processor_handles_[i], nullptr, kBufferLen); - DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_INVAL); - ASSERT_EQ(err, AMDSMI_STATUS_NOT_SUPPORTED); - } else { - CHK_ERR_ASRT(err) - IF_VERB(STANDARD) { std::cout << "\t**Device Vram Vendor name: " << buffer << std::endl; } - } - amdsmi_vram_info_t vram_info; DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_info", "gpu=" + std::to_string(i), VERB(STANDARD)); err = amdsmi_get_gpu_vram_info(processor_handles_[i], &vram_info); diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_memallocmode_read_write_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_memallocmode_read_write_test.cc index a48dca4aaa1c..463227f76531 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_memallocmode_read_write_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_memallocmode_read_write_test.cc @@ -61,11 +61,11 @@ void TestComputePartitionMemAllocModeReadWrite::Close() { TestBase::Close(); } -static std::string memAllocModeString(amdsmi_compute_partition_mem_alloc_mode_t mode) { +static std::string memAllocModeString(amdsmi_accelerator_partition_mem_alloc_mode_t mode) { switch (mode) { - case AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_CAPPING: + case AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING: return "CAPPING"; - case AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_ALL: + case AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL: return "ALL"; default: return "INVALID"; @@ -74,8 +74,8 @@ static std::string memAllocModeString(amdsmi_compute_partition_mem_alloc_mode_t void TestComputePartitionMemAllocModeReadWrite::Run(void) { amdsmi_status_t ret; - amdsmi_compute_partition_mem_alloc_mode_t original_mode; - amdsmi_compute_partition_mem_alloc_mode_t current_mode; + amdsmi_accelerator_partition_mem_alloc_mode_t original_mode; + amdsmi_accelerator_partition_mem_alloc_mode_t current_mode; if (num_monitor_devs() == 0) { return; @@ -83,37 +83,37 @@ void TestComputePartitionMemAllocModeReadWrite::Run(void) { // Invalid argument tests (run only when at least one device exists; // the handle is validated by SetUp). - ret = amdsmi_get_gpu_compute_partition_mem_alloc_mode(processor_handles_[0], nullptr); + ret = amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(processor_handles_[0], nullptr); EXPECT_EQ(ret, AMDSMI_STATUS_INVAL); - ret = amdsmi_set_gpu_compute_partition_mem_alloc_mode(processor_handles_[0], - AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_INVALID); + ret = amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( + processor_handles_[0], AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_INVALID); EXPECT_EQ(ret, AMDSMI_STATUS_INVAL); - ret = amdsmi_set_gpu_compute_partition_mem_alloc_mode( - processor_handles_[0], static_cast(99)); + ret = amdsmi_set_gpu_accelerator_partition_mem_alloc_mode( + processor_handles_[0], static_cast(99)); EXPECT_EQ(ret, AMDSMI_STATUS_INVAL); const bool isVerbose = (verbosity() > 0); - const std::vector modes_to_test = { - AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_CAPPING, - AMDSMI_COMPUTE_PARTITION_MEM_ALLOC_ALL, + const std::vector modes_to_test = { + AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_CAPPING, + AMDSMI_ACCELERATOR_PARTITION_MEM_ALLOC_ALL, }; for (uint32_t dv_ind = 0; dv_ind < num_monitor_devs(); dv_ind++) { PrintDeviceHeader(processor_handles_[dv_ind]); // Get original mode - DISPLAY_AMDSMI_API("amdsmi_get_gpu_compute_partition_mem_alloc_mode", + DISPLAY_AMDSMI_API("amdsmi_get_gpu_accelerator_partition_mem_alloc_mode", "gpu=" + std::to_string(dv_ind), isVerbose); - ret = - amdsmi_get_gpu_compute_partition_mem_alloc_mode(processor_handles_[dv_ind], &original_mode); + ret = amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(processor_handles_[dv_ind], + &original_mode); DISPLAY_AMDSMI_STATUS(isVerbose, __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); if (ret == AMDSMI_STATUS_NOT_SUPPORTED) { IF_VERB(STANDARD) { std::cout << "\t**Device " << dv_ind - << ": compute_partition_mem_alloc_mode not supported; skipping.\n"; + << ": accelerator_partition_mem_alloc_mode not supported; skipping.\n"; } continue; } @@ -124,7 +124,7 @@ void TestComputePartitionMemAllocModeReadWrite::Run(void) { } IF_VERB(STANDARD) { - std::cout << "\t**Device " << dv_ind << ": original compute_partition_mem_alloc_mode = " + std::cout << "\t**Device " << dv_ind << ": original accelerator_partition_mem_alloc_mode = " << memAllocModeString(original_mode) << "\n"; } @@ -135,10 +135,10 @@ void TestComputePartitionMemAllocModeReadWrite::Run(void) { << "\n"; } - DISPLAY_AMDSMI_API("amdsmi_set_gpu_compute_partition_mem_alloc_mode", + DISPLAY_AMDSMI_API("amdsmi_set_gpu_accelerator_partition_mem_alloc_mode", "gpu=" + std::to_string(dv_ind) + ", mode=" + memAllocModeString(mode), isVerbose); - ret = amdsmi_set_gpu_compute_partition_mem_alloc_mode(processor_handles_[dv_ind], mode); + ret = amdsmi_set_gpu_accelerator_partition_mem_alloc_mode(processor_handles_[dv_ind], mode); DISPLAY_AMDSMI_STATUS(isVerbose, __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); if (ret == AMDSMI_STATUS_NOT_SUPPORTED) { @@ -155,10 +155,10 @@ void TestComputePartitionMemAllocModeReadWrite::Run(void) { } // Verify the mode was applied - DISPLAY_AMDSMI_API("amdsmi_get_gpu_compute_partition_mem_alloc_mode", + DISPLAY_AMDSMI_API("amdsmi_get_gpu_accelerator_partition_mem_alloc_mode", "gpu=" + std::to_string(dv_ind) + " (verify)", isVerbose); - ret = amdsmi_get_gpu_compute_partition_mem_alloc_mode(processor_handles_[dv_ind], - ¤t_mode); + ret = amdsmi_get_gpu_accelerator_partition_mem_alloc_mode(processor_handles_[dv_ind], + ¤t_mode); DISPLAY_AMDSMI_STATUS(isVerbose, __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); EXPECT_EQ(ret, AMDSMI_STATUS_SUCCESS); if (ret == AMDSMI_STATUS_SUCCESS) { @@ -175,17 +175,17 @@ void TestComputePartitionMemAllocModeReadWrite::Run(void) { std::cout << "\t**Device " << dv_ind << ": restoring original mode = " << memAllocModeString(original_mode) << "\n"; } - DISPLAY_AMDSMI_API("amdsmi_set_gpu_compute_partition_mem_alloc_mode", + DISPLAY_AMDSMI_API("amdsmi_set_gpu_accelerator_partition_mem_alloc_mode", "gpu=" + std::to_string(dv_ind) + ", mode=" + memAllocModeString(original_mode) + " (restore)", isVerbose); - ret = - amdsmi_set_gpu_compute_partition_mem_alloc_mode(processor_handles_[dv_ind], original_mode); + ret = amdsmi_set_gpu_accelerator_partition_mem_alloc_mode(processor_handles_[dv_ind], + original_mode); DISPLAY_AMDSMI_STATUS(isVerbose, __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); EXPECT_EQ(ret, AMDSMI_STATUS_SUCCESS); if (ret != AMDSMI_STATUS_SUCCESS) { std::cerr << "\t**WARNING: failed to restore device " << dv_ind - << " to original compute_partition_mem_alloc_mode = " + << " to original accelerator_partition_mem_alloc_mode = " << memAllocModeString(original_mode) << "; device left in modified state.\n"; } } diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_read_write_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_read_write_test.cc index 4ae82c0b79b3..7e1f2a05acbc 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_read_write_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/computepartition_read_write_test.cc @@ -75,43 +75,45 @@ const uint32_t MAX_QPX_PARTITIONS = 4; // const uint32_t MAX_CPX_PARTITIONS = 8; static const std::string computePartitionString( - amdsmi_compute_partition_type_t computeParitionType) { + amdsmi_accelerator_partition_type_t computeParitionType) { /** * typedef enum { - * AMDSMI_COMPUTE_PARTITION_INVALID = 0, - * AMDSMI_COMPUTE_PARTITION_SPX, //!< Single GPU mode (SPX)- All XCCs work - * //!< together with shared memory - * AMDSMI_COMPUTE_PARTITION_DPX, //!< Dual GPU mode (DPX)- Half XCCs work - * //!< together with shared memory - * AMDSMI_COMPUTE_PARTITION_TPX, //!< Triple GPU mode (TPX)- One-third XCCs - * //!< work together with shared memory - * AMDSMI_COMPUTE_PARTITION_QPX, //!< Quad GPU mode (QPX)- Quarter XCCs - * //!< work together with shared memory - * AMDSMI_COMPUTE_PARTITION_CPX, //!< Core mode (CPX)- Per-chip XCC with - * //!< shared memory - * } amdsmi_compute_partition_type_t; + * AMDSMI_ACCELERATOR_PARTITION_INVALID = 0, + * AMDSMI_ACCELERATOR_PARTITION_SPX, //!< Single GPU mode (SPX)- All XCCs work + * //!< together with shared memory + * AMDSMI_ACCELERATOR_PARTITION_DPX, //!< Dual GPU mode (DPX)- Half XCCs work + * //!< together with shared memory + * AMDSMI_ACCELERATOR_PARTITION_TPX, //!< Triple GPU mode (TPX)- One-third XCCs + * //!< work together with shared memory + * AMDSMI_ACCELERATOR_PARTITION_QPX, //!< Quad GPU mode (QPX)- Quarter XCCs + * //!< work together with shared memory + * AMDSMI_ACCELERATOR_PARTITION_CPX, //!< Core mode (CPX)- Per-chip XCC with + * //!< shared memory + * } amdsmi_accelerator_partition_type_t; * */ switch (computeParitionType) { - case AMDSMI_COMPUTE_PARTITION_SPX: + case AMDSMI_ACCELERATOR_PARTITION_SPX: return "SPX"; - case AMDSMI_COMPUTE_PARTITION_DPX: + case AMDSMI_ACCELERATOR_PARTITION_DPX: return "DPX"; - case AMDSMI_COMPUTE_PARTITION_TPX: + case AMDSMI_ACCELERATOR_PARTITION_TPX: return "TPX"; - case AMDSMI_COMPUTE_PARTITION_QPX: + case AMDSMI_ACCELERATOR_PARTITION_QPX: return "QPX"; - case AMDSMI_COMPUTE_PARTITION_CPX: + case AMDSMI_ACCELERATOR_PARTITION_CPX: return "CPX"; default: return "N/A"; } } -static const std::map - mapStringToSMIComputePartitionTypes{ - {"SPX", AMDSMI_COMPUTE_PARTITION_SPX}, {"DPX", AMDSMI_COMPUTE_PARTITION_DPX}, - {"TPX", AMDSMI_COMPUTE_PARTITION_TPX}, {"QPX", AMDSMI_COMPUTE_PARTITION_QPX}, - {"CPX", AMDSMI_COMPUTE_PARTITION_CPX}, {"UNKNOWN", AMDSMI_COMPUTE_PARTITION_INVALID}}; +static const std::map + mapStringToSMIComputePartitionTypes{{"SPX", AMDSMI_ACCELERATOR_PARTITION_SPX}, + {"DPX", AMDSMI_ACCELERATOR_PARTITION_DPX}, + {"TPX", AMDSMI_ACCELERATOR_PARTITION_TPX}, + {"QPX", AMDSMI_ACCELERATOR_PARTITION_QPX}, + {"CPX", AMDSMI_ACCELERATOR_PARTITION_CPX}, + {"UNKNOWN", AMDSMI_ACCELERATOR_PARTITION_INVALID}}; static const std::map resource_types_map = { @@ -406,10 +408,10 @@ void TestComputePartitionReadWrite::Run(void) { if (ret == AMDSMI_STATUS_NOT_SUPPORTED) { continue; } - for (int partition = static_cast(AMDSMI_COMPUTE_PARTITION_SPX); - partition <= static_cast(AMDSMI_COMPUTE_PARTITION_CPX); partition++) { - amdsmi_compute_partition_type_t updatePartition = - static_cast(partition); + for (int partition = static_cast(AMDSMI_ACCELERATOR_PARTITION_SPX); + partition <= static_cast(AMDSMI_ACCELERATOR_PARTITION_CPX); partition++) { + amdsmi_accelerator_partition_type_t updatePartition = + static_cast(partition); IF_VERB(STANDARD) { std::cout << "\t**" @@ -419,7 +421,9 @@ void TestComputePartitionReadWrite::Run(void) { DISPLAY_AMDSMI_API("amdsmi_set_gpu_compute_partition", "gpu=" + std::to_string(dv_ind), VERB(STANDARD)); - auto ret_set = amdsmi_set_gpu_compute_partition(processor_handles_[dv_ind], updatePartition); + auto ret_set = amdsmi_set_gpu_compute_partition( + processor_handles_[dv_ind], + static_cast(updatePartition)); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret_set, AMDSMI_STATUS_SETTING_UNAVAILABLE, AMDSMI_STATUS_NO_PERM, AMDSMI_STATUS_SUCCESS, AMDSMI_STATUS_BUSY, AMDSMI_STATUS_NOT_SUPPORTED, @@ -475,11 +479,13 @@ void TestComputePartitionReadWrite::Run(void) { std::string(current_char_computePartition))); } } - amdsmi_compute_partition_type_t updatePartition = static_cast( - mapStringToSMIComputePartitionTypes.at(std::string(orig_char_computePartition))); + amdsmi_accelerator_partition_type_t updatePartition = + static_cast( + mapStringToSMIComputePartitionTypes.at(std::string(orig_char_computePartition))); DISPLAY_AMDSMI_API("amdsmi_set_gpu_compute_partition", "gpu=" + std::to_string(dv_ind), VERB(STANDARD)); - auto ret_set = amdsmi_set_gpu_compute_partition(processor_handles_[dv_ind], updatePartition); + auto ret_set = amdsmi_set_gpu_compute_partition( + processor_handles_[dv_ind], static_cast(updatePartition)); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret_set, AMDSMI_STATUS_SETTING_UNAVAILABLE, AMDSMI_STATUS_NO_PERM, AMDSMI_STATUS_SUCCESS, AMDSMI_STATUS_BUSY, AMDSMI_STATUS_NOT_SUPPORTED, @@ -964,7 +970,7 @@ void TestComputePartitionReadWrite::Run(void) { } for (int partition = static_cast( mapStringToSMIComputePartitionTypes.at(std::string(orig_char_computePartition))); - partition <= static_cast(AMDSMI_COMPUTE_PARTITION_CPX); partition++) { + partition <= static_cast(AMDSMI_ACCELERATOR_PARTITION_CPX); partition++) { uint32_t device_index2 = 0; amdsmi_processor_handle p_handle2 = {}; smi_amdgpu_get_device_count(¤t_num_devices); @@ -982,10 +988,11 @@ void TestComputePartitionReadWrite::Run(void) { std::cout << "\t=========== END INDEX/p_handle DEVICE INFO 2 =============\n"; } - amdsmi_compute_partition_type_t updatePartition = - static_cast(partition); + amdsmi_accelerator_partition_type_t updatePartition = + static_cast(partition); DISPLAY_AMDSMI_API("amdsmi_set_gpu_compute_partition", "", VERB(STANDARD)); - auto ret_set = amdsmi_set_gpu_compute_partition(p_handle2, updatePartition); + auto ret_set = amdsmi_set_gpu_compute_partition( + p_handle2, static_cast(updatePartition)); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret_set, AMDSMI_STATUS_SETTING_UNAVAILABLE, AMDSMI_STATUS_NO_PERM, AMDSMI_STATUS_SUCCESS, AMDSMI_STATUS_BUSY, AMDSMI_STATUS_NOT_SUPPORTED, @@ -1047,14 +1054,16 @@ void TestComputePartitionReadWrite::Run(void) { smi_amdgpu_get_processor_handle_by_index(dv_ind, &p_handle3); smi_amdgpu_get_device_index(p_handle3, &device_index3); - amdsmi_compute_partition_type_t updatePartition = static_cast( - mapStringToSMIComputePartitionTypes.at(std::string(orig_char_computePartition))); + amdsmi_accelerator_partition_type_t updatePartition = + static_cast( + mapStringToSMIComputePartitionTypes.at(std::string(orig_char_computePartition))); IF_VERB(STANDARD) { std::cout << "\t**ABOUT TO GO BACK TO ORIGINAL PARTITION (" << orig_char_computePartition << ")\n"; } DISPLAY_AMDSMI_API("amdsmi_set_gpu_compute_partition", "", VERB(STANDARD)); - auto ret_set = amdsmi_set_gpu_compute_partition(p_handle3, updatePartition); + auto ret_set = amdsmi_set_gpu_compute_partition( + p_handle3, static_cast(updatePartition)); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret_set, AMDSMI_STATUS_SUCCESS); checkPartitionIdChanges(processor_handles_, dv_ind, std::string(orig_char_computePartition), isVerbose, true); diff --git a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc index 3233f9e60684..8430c6c40fc0 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/gpu/partition/memorypartition_read_write_test.cc @@ -460,7 +460,7 @@ void TestMemoryPartitionReadWrite::Run(void) { } /****************************************/ - /* amdsmi_set_gpu_memory_partition(...) */ + /* amdsmi_set_gpu_memory_partition_mode(...) */ /****************************************/ // Verify api support checking functionality is working amdsmi_memory_partition_type_t null_memory_partition = {}; @@ -468,7 +468,7 @@ void TestMemoryPartitionReadWrite::Run(void) { VERB(STANDARD)); err = amdsmi_set_gpu_memory_partition_mode(processor_handles_[dv_ind], null_memory_partition); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_INVAL); - std::cout << "\t**amdsmi_set_gpu_memory_partition(amdsmi_set_gpu_memory_partition_mode" + std::cout << "\t**amdsmi_set_gpu_memory_partition_mode" << "(processor_handles_[" << dv_ind << "], nullptr): " << smi_amdgpu_get_status_string(err, false) << "\n"; // Note: new_memory_partition is not set @@ -665,13 +665,13 @@ void TestMemoryPartitionReadWrite::Run(void) { << "Returning memory partition to: " << memoryPartitionString(new_memory_partition) << std::endl; } - DISPLAY_AMDSMI_API("amdsmi_set_gpu_memory_partition", "gpu=" + std::to_string(dv_ind), + DISPLAY_AMDSMI_API("amdsmi_set_gpu_memory_partition_mode", "gpu=" + std::to_string(dv_ind), VERB(STANDARD)); - ret = amdsmi_set_gpu_memory_partition(processor_handles_[dv_ind], new_memory_partition); + ret = amdsmi_set_gpu_memory_partition_mode(processor_handles_[dv_ind], new_memory_partition); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_SUCCESS); IF_VERB(STANDARD) { std::cout << "\t**" - << "amdsmi_set_gpu_memory_partition(processor_handles_[" << dv_ind << "], " + << "amdsmi_set_gpu_memory_partition_mode(processor_handles_[" << dv_ind << "], " << orig_memory_partition << "): " << smi_amdgpu_get_status_string(ret, false) << std::endl; } diff --git a/projects/amdsmi/tests/amd_smi_test/functional/ifoe/fabric/fabric_read_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/ifoe/fabric/fabric_read_test.cc index 58636e66ec7b..c9873d4c9288 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/ifoe/fabric/fabric_read_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/ifoe/fabric/fabric_read_test.cc @@ -147,8 +147,8 @@ void TestFabricRead::Run(void) { "(no UALoE sysfs content); BDF may still be valid" << std::endl; } - const auto& v1 = fabric_info.fabric_info.fabric_version.v1; - std::cout << "\t\tversion: " << fabric_info.fabric_info.version << "\n" + const auto& v1 = fabric_info.fabric_info.v1; + std::cout << "\t\tversion: " << fabric_info.fabric_version << "\n" << "\t\taccelerator_id: " << v1.accelerator_id << "\n" << "\t\tfabric_type: " << v1.fabric_type << "\n" << "\t\tbandwidth: " << v1.bandwidth << " Mb/s" << "\n" @@ -216,6 +216,7 @@ void TestFabricRead::Run(void) { CHK_ERR_ASRT(err) IF_VERB(STANDARD) { + const char* name; // Walk datasets and print a sample of telemetry items for (uint32_t cat = 0; cat < AMDSMI_FABRIC_TELEMETRY_CATEGORY_MAX; ++cat) { if (!tel->datasets[cat]) continue; @@ -232,7 +233,12 @@ void TestFabricRead::Run(void) { // ── amdsmi_fabric_telem_id_to_string ───────────────────────────── for (uint32_t item = 0; item < in.item_count; ++item) { const auto& it = in.items[item]; - const char* name = amdsmi_fabric_telem_id_to_string(it.id); + err = amdsmi_fabric_telem_id_to_string(it.id, &name); + // Unmapped telemetry ids return NOT_FOUND with a "UNKNOWN" name; + // tolerate them so a single unknown id does not fail the test. + if (err != AMDSMI_STATUS_NOT_FOUND) { + CHK_ERR_ASRT(err) + } std::cout << "\t\t [" << item << "] id=0x" << std::hex << it.id << std::dec << " name=" << (name ? name : "NULL") << " value=" << it.value << "\n"; diff --git a/projects/amdsmi/tests/amd_smi_test/functional/ifoe/identity/ifoe_info_read_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/ifoe/identity/ifoe_info_read_test.cc index 6089da4f0129..06c68f1a6290 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/ifoe/identity/ifoe_info_read_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/ifoe/identity/ifoe_info_read_test.cc @@ -127,21 +127,6 @@ void TestIfoeInfoRead::Run(void) { DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_SUCCESS); CHK_ERR_ASRT(err) - DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_vendor", "gpu=" + std::to_string(i), VERB(STANDARD)); - err = amdsmi_get_gpu_vram_vendor(processor_handles_[i], buffer, kBufferLen); - DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_SUCCESS); - if (err == AMDSMI_STATUS_NOT_SUPPORTED) { - std::cout << "\t**Vram Vendor string not supported on this system." << std::endl; - DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_vendor(nullptr check)", "gpu=" + std::to_string(i), - VERB(STANDARD)); - err = amdsmi_get_gpu_vram_vendor(processor_handles_[i], nullptr, kBufferLen); - DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, err, AMDSMI_STATUS_NOT_SUPPORTED); - ASSERT_EQ(err, AMDSMI_STATUS_NOT_SUPPORTED); - } else { - CHK_ERR_ASRT(err) - IF_VERB(STANDARD) { std::cout << "\t**Device Vram Vendor name: " << buffer << std::endl; } - } - amdsmi_vram_info_t vram_info; DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_info", "gpu=" + std::to_string(i), VERB(STANDARD)); err = amdsmi_get_gpu_vram_info(processor_handles_[i], &vram_info); diff --git a/projects/amdsmi/tests/amd_smi_test/functional/system/mutual_exclusion_test.cc b/projects/amdsmi/tests/amd_smi_test/functional/system/mutual_exclusion_test.cc index a218b7411034..f51d76e1ac94 100644 --- a/projects/amdsmi/tests/amd_smi_test/functional/system/mutual_exclusion_test.cc +++ b/projects/amdsmi/tests/amd_smi_test/functional/system/mutual_exclusion_test.cc @@ -375,8 +375,9 @@ void TestMutualExclusion::Run(void) { DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_BUSY); CHECK_RET(ret, AMDSMI_STATUS_BUSY); - DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_vendor", "0", VERB(STANDARD)); - ret = amdsmi_get_gpu_vram_vendor(processor_handles_[0], dmy_str, 10); + amdsmi_vram_info_t vram_info; + DISPLAY_AMDSMI_API("amdsmi_get_gpu_vram_info", "0", VERB(STANDARD)); + ret = amdsmi_get_gpu_vram_info(processor_handles_[0], &vram_info); DISPLAY_AMDSMI_STATUS(VERB(STANDARD), __FILE__, __LINE__, ret, AMDSMI_STATUS_BUSY); CHECK_RET(ret, AMDSMI_STATUS_BUSY); @@ -488,7 +489,7 @@ void TestMutualExclusion::Run(void) { amdsmi_dev_firmware_version_get amdsmi_dev_name_get amdsmi_dev_brand_get - amdsmi_get_gpu_vram_vendor + amdsmi_get_gpu_vram_info amdsmi_get_gpu_subsystem_name amdsmi_get_gpu_vendor_name amdsmi_get_gpu_pci_bandwidth diff --git a/projects/amdsmi/tests/amd_smi_test/main.cc b/projects/amdsmi/tests/amd_smi_test/main.cc index 6a5f2a3ae09e..f22322204652 100644 --- a/projects/amdsmi/tests/amd_smi_test/main.cc +++ b/projects/amdsmi/tests/amd_smi_test/main.cc @@ -24,6 +24,7 @@ #include #include "amd_smi/impl/amd_smi_utils.h" +#include "functional/gpu/clock/clock_limit_read_write.h" #include "functional/gpu/clock/frequencies_read.h" #include "functional/gpu/clock/frequencies_read_write.h" #include "functional/gpu/events/evt_notif_read_write.h" @@ -175,6 +176,12 @@ TEST(GpuFunctionalReadWrite, TestFrequenciesReadWrite) { TestFrequenciesReadWrite tst; RunGenericTest(&tst); } +TEST(GpuFunctionalReadWrite, TestClockLimitReadWrite) { + if (std::getenv("AMDSMI_NON_PRIVILEGED")) GTEST_SKIP_("Skipped in non-privileged mode"); + if (!amd::smi::is_sudo_user()) GTEST_SKIP_("Invalid permission - Must run as super user"); + TestClockLimitReadWrite tst; + RunGenericTest(&tst); +} TEST(GpuFunctionalReadWrite, TestPciReadWrite) { if (std::getenv("AMDSMI_NON_PRIVILEGED")) GTEST_SKIP_("Skipped in non-privileged mode"); if (amd::smi::is_vm_guest()) GTEST_SKIP(); diff --git a/projects/amdsmi/tests/python/cli/base.py b/projects/amdsmi/tests/python/cli/base.py index f65275c83587..3dddd7b5b190 100644 --- a/projects/amdsmi/tests/python/cli/base.py +++ b/projects/amdsmi/tests/python/cli/base.py @@ -88,6 +88,10 @@ def setUpClass(cls): TestCliBase.sub_args = baseline["sub_args"] TestCliBase._initialized = True + # TODO: Remove this condition when CLI supports User automated input + # Commands that need User permission to run + cls.cmds_need_permission = {"set": ["--fan", "--memory-partition", "--compute-partition"]} + @classmethod def _build_baseline(cls): baseline = {} @@ -182,12 +186,12 @@ def __init__(self, *args, **kwargs): "DETERMINISM", ] self.profile_levels = [ - "CUSTOM_MASK", - "VIDEO_MASK", - "POWER_SAVING_MASK", - "COMPUTE_MASK", - "VR_MASK", - "THREE_D_FULL_SCR_MASK", + "CUSTOM", + "VIDEO", + "POWER_SAVING", + "COMPUTE", + "VR", + "3D_FULL_SCREEN", "BOOTUP_DEFAULT", ] self.compute_partition_modes = ["SPX", "DPX", "TPX", "QPX", "CPX"] @@ -538,7 +542,7 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): if clock_sys != "N/A" and len(clock_sys["frequency_levels"]): num = len(clock_sys["frequency_levels"]) level = f"Level {num - 1}" - clock_freq = int(clock_sys["frequency_levels"][level].split()[0].strip()) + clock_freq = int(clock_sys["frequency_levels"][level]["value"]) cmd = cmd.replace( "{perf_determinism}", f"--perf-determinism {clock_freq + 50}", 1 ) @@ -708,6 +712,21 @@ def CreateCmds(self, cmd_name, list1_name, list2_name, list3_name, list4_name): cmds[index] = (cmd, cond) + # Remove commands requiring input + for index, cmd_cond in enumerate(cmds): + cmd, cond = cmd_cond + if not cmd: + continue + items = cmd.split() + if len(items) < 3: + continue + if items[1] == "set": + if items[2] in self.cmds_need_permission["set"]: + cmd = "" + # Update cmds when cmd has changed + if not cmd: + cmds[index] = (cmd, cond) + # Remove empty (cmd,cond) arguments cmds = [cmd_cond for cmd_cond in cmds if cmd_cond[0] != ""] diff --git a/projects/amdsmi/tests/python/cli/test_set.py b/projects/amdsmi/tests/python/cli/test_set.py index d2906410f1dd..0b7faa92a01d 100644 --- a/projects/amdsmi/tests/python/cli/test_set.py +++ b/projects/amdsmi/tests/python/cli/test_set.py @@ -41,13 +41,6 @@ def test_command(self): msg = f"{self.tab}### amd-smi set" self.common.print(msg) - # TODO allow set commands to be executed - if not self.PrintCmdsOnly: - if self.common.TODO_SKIP_FAIL: - msg = f"{self.tab}Needs input" - # self.common.print(msg) - self.skipTest(msg) - # Get current settings power_profile = {} for index, gpu in enumerate(self.common.processors): @@ -99,7 +92,7 @@ def test_command(self): if clock_sys != "N/A": num = len(clock_sys["frequency_levels"]) level = f"Level {num - 1}" - clock_freq = int(clock_sys["frequency_levels"][level].split()[0].strip()) + clock_freq = int(clock_sys["frequency_levels"][level]["value"]) cmds.append( (f"amd-smi set --perf-determinism {clock_freq} --gpu {index}", self.PASS) ) diff --git a/projects/amdsmi/tests/python/common/common.py b/projects/amdsmi/tests/python/common/common.py index 3ee24a4bbe0a..07edda34e4dc 100644 --- a/projects/amdsmi/tests/python/common/common.py +++ b/projects/amdsmi/tests/python/common/common.py @@ -314,13 +314,13 @@ def build_type_lists(): (member.name, amdsmi.AmdSmiMemoryPartitionType(member.value), cond) ) - compute_partition_mem_alloc_mode_types = [] - for member in amdsmi.AmdSmiComputePartitionMemAllocModeType: + accelerator_partition_mem_alloc_mode_types = [] + for member in amdsmi.AmdSmiAcceleratorPartitionMemAllocModeType: cond = PASS if member.name in ["INVALID"]: cond = FAIL - compute_partition_mem_alloc_mode_types.append( - (member.name, amdsmi.AmdSmiComputePartitionMemAllocModeType(member.value), cond) + accelerator_partition_mem_alloc_mode_types.append( + (member.name, amdsmi.AmdSmiAcceleratorPartitionMemAllocModeType(member.value), cond) ) freq_inds = [] @@ -375,7 +375,7 @@ def build_type_lists(): "counter_commands": counter_commands, "compute_partition_types": compute_partition_types, "memory_partition_types": memory_partition_types, - "compute_partition_mem_alloc_mode_types": compute_partition_mem_alloc_mode_types, + "accelerator_partition_mem_alloc_mode_types": accelerator_partition_mem_alloc_mode_types, "freq_inds": freq_inds, "power_profile_preset_masks": power_profile_preset_masks, "processor_types": processor_types, @@ -406,7 +406,9 @@ def build_type_lists(): COUNTER_COMMANDS = _TYPE_LISTS["counter_commands"] COMPUTE_PARTITION_TYPES = _TYPE_LISTS["compute_partition_types"] MEMORY_PARTITION_TYPES = _TYPE_LISTS["memory_partition_types"] -COMPUTE_PARTITION_MEM_ALLOC_MODE_TYPES = _TYPE_LISTS["compute_partition_mem_alloc_mode_types"] +ACCELERATOR_PARTITION_MEM_ALLOC_MODE_TYPES = _TYPE_LISTS[ + "accelerator_partition_mem_alloc_mode_types" +] FREQ_INDS = _TYPE_LISTS["freq_inds"] POWER_PROFILE_PRESET_MASKS = _TYPE_LISTS["power_profile_preset_masks"] PROCESSOR_TYPES = _TYPE_LISTS["processor_types"] diff --git a/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py b/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py index 8b087895c456..a0e8e127cacb 100644 --- a/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_benchmark.py @@ -2054,33 +2054,6 @@ def test_performance_get_gpu_vram_usage(self): "amdsmi_get_gpu_vram_usage", "Processors", "get_gpu_vram_usage" ) - def test_performance_get_gpu_vram_vendor(self): - self.common.print_func_name("") - - for i, processor in enumerate(self.processors): - self._log_test_start("amdsmi_get_gpu_vram_vendor", "Processor", i) - - self._print_api_result(amdsmi.amdsmi_get_gpu_vram_vendor, i, processor) - - stats = self._measure_api_performance( - amdsmi.amdsmi_get_gpu_vram_vendor, f"get_gpu_vram_vendor_gpu_{i}", processor - ) - - self.perf_results[f"get_gpu_vram_vendor_gpu_{i}"] = stats - - if stats["successful_runs"] > 0: - self._print_performance_results(stats) - else: - self.common.print( - f" Processor {i}: All calls failed - {stats['errors'][0]['error_info'] if stats['errors'] else 'Unknown'}" - ) - - self._log_test_completion("Processor", i) - - self._log_performance_summary( - "amdsmi_get_gpu_vram_vendor", "Processors", "get_gpu_vram_vendor" - ) - def test_performance_get_gpu_xcd_counter(self): self.common.print_func_name("") if self.TODO_SKIP_FAIL: @@ -3362,46 +3335,6 @@ def test_performance_set_gpu_clk_limit(self): self._log_performance_summary("amdsmi_set_gpu_clk_limit", "Processors", "set_gpu_clk_limit") - def test_performance_set_gpu_clk_range(self): - self.common.print_func_name("") - - # Use TODO placeholder values like original test - min_clk_value = 100 - max_clk_value = 200 - - for i, processor in enumerate(self.processors): - for clk_type_name, clk_type, clk_cond in common.CLK_TYPES: - self._log_test_start( - "amdsmi_set_gpu_clk_range", - "Processor", - i, - min_clk_value=min_clk_value, - max_clk_value=max_clk_value, - clk_type=clk_type_name, - ) - - stats = self._measure_api_performance( - amdsmi.amdsmi_set_gpu_clk_range, - f"set_gpu_clk_range_processor_{i}_clk_{clk_type_name}", - processor, - min_clk_value, - max_clk_value, - clk_type, - ) - - self.perf_results[f"set_gpu_clk_range_processor_{i}_clk_{clk_type_name}"] = stats - - if stats["successful_runs"] > 0: - self._print_performance_results(stats) - else: - self.common.print( - f" Processor {i} clk={clk_type_name}: All calls failed - {stats['errors'][0]['error_info'] if stats['errors'] else 'Unknown'}" - ) - - self._log_test_completion("Processor", i, f"clk_type={clk_type_name}") - - self._log_performance_summary("amdsmi_set_gpu_clk_range", "Processors", "set_gpu_clk_range") - def test_performance_set_gpu_compute_partition(self): self.common.print_func_name("") diff --git a/projects/amdsmi/tests/python/functional/gpu/test_identity.py b/projects/amdsmi/tests/python/functional/gpu/test_identity.py index 450bfbb3fad9..72dc16633b07 100755 --- a/projects/amdsmi/tests/python/functional/gpu/test_identity.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_identity.py @@ -137,11 +137,6 @@ def test_get_gpu_vram_info(self): self.common.Test_API_Per_GPU(amdsmi_get_gpu_vram_info=amdsmi.amdsmi_get_gpu_vram_info) return - def test_get_gpu_vram_vendor(self): - self.common.print_func_name("") - self.common.Test_API_Per_GPU(amdsmi_get_gpu_vram_vendor=amdsmi.amdsmi_get_gpu_vram_vendor) - return - def test_get_lib_version(self): self.common.print_func_name("") self.common.Test_API(amdsmi_get_lib_version=amdsmi.amdsmi_get_lib_version) diff --git a/projects/amdsmi/tests/python/functional/gpu/test_overdrive.py b/projects/amdsmi/tests/python/functional/gpu/test_overdrive.py index 6464ce0affcc..dde38b81f7ee 100755 --- a/projects/amdsmi/tests/python/functional/gpu/test_overdrive.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_overdrive.py @@ -239,31 +239,6 @@ def test_set_gpu_clk_limit(self): raise self.raise_exception return - # out of order; min_clk_value, max_clk_value then clk_type - - def test_set_gpu_clk_range(self): - self.common.print_func_name("") - - # TODO Find better way to set min_clk_value, max_clk_value - min_clk_value = 100 - max_clk_value = 200 - - for i, gpu in enumerate(self.common.processors): - self.common.print_device_header(i) - for _, clk_type, clk_cond in common.CLK_TYPES: - msg = f"\t### amdsmi_set_gpu_clk_range(gpu={i}, min_clk_value={min_clk_value}, max_clk_value={max_clk_value}, clk_type={clk_type}):" - try: - amdsmi.amdsmi_set_gpu_clk_range(gpu, min_clk_value, max_clk_value, clk_type) - self.common.print(msg, "") - self.common.check_ret("", "", self.common.PASS) - except amdsmi.AmdSmiLibraryException as e: - if self.common.check_ret(msg, e, clk_cond): - self.raise_exception = e - self.common.print("") - if self.raise_exception: - raise self.raise_exception - return - def test_set_gpu_od_clk_info(self): self.common.print_func_name("") diff --git a/projects/amdsmi/tests/python/functional/gpu/test_partition.py b/projects/amdsmi/tests/python/functional/gpu/test_partition.py index 09620e74fbf1..8f85c1aa687f 100755 --- a/projects/amdsmi/tests/python/functional/gpu/test_partition.py +++ b/projects/amdsmi/tests/python/functional/gpu/test_partition.py @@ -277,10 +277,10 @@ def test_get_gpu_compute_partition(self): ) return - def test_get_gpu_compute_partition_mem_alloc_mode(self): + def test_get_gpu_accelerator_partition_mem_alloc_mode(self): self.common.print_func_name("") self.common.Test_API_Per_GPU( - amdsmi_get_gpu_compute_partition_mem_alloc_mode=amdsmi.amdsmi_get_gpu_compute_partition_mem_alloc_mode + amdsmi_get_gpu_accelerator_partition_mem_alloc_mode=amdsmi.amdsmi_get_gpu_accelerator_partition_mem_alloc_mode ) return @@ -328,83 +328,11 @@ def test_set_gpu_memory_partition_return_codes(self): """Verify that set accepts supported modes and rejects unsupported ones.""" self.common.print_func_name("") - NPS_NAME_TO_TYPE = { - "NPS1": amdsmi.AmdSmiMemoryPartitionType.NPS1, - "NPS2": amdsmi.AmdSmiMemoryPartitionType.NPS2, - "NPS4": amdsmi.AmdSmiMemoryPartitionType.NPS4, - "NPS8": amdsmi.AmdSmiMemoryPartitionType.NPS8, - } - - raise_exception = None - for i, gpu in enumerate(self.common.processors): - self.common.print_device_header(i) - - # Read current partition for restore. - original_partition = None - msg = f"\t### amdsmi_get_gpu_memory_partition(gpu={i}):" - try: - original_partition = amdsmi.amdsmi_get_gpu_memory_partition(gpu) - self.common.print(msg, original_partition) - except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: - if self.common.check_ret(msg, e, self.common.PASS): - raise_exception = e - continue - - # Read hardware capabilities to build per-GPU condition map. - supported_modes = set() - msg = f"\t### amdsmi_get_gpu_memory_partition_config(gpu={i}):" - try: - config = amdsmi.amdsmi_get_gpu_memory_partition_config(gpu) - caps = config.get("partition_caps", []) - self.common.print(msg, caps) - supported_modes = {c for c in caps if c in NPS_NAME_TO_TYPE} - except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: - # Older kernels may not support config query; fall through with empty set. - self.common.print(msg, e) - - # Attempt each NPS mode; condition is driven by hardware capabilities. - for mode_name, mode_type in NPS_NAME_TO_TYPE.items(): - cond = self.common.PASS if mode_name in supported_modes else self.common.FAIL - msg = f"\t### amdsmi_set_gpu_memory_partition(gpu={i}, mode={mode_name}):" - try: - amdsmi.amdsmi_set_gpu_memory_partition(gpu, mode_type) - self.common.print(msg, "SUCCESS") - if cond == self.common.FAIL: - self.common.print( - f"\t TEST FAILURE: expected failure for {mode_name} but call succeeded", - "", - ) - raise_exception = amdsmi.AmdSmiLibraryException(0) - else: - self.common.check_ret("", "", self.common.PASS) - self.common.print( - f"\t [info] Staged {mode_name}. Apply with: " - "sudo modprobe -r amdgpu && sudo modprobe amdgpu", - "", - ) - except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: - if self.common.check_ret(msg, e, cond): - raise_exception = e - - # Restore original partition (stages the write; driver reload applies it). - if original_partition and original_partition in NPS_NAME_TO_TYPE: - msg = f"\t### amdsmi_set_gpu_memory_partition restore(gpu={i}, mode={original_partition}):" - try: - amdsmi.amdsmi_set_gpu_memory_partition( - gpu, NPS_NAME_TO_TYPE[original_partition] - ) - self.common.print(msg, "SUCCESS") - self.common.print( - f"\t [info] Restore staged to {original_partition}. Apply with: " - "sudo modprobe -r amdgpu && sudo modprobe amdgpu", - "", - ) - except (amdsmi.AmdSmiLibraryException, amdsmi.AmdSmiParameterException) as e: - if self.common.check_ret(msg, e, self.common.PASS): - raise_exception = e - - if raise_exception: - raise raise_exception + self.common.Test_Per_GPU_With_One_Enum( + amdsmi_set_gpu_memory_partition_mode=amdsmi.amdsmi_set_gpu_memory_partition_mode, + memory_partition_type=common.MEMORY_PARTITION_TYPES, + ) + return def test_set_gpu_memory_partition_idempotent(self): """Setting the current partition mode must succeed and leave the read-back unchanged.""" @@ -480,12 +408,12 @@ def test_set_gpu_memory_partition_mode(self): ) return - def test_set_gpu_compute_partition_mem_alloc_mode(self): + def test_set_gpu_accelerator_partition_mem_alloc_mode(self): self.common.print_func_name("") self.common.Test_Per_GPU_With_One_Enum( - amdsmi_set_gpu_compute_partition_mem_alloc_mode=amdsmi.amdsmi_set_gpu_compute_partition_mem_alloc_mode, - compute_partition_mem_alloc_mode=common.COMPUTE_PARTITION_MEM_ALLOC_MODE_TYPES, + amdsmi_set_gpu_accelerator_partition_mem_alloc_mode=amdsmi.amdsmi_set_gpu_accelerator_partition_mem_alloc_mode, + accelerator_partition_mem_alloc_mode=common.ACCELERATOR_PARTITION_MEM_ALLOC_MODE_TYPES, ) return diff --git a/projects/amdsmi/tools/hw_compatibility_status.py b/projects/amdsmi/tools/hw_compatibility_status.py index 0560b41b9358..13e934f128f6 100644 --- a/projects/amdsmi/tools/hw_compatibility_status.py +++ b/projects/amdsmi/tools/hw_compatibility_status.py @@ -342,7 +342,7 @@ def run_tests(): # Get CPU socket handles (for CPU APIs) try: - cpu_socket_handles = amdsmi.amdsmi_get_cpusocket_handles() + cpu_socket_handles = amdsmi.amdsmi_get_cpu_handles() except: cpu_socket_handles = [] @@ -516,8 +516,6 @@ def run_tests(): test_api("amdsmi_get_gpu_id", lambda: amdsmi.amdsmi_get_gpu_id(gpu_handle)) - test_api("amdsmi_get_gpu_vram_vendor", lambda: amdsmi.amdsmi_get_gpu_vram_vendor(gpu_handle)) - test_api( "amdsmi_get_gpu_drm_render_minor", lambda: amdsmi.amdsmi_get_gpu_drm_render_minor(gpu_handle), @@ -710,12 +708,6 @@ def _utilization_counter_types(): lambda: amdsmi.amdsmi_get_gpu_target_frequency_range(gpu_handle), ) - test_api( - "amdsmi_set_gpu_clk_range", - lambda: amdsmi.amdsmi_set_gpu_clk_range(gpu_handle, 500, 2500, need("AmdSmiClkType").GFX), - requires_root=True, - ) - # set_gpu_clk_limit takes (handle, clk_type, limit_type, value). # Read the current target range and replay max as a no-op-ish write. def _set_clk_limit_noop(): @@ -1236,6 +1228,7 @@ def _link_topology_nearest(): lambda: amdsmi.amdsmi_get_gpu_memory_partition(gpu_handle), ) + # amdsmi_set_gpu_memory_partition deprecated, use amdsmi_set_gpu_memory_partition_mode instead test_api( "amdsmi_set_gpu_memory_partition", None, @@ -1429,7 +1422,7 @@ def _cpu_affinity_with_scope(): cpu_socket = cpu_socket_handles[0] if cpu_socket_handles else None cpu_core = cpu_core_handles[0] if cpu_core_handles else None - test_api("amdsmi_get_cpusocket_handles", lambda: amdsmi.amdsmi_get_cpusocket_handles()) + test_api("amdsmi_get_cpu_handles", lambda: amdsmi.amdsmi_get_cpu_handles()) test_api("amdsmi_get_cpucore_handles", lambda: amdsmi.amdsmi_get_cpucore_handles()) diff --git a/projects/rccl/src/include/amdsmi_wrap.h b/projects/rccl/src/include/amdsmi_wrap.h index f6636281b31e..83319f69ef9a 100644 --- a/projects/rccl/src/include/amdsmi_wrap.h +++ b/projects/rccl/src/include/amdsmi_wrap.h @@ -353,15 +353,16 @@ typedef struct { } amdsmi_fabric_telemetry_t; #define AMDSMI_FABRIC_ACTIVE_ACCELERATORS_BITMAP_SIZE 32 -#define AMDSMI_FABRIC_MAX_LOCAL_GPUS 8 +#define AMDSMI_FABRIC_MAX_LOCAL_GPUS 16 /** * @brief Fabric type */ typedef enum { AMDSMI_FABRIC_TYPE_UALOE, - AMDSMI_FABRIC_TYPE_UALLINK, - AMDSMI_FABRIC_TYPE_UNKNOWN + AMDSMI_FABRIC_TYPE_UALINK, + AMDSMI_FABRIC_TYPE_UNKNOWN, + AMDSMI_FABRIC_TYPE_UALLINK = AMDSMI_FABRIC_TYPE_UALINK //!< Deprecated, use AMDSMI_FABRIC_TYPE_UALINK instead } amdsmi_fabric_type_t; /** @@ -403,19 +404,15 @@ typedef struct { amdsmi_fabric_accelerator_vpod_state_t accel_state; //!< Accelerator vPoD State } amdsmi_fabric_info_v1_t; -typedef struct { - uint32_t version; - union fabric_info_ { - amdsmi_fabric_info_v1_t v1; - } fabric_version; -} amdsmi_fabric_info_ver_t; - /** * @brief Fabric device information structure */ typedef struct { amdsmi_bdf_t bdf; //!< BDF of the Fabric device - amdsmi_fabric_info_ver_t fabric_info; + uint32_t fabric_version; + union fabric_info_ { + amdsmi_fabric_info_v1_t v1; + } fabric_info; uint32_t reserved[15]; } amdsmi_fabric_info_t; @@ -449,18 +446,18 @@ amdsmi_status_t amdsmi_get_gpu_fabric_info(amdsmi_processor_handle processor_han * * @platform{gpu_bm_linux} @platform{host} * - * @details Given a telemetry item ID @p telem_id, - * this function returns a pointer to a string containing the human-readable name - * for the specified telemetry item. The returned string is statically allocated - * and should not be freed by the caller. - * + * @details Given a telemetry item ID @p telem_id, this function writes a pointer + * to a human-readable name for the specified telemetry item through @p telem_name. + * The returned string is statically allocated and should not be freed by the caller. * * @param[in] telem_id The telemetry item ID for which the name is requested * - * @return const char* | Pointer to string containing the telemetry item name, - * or UNKNOWN if the category or telemetry ID is not recognized + * @param[out] telem_name Receives a pointer to the telemetry item name string, + * or "UNKNOWN" if the category or telemetry ID is not recognized + * + * @return ::amdsmi_status_t | ::AMDSMI_STATUS_SUCCESS on success, non-zero on fail */ -const char* amdsmi_fabric_telem_id_to_string(uint64_t telem_id); +amdsmi_status_t amdsmi_fabric_telem_id_to_string(uint64_t telem_id, const char** telem_name); /** @} End rcclFabricCompat */ #endif // !AMDSMI_FABRIC_DIRECT diff --git a/projects/rccl/src/misc/amdsmi_wrap.cc b/projects/rccl/src/misc/amdsmi_wrap.cc index 6e76c6c95bf6..368b483fa020 100644 --- a/projects/rccl/src/misc/amdsmi_wrap.cc +++ b/projects/rccl/src/misc/amdsmi_wrap.cc @@ -113,7 +113,7 @@ RCCL_AMDSMI_FN(amdsmi_get_fabric_telemetry_data, amdsmi_status_t, (amdsmi_processor_handle processor_handle, amdsmi_fabric_telemetry_t* telemetry)) RCCL_AMDSMI_FN(amdsmi_free_fabric_telemetry, amdsmi_status_t, (amdsmi_processor_handle processor_handle, amdsmi_fabric_telemetry_t* telemetry)) -RCCL_AMDSMI_FN(amdsmi_fabric_telem_id_to_string, const char*, (uint64_t telem_id)) +RCCL_AMDSMI_FN(amdsmi_fabric_telem_id_to_string, amdsmi_status_t, (uint64_t telem_id, const char** telem_name)) // Firmware info RCCL_AMDSMI_FN(amdsmi_get_fw_info, amdsmi_status_t, (amdsmi_processor_handle processor_handle, amdsmi_fw_info_t* info)) } // namespace @@ -640,13 +640,13 @@ ncclResult_t amd_smi_ensureFabricInitialized() { devInfo->fabricSupported = false; continue; } - if (fabricInfo.fabric_info.version != AMDSMI_FABRIC_INFO_CURRENT_VERSION) { + if (fabricInfo.fabric_version != AMDSMI_FABRIC_INFO_CURRENT_VERSION) { WARN("AMD SMI fabric: unexpected fabric info version %u for device %u, expected %u", - fabricInfo.fabric_info.version, d, AMDSMI_FABRIC_INFO_CURRENT_VERSION); + fabricInfo.fabric_version, d, AMDSMI_FABRIC_INFO_CURRENT_VERSION); devInfo->fabricSupported = false; continue; } - const amdsmi_fabric_info_v1_t* v1 = &fabricInfo.fabric_info.fabric_version.v1; + const amdsmi_fabric_info_v1_t* v1 = &fabricInfo.fabric_info.v1; devInfo->fabricSupported = ((v1->fabric_type == AMDSMI_FABRIC_TYPE_UALOE || v1->fabric_type == AMDSMI_FABRIC_TYPE_UALLINK) && (v1->accel_state == AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_ACTIVE || @@ -762,5 +762,10 @@ const char* amd_smi_fabricTelemIdToString(uint64_t telemId) { if (pfn_amdsmi_fabric_telem_id_to_string == nullptr) { return ARSMI_fabric_telem_id_to_string(telemId); } - return pfn_amdsmi_fabric_telem_id_to_string(telemId); + const char* telemName = nullptr; + amdsmi_status_t ret = pfn_amdsmi_fabric_telem_id_to_string(telemId, &telemName); + if (ret != AMDSMI_STATUS_SUCCESS || telemName == nullptr) { + return "UNKNOWN"; + } + return telemName; } From 9030ac095c582f86c1a94c5c9815619673be7665 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Fri, 7 Aug 2026 12:02:54 -0500 Subject: [PATCH 4/5] docs(amdsmi): set changelog version to 10.0 Signed-off-by: Maisam Arif --- projects/amdsmi/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/amdsmi/CHANGELOG.md b/projects/amdsmi/CHANGELOG.md index 534f55a1648e..bea2b4e7d18f 100644 --- a/projects/amdsmi/CHANGELOG.md +++ b/projects/amdsmi/CHANGELOG.md @@ -4,7 +4,7 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr ***All information listed below is for reference and subject to change.*** -## amd_smi_lib for ROCm 7.15.0 +## amd_smi_lib for ROCm 10.0 ### Added From 94cec8fda6989d24a4f45f12988d04f10558879b Mon Sep 17 00:00:00 2001 From: Nilesh M Negi Date: Fri, 31 Jul 2026 15:22:23 -0500 Subject: [PATCH 5/5] fix(rccl): correct UALoE fabric detection and diagnostics on the amdsmi path (#9464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `RCCL_USE_AMD_SMI_LIB=1` silently disabled UALoE fabric on hardware where it is present and `ready`, while the default sysfs path worked. At the same time, the default path printed a warning telling users to set that very variable to "enable fabric support" — advice that would have made things worse. These two together sent readers (and agents) chasing a configuration problem that did not exist. ## Technical Details **Version check (`src/misc/amdsmi_wrap.cc`)** `amdsmi_get_gpu_fabric_info()` returns `SUCCESS` with a correct payload but never writes `fabric_info.version`, which stays at the sentinel `amdsmi.h` documents for fields it cannot source from sysfs. Strict equality against `AMDSMI_FABRIC_INFO_CURRENT_VERSION` could therefore never pass against `libamd_smi.so.26`, so every device was dropped with a warning. A probe showed `version = 0xFFFFFFFF` alongside `fabric_type = UALOE`, `accel_state = READY` and an `accel_id`/`vpod` matching sysfs; pre-setting version is overwritten, so it is not an input field. The sentinel now means "unreported", and v1 is assumed, while a version the library actually claims is still rejected. **False warning (`src/misc/amdsmi_wrap.cc`)** The "RCCL_USE_AMD_SMI_LIB not set, but HIP_FABRIC_API is defined" warning predates sysfs fabric discovery via `ARSMI_get_fabric_info()`, which populates `amdsmiFabricDevices` in the same way. With the env var defaulting to 0 and `HIP_FABRIC_API` auto-detected on modern ROCm, it printed on every rank of every job, and it conflated `HIP_FABRIC_API` (a `hipMemFabricHandle_t` build flag) with which library does discovery. Replaced by an `INFO` naming the active path. **Struct layout compatibility (`src/include/amdsmi_wrap.h`)** amd-smi 27.0 flattened `amdsmi_fabric_info_t`: the version moved from `fabric_info.version` to a top-level `fabric_version`, and the payload from `fabric_info.fabric_version.v1` to `fabric_info.v1`. Field offsets did not move (version at byte 8, v1 at 12), so this is a source-level rename only. A template-based shape detector (`amdSmiFabricInfoIsFlat`) with `if constexpr` accessor functions handles either layout at compile time. `static_assert` guards pin the struct sizes (244/320 bytes) and `reserved` offset (256) to catch future ABI drift. **Telemetry ABI (`src/misc/amdsmi_wrap.cc`)** `amdsmi_fabric_telem_id_to_string` changed from returning the name to writing it through an out-parameter in 27.0. The two are not call-compatible and we reach it through `dlopen`, so the choice follows the version the loaded runtime reports. An unknown version assumes the out-parameter form: calling a 27+ implementation the old way would leave it writing through an uninitialized pointer, whereas calling an older one the new way merely leaves the name unwritten. **Minor** amd-smi's version string was printed as a fourth dotted component, rendering as `version 26.5.0.26.5.0`. Now it will be printed as `version 26.5.0 (build 26.5.0)`. ## JIRA ID AICOMRCCL-1656 Signed-off-by: Maisam Arif Co-authored-by: Cursor Co-authored-by: Maisam Arif Signed-off-by: Maisam Arif --- projects/rccl/src/include/amdsmi_wrap.h | 149 ++++++++ projects/rccl/src/misc/amdsmi_wrap.cc | 107 ++++-- projects/rccl/test/AmdSmiFabricTests.cpp | 306 ++++++++++++++++ projects/rccl/test/AmdSmiLifecycleStub.cpp | 85 +++++ projects/rccl/test/AmdSmiWrapTests.cpp | 343 ++++++++++++++++++ projects/rccl/test/CMakeLists.txt | 93 +++++ .../rccl/test/test_categories_amdsmi.yaml | 62 ++++ projects/rccl/tools/topo_expl/Makefile | 24 +- 8 files changed, 1140 insertions(+), 29 deletions(-) create mode 100644 projects/rccl/test/AmdSmiFabricTests.cpp create mode 100644 projects/rccl/test/AmdSmiLifecycleStub.cpp create mode 100644 projects/rccl/test/AmdSmiWrapTests.cpp create mode 100644 projects/rccl/test/test_categories_amdsmi.yaml diff --git a/projects/rccl/src/include/amdsmi_wrap.h b/projects/rccl/src/include/amdsmi_wrap.h index 83319f69ef9a..c8f749582a9f 100644 --- a/projects/rccl/src/include/amdsmi_wrap.h +++ b/projects/rccl/src/include/amdsmi_wrap.h @@ -3,8 +3,12 @@ #ifndef AMDSMI_WRAP_H_ #define AMDSMI_WRAP_H_ +#include #include +#include #include +#include +#include #include "nccl.h" #include "device.h" @@ -462,6 +466,37 @@ amdsmi_status_t amdsmi_fabric_telem_id_to_string(uint64_t telem_id, const char** /** @} End rcclFabricCompat */ #endif // !AMDSMI_FABRIC_DIRECT +/************************************************************************* + * Fabric ABI guard + * + * amdsmi_get_gpu_fabric_info() is reached through dlopen, so it writes this + * struct using the shipped library's layout, not ours. If our declaration is + * smaller the library writes past the end of the caller's object, and any + * shift moves the fields we read. Neither shows up as a compiler diagnostic, + * so recognize the two shipped layouts here: a mismatch must be a deliberate + * update, not a silent memory bug. Some pre-release ROCm 7.14 snapshots used + * AMDSMI_FABRIC_MAX_LOCAL_GPUS=8, while final ROCm 7.14 and ROCm 7.15 use 16 + * without changing the amd_smi major. + ************************************************************************/ +constexpr size_t kAmdSmiFabricInfo8GpuSize = 288; +constexpr size_t kAmdSmiFabricInfo16GpuSize = 320; +constexpr size_t kAmdSmiFabricState8GpuOffset = 208; +constexpr size_t kAmdSmiFabricState16GpuOffset = 240; + +constexpr bool amdSmiFabricLayoutIs8Gpu = + sizeof(amdsmi_fabric_info_v1_t) == 212 && sizeof(amdsmi_fabric_info_t) == kAmdSmiFabricInfo8GpuSize && + offsetof(amdsmi_fabric_info_v1_t, addr_mode) == kAmdSmiFabricState8GpuOffset - sizeof(uint32_t) && + offsetof(amdsmi_fabric_info_v1_t, accel_state) == kAmdSmiFabricState8GpuOffset && + offsetof(amdsmi_fabric_info_t, reserved) == 224; + +constexpr bool amdSmiFabricLayoutIs16Gpu = + sizeof(amdsmi_fabric_info_v1_t) == 244 && sizeof(amdsmi_fabric_info_t) == kAmdSmiFabricInfo16GpuSize && + offsetof(amdsmi_fabric_info_v1_t, addr_mode) == kAmdSmiFabricState16GpuOffset - sizeof(uint32_t) && + offsetof(amdsmi_fabric_info_v1_t, accel_state) == kAmdSmiFabricState16GpuOffset && + offsetof(amdsmi_fabric_info_t, reserved) == 256; + +static_assert(amdSmiFabricLayoutIs8Gpu || amdSmiFabricLayoutIs16Gpu, "unsupported amdsmi fabric layout"); + /************************************************************************* * AMD SMI Fabric Info Cache * @@ -472,6 +507,120 @@ amdsmi_status_t amdsmi_fabric_telem_id_to_string(uint64_t telem_id, const char** #define AMDSMI_FABRIC_INFO_VERSION_1 1 #define AMDSMI_FABRIC_INFO_CURRENT_VERSION AMDSMI_FABRIC_INFO_VERSION_1 +// Sentinel amd_smi leaves in numeric fabric fields it could not source from sysfs +constexpr uint32_t kAmdSmiFabricVersionUnreported = 0xFFFFFFFFu; + +// amd_smi never populates amdsmi_fabric_info_t's version field, so an unreported value +// has to be read as v1 rather than as a reason to discard an otherwise valid payload. +// A version the library does claim, but that we cannot interpret, is still refused. +inline bool amdSmiFabricVersionUsable(uint32_t version) { + return version == kAmdSmiFabricVersionUnreported || version == AMDSMI_FABRIC_INFO_CURRENT_VERSION; +} + +// Fabric is usable only over a link type we drive, and only once the accelerator has +// reached a state where its vPoD can carry traffic. +// +// AMDSMI_FABRIC_TYPE_UALLINK is deprecated in favour of the correctly spelled +// AMDSMI_FABRIC_TYPE_UALINK, but only the former exists in amd_smi before 27.0 and +// the latter is an alias for it after, so the old name is the one that compiles +// against every header we support. +inline bool amdSmiFabricStateUsable(amdsmi_fabric_type_t type, amdsmi_fabric_accelerator_vpod_state_t state) { + return (type == AMDSMI_FABRIC_TYPE_UALOE || type == AMDSMI_FABRIC_TYPE_UALLINK) && + (state == AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_ACTIVE || state == AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY); +} + +/************************************************************************* + * amdsmi_fabric_info_t layout compatibility + * + * amd_smi 27.0 flattened this struct. The version moved from a nested + * fabric_info.version to a top-level fabric_version, and the payload from + * fabric_info.fabric_version.v1 to fabric_info.v1 -- note that fabric_version + * names different things either side of the change. The version and payload + * offsets did not move (bytes 8 and 12), so the shape change itself is only a + * source-level rename; the ABI guard above handles payload capacity separately. + * + * Detect the shape rather than test a version macro, so a header that carries + * the change under any version still resolves correctly. Only the flattened + * layout has a top-level integral fabric_version; in the nested layout that name + * exists solely inside fabric_info, as a union. + ************************************************************************/ +template +struct amdSmiFabricInfoIsFlat : std::false_type {}; + +template +struct amdSmiFabricInfoIsFlat().fabric_version)>> + : std::is_integral().fabric_version)>> {}; + +// Templates so that if constexpr discards the branch that does not compile against +// the header in use; in a non-template function both branches are type-checked. +template +inline uint32_t amdSmiFabricInfoVersion(const FabricInfoT& info) { + if constexpr (amdSmiFabricInfoIsFlat::value) { + return info.fabric_version; + } else { + return info.fabric_info.version; + } +} + +template +inline const amdsmi_fabric_info_v1_t* amdSmiFabricInfoV1(const FabricInfoT& info) { + if constexpr (amdSmiFabricInfoIsFlat::value) { + return &info.fabric_info.v1; + } else { + return &info.fabric_info.fabric_version.v1; + } +} + +// amd_smi 26.x used both supported payload sizes under the same SONAME. Use a +// maximum-sized canary buffer so either runtime can write safely, then identify +// how much it wrote. Both implementations value-initialize a local struct and +// assign the complete object, making the final 32 bytes zero only for the +// 16-GPU runtime. +constexpr unsigned char kAmdSmiFabricBufferCanary = 0xA5; + +struct alignas(amdsmi_fabric_info_t) amdSmiFabricInfoBuffer { + unsigned char bytes[kAmdSmiFabricInfo16GpuSize]; +}; + +enum class amdSmiFabricRuntimeLayout { + EightGpu, + SixteenGpu, + Unknown, +}; + +inline void amdSmiPrepareFabricInfoBuffer(amdSmiFabricInfoBuffer& buffer) { + memset(buffer.bytes, kAmdSmiFabricBufferCanary, sizeof(buffer.bytes)); +} + +inline amdsmi_fabric_info_t* amdSmiFabricInfoBufferAsInfo(amdSmiFabricInfoBuffer& buffer) { + return reinterpret_cast(buffer.bytes); +} + +inline amdSmiFabricRuntimeLayout amdSmiDetectFabricRuntimeLayout(const amdSmiFabricInfoBuffer& buffer) { + bool tailIsCanary = true; + bool tailIsZero = true; + for (size_t i = kAmdSmiFabricInfo8GpuSize; i < kAmdSmiFabricInfo16GpuSize; ++i) { + tailIsCanary &= buffer.bytes[i] == kAmdSmiFabricBufferCanary; + tailIsZero &= buffer.bytes[i] == 0; + } + if (tailIsCanary) return amdSmiFabricRuntimeLayout::EightGpu; + if (tailIsZero) return amdSmiFabricRuntimeLayout::SixteenGpu; + return amdSmiFabricRuntimeLayout::Unknown; +} + +// amd_smi 27.0 also changed amdsmi_fabric_telem_id_to_string from returning the name +// to writing it through an out-parameter. The two are not call-compatible and we reach +// it through dlopen, so the choice has to follow the version the loaded runtime +// reports. An unknown version must assume the out-parameter form: calling a 27+ +// implementation the old way leaves it writing through an uninitialized pointer, +// whereas calling an older one the new way merely leaves the name unwritten. +constexpr uint32_t kAmdSmiLibVersionUnknown = 0; +constexpr uint32_t kAmdSmiTelemOutParamMajor = 27; + +inline bool amdSmiTelemIdUsesOutParam(uint32_t libMajor) { + return libMajor == kAmdSmiLibVersionUnknown || libMajor >= kAmdSmiTelemOutParamMajor; +} + struct amdsmiFabricDeviceInfo { bool fabricSupported; //!< Whether UALoE fabric is available amdsmi_fabric_type_t fabricType; //!< Fabric type (UALOE or UALLINK) diff --git a/projects/rccl/src/misc/amdsmi_wrap.cc b/projects/rccl/src/misc/amdsmi_wrap.cc index 368b483fa020..7c2b27533f62 100644 --- a/projects/rccl/src/misc/amdsmi_wrap.cc +++ b/projects/rccl/src/misc/amdsmi_wrap.cc @@ -132,8 +132,17 @@ std::mutex fabricLock; // Thread safety for fabric operations bool fabricInitialized = false; thread_local bool threadFabricInitialized = false; ncclResult_t fabricInitResult = ncclSuccess; +std::mutex amdSmiInitLock; ncclResult_t amdSmiInitResult = ncclSuccess; -std::atomic amdSmiInitCalled{false}; // Track if amd_smi_init has been called +bool amdSmiInitCalled = false; +bool amdSmiLibInitialized = false; + +// Major version of the loaded amd_smi, as it reports itself. Selects the telemetry +// name call convention; see amdSmiTelemIdUsesOutParam(). +std::atomic amdSmiLibMajor{kAmdSmiLibVersionUnknown}; + +// Signature amdsmi_fabric_telem_id_to_string had before amd_smi 27.0 +using AmdSmiTelemIdToStringLegacyFn = const char* (*)(uint64_t); } // namespace /************************************************************************* @@ -141,7 +150,7 @@ std::atomic amdSmiInitCalled{false}; // Track if amd_smi_init has been cal ************************************************************************/ static ncclResult_t getProcessorHandle(uint32_t deviceIndex, amdsmi_processor_handle* procHandle) { if (!rcclParamUseAmdSmiLib()) { - return ncclSystemError; // Fabric only supported with amd_smi_lib + return ncclSystemError; // processor handles are amd_smi_lib-only } uint32_t socket_count = 0; @@ -179,10 +188,7 @@ static bool amd_smi_FabricFunctionsLoaded() { * Existing AMD SMI Wrapper Functions ************************************************************************/ -ncclResult_t amd_smi_init() { - // Ensure we only initialize once - if (amdSmiInitCalled.exchange(true)) return amdSmiInitResult; - +static ncclResult_t amd_smi_init_impl() { if (__atomic_load_n(&is_wsl2, __ATOMIC_ACQUIRE) == -1) __atomic_store_n(&is_wsl2, (access("/dev/dxg", F_OK) == -1) ? 0 : 1, __ATOMIC_RELEASE); if (__atomic_load_n(&is_wsl2, __ATOMIC_ACQUIRE)) { @@ -197,7 +203,6 @@ ncclResult_t amd_smi_init() { static void* libhandle = dlopen(RCCL_AMDSMI_LIBNAME, RTLD_NOW); if (libhandle == nullptr) { WARN("Failed to open %s: %s", RCCL_AMDSMI_LIBNAME, dlerror()); - amdSmiInitResult = ncclInternalError; return ncclInternalError; } @@ -235,32 +240,51 @@ ncclResult_t amd_smi_init() { // initialize amd-smi for AMD GPUs AMDSMITRY(amdsmi_init, AMDSMI_INIT_AMD_GPUS); + amdSmiLibInitialized = true; // get amd-smi version amdsmi_version_t version; AMDSMITRY(amdsmi_get_lib_version, &version); - INFO(NCCL_INIT, "amdsmi_lib: version %d.%d.%d.%s", version.major, version.minor, version.release, version.build); + INFO(NCCL_INIT, "amdsmi_lib: version %u.%u.%u (build %s)", version.major, version.minor, version.release, + version.build); + // Retained because the telemetry name call convention depends on it + amdSmiLibMajor.store(version.major, std::memory_order_release); } else { -#ifdef HIP_FABRIC_API - WARN("RCCL_USE_AMD_SMI_LIB not set, but HIP_FABRIC_API is defined. Fabric support is only available through AMD " - "SMI. Rerun with RCCL_USE_AMD_SMI_LIB=1 to enable AMD SMI and UALoE fabric support."); -#endif // initialize alternate rsmi ARSMICHECK(ARSMI_init()); - INFO(NCCL_INIT, "initialized internal alternative rsmi functionality"); + // RCCL_USE_AMD_SMI_LIB only selects who performs fabric *discovery*: amd_smi_lib, or the + // ualink sysfs nodes via ARSMI_get_fabric_info(). Both populate amdsmiFabricDevices + // identically, so UALoE/UALLink works on this path. + INFO(NCCL_INIT, "initialized internal alternative rsmi functionality; UALoE/UALLink fabric discovery uses sysfs " + "(set RCCL_USE_AMD_SMI_LIB=1 to use amd_smi_lib instead)"); } return ncclSuccess; } +ncclResult_t amd_smi_init() { + std::lock_guard lock(amdSmiInitLock); + if (amdSmiInitCalled) return amdSmiInitResult; + + // Cache failures too; shutdown explicitly resets this state for a retry. + amdSmiInitResult = amd_smi_init_impl(); + amdSmiInitCalled = true; + return amdSmiInitResult; +} + ncclResult_t amd_smi_shutdown() { - // Only shutdown if we actually initialized with amd_smi_lib - if (!rcclParamUseAmdSmiLib() || pfn_amdsmi_shut_down == nullptr) { - return ncclSuccess; + std::lock_guard lock(amdSmiInitLock); + + if (!amdSmiInitCalled) return ncclSuccess; + + // The backend may be initialized even if a later version query failed. + if (amdSmiLibInitialized) { + AMDSMITRY(amdsmi_shut_down); + amdSmiLibInitialized = false; } - amdSmiInitCalled.store(false); - amdSmiInitResult = ncclSuccess; - AMDSMITRY(amdsmi_shut_down); + amdSmiLibMajor.store(kAmdSmiLibVersionUnknown, std::memory_order_release); + amdSmiInitResult = ncclSuccess; + amdSmiInitCalled = false; return ncclSuccess; } @@ -602,6 +626,30 @@ ncclResult_t amd_smi_ensureFabricInitialized() { fabricInitResult = ncclInternalError; return fabricInitResult; } + + // amd_smi 26.x shipped both fabric payload sizes under the same SONAME, so + // the library version cannot identify a mixed header/runtime installation. + // Probe through an oversized canary buffer before using the typed structure. + // If the layouts differ (or cannot be identified), use the sysfs backend. + if (!useSysfs && amd_smi_FabricFunctionsLoaded()) { + amdsmi_processor_handle probeHandle; + if (getProcessorHandle(0, &probeHandle) == ncclSuccess) { + amdSmiFabricInfoBuffer probeBuffer; + amdSmiPrepareFabricInfoBuffer(probeBuffer); + pfn_amdsmi_get_gpu_fabric_info(probeHandle, amdSmiFabricInfoBufferAsInfo(probeBuffer)); + const amdSmiFabricRuntimeLayout runtimeLayout = amdSmiDetectFabricRuntimeLayout(probeBuffer); + const bool runtimeLayoutIs8Gpu = runtimeLayout == amdSmiFabricRuntimeLayout::EightGpu; + if (runtimeLayout == amdSmiFabricRuntimeLayout::Unknown) { + WARN("AMD SMI fabric: unable to verify the loaded library's fabric ABI; falling back to sysfs"); + useSysfs = true; + } else if (runtimeLayoutIs8Gpu != amdSmiFabricLayoutIs8Gpu) { + WARN("AMD SMI fabric ABI mismatch: RCCL was built for the %u-GPU layout, but the loaded library uses the " + "%u-GPU layout; falling back to sysfs", + amdSmiFabricLayoutIs8Gpu ? 8 : 16, runtimeLayoutIs8Gpu ? 8 : 16); + useSysfs = true; + } + } + } amdsmiFabricDeviceCount = (int)numDevs; for (uint32_t d = 0; d < numDevs; d++) { @@ -640,17 +688,15 @@ ncclResult_t amd_smi_ensureFabricInitialized() { devInfo->fabricSupported = false; continue; } - if (fabricInfo.fabric_version != AMDSMI_FABRIC_INFO_CURRENT_VERSION) { - WARN("AMD SMI fabric: unexpected fabric info version %u for device %u, expected %u", - fabricInfo.fabric_version, d, AMDSMI_FABRIC_INFO_CURRENT_VERSION); + const uint32_t fabricVersion = amdSmiFabricInfoVersion(fabricInfo); + if (!amdSmiFabricVersionUsable(fabricVersion)) { + WARN("AMD SMI fabric: unexpected fabric info version %u for device %u, expected %u", fabricVersion, d, + AMDSMI_FABRIC_INFO_CURRENT_VERSION); devInfo->fabricSupported = false; continue; } - const amdsmi_fabric_info_v1_t* v1 = &fabricInfo.fabric_info.v1; - devInfo->fabricSupported = - ((v1->fabric_type == AMDSMI_FABRIC_TYPE_UALOE || v1->fabric_type == AMDSMI_FABRIC_TYPE_UALLINK) && - (v1->accel_state == AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_ACTIVE || - v1->accel_state == AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY)); + const amdsmi_fabric_info_v1_t* v1 = amdSmiFabricInfoV1(fabricInfo); + devInfo->fabricSupported = amdSmiFabricStateUsable(v1->fabric_type, v1->accel_state); devInfo->fabricType = v1->fabric_type; devInfo->state = v1->accel_state; devInfo->acceleratorId = v1->accelerator_id; @@ -762,6 +808,13 @@ const char* amd_smi_fabricTelemIdToString(uint64_t telemId) { if (pfn_amdsmi_fabric_telem_id_to_string == nullptr) { return ARSMI_fabric_telem_id_to_string(telemId); } + if (!amdSmiTelemIdUsesOutParam(amdSmiLibMajor.load(std::memory_order_acquire))) { + // dlsym gave us an address, not a prototype: on a pre-27 runtime the symbol has + // to be called through its own signature or the extra argument is read as one. + auto legacyFn = reinterpret_cast(pfn_amdsmi_fabric_telem_id_to_string); + const char* legacyName = legacyFn(telemId); + return legacyName == nullptr ? "UNKNOWN" : legacyName; + } const char* telemName = nullptr; amdsmi_status_t ret = pfn_amdsmi_fabric_telem_id_to_string(telemId, &telemName); if (ret != AMDSMI_STATUS_SUCCESS || telemName == nullptr) { diff --git a/projects/rccl/test/AmdSmiFabricTests.cpp b/projects/rccl/test/AmdSmiFabricTests.cpp new file mode 100644 index 000000000000..fc59032188ab --- /dev/null +++ b/projects/rccl/test/AmdSmiFabricTests.cpp @@ -0,0 +1,306 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "amdsmi_wrap.h" + +#include +#include +#include + +namespace RcclUnitTesting +{ + +// --------------------------------------------------------------------------- +// Fabric info version acceptance +// +// amdsmi_get_gpu_fabric_info() returns SUCCESS with a fully populated payload +// but never writes the version field, leaving it at the sentinel amd_smi uses +// for anything it could not source from sysfs. Rejecting that sentinel made +// RCCL_USE_AMD_SMI_LIB=1 discard working UALoE fabric on every device. +// --------------------------------------------------------------------------- + +TEST(AmdSmiFabricVersion, AcceptsUnreportedSentinel) +{ + EXPECT_TRUE(amdSmiFabricVersionUsable(kAmdSmiFabricVersionUnreported)); +} + +TEST(AmdSmiFabricVersion, AcceptsCurrentVersion) +{ + EXPECT_TRUE(amdSmiFabricVersionUsable(AMDSMI_FABRIC_INFO_CURRENT_VERSION)); +} + +// A version the library actually claims, but whose union layout we do not +// implement, must still be refused rather than misparsed as v1. +TEST(AmdSmiFabricVersion, RejectsUnimplementedFutureVersion) +{ + EXPECT_FALSE(amdSmiFabricVersionUsable(AMDSMI_FABRIC_INFO_CURRENT_VERSION + 1)); +} + +// Distinct from the sentinel: zero means the library returned without writing +// the field at all, which contradicts a SUCCESS status and is worth surfacing. +TEST(AmdSmiFabricVersion, RejectsZero) +{ + EXPECT_FALSE(amdSmiFabricVersionUsable(0)); +} + +// --------------------------------------------------------------------------- +// Fabric type / accelerator state eligibility +// +// This is the decision the version check was gating: only a link type we drive +// combined with a state whose vPoD can carry traffic counts as usable. +// --------------------------------------------------------------------------- + +TEST(AmdSmiFabricState, UaloeReadyIsUsable) +{ + EXPECT_TRUE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UALOE, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY)); +} + +TEST(AmdSmiFabricState, UaloeActiveIsUsable) +{ + EXPECT_TRUE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UALOE, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_ACTIVE)); +} + +TEST(AmdSmiFabricState, UalinkReadyIsUsable) +{ + EXPECT_TRUE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UALLINK, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY)); +} + +TEST(AmdSmiFabricState, ConfiguredButNotReadyIsNotUsable) +{ + EXPECT_FALSE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UALOE, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_CONFIGURED)); +} + +TEST(AmdSmiFabricState, UnconfiguredIsNotUsable) +{ + EXPECT_FALSE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UALOE, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_UNCONFIGURED)); +} + +TEST(AmdSmiFabricState, ErrorStateIsNotUsable) +{ + EXPECT_FALSE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UALOE, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_ERROR)); +} + +TEST(AmdSmiFabricState, UnknownLinkTypeIsNotUsable) +{ + EXPECT_FALSE(amdSmiFabricStateUsable(AMDSMI_FABRIC_TYPE_UNKNOWN, + AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_ACTIVE)); +} + +// The combination this node actually reports, and that the version check used +// to throw away: UALoE + READY reached through the full struct. +TEST(AmdSmiFabricState, PayloadFromSysfsBackedDeviceIsUsable) +{ + amdsmi_fabric_info_v1_t v1{}; + v1.fabric_type = AMDSMI_FABRIC_TYPE_UALOE; + v1.accel_state = AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY; + v1.accelerator_id = 3; + v1.ppod_size = 8; + v1.vpod_id = 1; + v1.vpod_size = 4; + + EXPECT_TRUE(amdSmiFabricVersionUsable(kAmdSmiFabricVersionUnreported)); + EXPECT_TRUE(amdSmiFabricStateUsable(v1.fabric_type, v1.accel_state)); +} + +// --------------------------------------------------------------------------- +// Struct layout +// +// The library writes these structs through dlopen using its own layout, so a +// declaration that disagrees overflows the caller's object and shifts the +// fields we read. amdsmi_wrap.h accepts the coherent 8-GPU layout used by some +// pre-release ROCm 7.14 snapshots and the 16-GPU layout used by final ROCm 7.14 +// and ROCm 7.15, but rejects any other combination. +// --------------------------------------------------------------------------- + +TEST(AmdSmiFabricLayout, MatchesShippedLibraryAbi) +{ + EXPECT_TRUE(amdSmiFabricLayoutIs8Gpu || amdSmiFabricLayoutIs16Gpu); + EXPECT_NE(amdSmiFabricLayoutIs8Gpu, amdSmiFabricLayoutIs16Gpu); +} + +TEST(AmdSmiFabricLayout, TrailingFieldsAreNotShifted) +{ + const size_t expectedAddrMode = amdSmiFabricLayoutIs8Gpu ? 204u : 236u; + EXPECT_EQ(offsetof(amdsmi_fabric_info_v1_t, addr_mode), expectedAddrMode); + EXPECT_EQ(offsetof(amdsmi_fabric_info_v1_t, accel_state), expectedAddrMode + sizeof(uint32_t)); +} + +TEST(AmdSmiFabricRuntimeLayout, DetectsEightGpuWriter) +{ + amdSmiFabricInfoBuffer buffer; + amdSmiPrepareFabricInfoBuffer(buffer); + memset(buffer.bytes, 0, kAmdSmiFabricInfo8GpuSize); + + EXPECT_EQ(amdSmiDetectFabricRuntimeLayout(buffer), amdSmiFabricRuntimeLayout::EightGpu); +} + +TEST(AmdSmiFabricRuntimeLayout, DetectsSixteenGpuWriter) +{ + amdSmiFabricInfoBuffer buffer; + amdSmiPrepareFabricInfoBuffer(buffer); + memset(buffer.bytes, 0, kAmdSmiFabricInfo16GpuSize); + + EXPECT_EQ(amdSmiDetectFabricRuntimeLayout(buffer), amdSmiFabricRuntimeLayout::SixteenGpu); +} + +TEST(AmdSmiFabricRuntimeLayout, RejectsUnknownWriter) +{ + amdSmiFabricInfoBuffer buffer; + amdSmiPrepareFabricInfoBuffer(buffer); + buffer.bytes[kAmdSmiFabricInfo8GpuSize] = 0; + + EXPECT_EQ(amdSmiDetectFabricRuntimeLayout(buffer), amdSmiFabricRuntimeLayout::Unknown); +} + +// --------------------------------------------------------------------------- +// Both amdsmi_fabric_info_t shapes +// +// amd_smi 27.0 flattened the struct, so RCCL has to read a version and a v1 +// payload out of either shape. Only one of them exists in any given build, so +// stand both up as mocks here: that is the only way to compile and exercise the +// branch the installed header does not select. +// --------------------------------------------------------------------------- + +namespace +{ +union MockFabricPayload +{ + amdsmi_fabric_info_v1_t v1; +}; + +// Pre-27: version and payload sit inside fabric_info, where the union carries +// the fabric_version name. +struct MockNestedFabricVer +{ + uint32_t version; + MockFabricPayload fabric_version; +}; + +struct MockNestedFabricInfo +{ + amdsmi_bdf_t bdf; + MockNestedFabricVer fabric_info; + uint32_t reserved[15]; +}; + +// 27.0 and later: version hoisted to the top level, payload reachable directly. +struct MockFlatFabricInfo +{ + amdsmi_bdf_t bdf; + uint32_t fabric_version; + MockFabricPayload fabric_info; + uint32_t reserved[15]; +}; + +// A top-level fabric_version that is not the version number, which is what the +// name means in the nested shape. Detection keys on the type, not the name. +struct MockHoistedUnionFabricInfo +{ + amdsmi_bdf_t bdf; + MockFabricPayload fabric_version; +}; +} // namespace + +TEST(AmdSmiFabricLayoutCompat, DetectsEachShape) +{ + EXPECT_FALSE(amdSmiFabricInfoIsFlat::value); + EXPECT_TRUE(amdSmiFabricInfoIsFlat::value); + EXPECT_FALSE(amdSmiFabricInfoIsFlat::value); +} + +TEST(AmdSmiFabricLayoutCompat, ReadsVersionFromNestedShape) +{ + MockNestedFabricInfo info{}; + info.fabric_info.version = kAmdSmiFabricVersionUnreported; + + EXPECT_EQ(amdSmiFabricInfoVersion(info), kAmdSmiFabricVersionUnreported); + EXPECT_TRUE(amdSmiFabricVersionUsable(amdSmiFabricInfoVersion(info))); +} + +TEST(AmdSmiFabricLayoutCompat, ReadsVersionFromFlatShape) +{ + MockFlatFabricInfo info{}; + info.fabric_version = kAmdSmiFabricVersionUnreported; + + EXPECT_EQ(amdSmiFabricInfoVersion(info), kAmdSmiFabricVersionUnreported); + EXPECT_TRUE(amdSmiFabricVersionUsable(amdSmiFabricInfoVersion(info))); +} + +TEST(AmdSmiFabricLayoutCompat, ReadsPayloadFromNestedShape) +{ + MockNestedFabricInfo info{}; + info.fabric_info.fabric_version.v1.fabric_type = AMDSMI_FABRIC_TYPE_UALOE; + info.fabric_info.fabric_version.v1.accel_state = AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY; + info.fabric_info.fabric_version.v1.accelerator_id = 3; + + const amdsmi_fabric_info_v1_t* v1 = amdSmiFabricInfoV1(info); + ASSERT_NE(v1, nullptr); + EXPECT_EQ(v1->accelerator_id, 3u); + EXPECT_TRUE(amdSmiFabricStateUsable(v1->fabric_type, v1->accel_state)); +} + +TEST(AmdSmiFabricLayoutCompat, ReadsPayloadFromFlatShape) +{ + MockFlatFabricInfo info{}; + info.fabric_info.v1.fabric_type = AMDSMI_FABRIC_TYPE_UALOE; + info.fabric_info.v1.accel_state = AMDSMI_FABRIC_ACCELERATOR_VPOD_STATE_READY; + info.fabric_info.v1.accelerator_id = 3; + + const amdsmi_fabric_info_v1_t* v1 = amdSmiFabricInfoV1(info); + ASSERT_NE(v1, nullptr); + EXPECT_EQ(v1->accelerator_id, 3u); + EXPECT_TRUE(amdSmiFabricStateUsable(v1->fabric_type, v1->accel_state)); +} + +// The rename did not move anything, which is why one set of ABI asserts covers +// both headers. If a future layout change breaks this, the accessors are no +// longer enough on their own. +TEST(AmdSmiFabricLayoutCompat, BothShapesShareOneAbi) +{ + EXPECT_EQ(sizeof(MockNestedFabricInfo), sizeof(MockFlatFabricInfo)); + EXPECT_EQ(sizeof(MockFlatFabricInfo), sizeof(amdsmi_fabric_info_t)); + + EXPECT_EQ(offsetof(MockNestedFabricInfo, fabric_info) + offsetof(MockNestedFabricVer, version), + offsetof(MockFlatFabricInfo, fabric_version)); + EXPECT_EQ(offsetof(MockNestedFabricInfo, fabric_info) + offsetof(MockNestedFabricVer, fabric_version), + offsetof(MockFlatFabricInfo, fabric_info)); + EXPECT_EQ(offsetof(MockNestedFabricInfo, reserved), offsetof(MockFlatFabricInfo, reserved)); +} + +// --------------------------------------------------------------------------- +// Telemetry name call convention +// +// amdsmi_fabric_telem_id_to_string swapped its return value for an out-parameter +// in 27.0, and we reach it through dlopen, so the version the loaded runtime +// reports decides how it may be called. +// --------------------------------------------------------------------------- + +TEST(AmdSmiFabricTelemAbi, PreflattenRuntimeUsesReturnValue) +{ + EXPECT_FALSE(amdSmiTelemIdUsesOutParam(26)); +} + +TEST(AmdSmiFabricTelemAbi, FlattenedRuntimeUsesOutParam) +{ + EXPECT_TRUE(amdSmiTelemIdUsesOutParam(kAmdSmiTelemOutParamMajor)); + EXPECT_TRUE(amdSmiTelemIdUsesOutParam(kAmdSmiTelemOutParamMajor + 1)); +} + +// Guessing wrong is not symmetric: the out-parameter form called on an older +// runtime just leaves the name unwritten, while the older form called on a newer +// runtime hands it an uninitialized pointer to write through. +TEST(AmdSmiFabricTelemAbi, UnknownVersionTakesTheSaferConvention) +{ + EXPECT_TRUE(amdSmiTelemIdUsesOutParam(kAmdSmiLibVersionUnknown)); +} + +} // namespace RcclUnitTesting diff --git a/projects/rccl/test/AmdSmiLifecycleStub.cpp b/projects/rccl/test/AmdSmiLifecycleStub.cpp new file mode 100644 index 000000000000..f6d50fedf941 --- /dev/null +++ b/projects/rccl/test/AmdSmiLifecycleStub.cpp @@ -0,0 +1,85 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include +#include +#include +#include +#include + +namespace +{ + +constexpr int kAmdSmiSuccess = 0; +constexpr int kAmdSmiUnknownError = -1; + +struct AmdSmiVersion +{ + uint32_t major; + uint32_t minor; + uint32_t release; + const char* build; +}; + +std::atomic initCount{0}; +std::atomic versionCount{0}; +std::atomic shutdownCount{0}; + +bool envEnabled(const char* name) +{ + const char* value = std::getenv(name); + return value != nullptr && value[0] == '1'; +} + +} // namespace + +extern "C" int amdsmi_init(uint64_t) +{ + ++initCount; + if(envEnabled("RCCL_AMDSMI_TEST_DELAY_INIT")) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + return envEnabled("RCCL_AMDSMI_TEST_FAIL_INIT") ? kAmdSmiUnknownError : kAmdSmiSuccess; +} + +extern "C" int amdsmi_get_lib_version(AmdSmiVersion* version) +{ + ++versionCount; + if(envEnabled("RCCL_AMDSMI_TEST_FAIL_VERSION")) + return kAmdSmiUnknownError; + + version->major = 0; + version->minor = 0; + version->release = 0; + version->build = "test"; + return kAmdSmiSuccess; +} + +extern "C" int amdsmi_shut_down() +{ + ++shutdownCount; + return kAmdSmiSuccess; +} + +extern "C" int amdsmi_status_code_to_string(int, const char** statusString) +{ + *statusString = "injected amd_smi failure"; + return kAmdSmiSuccess; +} + +extern "C" unsigned amdsmi_test_init_count() +{ + return initCount.load(); +} + +extern "C" unsigned amdsmi_test_version_count() +{ + return versionCount.load(); +} + +extern "C" unsigned amdsmi_test_shutdown_count() +{ + return shutdownCount.load(); +} diff --git a/projects/rccl/test/AmdSmiWrapTests.cpp b/projects/rccl/test/AmdSmiWrapTests.cpp new file mode 100644 index 000000000000..3700d0d186ff --- /dev/null +++ b/projects/rccl/test/AmdSmiWrapTests.cpp @@ -0,0 +1,343 @@ +/************************************************************************* + * Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Exercises the amdsmi_wrap API against the machine's real GPUs. +// +// These call into librccl rather than recompiling the wrapper, so they only +// link in Debug: src/CMakeLists.txt exports the internal amd_smi_* symbols +// under -fvisibility=default there and hides them everywhere else. +// +// Every case skips rather than fails when what it needs is absent, so the +// suite stays meaningful on a machine with no GPUs, no amd_smi library, or no +// fabric. + +#include "amdsmi_wrap.h" +#include "alt_rsmi.h" +#include "common/ProcessIsolatedTestRunner.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace RcclUnitTesting +{ + +namespace +{ + +using StubCounterFn = unsigned (*)(); + +std::string lifecycleStubLibraryPath() +{ + std::string path = AMDSMI_TEST_STUB_DIR; + if(const char* inheritedPath = std::getenv("LD_LIBRARY_PATH")) + path += ":" + std::string(inheritedPath); + return path; +} + +unsigned lifecycleStubCounter(const char* symbol) +{ + void* handle = dlopen(AMDSMI_TEST_STUB_SONAME, RTLD_NOW | RTLD_NOLOAD); + if(handle == nullptr) + { + ADD_FAILURE() << "lifecycle stub is not loaded: " << dlerror(); + return 0; + } + + dlerror(); + auto counter = reinterpret_cast(dlsym(handle, symbol)); + if(const char* error = dlerror()) + { + ADD_FAILURE() << "failed to resolve " << symbol << ": " << error; + dlclose(handle); + return 0; + } + + const unsigned value = counter(); + dlclose(handle); + return value; +} + +// amd_smi_init() succeeds on the sysfs fallback path too, so a failure here +// means neither backend is usable and there is nothing to assert against. +class AmdSmiWrapTest : public ::testing::Test +{ +protected: + void SetUp() override + { + if(amd_smi_init() != ncclSuccess) + GTEST_SKIP() << "amd_smi_init() failed: no usable SMI backend"; + initialized_ = true; + + if(amd_smi_getNumDevice(&numDevices_) != ncclSuccess) + GTEST_SKIP() << "amd_smi_getNumDevice() failed"; + } + + void TearDown() override + { + if(initialized_) + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); + } + + void requireDevices(uint32_t count) + { + if(numDevices_ < count) + GTEST_SKIP() << "needs " << count << " device(s), found " << numDevices_; + } + + uint32_t numDevices_ = 0; + bool initialized_ = false; +}; + +} // namespace + +TEST(AmdSmiWrapLifecycle, InitAndShutdownSucceed) +{ + if(amd_smi_init() != ncclSuccess) + GTEST_SKIP() << "amd_smi_init() failed: no usable SMI backend"; + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); +} + +// Re-initializing after a shutdown has to work: RCCL brings the wrapper up per +// communicator, so a one-shot init would break every rank after the first. +TEST(AmdSmiWrapLifecycle, InitIsRepeatable) +{ + if(amd_smi_init() != ncclSuccess) + GTEST_SKIP() << "amd_smi_init() failed: no usable SMI backend"; + ASSERT_EQ(amd_smi_shutdown(), ncclSuccess); + + EXPECT_EQ(amd_smi_init(), ncclSuccess); + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); +} + +TEST(AmdSmiWrapLifecycle, ConcurrentCallersReceiveCachedFailure) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "ConcurrentCallersReceiveCachedFailure", + []() { + constexpr size_t kThreadCount = 16; + std::vector results(kThreadCount, ncclSuccess); + std::vector threads; + threads.reserve(kThreadCount); + + for(size_t i = 0; i < kThreadCount; ++i) + threads.emplace_back([&, i]() { results[i] = amd_smi_init(); }); + for(auto& thread : threads) + thread.join(); + + for(size_t i = 0; i < results.size(); ++i) + EXPECT_EQ(results[i], ncclInternalError) << "caller " << i; + + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_init_count"), 1u); + EXPECT_EQ(amd_smi_init(), ncclInternalError); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_init_count"), 1u); + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_shutdown_count"), 0u); + }, + {{"RCCL_USE_AMD_SMI_LIB", "1"}, + {"RCCL_AMDSMI_TEST_DELAY_INIT", "1"}, + {"RCCL_AMDSMI_TEST_FAIL_INIT", "1"}, + {"LD_LIBRARY_PATH", lifecycleStubLibraryPath()}} + ); +} + +TEST(AmdSmiWrapLifecycle, ShutdownCleansUpAfterVersionFailure) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "ShutdownCleansUpAfterVersionFailure", + []() { + EXPECT_EQ(amd_smi_init(), ncclInternalError); + EXPECT_EQ(amd_smi_init(), ncclInternalError); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_init_count"), 1u); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_version_count"), 1u); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_shutdown_count"), 0u); + + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_shutdown_count"), 1u); + + // Shutdown clears the cached failure and permits a fresh attempt. + EXPECT_EQ(amd_smi_init(), ncclInternalError); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_init_count"), 2u); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_version_count"), 2u); + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); + EXPECT_EQ(lifecycleStubCounter("amdsmi_test_shutdown_count"), 2u); + }, + {{"RCCL_USE_AMD_SMI_LIB", "1"}, + {"RCCL_AMDSMI_TEST_FAIL_VERSION", "1"}, + {"LD_LIBRARY_PATH", lifecycleStubLibraryPath()}} + ); +} + +TEST_F(AmdSmiWrapTest, PciBusIdIsPopulatedForEveryDevice) +{ + requireDevices(1); + + for(uint32_t i = 0; i < numDevices_; i++) + { + char busId[32] = {0}; + ASSERT_EQ(amd_smi_getDevicePciBusIdString(i, busId, sizeof(busId)), ncclSuccess) + << "device " << i; + EXPECT_GT(strlen(busId), 0u) << "device " << i << " reported an empty bus ID"; + } +} + +// The topology code identifies GPUs by bus ID, so the mapping has to be a +// bijection: every device's own bus ID must resolve back to that same index. +TEST_F(AmdSmiWrapTest, PciBusIdRoundTripsToDeviceIndex) +{ + requireDevices(1); + + for(uint32_t i = 0; i < numDevices_; i++) + { + char busId[32] = {0}; + ASSERT_EQ(amd_smi_getDevicePciBusIdString(i, busId, sizeof(busId)), ncclSuccess); + + uint32_t deviceIndex = 0xFFFFFFFFu; + ASSERT_EQ(amd_smi_getDeviceIndexByPciBusId(busId, &deviceIndex), ncclSuccess) + << "bus ID " << busId; + EXPECT_EQ(deviceIndex, i) << "bus ID " << busId << " resolved to the wrong device"; + } +} + +TEST_F(AmdSmiWrapTest, LinkInfoIsReportedBetweenTwoDevices) +{ + requireDevices(2); + + amdsmi_link_type_t linkType; + int hops = -1; + int count = -1; + ASSERT_EQ(amd_smi_getLinkInfo(0, 1, &linkType, &hops, &count), ncclSuccess); + + EXPECT_GE(hops, 0); + EXPECT_GE(count, 0); +} + +TEST_F(AmdSmiWrapTest, FirmwareVersionQuerySucceeds) +{ + requireDevices(1); + + // Reported as 0 when the backend cannot source it; only the call contract + // is under test here. + uint64_t fwVersion = 0; + EXPECT_EQ(amd_smi_getFirmwareVersion(0, &fwVersion), ncclSuccess); +} + +// amd_smi_isFabricSupported() and amd_smi_getFabricBandwidth() both read the +// cache that amd_smi_ensureFabricInitialized() fills, so they must agree with +// the cached record rather than each deriving their own answer. +TEST_F(AmdSmiWrapTest, FabricViewsAgreeWithCachedDeviceInfo) +{ + requireDevices(1); + ASSERT_EQ(amd_smi_ensureFabricInitialized(), ncclSuccess); + + for(uint32_t i = 0; i < numDevices_; i++) + { + struct amdsmiFabricDeviceInfo devInfo; + memset(&devInfo, 0, sizeof(devInfo)); + ASSERT_EQ(amd_smi_getFabricDeviceInfo(i, &devInfo), ncclSuccess) << "device " << i; + + bool supported = false; + ASSERT_EQ(amd_smi_isFabricSupported(i, &supported), ncclSuccess) << "device " << i; + EXPECT_EQ(supported, devInfo.fabricSupported) << "device " << i; + + uint32_t bandwidth = 0; + ASSERT_EQ(amd_smi_getFabricBandwidth(i, &bandwidth), ncclSuccess) << "device " << i; + + if(!supported) + { + // Documented contract: 0 tells the caller to fall back to + // arch-based defaults instead of trusting a fabric number. + EXPECT_EQ(bandwidth, 0u) << "device " << i << " reported bandwidth without fabric"; + continue; + } + + EXPECT_EQ(bandwidth, devInfo.bandwidth) << "device " << i; + // A device is only marked supported when it passed this predicate, so + // the cached type/state must still satisfy it. + EXPECT_TRUE(amdSmiFabricStateUsable(devInfo.fabricType, devInfo.state)) + << "device " << i << " marked supported with type " << devInfo.fabricType + << " state " << devInfo.state; + EXPECT_GT(devInfo.vpodSize, 0u) << "device " << i; + } +} + +// amd_smi_fabricTelemIdToString dispatches on the loaded runtime's major +// version between the pre-27 return-value signature and the 27+ out-param +// signature (see amdSmiTelemIdUsesOutParam in amdsmi_wrap.h). The predicate +// itself is unit-tested in AmdSmiFabricTests; this drives the actual dispatch +// against whichever runtime is really loaded and pins the contract its callers +// depend on: for any id it returns a printable string, never nullptr. A +// regression in the reinterpret_cast legacy path (or the out-param path) would +// otherwise only surface as a crash or garbage log line on real hardware. +TEST_F(AmdSmiWrapTest, FabricTelemIdToStringNeverReturnsNull) +{ + requireDevices(1); + if(amd_smi_ensureFabricInitialized() != ncclSuccess) + GTEST_SKIP() << "fabric not available on this machine"; + + // An id the runtime does not map still has to come back as a printable + // string, because callers log it unconditionally. Whichever call + // convention matched the loaded library, both paths collapse a missing + // name to "UNKNOWN" rather than nullptr. + const char* unknownName = amd_smi_fabricTelemIdToString(~static_cast(0)); + ASSERT_NE(unknownName, nullptr); + EXPECT_GT(std::strlen(unknownName), 0u); +} + +// The regression this suite exists for: RCCL_USE_AMD_SMI_LIB selects between +// amdsmi_get_gpu_fabric_info() and the ualink sysfs nodes, and the two are +// meant to be interchangeable views of one fabric. They silently diverged when +// the library returned SUCCESS without ever writing fabric_info.version and +// RCCL rejected the payload on that basis, so the library path reported no +// fabric on hardware where sysfs plainly showed it. +// +// Runs isolated because RCCL_PARAM caches the env var in a static on first +// read, so the value has to be set before this process starts. +TEST(AmdSmiWrapFabricPaths, LibraryPathMatchesSysfs) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "LibraryPathMatchesSysfs", + []() { + if(amd_smi_init() != ncclSuccess) + GTEST_SKIP() << "amd_smi_init() failed: no usable SMI backend"; + + // Read sysfs first: it decides whether this machine has fabric at + // all, and so whether there is anything to compare against. + ASSERT_EQ(ARSMI_init(), 0); + struct ARSMI_fabricInfo sysfsInfo; + memset(&sysfsInfo, 0, sizeof(sysfsInfo)); + const int sysfsResult = ARSMI_get_fabric_info(0, &sysfsInfo); + if(sysfsResult == ENODEV) + GTEST_SKIP() << "device 0 has no ualink sysfs node: no fabric on this machine"; + ASSERT_EQ(sysfsResult, 0); + + ASSERT_EQ(amd_smi_ensureFabricInitialized(), ncclSuccess); + struct amdsmiFabricDeviceInfo libInfo; + memset(&libInfo, 0, sizeof(libInfo)); + ASSERT_EQ(amd_smi_getFabricDeviceInfo(0, &libInfo), ncclSuccess); + + EXPECT_EQ(libInfo.fabricSupported, sysfsInfo.supported == 1) + << "amd_smi and sysfs disagree on whether device 0 has fabric"; + EXPECT_EQ(static_cast(libInfo.fabricType), static_cast(sysfsInfo.fabric_type)); + EXPECT_EQ(static_cast(libInfo.state), static_cast(sysfsInfo.accel_state)); + EXPECT_EQ(libInfo.acceleratorId, sysfsInfo.accel_id); + EXPECT_EQ(libInfo.ppodSize, sysfsInfo.ppod_size); + EXPECT_EQ(libInfo.cliqueId, sysfsInfo.vpod_id); + EXPECT_EQ(libInfo.vpodSize, sysfsInfo.vpod_size); + + EXPECT_EQ(amd_smi_shutdown(), ncclSuccess); + }, + {{"RCCL_USE_AMD_SMI_LIB", "1"}} + ); +} + +} // namespace RcclUnitTesting diff --git a/projects/rccl/test/CMakeLists.txt b/projects/rccl/test/CMakeLists.txt index 5d8192745940..5bb18c7fe9f4 100644 --- a/projects/rccl/test/CMakeLists.txt +++ b/projects/rccl/test/CMakeLists.txt @@ -414,6 +414,82 @@ if(BUILD_TESTS) endif() endif() + # rccl-UnitTestsAmdSmi: tests for the amdsmi_wrap layer. + # + # AmdSmiFabricTests.cpp covers the fabric predicates and struct layout pinned in + # amdsmi_wrap.h. Those are header-inline, so they need no librccl.so symbols and + # run in both Release and Debug. + # + # AmdSmiWrapTests.cpp drives the amd_smi_* API itself, so it needs symbols that + # src/CMakeLists.txt only exports under -fvisibility=default in Debug. Adding it + # in Release would fail to link, so it joins the target only where they exist. + if (ROCM_VERSION VERSION_GREATER_EQUAL "60400") + list(APPEND RCCL_TEST_EXECUTABLES rccl-UnitTestsAmdSmi) + + # main_altrsmi.cpp is a plain InitGoogleTest/RUN_ALL_TESTS entry point, reused here. + set(TEST_AMDSMI_SOURCE_FILES + AmdSmiFabricTests.cpp + common/main_altrsmi.cpp + ) + + if(CMAKE_BUILD_TYPE MATCHES "Debug") + list(APPEND TEST_AMDSMI_SOURCE_FILES + AmdSmiWrapTests.cpp + common/ProcessIsolatedTestRunner.cpp + ) + + # Loaded only by isolated lifecycle tests, never by the normal test process. + add_library(rccl-AmdSmiLifecycleStub SHARED AmdSmiLifecycleStub.cpp) + set_target_properties(rccl-AmdSmiLifecycleStub PROPERTIES + OUTPUT_NAME "amd_smi" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/amdsmi_lifecycle_stub" + ) + + # Match the versioned name used by amdsmi_wrap when headers are present. + # Older ROCm configurations may not define SMI_INCLUDE_DIR, so also check + # ROCM_PATH; without either header amdsmi_wrap uses the unversioned name. + if(EXISTS "${SMI_INCLUDE_DIR}/amd_smi/amdsmi.h") + set(AMDSMI_TEST_HEADER "${SMI_INCLUDE_DIR}/amd_smi/amdsmi.h") + elseif(EXISTS "${ROCM_PATH}/include/amd_smi/amdsmi.h") + set(AMDSMI_TEST_HEADER "${ROCM_PATH}/include/amd_smi/amdsmi.h") + endif() + if(AMDSMI_TEST_HEADER) + file(STRINGS "${AMDSMI_TEST_HEADER}" AMDSMI_VERSION_MAJOR_LINE + REGEX "^#define AMDSMI_LIB_VERSION_MAJOR [0-9]+" + LIMIT_COUNT 1 + ) + string(REGEX REPLACE ".* ([0-9]+)$" "\\1" + AMDSMI_TEST_SOVERSION "${AMDSMI_VERSION_MAJOR_LINE}" + ) + set_target_properties(rccl-AmdSmiLifecycleStub PROPERTIES + SOVERSION "${AMDSMI_TEST_SOVERSION}" + ) + endif() + endif() + + add_executable(rccl-UnitTestsAmdSmi ${TEST_AMDSMI_SOURCE_FILES}) + + if(CMAKE_BUILD_TYPE MATCHES "Debug") + add_dependencies(rccl-UnitTestsAmdSmi rccl-AmdSmiLifecycleStub) + target_compile_definitions(rccl-UnitTestsAmdSmi PRIVATE + AMDSMI_TEST_STUB_DIR="$" + AMDSMI_TEST_STUB_SONAME="$" + ) + endif() + + # amdsmi_wrap.h needs both halves of this pairing: the include path so its + # __has_include finds amdsmi, and AMDSMI_FABRIC_DIRECT, which is PRIVATE to the + # rccl target and so does not reach test TUs. With the headers reachable but the + # define absent, the header's fabric compat block compiles on top of the real + # declarations and every fabric type collides. + if(DEFINED SMI_INCLUDE_DIR) + target_include_directories(rccl-UnitTestsAmdSmi PRIVATE ${SMI_INCLUDE_DIR}) + endif() + if(AMDSMI_FABRIC_API) + target_compile_definitions(rccl-UnitTestsAmdSmi PRIVATE AMDSMI_FABRIC_DIRECT) + endif() + endif() + # Create separate MPI test binary if MPI tests are enabled if(ENABLE_MPI_TESTS) set(MPI_TEST_SOURCE_FILES @@ -575,6 +651,23 @@ if(BUILD_TESTS) else() message(WARNING "Skipping test categories for rccl-UnitTestsFixtures: missing ${_rccl_test_categories_fixtures_yaml}") endif() + + # Registered outside the Debug-only block below: this target's fabric predicate + # and layout cases need no librccl symbols and so run in Release too. The + # AmdSmiWrap* patterns in the yaml match nothing in a Release binary, where + # that file is left out for lack of exported amd_smi_* symbols. + set(_rccl_test_categories_amdsmi_yaml "${CMAKE_CURRENT_SOURCE_DIR}/test_categories_amdsmi.yaml") + if(EXISTS "${_rccl_test_categories_amdsmi_yaml}") + message(STATUS "Applying test categories for rccl-UnitTestsAmdSmi") + apply_test_category_labels( + rccl-UnitTestsAmdSmi + "${_rccl_test_categories_amdsmi_yaml}" + "${PROJECT_BINARY_DIR}" + "${INSTALL_TEST_FILE}" + ) + else() + message(WARNING "Skipping test categories for rccl-UnitTestsAmdSmi: missing ${_rccl_test_categories_amdsmi_yaml}") + endif() endif() if(ROCM_VERSION VERSION_GREATER_EQUAL "60400" AND CMAKE_BUILD_TYPE MATCHES "Debug") diff --git a/projects/rccl/test/test_categories_amdsmi.yaml b/projects/rccl/test/test_categories_amdsmi.yaml new file mode 100644 index 000000000000..3cdaadcde4ea --- /dev/null +++ b/projects/rccl/test/test_categories_amdsmi.yaml @@ -0,0 +1,62 @@ +# rccl-UnitTestsAmdSmi — shared/ctest schema (ROCm 6.4+, Release and Debug). +# RCCL does not support Windows: exclude_windows matches all tests so generated suites are empty on Windows. +# +# Suites: +# AmdSmiFabric* — pure predicate and struct layout checks, no hardware needed. +# Present in every build type. +# AmdSmiWrap* — drives the amd_smi_* API against real GPUs; each case skips +# when the devices, amd_smi library, or fabric it needs is absent. +# Debug builds only: the internal amd_smi_* symbols are hidden in +# Release, so these patterns match nothing there. +# +# Tiers: +# quick — canonical anchor; NCCL_DEBUG=INFO for library logging. +# standard — merge quick; override labels. +# comprehensive — merge quick; override labels. +# full — merge quick; override labels. + +test_categories: + quick: &amdsmi_quick + description: "Smoke — amdsmi_wrap fabric predicates, struct layout, and wrapper API, short timeout." + test_patterns: + - "AmdSmiFabric*.*" + - "AmdSmiWrap*.*" + exclude_windows: + - "*" + labels: + - "rccl" + - "quick" + - "smoke" + - "precheck" + - "amdsmi" + env_variables: + - "NCCL_DEBUG=INFO" + standard: + <<: *amdsmi_quick + labels: + - "rccl" + - "standard" + - "regression" + - "amdsmi" + comprehensive: + <<: *amdsmi_quick + labels: + - "rccl" + - "comprehensive" + - "comprehensive_regression" + - "amdsmi" + full: + <<: *amdsmi_quick + labels: + - "rccl" + - "full" + - "full_regression" + - "amdsmi" + +execution_settings: + timeout_multiplier: 1 + category_timeouts: + quick: 600 + standard: 600 + comprehensive: 600 + full: 600 diff --git a/projects/rccl/tools/topo_expl/Makefile b/projects/rccl/tools/topo_expl/Makefile index 7ac28afb0435..d15c634fdbef 100644 --- a/projects/rccl/tools/topo_expl/Makefile +++ b/projects/rccl/tools/topo_expl/Makefile @@ -28,14 +28,34 @@ else NCCL_OS_DEFINE := -DNCCL_OS_WINDOWS endif +# topo_expl bypasses RCCL's CMake configuration, so reproduce the two +# compile-time feature probes needed by headers it imports. +HAVE_HIP_FABRIC_API := $(shell printf '%s\n' \ + '#include ' \ + 'int main() { hipMemFabricHandle_t h{}; (void)h; auto fn = &hipMemImportFromShareableHandle; (void)fn; }' | \ + $(HIPCC) -x c++ -std=c++17 -fsyntax-only -D__HIP_PLATFORM_AMD__ -I$(ROCM_PATH)/include - \ + >/dev/null 2>&1 && echo yes || echo no) +ifeq ($(HAVE_HIP_FABRIC_API),yes) +FEATURE_DEFINES += -DHIP_FABRIC_API +endif + +HAVE_AMDSMI_FABRIC_API := $(shell printf '%s\n' \ + '#include ' \ + 'int main() { amdsmi_fabric_info_t info{}; (void)info; }' | \ + $(HIPCC) -x c++ -std=c++17 -fsyntax-only -I$(ROCM_PATH)/include - \ + >/dev/null 2>&1 && echo yes || echo no) +ifeq ($(HAVE_AMDSMI_FABRIC_API),yes) +FEATURE_DEFINES += -DAMDSMI_FABRIC_DIRECT +endif + CXXFLAGS = -g -ffunction-sections -fdata-sections -DROCM_VERSION=$(ROCM_VERSION) \ -std=c++17 \ -Wl,--gc-sections -fgpu-rdc \ -Iinclude -Itest -Ihipify_rccl/include -Ihipify_rccl/include/plugin \ -Ihipify_rccl/include/nccl_device \ - -Ihipify_rccl/src/device/include -Ihipify_rccl/graph -I/opt/rocm/include/ \ + -Ihipify_rccl/src/device/include -Ihipify_rccl/graph -I$(ROCM_PATH)/include/ \ -DTOPO_EXPL -DENABLE_TRACE -DENABLE_LL128 -DNVTX_NO_IMPL -DRCCL_EXPOSE_STATIC \ - $(NCCL_OS_DEFINE) \ + $(NCCL_OS_DEFINE) $(FEATURE_DEFINES) \ -lpthread TOPO_EXPL_FILES = src/$(EXE).cpp src/topo_expl_api.cpp src/model.cpp src/topo_expl_impl.cpp src/utils.cpp src/stubs.cc test/topo_expl_tests.cpp