ci(amdsmi): [ROCM-3941] Python packaging rework

.github/workflows/amdsmi-build.yml

@@ -50,7 +50,6 @@ jobs:
     runs-on:
       - self-hosted
       - ${{ vars.RUNNER_TYPE }}
-    continue-on-error: true
     strategy:
       max-parallel: 10
       fail-fast: false
@@ -70,7 +69,13 @@ jobs:
       options: --rm --privileged --device=/dev/kfd --device=/dev/dri --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --shm-size=64G --cap-add=SYS_MODULE -v /lib/modules:/lib/modules -u root
 
     steps:
+      # rocm-systems is a large monorepo; only the amdsmi project and this
+      # workflow are needed, so sparse-checkout keeps the fetch cheap.
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          sparse-checkout: |
+            projects/amdsmi
+            .github/workflows
 
       - name: Mark workspace safe for git
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
@@ -85,26 +90,16 @@ jobs:
       - name: Set Project Directory
         run: |
           set -euo pipefail
-          TARGET_DIR=$(find $GITHUB_WORKSPACE -path "*/projects/amdsmi/CMakeLists.txt" -exec dirname {} \;)
-          if [ -z "$TARGET_DIR" ]; then
-            TARGET_DIR=$(find $GITHUB_WORKSPACE -maxdepth 2 -name "CMakeLists.txt" -exec dirname {} \; | head -n 1)
-          fi
-          echo "PROJECT_DIR=$TARGET_DIR" >> $GITHUB_ENV
+          # sparse-checkout places the project at a fixed path.
+          echo "PROJECT_DIR=$GITHUB_WORKSPACE/projects/amdsmi" >> $GITHUB_ENV
 
       - name: Run AMDSMI Build and Install
         run: |
           set -euo pipefail
-          SCRIPT=$(find $GITHUB_WORKSPACE -path "*/projects/amdsmi/tests/amdsmi_build/run_amdsmi_build.py" | head -1)
-          if [ -z "$SCRIPT" ]; then
-            echo "ERROR: run_amdsmi_build.py not found"
-            exit 1
-          fi
-          echo "Using runner script: $SCRIPT"
-
           # The script autodetects package manager, format, and distro
           # quirks (Debian10 archived repos, RHEL10/AlmaLinux8 QA_RPATHS,
           # AzureLinux3 more_itertools, etc.) from /etc/os-release.
-          python3 "$SCRIPT" \
+          python3 "${{ env.PROJECT_DIR }}/tests/amdsmi_build/run_amdsmi_build.py" \
             --project-dir "${{ env.PROJECT_DIR }}" \
             --os-label "${{ matrix.os }}" \
             --test-results-dir "/tmp/test-results-${{ matrix.os }}"
@@ -126,10 +121,14 @@ jobs:
           done
 
       - name: Upload Package Artifact
+        # Not consumed by any downstream job; kept so a reviewer can download
+        # the built .deb/.rpm from a PR run for manual testing.
         if: github.event_name == 'pull_request'
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: amd-smi-lib-${{ matrix.package_format }}-${{ matrix.os }}-${{ env.PR_NUMBER }}-${{ env.BUILD_DATE }}
+          # 99999 is the local (non-release) package version CPack stamps when
+          # ROCM_LIBPATCH_VERSION is unset, so it matches every dev-built package.
           path: |
             ${{ env.PROJECT_DIR }}/build/amd-smi-lib*99999-local${{ matrix.arch_suffix }}
             ${{ env.PROJECT_DIR }}/build/amd-smi-lib-*99999-local*.rpm
@@ -192,6 +191,7 @@ jobs:
           cd /opt/rocm/share/amd_smi/tests/python_unittest
           ./integration_test.py -v > "$RESULTS_DIR/integration_test_output.txt" 2>&1
           ./unit_tests.py -v > "$RESULTS_DIR/unit_test_output.txt" 2>&1
+          python3 test_abi_compat.py -v > "$RESULTS_DIR/abi_compat_output.txt" 2>&1
           if [ -f ./perf_tests.py ]; then
             ./perf_tests.py -v > "$RESULTS_DIR/perf_test_output.txt" 2>&1 || echo 'perf_tests.py failed'
           else
@@ -222,6 +222,10 @@ jobs:
         if: always()
         run: cat /tmp/test-results-${{ matrix.os }}/perf_test_output.txt || echo "No perf test results"
 
+      - name: ABI Compat Test Results
+        if: always()
+        run: cat /tmp/test-results-${{ matrix.os }}/abi_compat_output.txt || echo "No ABI compat results"
+
       - name: Example DRM Test Results
         if: always()
         run: cat /tmp/test-results-${{ matrix.os }}/amd_smi_drm_ex.log || echo "No DRM example results"
@@ -234,26 +238,7 @@ jobs:
         if: always()
         run: |
           set -euo pipefail
-          SCRIPT=$(find $GITHUB_WORKSPACE -path "*/projects/amdsmi/tests/amdsmi_build/run_amdsmi_build.py" | head -1)
-          python3 "$SCRIPT" summarize \
+          python3 "${{ env.PROJECT_DIR }}/tests/amdsmi_build/run_amdsmi_build.py" summarize \
             --results-dir "/tmp/test-results-${{ matrix.os }}" \
             --os-label "${{ matrix.os }}" \
             --summary-file "$GITHUB_STEP_SUMMARY"
-
-      - name: Uninstall
-        if: always()
-        run: |
-          set -euo pipefail
-          case "${{ matrix.package_format }}" in
-            deb) apt remove -y amd-smi-lib || true ;;
-            rpm)
-              case "${{ matrix.os }}" in
-                SLES) zypper remove -y amd-smi-lib || true ;;
-                *) dnf remove -y amd-smi-lib || true ;;
-              esac
-              ;;
-          esac
-          rm -f /usr/local/bin/amd-smi
-          if [ -d /opt/rocm/share/amd_smi ]; then
-            rm -rf /opt/rocm/share/amd_smi
-          fi

projects/amdsmi/.gitignore

@@ -36,6 +36,11 @@ device/
 # misc
 esmi_ib_library/
 
+# editor / tooling backup files (keep accidental cruft out of the index)
+*.bak
+*.backup
+*.orig
+
 # do NOT ignore these files
 !.clang-format
 !.clang-tidy

projects/amdsmi/CHANGELOG.md

@@ -97,6 +97,11 @@ Full documentation for amd_smi_lib is available at [https://rocm.docs.amd.com/pr
 
 ### Changed
 
+- **Reworked Python package install layout for system packages and pip wheels**.
+  - System `.deb` / `.rpm` packages no longer run `pip install` during postinst; the `amdsmi` Python package is installed directly into the default system Python's `site-packages`/`dist-packages` (detected from the distro-default interpreter) as part of the package payload, so plain `import amdsmi` works after `apt install` / `dnf install`. Removing the package removes those files. `pip list` will no longer show `amdsmi` for the system package.
+  - The module is additionally staged under `share/amd_smi` (its historical location) for consumers that resolve it via `ROCM_PATH/share/amd_smi` rather than `site-packages` (e.g. rocprofiler-compute, omnistat) and for the TheRock artifact layout, whose manifest captures `share/amd_smi` but not the interpreter's `site-packages`. `site-packages` remains the primary importable location.
+  - Added new CMake option `-DBUILD_PYTHON_WHEEL=ON` (default `OFF`) which builds the standalone Python wheel and an isolated `libamd_smi_python.so` (distinct SONAME) bundled inside it, so the wheel-shipped library can coexist in-process with the system `libamd_smi.so` without symbol collisions. With `-DBUILD_PYTHON_WHEEL=OFF` (the default used by ROCm CI) only the system-package layout is built; no wheel artifact is produced.
+  - `py-interface/amdsmi_wrapper.py` now loads the shared library in this order: the `AMDSMI_LIB_OVERRIDE` env var (development / ABI-test escape hatch), a `libamd_smi_python.so` bundled next to the wrapper (pip wheel), then the system `libamd_smi.so` via the dynamic linker. A `_MissingLibrary` sentinel defers `OSError` to the first API call when no candidate is loadable, so import-time tooling (docs, lint) still works without a runtime library.
 - **Normalized JSON/CSV key casing in `amd-smi metric` clock and temperature sections**.  
   - The `uclk_aid`, `socclks_mid`, and temperature `xcd` keys are now lowercase (`aid_<N>`, `mid_<N>`, `xcp_<N>`) in JSON and CSV output, matching the existing `xcp_<N>` usage keys; they were previously uppercase (`AID_<N>`, `MID_<N>`, `XCP_<N>`).
   - Human-readable output is unchanged, since it uppercases all keys.

projects/amdsmi/CMakeLists.txt

@@ -119,6 +119,17 @@ else()
 endif()
 
 option(BUILD_EXAMPLES "Build examples" OFF)
+option(
+    BUILD_PYTHON_WHEEL
+    "Build the standalone Python wheel and the isolated libamd_smi_python.so it bundles. OFF means only the system-package layout is built (no .whl artifact)."
+    OFF
+)
+if(BUILD_PYTHON_WHEEL AND NOT BUILD_SHARED_LIBS)
+    message(
+        FATAL_ERROR
+        "BUILD_PYTHON_WHEEL=ON requires BUILD_SHARED_LIBS=ON: the wheel ships a .so (libamd_smi_python.so) and the py-interface subdirectory only enters when BUILD_SHARED_LIBS=ON."
+    )
+endif()
 
 # If amdsmi is built as a static library, it should support being embedded in other programs. The setting below essentially enables the -fPIC flag.
 set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "Enable position independent code for all targets")
@@ -467,11 +478,10 @@ add_subdirectory(goamdsmi_shim)
 set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "python3-argcomplete, libdrm-amdgpu-dev")
 set(CPACK_DEBIAN_ASAN_PACKAGE_RECOMMENDS ${CPACK_DEBIAN_PACKAGE_RECOMMENDS})
 set(CPACK_DEBIAN_DEV_PACKAGE_RECOMMENDS ${CPACK_DEBIAN_PACKAGE_RECOMMENDS})
-set(CPACK_DEBIAN_PACKAGE_DEPENDS
-    "libdrm-dev, sudo, libc6, python3 (>= 3.6.8), python3-pip, python3-setuptools, python3-wheel"
-)
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, sudo, libc6, python3 (>= 3.6.8)")
 set(CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS})
 set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS})
+set(CPACK_DEBIAN_TESTS_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS})
 
 # $CURRENT_YEAR is used by copyright.in
 string(TIMESTAMP CURRENT_YEAR "%Y")
@@ -518,7 +528,8 @@ install(
 
 # RPM package specific variables
 set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
-    "${CPACK_PACKAGING_INSTALL_PREFIX} ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}"
+    "${CPACK_PACKAGING_INSTALL_PREFIX}"
+    "${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}"
 )
 #Set rpm distro
 if(CPACK_RPM_PACKAGE_RELEASE)
@@ -529,11 +540,57 @@ endif()
 set(CPACK_RPM_PACKAGE_SUGGESTS "python3-argcomplete, libdrm-amdgpu-devel")
 set(CPACK_RPM_DEV_PACKAGE_SUGGESTS ${CPACK_RPM_PACKAGE_SUGGESTS})
 set(CPACK_RPM_ASAN_PACKAGE_SUGGESTS ${CPACK_RPM_PACKAGE_SUGGESTS})
-# python version gated by rhel8 :(
 # note that RPM based systems have "devel" suffix instead of "dev" in debian based systems
-set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, sudo, python3 >= 3.6.8, python3-pip, python3-wheel, python3-setuptools")
+#
+# The amdsmi Python module installs into a version-specific site-packages path
+# on RPM distros (e.g. /usr/lib64/python3.9/site-packages). py-interface/
+# CMakeLists.txt detects that path from the system's DEFAULT python3
+# (platform-python on RHEL), so the module lands where a plain `import amdsmi`
+# and the `amd-smi` CLI (#!/usr/bin/python3) actually look. This dependency
+# pins the RPM to that same default interpreter's ABI as defense-in-depth: it
+# makes the interpreter dependency explicit and keeps the RPM from installing
+# onto a host missing that interpreter.
+#
+# On RPM distros there is NO version-independent site-packages location --
+# /usr/lib/python3/site-packages (Debian's version-agnostic dir) is not on any
+# RHEL interpreter's sys.path -- so the install path is intrinsically tied to a
+# python minor and the dependency must be too. The canonical RPM expression is
+# `python(abi) = X.Y` (per the Fedora Python Packaging Guidelines): normally
+# rpmbuild's dependency generator emits this automatically for files under
+# %{python3_sitelib}. That auto-generator does NOT fire here -- CPack installs
+# the module as raw .py files (no .dist-info metadata) and the RHEL8 build image
+# lacks the path-based python.attr generator -- so we set the same requirement
+# by hand. The token that expresses "this python minor" differs by distro
+# family, so pick it from /etc/os-release:
+#   * RHEL/CentOS/Fedora/AlmaLinux/AzureLinux: `python(abi) = X.Y`, provided by
+#     the pythonX.Y packages (dnf pulls the matching interpreter in on install).
+#   * SLES/openSUSE: no python(abi) provide; the versioned package is `pythonXY`
+#     (no dot), e.g. python311 (zypper pulls it in on install).
+# Fall back to the loose `python3 >= 3.6.8` when the ABI could not be derived
+# (version-agnostic Debian path) or the distro family is unrecognized.
+set(_amdsmi_py_req "python3 >= 3.6.8")
+if(AMDSMI_SYSTEM_PYTHON_ABI)
+    set(_amdsmi_os_id "")
+    if(EXISTS "/etc/os-release")
+        file(READ "/etc/os-release" _amdsmi_os_release)
+        # ID_LIKE covers derivatives (e.g. almalinux -> "rhel centos fedora").
+        string(REGEX MATCH "ID_LIKE=[\"']?([^\"'\n]+)" _ "${_amdsmi_os_release}")
+        set(_amdsmi_os_id "${CMAKE_MATCH_1}")
+        string(REGEX MATCH "\nID=[\"']?([^\"'\n]+)" _ "\n${_amdsmi_os_release}")
+        string(APPEND _amdsmi_os_id " ${CMAKE_MATCH_1}")
+    endif()
+    if(_amdsmi_os_id MATCHES "rhel|centos|fedora|almalinux|rocky|mariner|azurelinux")
+        set(_amdsmi_py_req "python(abi) = ${AMDSMI_SYSTEM_PYTHON_ABI}")
+    elseif(_amdsmi_os_id MATCHES "suse|sles")
+        string(REPLACE "." "" _amdsmi_abi_nodot "${AMDSMI_SYSTEM_PYTHON_ABI}")
+        set(_amdsmi_py_req "python${_amdsmi_abi_nodot}")
+    endif()
+endif()
+message(STATUS "amdsmi RPM python interpreter dependency: ${_amdsmi_py_req}")
+set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, sudo, ${_amdsmi_py_req}")
 set(CPACK_RPM_DEV_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES})
 set(CPACK_RPM_ASAN_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES})
+set(CPACK_RPM_TESTS_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES})
 
 # don't terminate if bytecompile of python files fails
 set(CPACK_RPM_SPEC_MORE_DEFINE "%define _python_bytecompile_errors_terminate_build 0")

projects/amdsmi/DEBIAN/postinst.in

@@ -8,7 +8,11 @@ do_configureLogrotate() {
 
   mkdir -p "${logPath}"
   touch "${logFile}"
-  chmod -R a+rw "${logPath}"
+  # Relax permissions on the operational log file only so non-root
+  # processes can append to it. Do NOT use chmod -R here: recursing
+  # would also make the log directory and sibling files (e.g.
+  # postinst.log) world-writable, which would let a local user create
+  # or tamper with files under a root-owned log directory (CWE-732).
   chmod a+rw "${logFile}"
 
   if ! command -v logrotate &>/dev/null; then
@@ -17,7 +21,7 @@ do_configureLogrotate() {
     return
   fi
 
-  if [ ! -f $logrotateConfFile ]; then
+  if [ ! -f "$logrotateConfFile" ]; then
     touch "${logrotateConfFile}"
     chmod 644 "${logrotateConfFile}" # root r/w, all others read
     # AMD SMI logging rotation, rotates files using root user/group
@@ -67,101 +71,25 @@ do_ldconfig() {
   fi
 }
 
-do_install_amdsmi_python_lib() {
-  # get python version
-  local python3_minor_version
-  python3_minor_version=$(python3 -c 'import sys;print(sys.version_info.minor)')
-  if [ $? -ne 0 ]; then
-    echo "[WARNING] Could not determine python version. "\
-      "AMD-SMI python library will not be installed."
-    return
-  fi
-
-  # check if python version is supported
-  if [ "$python3_minor_version" -lt 6 ]; then
-    echo "[WARNING] AMD-SMI python library is not "\
-      "supported on python version 3.$python3_minor_version. "\
-      "AMD-SMI python library will not be installed."
-    return
-  fi
-
-  local PREVIOUS_PIP_ROOT_USER_ACTION="$PIP_ROOT_USER_ACTION"
-  export PIP_ROOT_USER_ACTION=ignore
-  # python3.11 requires --break-system-packages
-  local PREVIOUS_PIP_BREAK_SYSTEM_PACKAGES="$PIP_BREAK_SYSTEM_PACKAGES"
-  export PIP_BREAK_SYSTEM_PACKAGES=1
-
-  # Remove old python library
-  local amdsmi_pip_list_output
-  amdsmi_pip_list_output=$(python3 -m pip list --format=columns --disable-pip-version-check)
-  # check pip list output for amdsmi
-  if [[ $amdsmi_pip_list_output == *"amdsmi"* ]]; then
-    echo "Detected old AMD-SMI python library (amdsmi)..."
-    python3 -m pip uninstall amdsmi --yes --quiet --disable-pip-version-check
-    echo "Removed old AMD-SMI python library (amdsmi)..."
-  fi
-
-  # static builds don't include python lib
-  if [ "@BUILD_SHARED_LIBS@" != "ON" ]; then
-    return
-  fi
-
- check_and_install_amdsmi() {
-  local setuptools_version
-  setuptools_version=$(python3 -c 'import setuptools; print(setuptools.__version__)')
-  if [ $? -ne 0 ]; then
-    echo "[WARNING] Could not determine setuptools version. "\
-      "AMD-SMI python library will not be installed."
-    return
-  fi
-
-  # install python library at @CPACK_PACKAGING_INSTALL_PREFIX@/@SHARE_INSTALL_PREFIX@/amdsmi
-  local python_lib_path=@CPACK_PACKAGING_INSTALL_PREFIX@/@SHARE_INSTALL_PREFIX@
-  local amdsmi_python_lib_path="$python_lib_path"
-  local amdsmi_setup_py_path="$python_lib_path/setup.py"
-
-  # Decide installation method based on setuptools version
-  if [[ "$(printf '%s\n' "$setuptools_version" "28.5" | sort -V | head -n1)" == "$setuptools_version" ]]; then
-    echo "[WARNING] Setuptools version is less than 28.5. AMD-SMI will not be installed."
-  elif [[ "$(printf '%s\n' "$setuptools_version" "41.0.1" | sort -V | head -n1)" != "41.0.1" ]]; then
-    echo "Using setup.py for installation due to setuptools version $setuptools_version"
-    python3 "$amdsmi_setup_py_path" install
-  else
-    echo "Using pyproject.toml for installation due to setuptools version $setuptools_version"
-    python3 -m pip install "$amdsmi_python_lib_path" --quiet --disable-pip-version-check --no-build-isolation --no-index
-  fi
-}
-
-  # Call the function
-  check_and_install_amdsmi
-
-  export PIP_ROOT_USER_ACTION="$PREVIOUS_PIP_ROOT_USER_ACTION"
-  export PIP_BREAK_SYSTEM_PACKAGES="$PREVIOUS_PIP_BREAK_SYSTEM_PACKAGES"
-
-  # only try to activate argcomplete if such command exists
-  # python3-argcomplete is recommended but optional, we handle its absence gracefully
+do_activate_argcomplete() {
+  # python3-argcomplete is recommended but optional; activate global bash
+  # completion when its helper is present, otherwise continue gracefully.
   if command -v activate-global-python-argcomplete &>/dev/null; then
-    activate-global-python-argcomplete 2>/dev/null || {
-      echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete"
-    }
+    activate-global-python-argcomplete 2>/dev/null || \
+      echo "[INFO] Bash completion activation skipped. Enable it manually with: activate-global-python-argcomplete"
+  elif command -v activate-global-python-argcomplete3 &>/dev/null; then
+    activate-global-python-argcomplete3 2>/dev/null || \
+      echo "[INFO] Bash completion activation skipped. Enable it manually with: activate-global-python-argcomplete3"
   else
-    # try older argcomplete3 version
-    if command -v activate-global-python-argcomplete3 &>/dev/null; then
-      activate-global-python-argcomplete3 2>/dev/null || {
-        echo "[INFO] Bash completion activation skipped. You can manually enable it with: activate-global-python-argcomplete3"
-      }
-    else
-      echo "[WARNING] Could not find argcomplete activation command. "\
-        "Argument completion will not work. Install python3-argcomplete package to enable it."
-    fi
+    echo "[WARNING] Could not find argcomplete activation command."\
+    "Argument completion will not work. Install python3-argcomplete to enable it."
   fi
 }
 
-
 case "$1" in
   ( configure )
-    do_install_amdsmi_python_lib
     do_ldconfig
+    do_activate_argcomplete
     do_configureLogrotate || exit 0
   ;;
   ( abort-upgrade | abort-remove | abort-deconfigure )