Skip to content
2 changes: 1 addition & 1 deletion projects/composablekernel/script/tools/ck-docker
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ cmd_status() {
docker ps --filter "name=^${name}$" --format "table {{.Names}}\t{{.Status}}\t{{.Image}}"
echo ""
echo "GPU Information:"
docker exec "${name}" bash -c "rocm-smi --showproductname 2>/dev/null | head -10 || echo 'No GPU detected'"
ck_smi_show_gpu_info_in_container "${name}" 10
elif container_exists "${name}"; then
echo "Container '${name}' exists but is STOPPED"
echo "Start with: ck-docker start ${name}"
Expand Down
10 changes: 8 additions & 2 deletions projects/composablekernel/script/tools/ck-exec
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,15 @@ Environment:
CK_CONTAINER_NAME - Override default container name

Examples:
ck-exec rocm-smi # Run rocm-smi
ck-exec ck_smi_show_gpu_info # GPU info (prefers amd-smi; falls back to rocm-smi)
ck-exec rocminfo # Run rocminfo
ck-exec ls -la build/bin # List build binaries
ck-exec -w /workspace/build ninja -t commands # Run ninja commands
ck-exec --interactive python3 # Interactive Python session

Common Commands:
ck-exec rocm-smi # Check GPU status
ck-exec ck_smi_show_gpu_info # GPU status via SMI wrapper
ck-exec ck_smi_count_gpus # Check GPU count via SMI wrapper
ck-exec rocminfo \| grep gfx # Check GPU architecture
ck-exec hipcc --version # Check HIP compiler version
ck-exec cmake --version # Check CMake version
Expand Down Expand Up @@ -103,6 +104,11 @@ for arg in "${command_args[@]}"; do
cmd_string="${cmd_string} $(printf '%q' "$arg")"
done

# Source common.sh inside the container (when the project is mounted) so that
# ck_smi_* and other shell helper functions are available to the command.
common_sh="/workspace/script/tools/common.sh"
cmd_string="if [ -f ${common_sh} ]; then source ${common_sh}; fi;${cmd_string}"

# Auto-detect Python commands and enable unbuffered output for live streaming
env_flags=""
for arg in "${command_args[@]}"; do
Expand Down
2 changes: 1 addition & 1 deletion projects/composablekernel/script/tools/ck-rocprof.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ Results stored in `workloads/<name>/`:

- `ck-docker`: Container management
- `rocprof-compute`: AMD GPU profiler v2
- `rocm-smi`: System monitoring
- `ck_smi_*` / `smi_cli.py`: System monitoring

## License

Expand Down
7 changes: 5 additions & 2 deletions projects/composablekernel/script/tools/ck-shell
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ Environment:
Examples:
ck-shell # Open interactive shell
ck-shell my_container # Open shell in specific container
ck-shell -c "rocm-smi" # Execute single command
ck-shell -c "ck_smi_show_gpu_info" # GPU info via SMI wrapper (rocm-smi or amd-smi)
ck-shell -c "cd build && ls bin" # Execute command in build directory

EOF
Expand Down Expand Up @@ -77,7 +77,10 @@ fi
if [ -n "$command" ]; then
echo "Executing: ${command}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
docker exec "${CONTAINER_NAME}" bash -c "${command}"
# Source common.sh inside the container (when the project is mounted) so that
# ck_smi_* and other shell helper functions are available to the command.
common_sh="/workspace/script/tools/common.sh"
docker exec "${CONTAINER_NAME}" bash -c "if [ -f ${common_sh} ]; then source ${common_sh}; fi;${command}"
else
echo "Opening shell in '${CONTAINER_NAME}' (type 'exit' to leave)..."
docker exec -it "${CONTAINER_NAME}" bash
Expand Down
2 changes: 1 addition & 1 deletion projects/composablekernel/script/tools/ck-start
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,4 @@ docker exec "${CONTAINER_NAME}" bash -c "echo 'Working directory:' && pwd"
# Show GPU info
echo ""
echo "GPU Information:"
docker exec "${CONTAINER_NAME}" bash -c "rocm-smi --showproductname 2>/dev/null | head -5 || echo 'No GPU detected'"
ck_smi_show_gpu_info_in_container "${CONTAINER_NAME}" 5
2 changes: 1 addition & 1 deletion projects/composablekernel/script/tools/ck-status
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ Mounts: {{range .Mounts}}
echo ""
echo "GPU Information:"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
docker exec "${CONTAINER_NAME}" bash -c "rocm-smi --showproductname 2>/dev/null | head -10 || echo 'No GPU detected'"
ck_smi_show_gpu_info_in_container "${CONTAINER_NAME}" 10

if [ "$verbose" = true ]; then
echo ""
Expand Down
37 changes: 37 additions & 0 deletions projects/composablekernel/script/tools/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -179,3 +179,40 @@ list_cmake_presets() {
sed -n 's/.*"name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$presets_file" | grep -v '^use-'
fi
}

# ============================================================================
# GPU SMI wrappers (delegate to tile_engine/ops/common/smi_cli.py)
# ============================================================================

_ck_smi_cli() {
local project_root
project_root="$(get_project_root "$(dirname "${BASH_SOURCE[0]}")")"
PYTHONPATH="${project_root}/tile_engine/ops/common:${PYTHONPATH:-}" \
python3 "${project_root}/tile_engine/ops/common/smi_cli.py" "$@"
}

ck_smi_list_gpu_ids() { _ck_smi_cli list-ids; }
ck_smi_count_gpus() { _ck_smi_cli count; }
ck_smi_show_gpu_info() { _ck_smi_cli show-info --head "${1:-10}"; }
ck_smi_check_gpu_available() { _ck_smi_cli check; }
ck_smi_show_version() { _ck_smi_cli show-version; }

# Run ck_smi_show_gpu_info inside a Docker container (project mounted at /workspace)
ck_smi_show_gpu_info_in_container() {
local container="$1"
local head="${2:-10}"

# Prevent command injection via the bash -c string.
if ! [[ "${head}" =~ ^[0-9]+$ ]]; then
head=10
fi

docker exec "${container}" bash -c '
if [ -f /workspace/script/tools/common.sh ]; then
source /workspace/script/tools/common.sh
ck_smi_show_gpu_info "$1" 2>/dev/null || echo "No GPU detected"
else
echo "No GPU detected (project not mounted at /workspace)"
fi
' -- "${head}"
}
Comment thread
Copilot marked this conversation as resolved.
Resolved
119 changes: 119 additions & 0 deletions projects/composablekernel/script/tools/test_ck_smi_helpers.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
#!/bin/bash
# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
# SPDX-License-Identifier: MIT
#
# Live test (pure bash): rocm-smi vs amd-smi normalized output must agree.

set -euo pipefail

failures=0

require_tools() {
local missing=0
for tool in rocm-smi amd-smi; do
if ! command -v "${tool}" &>/dev/null; then
echo "ERROR: ${tool} not found on PATH"
missing=1
fi
done
if [ "${missing}" -ne 0 ]; then
exit 1
fi
}

run_smi() {
local tool=$1
shift
"${tool}" "$@" 2>/dev/null
}

# Parse GPU ids from rocm-smi --showid (GPU[0], GPU[1], ...)
parse_rocm_gpu_ids() {
run_smi rocm-smi --showid \
| grep -oE 'GPU\[[0-9]+\]' \
| sed 's/GPU\[//;s/\]//' \
| sort -nu \
| paste -sd, -
}

# Parse GPU ids from amd-smi list (GPU: 0, GPU: 1, ...)
parse_amd_gpu_ids() {
run_smi amd-smi list \
| grep '^GPU:' \
| sed 's/^GPU:[[:space:]]*//' \
| sort -nu \
| paste -sd, -
}

parse_rocm_product() {
run_smi rocm-smi --showproductname \
| sed -n 's/.*Card Series:[[:space:]]*\(.*\)/\1/p' \
| head -1
}

parse_amd_product() {
run_smi amd-smi static \
| sed -n 's/.*MARKET_NAME:[[:space:]]*\(.*\)/\1/p' \
| head -1
}

parse_rocm_gfx() {
run_smi rocm-smi --showproductname \
| sed -n 's/.*GFX Version:[[:space:]]*\([^[:space:]]*\).*/\1/p' \
| head -1
}

parse_amd_gfx() {
run_smi amd-smi static \
| sed -n 's/.*TARGET_GRAPHICS_VERSION:[[:space:]]*\([^[:space:]]*\).*/\1/p' \
| head -1
}

parse_rocm_driver() {
run_smi rocm-smi --showdriverversion \
| sed -n 's/.*Driver version:[[:space:]]*\([^[:space:]]*\).*/\1/p' \
| head -1
}

parse_amd_driver() {
local ver
ver=$(run_smi amd-smi version \
| sed -n 's/.*amdgpu version:[[:space:]]*\([^[:space:]|]*\).*/\1/p' \
| head -1)
if [ -n "${ver}" ]; then
echo "${ver}"
return
fi
run_smi amd-smi static -d \
| sed -n 's/.*VERSION:[[:space:]]*\([^[:space:]]*\).*/\1/p' \
| head -1
}

compare_field() {
local name=$1
local rocm_val=$2
local amd_val=$3
if [ "${rocm_val}" = "${amd_val}" ]; then
printf ' %-8s rocm-smi=%s amd-smi=%s OK\n' "${name}" "${rocm_val}" "${amd_val}"
else
printf ' %-8s rocm-smi=%s amd-smi=%s FAIL\n' "${name}" "${rocm_val}" "${amd_val}"
failures=$((failures + 1))
fi
}

require_tools

echo "rocm-smi vs amd-smi (live comparison, pure bash):"
echo "---------------------------------------------------"

compare_field "gpu_ids" "$(parse_rocm_gpu_ids)" "$(parse_amd_gpu_ids)"
compare_field "product" "$(parse_rocm_product)" "$(parse_amd_product)"
compare_field "gfx" "$(parse_rocm_gfx)" "$(parse_amd_gfx)"
compare_field "driver" "$(parse_rocm_driver)" "$(parse_amd_driver)"

echo ""
if [ "${failures}" -ne 0 ]; then
echo "${failures} field(s) mismatched."
exit 1
fi
echo "All fields match."
26 changes: 15 additions & 11 deletions projects/composablekernel/test_data/generate_test_dataset.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
set -e # Exit on error
set +x # Disable command echo (even if called with bash -x)

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../script/tools/common.sh
source "${SCRIPT_DIR}/../script/tools/common.sh"

# Trap to kill all background jobs on script exit/interruption
cleanup() {
echo ""
Expand Down Expand Up @@ -115,7 +119,7 @@ done
# Setup GPU array if GPUs are requested
if [ $NUM_GPUS -gt 0 ]; then
# Auto-detect available GPUs
AVAILABLE_GPUS_COUNT=$(rocm-smi --showid 2>/dev/null | grep -oP 'GPU\[\K[0-9]+' | wc -l)
AVAILABLE_GPUS_COUNT=$(ck_smi_count_gpus 2>/dev/null || echo 0)
if [ "$AVAILABLE_GPUS_COUNT" -gt 0 ]; then
MAX_AVAILABLE=$AVAILABLE_GPUS_COUNT
else
Expand Down Expand Up @@ -175,28 +179,28 @@ fi


# Check if running on GPU
if ! command -v rocm-smi &> /dev/null; then
echo "ERROR: ROCm not detected. Cannot generate MIOpen commands without GPU."
echo "This script requires an AMD GPU with ROCm installed."
if ! ck_smi_check_gpu_available; then
echo "ERROR: ROCm GPU not accessible. Cannot generate MIOpen commands without GPU."
echo "This script requires an AMD GPU with ROCm installed (rocm-smi or amd-smi)."
echo "Creating empty CSV files as placeholder..."
echo "# 2D Convolution Test Cases (No GPU available)" > conv_test_set_2d_dataset.csv
echo "# 3D Convolution Test Cases (No GPU available)" > conv_test_set_3d_dataset.csv
exit 1
fi

# Check if GPU is actually accessible
if ! rocm-smi &> /dev/null; then
echo "ERROR: rocm-smi failed. GPU may not be accessible."
echo "GPU detected. Driver / ROCm info:"
ck_smi_show_version || true

# Confirm at least one GPU is accessible via SMI before running workloads
GPU_COUNT=$(ck_smi_count_gpus 2>/dev/null || echo 0)
if [ "${GPU_COUNT}" -le 0 ] || ! ck_smi_check_gpu_available; then
echo "ERROR: GPU not accessible. Cannot generate MIOpen commands without a working GPU."
echo "Creating empty CSV files as placeholder..."
echo "# 2D Convolution Test Cases (GPU not accessible)" > conv_test_set_2d_dataset.csv
echo "# 3D Convolution Test Cases (GPU not accessible)" > conv_test_set_3d_dataset.csv
exit 1
fi

echo "GPU detected. ROCm version:"
rocm-smi --showdriverversion || true


echo ""
echo "Step 2: Running 2D/3D models and capturing MIOpen commands"
echo "-----------------------------------------"
Expand Down
66 changes: 66 additions & 0 deletions projects/composablekernel/tile_engine/ops/common/smi_cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
# SPDX-License-Identifier: MIT
"""CLI entry point for shell ck_smi_* delegates."""

import argparse
import sys

from smi_utils import (
check_gpu_available,
count_gpus,
detect_gpu_ids,
show_gpu_info,
show_version,
)


def main() -> int:
parser = argparse.ArgumentParser(description="CK GPU SMI wrapper CLI")
sub = parser.add_subparsers(dest="command", required=True)

sub.add_parser("list-ids", help="Print GPU ids one per line")

sub.add_parser("count", help="Print GPU count")

show_info = sub.add_parser("show-info", help="Print GPU product/static info")
show_info.add_argument(
"--head",
type=int,
default=10,
help="Max lines to print (0 for full output)",
)

sub.add_parser("check", help="Exit 0 if GPU is accessible via SMI")

sub.add_parser("show-version", help="Print driver version info")

args = parser.parse_args()

if args.command == "list-ids":
for gpu_id in detect_gpu_ids():
print(gpu_id)
return 0

if args.command == "count":
print(count_gpus())
return 0

if args.command == "show-info":
head = None if args.head == 0 else args.head
print(show_gpu_info(head=head))
return 0

if args.command == "check":
return 0 if check_gpu_available() else 1

if args.command == "show-version":
print(show_version())
return 0

parser.error(f"unknown command: {args.command}")
return 2


if __name__ == "__main__":
sys.exit(main())
Loading
Loading