diff --git a/projects/rocprofiler-compute/skills/README.md b/projects/rocprofiler-compute/skills/README.md index 9df9edaf74af..4b101070e4e0 100644 --- a/projects/rocprofiler-compute/skills/README.md +++ b/projects/rocprofiler-compute/skills/README.md @@ -45,7 +45,14 @@ folders, so install for one agent only if you use Cursor alongside them. /share/rocprofiler-compute/skills/install-skills.sh --agent claude --uninstall ``` -## Structure +## Developing skills + +The rest of this file is for people who edit these skills. The skills are also +published in [amd/skills](https://github.com/amd/skills), the AMD skills +catalog, so a change here must work in this repo and in that catalog. See +[Publishing to amd/skills](#publishing-to-amdskills). + +### Structure Every skill directory holds: @@ -54,9 +61,10 @@ Every skill directory holds: ├── SKILL.md # frontmatter (name, description) plus the instructions ├── skill-card.md # human-facing card: description, owner, license └── evals/ - ├── evals.json # prompts that must and must not route to this skill - ├── machine.yml # hardware the behavioral cases need - └── samples/ # copies of the samples the cases use + ├── evals.json # cases this repo and amd/skills run + ├── extended_evals.json # optional; cases only this repo runs + ├── machine.yml # hardware amd/skills runs evals.json on + └── samples/ # copies of the samples the cases use ``` `SKILL.md` states the decision: which command to run, when, and where the @@ -65,20 +73,32 @@ same fact is not maintained in two places. Link docs as `../../docs/...`: `install-skills.sh` rewrites those links to the docs installed next to the skills, so they keep working offline. -## Tests +Both dataset files use the +[skillscope format](https://github.com/amd/skillscope/blob/main/docs/authoring-evals.md). +A case that needs an application sets `workspace` to `evals/samples`, and its +prompt names the application. The contents of that directory are copied to the +workspace root when the case runs. Each copy in `evals/samples/` and its +original under `sample/` name each other in a header comment; change them +together. -CI runs the [skillscope](https://github.com/amd/skillscope) structural checks -on every pull request that changes the skills or `docs/`, the same checks the -AMD skills repo runs on its copy. CI does not grade routing or behavior: -routing needs a model API key, and behavior needs a supported GPU. +### Tests -`run_evals.py` grades both and is the release gate: +CI runs the [skillscope](https://github.com/amd/skillscope) structural checks, +the same checks amd/skills runs, on every pull request that changes the skills +or `docs/`. They cover both dataset files. CI does not grade routing or +behavior: routing needs a model API key, and behavior needs a supported GPU. + +`run_evals.py` grades both and is the release gate. It runs `evals.json` and +`extended_evals.json` together. `evals.json` targets Strix Halo, except for +pc-sampling, and most extended cases need an Instinct GPU, so pick the cases +for your GPU with `--skill` or `--only`: ```bash ./skills/run_evals.py --list # validate the datasets ./skills/run_evals.py # does the right skill load? ./skills/run_evals.py --mode behavioral # also grade the agent ./skills/run_evals.py --skill rocprof-compute-memory # one skill +./skills/run_evals.py --only mem-cache-hit-rates # one case ``` Routing mode denies the agent its tools and only checks which skill it chose, @@ -93,23 +113,7 @@ install script and its docs links. Both drive the `claude` CLI. The script exits 77 when that CLI is absent, so a caller can treat it as skipped rather than failed. -Each dataset needs at least three cases that should trigger the skill, two that -should not, and one triggering case with judged behavior. Negative cases carry -only an id, a prompt, and an optional note. - -A triggering case that needs an application sets `workspace` to -`evals/samples`, and its prompt names the application. The contents of the -workspace directory are copied to the workspace root when the case runs, as -skillscope does. The AMD skills repo copies only the skill directory, so -`evals/samples/` holds copies of the samples from `sample/`. Each copy and its -original name each other in a header comment; change them together. - -`evals/machine.yml` tells the AMD skills repo where to run the behavioral -cases. Every skill asks for Linux. Only a skill whose cases need an Instinct GPU -also asks for one; the others run on that repo's default GPU runners, which -cost less. - -## Keeping skills correct +### Keeping skills correct CI checks only structure and links, so keeping the content correct falls to each change. A PR that alters CLI options, output layout, supported @@ -124,3 +128,26 @@ skill needs in that skill's own steps. Skills must also serve users on the two previous ROCm releases. When a change alters how a feature behaves, keep a short note in the skill saying how it worked before and in which release. + +## Publishing to amd/skills + +amd/skills imports each skill directory from `develop`. It copies only the skill +directory and rewrites links that leave it, such as `../../docs/...`, to GitHub +URLs. Edit the skills here; the next import overwrites edits made there. + +amd/skills grades each skill with skillscope under these rules: + +- It runs only `evals/evals.json`. That file needs at least three cases that + should trigger the skill, two that should not, and one triggering case with + `expected_behavior` or `unexpected_behavior`. `run_evals.py` checks the same + minimum. +- It runs `evals.json` on Strix Halo (gfx1151) unless `evals/machine.yml` asks + for other hardware. Every skill sets `os: [Linux]` there to skip the Windows + runners. Only pc-sampling also sets `labels`, because it needs an MI300X; that + sends it to the catalog's gated Instinct runners. +- Put cases that cannot pass on those runners in `evals/extended_evals.json`: + cases that need an Instinct GPU, and the torch-trace cases that profile a + model, since the catalog runners may not have PyTorch. The catalog skips this + file. +- It copies only the skill directory, so `evals/samples/` holds copies of the + samples from `sample/`. diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json index 51403b1a4378..e3e1d8c37b96 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json @@ -53,9 +53,9 @@ ] }, { - "id": "kb-kernel-name-filter", + "id": "kb-kernel-name-filter-gfx115x", "skill_should_trigger": true, - "prompt": "profile only the kernels whose names end in bound in my ROCm application\n\nThe application is ./occupancy.hip.", + "prompt": "profile only the kernels whose names start with global in my ROCm application\n\nThe application is ./vmem.hip.", "workspace": "evals/samples", "expected_behavior": [ "uses rocprof-compute profile -k to filter by kernel name", @@ -66,13 +66,11 @@ { "id": "kb-cuda-application", "skill_should_trigger": false, - "note": "Profiling request, but CUDA tooling is out of scope.", "prompt": "profile my CUDA application with nsys" }, { "id": "kb-windows-request", "skill_should_trigger": false, - "note": "rocprof-compute is named, but the platform is unsupported.", "prompt": "profile my application on Windows with rocprof-compute" }, { @@ -83,7 +81,6 @@ { "id": "kb-write-a-kernel", "skill_should_trigger": false, - "note": "GPU performance adjacent, but this skill does not author kernel source.", "prompt": "write me a faster HIP matrix multiply kernel" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json new file mode 100644 index 000000000000..2567269145e4 --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json @@ -0,0 +1,15 @@ +{ + "evaluations": [ + { + "id": "kb-kernel-name-filter", + "skill_should_trigger": true, + "prompt": "profile only the kernels whose names end in bound in my ROCm application\n\nThe application is ./occupancy.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "uses rocprof-compute profile -k to filter by kernel name", + "treats the profile-time -k value as a regex matched against kernel names" + ], + "logs_contain": ["-k"] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json index 30202b2914c2..d38fe953dbd3 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json @@ -16,57 +16,37 @@ "logs_contain": ["-b memchart"] }, { - "id": "mem-cache-hit-rates", + "id": "mem-cache-hit-rates-gfx115x", "skill_should_trigger": true, - "prompt": "what are my L2 and vL1D hit rates for this kernel and are they bad?\n\nThe application is ./vsequential_access.cpp.", + "prompt": "what are my GL1 and GL2 hit rates for this kernel and are they bad?\n\nThe application is ./vsequential_access.cpp.", "workspace": "evals/samples", "expected_behavior": [ - "opens the l2 and vl1d blocks for a single kernel id", + "checks the cache block names this GPU uses before choosing -b", + "opens the gl1 and gl2 blocks for a single kernel id", "judges the rates against what the algorithm should do rather than a fixed cutoff" ], "unexpected_behavior": [ "applies a remembered threshold such as L2 below 50% being bad, as if the tool defined one" ], - "logs_contain": ["-b"] + "logs_contain": ["gl2"] }, { - "id": "mem-lds-bank-conflicts", - "skill_should_trigger": true, - "prompt": "does my kernel have LDS bank conflicts?\n\nThe application is ./lds.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "opens the lds block for the selected kernel", - "reads the bank conflict counters from the report" - ], - "logs_contain": ["lds"] - }, - { - "id": "mem-bandwidth-bound", + "id": "mem-bandwidth-bound-gfx115x", "skill_should_trigger": true, "prompt": "this kernel looks memory bound, where is the bandwidth going?\n\nThe application is ./vcopy.cpp.", "workspace": "evals/samples", "expected_behavior": [ "walks the hierarchy from the memory chart down to the level responsible", - "distinguishes traffic that caches absorbed from traffic that reached HBM" + "distinguishes traffic that caches absorbed from traffic that reached system memory" ], - "logs_contain": ["memchart"] - }, - { - "id": "mem-membw-analysis-gfx950", - "skill_should_trigger": true, - "prompt": "run the guided memory bandwidth analysis on my MI350 workload\n\nThe application is ./gl2_backpressure.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "profiles only blocks 3 and 30 with --experimental --membw-analysis", - "notes that the analysis is experimental and shipped for gfx950", - "reports the guidance the tool produced rather than substituting its own cutoffs" + "unexpected_behavior": [ + "describes the GPU memory as HBM" ], - "logs_contain": ["--membw-analysis"] + "logs_contain": ["memchart"] }, { "id": "mem-host-memory", "skill_should_trigger": false, - "note": "Memory question, but host-side allocation is not GPU counter analysis.", "prompt": "my application is leaking host memory, help me find it" }, { @@ -77,7 +57,6 @@ { "id": "mem-gpu-oom", "skill_should_trigger": false, - "note": "About allocation failure, not bandwidth or cache behaviour.", "prompt": "why does my model run out of GPU memory during training?" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json new file mode 100644 index 000000000000..65dc47e2ded1 --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json @@ -0,0 +1,52 @@ +{ + "evaluations": [ + { + "id": "mem-cache-hit-rates", + "skill_should_trigger": true, + "prompt": "what are my L2 and vL1D hit rates for this kernel and are they bad?\n\nThe application is ./vsequential_access.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "opens the l2 and vl1d blocks for a single kernel id", + "judges the rates against what the algorithm should do rather than a fixed cutoff" + ], + "unexpected_behavior": [ + "applies a remembered threshold such as L2 below 50% being bad, as if the tool defined one" + ], + "logs_contain": ["-b"] + }, + { + "id": "mem-lds-bank-conflicts", + "skill_should_trigger": true, + "prompt": "does my kernel have LDS bank conflicts?\n\nThe application is ./lds.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "opens the lds block for the selected kernel", + "reads the bank conflict counters from the report" + ], + "logs_contain": ["lds"] + }, + { + "id": "mem-bandwidth-bound", + "skill_should_trigger": true, + "prompt": "this kernel looks memory bound, where is the bandwidth going?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "walks the hierarchy from the memory chart down to the level responsible", + "distinguishes traffic that caches absorbed from traffic that reached HBM" + ], + "logs_contain": ["memchart"] + }, + { + "id": "mem-membw-analysis-gfx950", + "skill_should_trigger": true, + "prompt": "run the guided memory bandwidth analysis on my MI350 workload\n\nThe application is ./gl2_backpressure.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "profiles only blocks 3 and 30 with --experimental --membw-analysis", + "notes that the analysis is experimental and shipped for gfx950", + "reports the guidance the tool produced rather than substituting its own cutoffs" + ], + "logs_contain": ["--membw-analysis"] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml index 4d7efd456e93..78cd19714df9 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml @@ -1,2 +1 @@ os: [Linux] -labels: [mi300x, gpu, rocm] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json index 275e862c4699..d589f0934736 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json @@ -65,7 +65,6 @@ { "id": "pcs-rank-kernels-by-time", "skill_should_trigger": false, - "note": "Hotspot wording, but ranking whole kernels is counter work, not instruction sampling.", "prompt": "which of my GPU kernels accounts for the most total time?" }, { diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json index f7ab8e2bf205..60a72d51135f 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json @@ -66,7 +66,6 @@ { "id": "roof-concept-question", "skill_should_trigger": false, - "note": "Asks what a roofline is in the abstract, with no workload to analyze.", "prompt": "explain the roofline performance model to me" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml index 4d7efd456e93..78cd19714df9 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml +++ b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml @@ -1,2 +1 @@ os: [Linux] -labels: [mi300x, gpu, rocm] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json index 3c63e4793fff..628f8c8ddfc4 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json @@ -56,7 +56,6 @@ { "id": "sol-instruction-level", "skill_should_trigger": false, - "note": "Peak comparison is not the question; this is per-instruction work.", "prompt": "which individual instruction in this kernel stalls the most?" }, { @@ -67,7 +66,6 @@ { "id": "sol-hardware-spec-lookup", "skill_should_trigger": false, - "note": "Asks for a datasheet number, not a reading from a profile.", "prompt": "what is the theoretical peak FP64 throughput of an MI300X?" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json index 1fd3a5a7ff23..82c6975fddd6 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json @@ -1,31 +1,5 @@ { "evaluations": [ - { - "id": "tt-which-operator-is-slow", - "skill_should_trigger": true, - "prompt": "which PyTorch operator is launching the slow GPU kernels in my model?\n\nThe application is ./simple_net.py.", - "workspace": "evals/samples", - "expected_behavior": [ - "collects with --experimental --torch-trace", - "lists the captured operators to find the slow ones", - "checks the documented PyTorch version requirement" - ], - "unexpected_behavior": [ - "guesses an operator pattern without listing the captured operators first" - ], - "logs_contain": ["--torch-trace", "--list-torch-operators"] - }, - { - "id": "tt-map-kernels-to-model-code", - "skill_should_trigger": true, - "prompt": "profile my training script and tell me which layer the GPU time belongs to\n\nThe application is ./simple_net.py.", - "workspace": "evals/samples", - "expected_behavior": [ - "profiles the Python script with torch tracing enabled", - "advises profiling a few steady-state iterations rather than a full run" - ], - "logs_contain": ["--torch-trace"] - }, { "id": "tt-triton-inductor-kernels", "skill_should_trigger": true, @@ -80,7 +54,6 @@ { "id": "tt-model-accuracy", "skill_should_trigger": false, - "note": "PyTorch question with no performance profiling in it.", "prompt": "why is my PyTorch model's validation accuracy dropping?" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json new file mode 100644 index 000000000000..21a5c654e905 --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json @@ -0,0 +1,30 @@ +{ + "evaluations": [ + { + "id": "tt-which-operator-is-slow", + "skill_should_trigger": true, + "prompt": "which PyTorch operator is launching the slow GPU kernels in my model?\n\nThe application is ./simple_net.py.", + "workspace": "evals/samples", + "expected_behavior": [ + "collects with --experimental --torch-trace", + "lists the captured operators to find the slow ones", + "checks the documented PyTorch version requirement" + ], + "unexpected_behavior": [ + "guesses an operator pattern without listing the captured operators first" + ], + "logs_contain": ["--torch-trace", "--list-torch-operators"] + }, + { + "id": "tt-map-kernels-to-model-code", + "skill_should_trigger": true, + "prompt": "profile my training script and tell me which layer the GPU time belongs to\n\nThe application is ./simple_net.py.", + "workspace": "evals/samples", + "expected_behavior": [ + "profiles the Python script with torch tracing enabled", + "advises profiling a few steady-state iterations rather than a full run" + ], + "logs_contain": ["--torch-trace"] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/run_evals.py b/projects/rocprofiler-compute/skills/run_evals.py index 93abbdbbc4d0..9688d2ed77a0 100755 --- a/projects/rocprofiler-compute/skills/run_evals.py +++ b/projects/rocprofiler-compute/skills/run_evals.py @@ -78,15 +78,28 @@ def discover_skills(only: Optional[str]) -> list[Path]: def load_cases(skill: Path) -> list[dict[str, Any]]: - """Parse and validate one dataset, erroring on anything malformed.""" - dataset = json.loads((skill / "evals" / "evals.json").read_text()) - cases = dataset["evaluations"] + """Return the cases from both of a skill's datasets, erroring on bad ones. + + The minimum applies to evals.json only, the one file amd/skills runs. + """ + required = _read_evaluations(skill / "evals" / "evals.json") + extended = _read_evaluations(skill / "evals" / "extended_evals.json") + cases = required + extended + _check_cases(skill, cases) + _require_minimum(skill, required) + return cases - positive = 0 - negative = 0 - judged = 0 - seen: set[str] = set() +def _read_evaluations(path: Path) -> list[dict[str, Any]]: + """Return the evaluations in one dataset file, or none when it is absent.""" + if not path.is_file(): + return [] + return json.loads(path.read_text(encoding="utf-8"))["evaluations"] + + +def _check_cases(skill: Path, cases: list[dict[str, Any]]) -> None: + """Exit on the first malformed case.""" + seen: set[str] = set() for case in cases: unknown = set(case) - KNOWN_KEYS if unknown: @@ -102,25 +115,21 @@ def load_cases(skill: Path) -> list[dict[str, Any]]: seen.add(case["id"]) if "workspace" in case and not workspace_dir(skill, case["workspace"]).is_dir(): sys.exit(f"{skill.name}: {case['id']} names a missing workspace") - if case["skill_should_trigger"]: - positive += 1 - judged += any(case.get(key) for key in JUDGED_KEYS) - else: - negative += 1 - extra = set(case) - {"id", "prompt", "skill_should_trigger", "note"} - if extra: - sys.exit( - f"{skill.name}: {case['id']} is negative but sets {sorted(extra)}" - ) - - if positive < 3 or negative < 2: + continue + extra = set(case) - {"id", "prompt", "skill_should_trigger", "note"} + if extra: + sys.exit(f"{skill.name}: {case['id']} is negative but sets {sorted(extra)}") + + +def _require_minimum(skill: Path, cases: list[dict[str, Any]]) -> None: + """Exit unless the cases meet the minimum amd/skills requires.""" + positive = [case for case in cases if case["skill_should_trigger"]] + if len(positive) < 3 or len(cases) - len(positive) < 2: sys.exit(f"{skill.name}: needs at least 3 positive and 2 negative cases") - if not judged: + if not any(case.get(key) for case in positive for key in JUDGED_KEYS): sys.exit(f"{skill.name}: needs at least one positive case with judged behavior") - return cases - def workspace_dir(skill: Path, workspace: str) -> Path: """Join ``workspace`` onto the skill directory the way skillscope does."""