From e53c7815c874648f3abd16528755568e07d8383d Mon Sep 17 00:00:00 2001 From: "fei.zheng" Date: Wed, 4 Nov 2026 19:56:16 -0600 Subject: [PATCH 1/2] feat(rocprofiler-compute): Keep catalog skill evals short The AMD skills catalog grades evals.json only. Move the sample profile cases into extended_evals.json so that run stays a trigger check, and grade the long cases here with --extended. Co-authored-by: Cursor --- projects/rocprofiler-compute/skills/README.md | 30 ++++--- .../evals/evals.json | 57 +++---------- .../evals/extended_evals.json | 65 +++++++++++++++ .../rocprof-compute-memory/evals/evals.json | 65 ++++----------- .../evals/extended_evals.json | 78 +++++++++++++++++ .../evals/evals.json | 50 +++-------- .../evals/extended_evals.json | 49 +++++++++++ .../rocprof-compute-roofline/evals/evals.json | 54 +++--------- .../evals/extended_evals.json | 52 ++++++++++++ .../evals/evals.json | 36 ++------ .../evals/extended_evals.json | 38 +++++++++ .../evals/evals.json | 35 ++------ .../evals/extended_evals.json | 36 ++++++++ .../rocprofiler-compute/skills/run_evals.py | 83 +++++++++++++++---- 14 files changed, 464 insertions(+), 264 deletions(-) create mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json create mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json create mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json create mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json create mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json create mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json diff --git a/projects/rocprofiler-compute/skills/README.md b/projects/rocprofiler-compute/skills/README.md index 9df9edaf74af..2d83d46e147a 100644 --- a/projects/rocprofiler-compute/skills/README.md +++ b/projects/rocprofiler-compute/skills/README.md @@ -54,9 +54,10 @@ Every skill directory holds: ├── SKILL.md # frontmatter (name, description) plus the instructions ├── skill-card.md # human-facing card: description, owner, license └── evals/ - ├── evals.json # prompts that must and must not route to this skill - ├── machine.yml # hardware the behavioral cases need - └── samples/ # copies of the samples the cases use + ├── evals.json # minimum set the AMD skills catalog grades + ├── extended_evals.json # longer profile cases, run here with --extended + ├── machine.yml # hardware the behavioral cases need + └── samples/ # copies of the samples the cases use ``` `SKILL.md` states the decision: which command to run, when, and where the @@ -77,10 +78,23 @@ routing needs a model API key, and behavior needs a supported GPU. ```bash ./skills/run_evals.py --list # validate the datasets ./skills/run_evals.py # does the right skill load? -./skills/run_evals.py --mode behavioral # also grade the agent +./skills/run_evals.py --mode behavioral # grade the minimum set +./skills/run_evals.py --extended --mode behavioral # also grade the long profiles ./skills/run_evals.py --skill rocprof-compute-memory # one skill ``` +`evals/evals.json` is the minimum set. The AMD skills catalog grades that file +and no other. It keeps at least three prompts that should trigger the skill, +two that should not, and one triggering case with judged behavior. Negative +cases carry only an id, a prompt, and an optional note. The judged cases stay +short: they check that the right workflow is chosen, and they do not profile +a sample. + +Cases that build a sample, profile it, and read the report live in +`evals/extended_evals.json`, beside the minimum file and in the same format. +Add `--extended` to run them. The catalog workflow sets `extended: false`, so +moving a case into that file takes it out of the catalog run. + Routing mode denies the agent its tools and only checks which skill it chose, so it is quick and needs no GPU. Behavioral mode lets the agent work and grades `logs_contain`, `files_exist`, and the plain-language `expected_behavior` and @@ -93,12 +107,8 @@ install script and its docs links. Both drive the `claude` CLI. The script exits 77 when that CLI is absent, so a caller can treat it as skipped rather than failed. -Each dataset needs at least three cases that should trigger the skill, two that -should not, and one triggering case with judged behavior. Negative cases carry -only an id, a prompt, and an optional note. - -A triggering case that needs an application sets `workspace` to -`evals/samples`, and its prompt names the application. The contents of the +A case that profiles an application lives in `extended_evals.json`. It sets +`workspace` to `evals/samples`, and its prompt names the application. The contents of the workspace directory are copied to the workspace root when the case runs, as skillscope does. The AMD skills repo copies only the skill directory, so `evals/samples/` holds copies of the samples from `sample/`. Each copy and its diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json index 51403b1a4378..7020fa59fb91 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json @@ -1,44 +1,6 @@ { + "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ - { - "id": "kb-profile-hip-application", - "skill_should_trigger": true, - "prompt": "profile my HIP application with rocprof-compute and tell me where the time goes\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "asks for the launch command and a workload name before profiling", - "runs rocprof-compute profile --name -- ", - "runs analyze with --list-stats to find the hottest kernel" - ], - "unexpected_behavior": [ - "invents a profiling flag instead of checking --help" - ], - "logs_contain": ["rocprof-compute profile", "--list-stats"] - }, - { - "id": "kb-which-kernel-is-slow", - "skill_should_trigger": true, - "prompt": "which kernel is the hotspot in ./workloads/my_app and why is it slow?\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "points --path at the directory containing profiling_config.yaml rather than a parent directory", - "uses --list-stats first, then narrows to one kernel id with -k", - "reads the System Speed-of-Light panel before opening a detailed block" - ], - "logs_contain": ["--list-stats", "-k"] - }, - { - "id": "kb-expensive-replay-fallback", - "skill_should_trigger": true, - "prompt": "a full rocprof-compute replay of this workload is too expensive, collect something cheaper\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "offers a single-pass option such as --set, a narrowed -b, or --iteration-multiplexing", - "runs --list-sets before selecting a set", - "warns which analysis panels will be empty afterwards" - ], - "logs_contain": ["--list-sets"] - }, { "id": "kb-unsafe-to-replay", "skill_should_trigger": true, @@ -53,15 +15,16 @@ ] }, { - "id": "kb-kernel-name-filter", + "id": "kb-where-time-goes", "skill_should_trigger": true, - "prompt": "profile only the kernels whose names end in bound in my ROCm application\n\nThe application is ./occupancy.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "uses rocprof-compute profile -k to filter by kernel name", - "treats the profile-time -k value as a regex matched against kernel names" - ], - "logs_contain": ["-k"] + "note": "Catalog trigger check. The profiled version is kb-profile-hip-application in extended_evals.json.", + "prompt": "profile my HIP application with rocprof-compute and tell me where the time goes" + }, + { + "id": "kb-cheaper-collection", + "skill_should_trigger": true, + "note": "Catalog trigger check. The profiled version is kb-expensive-replay-fallback in extended_evals.json.", + "prompt": "a full rocprof-compute replay of this workload is too expensive, what cheaper collection should I use?" }, { "id": "kb-cuda-application", diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json new file mode 100644 index 000000000000..88e67388d630 --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json @@ -0,0 +1,65 @@ +{ + "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", + "evaluations": [ + { + "id": "kb-profile-hip-application", + "skill_should_trigger": true, + "prompt": "profile my HIP application with rocprof-compute and tell me where the time goes\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "asks for the launch command and a workload name before profiling", + "runs rocprof-compute profile --name -- ", + "runs analyze with --list-stats to find the hottest kernel" + ], + "unexpected_behavior": [ + "invents a profiling flag instead of checking --help" + ], + "logs_contain": [ + "rocprof-compute profile", + "--list-stats" + ] + }, + { + "id": "kb-which-kernel-is-slow", + "skill_should_trigger": true, + "prompt": "which kernel is the hotspot in ./workloads/my_app and why is it slow?\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "points --path at the directory containing profiling_config.yaml rather than a parent directory", + "uses --list-stats first, then narrows to one kernel id with -k", + "reads the System Speed-of-Light panel before opening a detailed block" + ], + "logs_contain": [ + "--list-stats", + "-k" + ] + }, + { + "id": "kb-expensive-replay-fallback", + "skill_should_trigger": true, + "prompt": "a full rocprof-compute replay of this workload is too expensive, collect something cheaper\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "offers a single-pass option such as --set, a narrowed -b, or --iteration-multiplexing", + "runs --list-sets before selecting a set", + "warns which analysis panels will be empty afterwards" + ], + "logs_contain": [ + "--list-sets" + ] + }, + { + "id": "kb-kernel-name-filter", + "skill_should_trigger": true, + "prompt": "profile only the kernels whose names end in bound in my ROCm application\n\nThe application is ./occupancy.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "uses rocprof-compute profile -k to filter by kernel name", + "treats the profile-time -k value as a regex matched against kernel names" + ], + "logs_contain": [ + "-k" + ] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json index 30202b2914c2..eb051217985a 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json @@ -1,67 +1,30 @@ { + "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ { - "id": "mem-access-inefficiency", + "id": "mem-which-blocks", "skill_should_trigger": true, - "prompt": "find the memory access inefficiencies in this rocprofiler-compute capture\n\nThe application is ./vrandom_access.cpp.", - "workspace": "evals/samples", + "note": "Catalog behavioral check. Asks which blocks to collect and does not stage a sample.", + "prompt": "which rocprof-compute blocks should I collect to inspect cache hit rates and LDS bank conflicts, without a full profile?", "expected_behavior": [ - "reads the memory chart before opening an individual cache level", - "cites hit rates or request counts from the report", - "ties any coalescing or tiling advice to those numbers" + "names the memory chart, or -b memchart, as the place to start", + "collects only the memory blocks the question needs, such as memchart, vl1d, l2, or lds" ], "unexpected_behavior": [ - "recommends an optimization without citing a metric from the report" - ], - "logs_contain": ["-b memchart"] + "starts a full multi-pass profile or the roofline benchmarks" + ] }, { - "id": "mem-cache-hit-rates", + "id": "mem-where-hit-rates", "skill_should_trigger": true, - "prompt": "what are my L2 and vL1D hit rates for this kernel and are they bad?\n\nThe application is ./vsequential_access.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "opens the l2 and vl1d blocks for a single kernel id", - "judges the rates against what the algorithm should do rather than a fixed cutoff" - ], - "unexpected_behavior": [ - "applies a remembered threshold such as L2 below 50% being bad, as if the tool defined one" - ], - "logs_contain": ["-b"] + "note": "Catalog trigger check. The profiled version is mem-cache-hit-rates in extended_evals.json.", + "prompt": "where do I read L2 and vL1D hit rates for one kernel in rocprof-compute?" }, { - "id": "mem-lds-bank-conflicts", + "id": "mem-where-bandwidth-goes", "skill_should_trigger": true, - "prompt": "does my kernel have LDS bank conflicts?\n\nThe application is ./lds.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "opens the lds block for the selected kernel", - "reads the bank conflict counters from the report" - ], - "logs_contain": ["lds"] - }, - { - "id": "mem-bandwidth-bound", - "skill_should_trigger": true, - "prompt": "this kernel looks memory bound, where is the bandwidth going?\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "walks the hierarchy from the memory chart down to the level responsible", - "distinguishes traffic that caches absorbed from traffic that reached HBM" - ], - "logs_contain": ["memchart"] - }, - { - "id": "mem-membw-analysis-gfx950", - "skill_should_trigger": true, - "prompt": "run the guided memory bandwidth analysis on my MI350 workload\n\nThe application is ./gl2_backpressure.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "profiles only blocks 3 and 30 with --experimental --membw-analysis", - "notes that the analysis is experimental and shipped for gfx950", - "reports the guidance the tool produced rather than substituting its own cutoffs" - ], - "logs_contain": ["--membw-analysis"] + "note": "Catalog trigger check. The profiled version is mem-bandwidth-bound in extended_evals.json.", + "prompt": "this AMD GPU kernel looks memory bound, which rocprof-compute view shows where the bandwidth goes?" }, { "id": "mem-host-memory", diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json new file mode 100644 index 000000000000..16cb9af9b03f --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json @@ -0,0 +1,78 @@ +{ + "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", + "evaluations": [ + { + "id": "mem-access-inefficiency", + "skill_should_trigger": true, + "prompt": "find the memory access inefficiencies in this rocprofiler-compute capture\n\nThe application is ./vrandom_access.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "reads the memory chart before opening an individual cache level", + "cites hit rates or request counts from the report", + "ties any coalescing or tiling advice to those numbers" + ], + "unexpected_behavior": [ + "recommends an optimization without citing a metric from the report" + ], + "logs_contain": [ + "-b memchart" + ] + }, + { + "id": "mem-cache-hit-rates", + "skill_should_trigger": true, + "prompt": "what are my L2 and vL1D hit rates for this kernel and are they bad?\n\nThe application is ./vsequential_access.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "opens the l2 and vl1d blocks for a single kernel id", + "judges the rates against what the algorithm should do rather than a fixed cutoff" + ], + "unexpected_behavior": [ + "applies a remembered threshold such as L2 below 50% being bad, as if the tool defined one" + ], + "logs_contain": [ + "-b" + ] + }, + { + "id": "mem-lds-bank-conflicts", + "skill_should_trigger": true, + "prompt": "does my kernel have LDS bank conflicts?\n\nThe application is ./lds.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "opens the lds block for the selected kernel", + "reads the bank conflict counters from the report" + ], + "logs_contain": [ + "lds" + ] + }, + { + "id": "mem-bandwidth-bound", + "skill_should_trigger": true, + "prompt": "this kernel looks memory bound, where is the bandwidth going?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "walks the hierarchy from the memory chart down to the level responsible", + "distinguishes traffic that caches absorbed from traffic that reached HBM" + ], + "logs_contain": [ + "memchart" + ] + }, + { + "id": "mem-membw-analysis-gfx950", + "skill_should_trigger": true, + "prompt": "run the guided memory bandwidth analysis on my MI350 workload\n\nThe application is ./gl2_backpressure.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "profiles only blocks 3 and 30 with --experimental --membw-analysis", + "notes that the analysis is experimental and shipped for gfx950", + "reports the guidance the tool produced rather than substituting its own cutoffs" + ], + "logs_contain": [ + "--membw-analysis" + ] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json index 275e862c4699..00d0cb8a7146 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json @@ -1,33 +1,6 @@ { + "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ - { - "id": "pcs-which-instruction-stalls", - "skill_should_trigger": true, - "prompt": "which instruction in my MI300 kernel stalls the most?\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "queries the device for supported PC sampling configurations before choosing an interval", - "collects with --experimental --pc-sampling", - "sorts the analysis by sample count and reads the stall reason column" - ], - "unexpected_behavior": [ - "hardcodes a sampling interval without querying the device" - ], - "logs_contain": ["--pc-sampling", "rocprofv3-avail"] - }, - { - "id": "pcs-source-line-attribution", - "skill_should_trigger": true, - "prompt": "map the hot GPU instructions back to my HIP source lines\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "states that source line mapping requires the application to be built with debug info", - "notes that samples map to assembly even without debug info" - ], - "unexpected_behavior": [ - "promises source line attribution for a build with no debug info" - ] - }, { "id": "pcs-stochastic-vs-host-trap", "skill_should_trigger": true, @@ -37,18 +10,9 @@ "notes stochastic needs MI300 or newer and host_trap is the fallback", "checks what the installed device actually supports" ], - "logs_contain": ["stochastic"] - }, - { - "id": "pcs-export-disassembly", - "skill_should_trigger": true, - "prompt": "export the annotated disassembly with sample counts from my pc sampling workload\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "uses --output-format csv or db", - "notes that the per-kernel disassembly is written under per_kernel_pc_sampling" - ], - "logs_contain": ["--output-format"] + "logs_contain": [ + "stochastic" + ] }, { "id": "pcs-host-trap-stall-reason", @@ -62,6 +26,12 @@ "interprets a stall reason from host-trap data" ] }, + { + "id": "pcs-stall-reason-question", + "skill_should_trigger": true, + "note": "Catalog trigger check. The profiled version is pcs-which-instruction-stalls in extended_evals.json.", + "prompt": "which instruction in my AMD GPU kernel stalls the most, and what is the stall reason?" + }, { "id": "pcs-rank-kernels-by-time", "skill_should_trigger": false, diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json new file mode 100644 index 000000000000..ff20f340bd48 --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json @@ -0,0 +1,49 @@ +{ + "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", + "evaluations": [ + { + "id": "pcs-which-instruction-stalls", + "skill_should_trigger": true, + "prompt": "which instruction in my MI300 kernel stalls the most?\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "queries the device for supported PC sampling configurations before choosing an interval", + "collects with --experimental --pc-sampling", + "sorts the analysis by sample count and reads the stall reason column" + ], + "unexpected_behavior": [ + "hardcodes a sampling interval without querying the device" + ], + "logs_contain": [ + "--pc-sampling", + "rocprofv3-avail" + ] + }, + { + "id": "pcs-source-line-attribution", + "skill_should_trigger": true, + "prompt": "map the hot GPU instructions back to my HIP source lines\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "states that source line mapping requires the application to be built with debug info", + "notes that samples map to assembly even without debug info" + ], + "unexpected_behavior": [ + "promises source line attribution for a build with no debug info" + ] + }, + { + "id": "pcs-export-disassembly", + "skill_should_trigger": true, + "prompt": "export the annotated disassembly with sample counts from my pc sampling workload\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "uses --output-format csv or db", + "notes that the per-kernel disassembly is written under per_kernel_pc_sampling" + ], + "logs_contain": [ + "--output-format" + ] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json index f7ab8e2bf205..9f59aabca83b 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json @@ -1,46 +1,6 @@ { + "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ - { - "id": "roof-compute-or-memory-bound", - "skill_should_trigger": true, - "prompt": "is this rocprof-compute kernel compute bound or memory bound?\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "analyzes the roof block for a single kernel id", - "compares arithmetic intensity against the empirical peaks printed in the table", - "says when the kernel is below both ceilings instead of forcing a bound" - ], - "unexpected_behavior": [ - "opens the generated roofline HTML chart instead of reading the table", - "calculates a peak instead of using the measured one" - ], - "logs_contain": ["-b roof"] - }, - { - "id": "roof-arithmetic-intensity", - "skill_should_trigger": true, - "prompt": "what is the arithmetic intensity of my hottest kernel and how much performance am I leaving on the table?\n\nThe application is ./mat_mul_max.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "reads FLOPs per byte and the peak values from the roofline table", - "narrows to one kernel id first" - ], - "logs_contain": ["roof"] - }, - { - "id": "roof-precision-selection", - "skill_should_trigger": true, - "prompt": "my matrix multiply sits far below the top ceiling on the roofline, is it slow?\n\nThe application is ./mat_mul_max.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "reads the precision from the instruction mix rather than assuming it", - "compares against the FP32 peak, not the highest matrix or FP16 peak" - ], - "unexpected_behavior": [ - "calls the kernel slow because it is below the highest ceiling in the table" - ], - "logs_contain": ["cu_ins"] - }, { "id": "roof-missing-data", "skill_should_trigger": true, @@ -53,6 +13,18 @@ "tells the user to re-profile without --no-roof when the architecture is the actual reason" ] }, + { + "id": "roof-which-bound", + "skill_should_trigger": true, + "note": "Catalog trigger check. The profiled version is roof-compute-or-memory-bound in extended_evals.json.", + "prompt": "is my AMD GPU kernel compute-bound or memory-bound on the rocprof-compute roofline?" + }, + { + "id": "roof-left-on-table", + "skill_should_trigger": true, + "note": "Catalog trigger check. The profiled version is roof-arithmetic-intensity in extended_evals.json.", + "prompt": "how much performance is my kernel leaving on the table according to the rocprof-compute roofline?" + }, { "id": "roof-generic-optimization", "skill_should_trigger": false, diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json new file mode 100644 index 000000000000..363380eb1b32 --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json @@ -0,0 +1,52 @@ +{ + "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", + "evaluations": [ + { + "id": "roof-compute-or-memory-bound", + "skill_should_trigger": true, + "prompt": "is this rocprof-compute kernel compute bound or memory bound?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "analyzes the roof block for a single kernel id", + "compares arithmetic intensity against the empirical peaks printed in the table", + "says when the kernel is below both ceilings instead of forcing a bound" + ], + "unexpected_behavior": [ + "opens the generated roofline HTML chart instead of reading the table", + "calculates a peak instead of using the measured one" + ], + "logs_contain": [ + "-b roof" + ] + }, + { + "id": "roof-arithmetic-intensity", + "skill_should_trigger": true, + "prompt": "what is the arithmetic intensity of my hottest kernel and how much performance am I leaving on the table?\n\nThe application is ./mat_mul_max.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "reads FLOPs per byte and the peak values from the roofline table", + "narrows to one kernel id first" + ], + "logs_contain": [ + "roof" + ] + }, + { + "id": "roof-precision-selection", + "skill_should_trigger": true, + "prompt": "my matrix multiply sits far below the top ceiling on the roofline, is it slow?\n\nThe application is ./mat_mul_max.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "reads the precision from the instruction mix rather than assuming it", + "compares against the FP32 peak, not the highest matrix or FP16 peak" + ], + "unexpected_behavior": [ + "calls the kernel slow because it is below the highest ceiling in the table" + ], + "logs_contain": [ + "cu_ins" + ] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json index 3c63e4793fff..f3f27bddec8a 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json @@ -1,34 +1,6 @@ { + "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ - { - "id": "sol-percent-of-peak", - "skill_should_trigger": true, - "prompt": "how close to peak is this kernel running in my rocprof-compute workload?\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "analyzes the sol block for a single kernel id", - "reads the Percent of Peak column from the report", - "compares against the Peak value the report prints rather than a remembered threshold" - ], - "unexpected_behavior": [ - "calculates a theoretical peak instead of using the one in the report" - ], - "logs_contain": ["-b sol"] - }, - { - "id": "sol-overall-efficiency", - "skill_should_trigger": true, - "prompt": "give me an overall efficiency summary of this rocprofiler-compute capture\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "selects one kernel with -k before reading Speed-of-Light", - "summarizes using the printed peak and percent of peak values" - ], - "unexpected_behavior": [ - "reports a Speed-of-Light table averaged across unrelated kernels" - ], - "logs_contain": ["-b sol"] - }, { "id": "sol-mfma-zero", "skill_should_trigger": true, @@ -53,6 +25,12 @@ "states a universal threshold such as 70% as if the tool defined one" ] }, + { + "id": "sol-how-close-to-peak", + "skill_should_trigger": true, + "note": "Catalog trigger check. The profiled version is sol-percent-of-peak in extended_evals.json.", + "prompt": "how close to hardware peak is this AMD GPU kernel in rocprof-compute?" + }, { "id": "sol-instruction-level", "skill_should_trigger": false, diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json new file mode 100644 index 000000000000..fa4d6250470e --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json @@ -0,0 +1,38 @@ +{ + "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", + "evaluations": [ + { + "id": "sol-percent-of-peak", + "skill_should_trigger": true, + "prompt": "how close to peak is this kernel running in my rocprof-compute workload?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "analyzes the sol block for a single kernel id", + "reads the Percent of Peak column from the report", + "compares against the Peak value the report prints rather than a remembered threshold" + ], + "unexpected_behavior": [ + "calculates a theoretical peak instead of using the one in the report" + ], + "logs_contain": [ + "-b sol" + ] + }, + { + "id": "sol-overall-efficiency", + "skill_should_trigger": true, + "prompt": "give me an overall efficiency summary of this rocprofiler-compute capture\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "selects one kernel with -k before reading Speed-of-Light", + "summarizes using the printed peak and percent of peak values" + ], + "unexpected_behavior": [ + "reports a Speed-of-Light table averaged across unrelated kernels" + ], + "logs_contain": [ + "-b sol" + ] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json index 1fd3a5a7ff23..23ff37726945 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json @@ -1,31 +1,6 @@ { + "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ - { - "id": "tt-which-operator-is-slow", - "skill_should_trigger": true, - "prompt": "which PyTorch operator is launching the slow GPU kernels in my model?\n\nThe application is ./simple_net.py.", - "workspace": "evals/samples", - "expected_behavior": [ - "collects with --experimental --torch-trace", - "lists the captured operators to find the slow ones", - "checks the documented PyTorch version requirement" - ], - "unexpected_behavior": [ - "guesses an operator pattern without listing the captured operators first" - ], - "logs_contain": ["--torch-trace", "--list-torch-operators"] - }, - { - "id": "tt-map-kernels-to-model-code", - "skill_should_trigger": true, - "prompt": "profile my training script and tell me which layer the GPU time belongs to\n\nThe application is ./simple_net.py.", - "workspace": "evals/samples", - "expected_behavior": [ - "profiles the Python script with torch tracing enabled", - "advises profiling a few steady-state iterations rather than a full run" - ], - "logs_contain": ["--torch-trace"] - }, { "id": "tt-triton-inductor-kernels", "skill_should_trigger": true, @@ -36,7 +11,9 @@ "unexpected_behavior": [ "relies on --triton-trace alone to capture torch.compile kernels" ], - "logs_contain": ["--torch-trace"] + "logs_contain": [ + "--torch-trace" + ] }, { "id": "tt-filter-by-operator", @@ -46,7 +23,9 @@ "filters with --torch-operator", "treats the pattern as a shell-style glob rather than a regex" ], - "logs_contain": ["--torch-operator"] + "logs_contain": [ + "--torch-operator" + ] }, { "id": "tt-custom-roctx-unsupported", diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json new file mode 100644 index 000000000000..681a3fe9f15d --- /dev/null +++ b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json @@ -0,0 +1,36 @@ +{ + "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", + "evaluations": [ + { + "id": "tt-which-operator-is-slow", + "skill_should_trigger": true, + "prompt": "which PyTorch operator is launching the slow GPU kernels in my model?\n\nThe application is ./simple_net.py.", + "workspace": "evals/samples", + "expected_behavior": [ + "collects with --experimental --torch-trace", + "lists the captured operators to find the slow ones", + "checks the documented PyTorch version requirement" + ], + "unexpected_behavior": [ + "guesses an operator pattern without listing the captured operators first" + ], + "logs_contain": [ + "--torch-trace", + "--list-torch-operators" + ] + }, + { + "id": "tt-map-kernels-to-model-code", + "skill_should_trigger": true, + "prompt": "profile my training script and tell me which layer the GPU time belongs to\n\nThe application is ./simple_net.py.", + "workspace": "evals/samples", + "expected_behavior": [ + "profiles the Python script with torch tracing enabled", + "advises profiling a few steady-state iterations rather than a full run" + ], + "logs_contain": [ + "--torch-trace" + ] + } + ] +} diff --git a/projects/rocprofiler-compute/skills/run_evals.py b/projects/rocprofiler-compute/skills/run_evals.py index 93abbdbbc4d0..b2da14ecadbc 100755 --- a/projects/rocprofiler-compute/skills/run_evals.py +++ b/projects/rocprofiler-compute/skills/run_evals.py @@ -12,6 +12,7 @@ Usage: ./skills/run_evals.py # routing only, no tools ./skills/run_evals.py --mode behavioral # also grade what the agent did + ./skills/run_evals.py --extended --mode behavioral # include the long profile cases ./skills/run_evals.py --skill rocprof-compute-memory # one skill ./skills/run_evals.py --only kb-kernel-name-filter # one case ./skills/run_evals.py --list # validate datasets, run nothing @@ -77,16 +78,39 @@ def discover_skills(only: Optional[str]) -> list[Path]: return found -def load_cases(skill: Path) -> list[dict[str, Any]]: - """Parse and validate one dataset, erroring on anything malformed.""" - dataset = json.loads((skill / "evals" / "evals.json").read_text()) - cases = dataset["evaluations"] +def load_cases(skill: Path, *, extended: bool) -> tuple[list[dict[str, Any]], int]: + """Parse and validate one skill's datasets. - positive = 0 - negative = 0 - judged = 0 - seen: set[str] = set() + ``evals/evals.json`` is the catalog minimum and always runs. The optional + ``evals/extended_evals.json`` is validated whenever it exists and is run + only when `extended` is set. Returns the cases to run and how many of + them came from the extended file. + """ + required = _read_cases(skill, "evals.json") + _require_minimum(skill, required) + extra_path = skill / "evals" / "extended_evals.json" + extra = _read_cases(skill, "extended_evals.json") if extra_path.is_file() else [] + seen = {case["id"] for case in required} + for case in extra: + if case["id"] in seen: + sys.exit(f"{skill.name}: duplicate case id {case['id']!r}") + seen.add(case["id"]) + if extended: + return required + extra, len(extra) + return required, len(extra) + +def _read_cases(skill: Path, filename: str) -> list[dict[str, Any]]: + """Parse one dataset file, erroring on anything malformed.""" + dataset = json.loads((skill / "evals" / filename).read_text()) + unknown_top = set(dataset) - {"evaluations", "comment"} + if unknown_top: + sys.exit(f"{skill.name}: unknown key(s) {sorted(unknown_top)} in {filename}") + cases = dataset.get("evaluations") + if not isinstance(cases, list): + sys.exit(f"{skill.name}: {filename} needs an evaluations array") + + seen: set[str] = set() for case in cases: unknown = set(case) - KNOWN_KEYS if unknown: @@ -102,25 +126,28 @@ def load_cases(skill: Path) -> list[dict[str, Any]]: seen.add(case["id"]) if "workspace" in case and not workspace_dir(skill, case["workspace"]).is_dir(): sys.exit(f"{skill.name}: {case['id']} names a missing workspace") - - if case["skill_should_trigger"]: - positive += 1 - judged += any(case.get(key) for key in JUDGED_KEYS) - else: - negative += 1 + if not case["skill_should_trigger"]: extra = set(case) - {"id", "prompt", "skill_should_trigger", "note"} if extra: sys.exit( f"{skill.name}: {case['id']} is negative but sets {sorted(extra)}" ) + return cases + +def _require_minimum(skill: Path, cases: list[dict[str, Any]]) -> None: + """The catalog bar applies to evals.json only, not the extended file.""" + positive = sum(case["skill_should_trigger"] for case in cases) + negative = len(cases) - positive + judged = sum( + case["skill_should_trigger"] and any(case.get(key) for key in JUDGED_KEYS) + for case in cases + ) if positive < 3 or negative < 2: sys.exit(f"{skill.name}: needs at least 3 positive and 2 negative cases") if not judged: sys.exit(f"{skill.name}: needs at least one positive case with judged behavior") - return cases - def workspace_dir(skill: Path, workspace: str) -> Path: """Join ``workspace`` onto the skill directory the way skillscope does.""" @@ -341,6 +368,14 @@ def main() -> int: parser.add_argument("--skill", help="Run one skill instead of all of them.") parser.add_argument("--only", help="Comma-separated case ids to run.") parser.add_argument("--list", action="store_true", help="Validate datasets only.") + parser.add_argument( + "--extended", + action="store_true", + help=( + "Also run evals/extended_evals.json. Those cases profile a sample " + "and are not graded by the AMD skills catalog." + ), + ) parser.add_argument( "--jobs", type=int, @@ -350,12 +385,24 @@ def main() -> int: args = parser.parse_args() skills = discover_skills(args.skill) - datasets = {skill.name: load_cases(skill) for skill in skills} + datasets = {} + extended_counts = {} + for skill in skills: + cases, extended_count = load_cases(skill, extended=args.extended) + datasets[skill.name] = cases + extended_counts[skill.name] = extended_count if args.list: for name, cases in datasets.items(): positive = sum(case["skill_should_trigger"] for case in cases) - print(f"{name}: {len(cases)} cases ({positive} positive)") + extra = extended_counts[name] + if args.extended and extra: + detail = f"{positive} positive, including {extra} extended" + elif extra: + detail = f"{positive} positive; {extra} extended not selected" + else: + detail = f"{positive} positive" + print(f"{name}: {len(cases)} cases ({detail})") return 0 if not shutil.which("claude"): From 031a4736a0bc372c7eb6554167f9010b81bc9acf Mon Sep 17 00:00:00 2001 From: Vignesh Edithal Date: Thu, 5 Nov 2026 20:02:11 +0000 Subject: [PATCH 2/2] [rocprofiler-compute] Run catalog skill evals on Strix Halo - evals.json keeps every case that passes on Strix Halo, where amd/skills runs it; extended_evals.json holds the MI300X, gfx950, and PyTorch cases - Add gfx115x memory and kernel-name-filter cases; only pc-sampling still asks for MI300X in machine.yml - run_evals.py runs both datasets; the README separates this repo's workflow from the rules amd/skills sets Co-Authored-By: Claude Opus 5.5 (1M context) --- projects/rocprofiler-compute/skills/README.md | 91 ++++++++++-------- .../evals/evals.json | 60 +++++++++--- .../evals/extended_evals.json | 52 +---------- .../rocprof-compute-memory/evals/evals.json | 48 ++++++---- .../evals/extended_evals.json | 34 +------ .../rocprof-compute-memory/evals/machine.yml | 1 - .../evals/evals.json | 51 +++++++--- .../evals/extended_evals.json | 49 ---------- .../rocprof-compute-roofline/evals/evals.json | 55 ++++++++--- .../evals/extended_evals.json | 52 ----------- .../evals/machine.yml | 1 - .../evals/evals.json | 38 ++++++-- .../evals/extended_evals.json | 38 -------- .../evals/evals.json | 10 +- .../evals/extended_evals.json | 10 +- .../rocprofiler-compute/skills/run_evals.py | 92 ++++++------------- 16 files changed, 279 insertions(+), 403 deletions(-) delete mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json delete mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json delete mode 100644 projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json diff --git a/projects/rocprofiler-compute/skills/README.md b/projects/rocprofiler-compute/skills/README.md index 2d83d46e147a..4b101070e4e0 100644 --- a/projects/rocprofiler-compute/skills/README.md +++ b/projects/rocprofiler-compute/skills/README.md @@ -45,7 +45,14 @@ folders, so install for one agent only if you use Cursor alongside them. /share/rocprofiler-compute/skills/install-skills.sh --agent claude --uninstall ``` -## Structure +## Developing skills + +The rest of this file is for people who edit these skills. The skills are also +published in [amd/skills](https://github.com/amd/skills), the AMD skills +catalog, so a change here must work in this repo and in that catalog. See +[Publishing to amd/skills](#publishing-to-amdskills). + +### Structure Every skill directory holds: @@ -54,9 +61,9 @@ Every skill directory holds: ├── SKILL.md # frontmatter (name, description) plus the instructions ├── skill-card.md # human-facing card: description, owner, license └── evals/ - ├── evals.json # minimum set the AMD skills catalog grades - ├── extended_evals.json # longer profile cases, run here with --extended - ├── machine.yml # hardware the behavioral cases need + ├── evals.json # cases this repo and amd/skills run + ├── extended_evals.json # optional; cases only this repo runs + ├── machine.yml # hardware amd/skills runs evals.json on └── samples/ # copies of the samples the cases use ``` @@ -66,35 +73,34 @@ same fact is not maintained in two places. Link docs as `../../docs/...`: `install-skills.sh` rewrites those links to the docs installed next to the skills, so they keep working offline. -## Tests +Both dataset files use the +[skillscope format](https://github.com/amd/skillscope/blob/main/docs/authoring-evals.md). +A case that needs an application sets `workspace` to `evals/samples`, and its +prompt names the application. The contents of that directory are copied to the +workspace root when the case runs. Each copy in `evals/samples/` and its +original under `sample/` name each other in a header comment; change them +together. -CI runs the [skillscope](https://github.com/amd/skillscope) structural checks -on every pull request that changes the skills or `docs/`, the same checks the -AMD skills repo runs on its copy. CI does not grade routing or behavior: -routing needs a model API key, and behavior needs a supported GPU. +### Tests -`run_evals.py` grades both and is the release gate: +CI runs the [skillscope](https://github.com/amd/skillscope) structural checks, +the same checks amd/skills runs, on every pull request that changes the skills +or `docs/`. They cover both dataset files. CI does not grade routing or +behavior: routing needs a model API key, and behavior needs a supported GPU. + +`run_evals.py` grades both and is the release gate. It runs `evals.json` and +`extended_evals.json` together. `evals.json` targets Strix Halo, except for +pc-sampling, and most extended cases need an Instinct GPU, so pick the cases +for your GPU with `--skill` or `--only`: ```bash ./skills/run_evals.py --list # validate the datasets ./skills/run_evals.py # does the right skill load? -./skills/run_evals.py --mode behavioral # grade the minimum set -./skills/run_evals.py --extended --mode behavioral # also grade the long profiles +./skills/run_evals.py --mode behavioral # also grade the agent ./skills/run_evals.py --skill rocprof-compute-memory # one skill +./skills/run_evals.py --only mem-cache-hit-rates # one case ``` -`evals/evals.json` is the minimum set. The AMD skills catalog grades that file -and no other. It keeps at least three prompts that should trigger the skill, -two that should not, and one triggering case with judged behavior. Negative -cases carry only an id, a prompt, and an optional note. The judged cases stay -short: they check that the right workflow is chosen, and they do not profile -a sample. - -Cases that build a sample, profile it, and read the report live in -`evals/extended_evals.json`, beside the minimum file and in the same format. -Add `--extended` to run them. The catalog workflow sets `extended: false`, so -moving a case into that file takes it out of the catalog run. - Routing mode denies the agent its tools and only checks which skill it chose, so it is quick and needs no GPU. Behavioral mode lets the agent work and grades `logs_contain`, `files_exist`, and the plain-language `expected_behavior` and @@ -107,19 +113,7 @@ install script and its docs links. Both drive the `claude` CLI. The script exits 77 when that CLI is absent, so a caller can treat it as skipped rather than failed. -A case that profiles an application lives in `extended_evals.json`. It sets -`workspace` to `evals/samples`, and its prompt names the application. The contents of the -workspace directory are copied to the workspace root when the case runs, as -skillscope does. The AMD skills repo copies only the skill directory, so -`evals/samples/` holds copies of the samples from `sample/`. Each copy and its -original name each other in a header comment; change them together. - -`evals/machine.yml` tells the AMD skills repo where to run the behavioral -cases. Every skill asks for Linux. Only a skill whose cases need an Instinct GPU -also asks for one; the others run on that repo's default GPU runners, which -cost less. - -## Keeping skills correct +### Keeping skills correct CI checks only structure and links, so keeping the content correct falls to each change. A PR that alters CLI options, output layout, supported @@ -134,3 +128,26 @@ skill needs in that skill's own steps. Skills must also serve users on the two previous ROCm releases. When a change alters how a feature behaves, keep a short note in the skill saying how it worked before and in which release. + +## Publishing to amd/skills + +amd/skills imports each skill directory from `develop`. It copies only the skill +directory and rewrites links that leave it, such as `../../docs/...`, to GitHub +URLs. Edit the skills here; the next import overwrites edits made there. + +amd/skills grades each skill with skillscope under these rules: + +- It runs only `evals/evals.json`. That file needs at least three cases that + should trigger the skill, two that should not, and one triggering case with + `expected_behavior` or `unexpected_behavior`. `run_evals.py` checks the same + minimum. +- It runs `evals.json` on Strix Halo (gfx1151) unless `evals/machine.yml` asks + for other hardware. Every skill sets `os: [Linux]` there to skip the Windows + runners. Only pc-sampling also sets `labels`, because it needs an MI300X; that + sends it to the catalog's gated Instinct runners. +- Put cases that cannot pass on those runners in `evals/extended_evals.json`: + cases that need an Instinct GPU, and the torch-trace cases that profile a + model, since the catalog runners may not have PyTorch. The catalog skips this + file. +- It copies only the skill directory, so `evals/samples/` holds copies of the + samples from `sample/`. diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json index 7020fa59fb91..e3e1d8c37b96 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/evals.json @@ -1,6 +1,44 @@ { - "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ + { + "id": "kb-profile-hip-application", + "skill_should_trigger": true, + "prompt": "profile my HIP application with rocprof-compute and tell me where the time goes\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "asks for the launch command and a workload name before profiling", + "runs rocprof-compute profile --name -- ", + "runs analyze with --list-stats to find the hottest kernel" + ], + "unexpected_behavior": [ + "invents a profiling flag instead of checking --help" + ], + "logs_contain": ["rocprof-compute profile", "--list-stats"] + }, + { + "id": "kb-which-kernel-is-slow", + "skill_should_trigger": true, + "prompt": "which kernel is the hotspot in ./workloads/my_app and why is it slow?\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "points --path at the directory containing profiling_config.yaml rather than a parent directory", + "uses --list-stats first, then narrows to one kernel id with -k", + "reads the System Speed-of-Light panel before opening a detailed block" + ], + "logs_contain": ["--list-stats", "-k"] + }, + { + "id": "kb-expensive-replay-fallback", + "skill_should_trigger": true, + "prompt": "a full rocprof-compute replay of this workload is too expensive, collect something cheaper\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "offers a single-pass option such as --set, a narrowed -b, or --iteration-multiplexing", + "runs --list-sets before selecting a set", + "warns which analysis panels will be empty afterwards" + ], + "logs_contain": ["--list-sets"] + }, { "id": "kb-unsafe-to-replay", "skill_should_trigger": true, @@ -15,27 +53,24 @@ ] }, { - "id": "kb-where-time-goes", + "id": "kb-kernel-name-filter-gfx115x", "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is kb-profile-hip-application in extended_evals.json.", - "prompt": "profile my HIP application with rocprof-compute and tell me where the time goes" - }, - { - "id": "kb-cheaper-collection", - "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is kb-expensive-replay-fallback in extended_evals.json.", - "prompt": "a full rocprof-compute replay of this workload is too expensive, what cheaper collection should I use?" + "prompt": "profile only the kernels whose names start with global in my ROCm application\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "uses rocprof-compute profile -k to filter by kernel name", + "treats the profile-time -k value as a regex matched against kernel names" + ], + "logs_contain": ["-k"] }, { "id": "kb-cuda-application", "skill_should_trigger": false, - "note": "Profiling request, but CUDA tooling is out of scope.", "prompt": "profile my CUDA application with nsys" }, { "id": "kb-windows-request", "skill_should_trigger": false, - "note": "rocprof-compute is named, but the platform is unsupported.", "prompt": "profile my application on Windows with rocprof-compute" }, { @@ -46,7 +81,6 @@ { "id": "kb-write-a-kernel", "skill_should_trigger": false, - "note": "GPU performance adjacent, but this skill does not author kernel source.", "prompt": "write me a faster HIP matrix multiply kernel" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json index 88e67388d630..2567269145e4 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-kernel-bottleneck/evals/extended_evals.json @@ -1,53 +1,5 @@ { - "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", "evaluations": [ - { - "id": "kb-profile-hip-application", - "skill_should_trigger": true, - "prompt": "profile my HIP application with rocprof-compute and tell me where the time goes\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "asks for the launch command and a workload name before profiling", - "runs rocprof-compute profile --name -- ", - "runs analyze with --list-stats to find the hottest kernel" - ], - "unexpected_behavior": [ - "invents a profiling flag instead of checking --help" - ], - "logs_contain": [ - "rocprof-compute profile", - "--list-stats" - ] - }, - { - "id": "kb-which-kernel-is-slow", - "skill_should_trigger": true, - "prompt": "which kernel is the hotspot in ./workloads/my_app and why is it slow?\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "points --path at the directory containing profiling_config.yaml rather than a parent directory", - "uses --list-stats first, then narrows to one kernel id with -k", - "reads the System Speed-of-Light panel before opening a detailed block" - ], - "logs_contain": [ - "--list-stats", - "-k" - ] - }, - { - "id": "kb-expensive-replay-fallback", - "skill_should_trigger": true, - "prompt": "a full rocprof-compute replay of this workload is too expensive, collect something cheaper\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "offers a single-pass option such as --set, a narrowed -b, or --iteration-multiplexing", - "runs --list-sets before selecting a set", - "warns which analysis panels will be empty afterwards" - ], - "logs_contain": [ - "--list-sets" - ] - }, { "id": "kb-kernel-name-filter", "skill_should_trigger": true, @@ -57,9 +9,7 @@ "uses rocprof-compute profile -k to filter by kernel name", "treats the profile-time -k value as a regex matched against kernel names" ], - "logs_contain": [ - "-k" - ] + "logs_contain": ["-k"] } ] } diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json index eb051217985a..d38fe953dbd3 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/evals.json @@ -1,35 +1,52 @@ { - "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ { - "id": "mem-which-blocks", + "id": "mem-access-inefficiency", "skill_should_trigger": true, - "note": "Catalog behavioral check. Asks which blocks to collect and does not stage a sample.", - "prompt": "which rocprof-compute blocks should I collect to inspect cache hit rates and LDS bank conflicts, without a full profile?", + "prompt": "find the memory access inefficiencies in this rocprofiler-compute capture\n\nThe application is ./vrandom_access.cpp.", + "workspace": "evals/samples", "expected_behavior": [ - "names the memory chart, or -b memchart, as the place to start", - "collects only the memory blocks the question needs, such as memchart, vl1d, l2, or lds" + "reads the memory chart before opening an individual cache level", + "cites hit rates or request counts from the report", + "ties any coalescing or tiling advice to those numbers" ], "unexpected_behavior": [ - "starts a full multi-pass profile or the roofline benchmarks" - ] + "recommends an optimization without citing a metric from the report" + ], + "logs_contain": ["-b memchart"] }, { - "id": "mem-where-hit-rates", + "id": "mem-cache-hit-rates-gfx115x", "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is mem-cache-hit-rates in extended_evals.json.", - "prompt": "where do I read L2 and vL1D hit rates for one kernel in rocprof-compute?" + "prompt": "what are my GL1 and GL2 hit rates for this kernel and are they bad?\n\nThe application is ./vsequential_access.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "checks the cache block names this GPU uses before choosing -b", + "opens the gl1 and gl2 blocks for a single kernel id", + "judges the rates against what the algorithm should do rather than a fixed cutoff" + ], + "unexpected_behavior": [ + "applies a remembered threshold such as L2 below 50% being bad, as if the tool defined one" + ], + "logs_contain": ["gl2"] }, { - "id": "mem-where-bandwidth-goes", + "id": "mem-bandwidth-bound-gfx115x", "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is mem-bandwidth-bound in extended_evals.json.", - "prompt": "this AMD GPU kernel looks memory bound, which rocprof-compute view shows where the bandwidth goes?" + "prompt": "this kernel looks memory bound, where is the bandwidth going?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "walks the hierarchy from the memory chart down to the level responsible", + "distinguishes traffic that caches absorbed from traffic that reached system memory" + ], + "unexpected_behavior": [ + "describes the GPU memory as HBM" + ], + "logs_contain": ["memchart"] }, { "id": "mem-host-memory", "skill_should_trigger": false, - "note": "Memory question, but host-side allocation is not GPU counter analysis.", "prompt": "my application is leaking host memory, help me find it" }, { @@ -40,7 +57,6 @@ { "id": "mem-gpu-oom", "skill_should_trigger": false, - "note": "About allocation failure, not bandwidth or cache behaviour.", "prompt": "why does my model run out of GPU memory during training?" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json index 16cb9af9b03f..65dc47e2ded1 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/extended_evals.json @@ -1,23 +1,5 @@ { - "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", "evaluations": [ - { - "id": "mem-access-inefficiency", - "skill_should_trigger": true, - "prompt": "find the memory access inefficiencies in this rocprofiler-compute capture\n\nThe application is ./vrandom_access.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "reads the memory chart before opening an individual cache level", - "cites hit rates or request counts from the report", - "ties any coalescing or tiling advice to those numbers" - ], - "unexpected_behavior": [ - "recommends an optimization without citing a metric from the report" - ], - "logs_contain": [ - "-b memchart" - ] - }, { "id": "mem-cache-hit-rates", "skill_should_trigger": true, @@ -30,9 +12,7 @@ "unexpected_behavior": [ "applies a remembered threshold such as L2 below 50% being bad, as if the tool defined one" ], - "logs_contain": [ - "-b" - ] + "logs_contain": ["-b"] }, { "id": "mem-lds-bank-conflicts", @@ -43,9 +23,7 @@ "opens the lds block for the selected kernel", "reads the bank conflict counters from the report" ], - "logs_contain": [ - "lds" - ] + "logs_contain": ["lds"] }, { "id": "mem-bandwidth-bound", @@ -56,9 +34,7 @@ "walks the hierarchy from the memory chart down to the level responsible", "distinguishes traffic that caches absorbed from traffic that reached HBM" ], - "logs_contain": [ - "memchart" - ] + "logs_contain": ["memchart"] }, { "id": "mem-membw-analysis-gfx950", @@ -70,9 +46,7 @@ "notes that the analysis is experimental and shipped for gfx950", "reports the guidance the tool produced rather than substituting its own cutoffs" ], - "logs_contain": [ - "--membw-analysis" - ] + "logs_contain": ["--membw-analysis"] } ] } diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml index 4d7efd456e93..78cd19714df9 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml +++ b/projects/rocprofiler-compute/skills/rocprof-compute-memory/evals/machine.yml @@ -1,2 +1 @@ os: [Linux] -labels: [mi300x, gpu, rocm] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json index 00d0cb8a7146..d589f0934736 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/evals.json @@ -1,6 +1,33 @@ { - "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ + { + "id": "pcs-which-instruction-stalls", + "skill_should_trigger": true, + "prompt": "which instruction in my MI300 kernel stalls the most?\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "queries the device for supported PC sampling configurations before choosing an interval", + "collects with --experimental --pc-sampling", + "sorts the analysis by sample count and reads the stall reason column" + ], + "unexpected_behavior": [ + "hardcodes a sampling interval without querying the device" + ], + "logs_contain": ["--pc-sampling", "rocprofv3-avail"] + }, + { + "id": "pcs-source-line-attribution", + "skill_should_trigger": true, + "prompt": "map the hot GPU instructions back to my HIP source lines\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "states that source line mapping requires the application to be built with debug info", + "notes that samples map to assembly even without debug info" + ], + "unexpected_behavior": [ + "promises source line attribution for a build with no debug info" + ] + }, { "id": "pcs-stochastic-vs-host-trap", "skill_should_trigger": true, @@ -10,9 +37,18 @@ "notes stochastic needs MI300 or newer and host_trap is the fallback", "checks what the installed device actually supports" ], - "logs_contain": [ - "stochastic" - ] + "logs_contain": ["stochastic"] + }, + { + "id": "pcs-export-disassembly", + "skill_should_trigger": true, + "prompt": "export the annotated disassembly with sample counts from my pc sampling workload\n\nThe application is ./vmem.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "uses --output-format csv or db", + "notes that the per-kernel disassembly is written under per_kernel_pc_sampling" + ], + "logs_contain": ["--output-format"] }, { "id": "pcs-host-trap-stall-reason", @@ -26,16 +62,9 @@ "interprets a stall reason from host-trap data" ] }, - { - "id": "pcs-stall-reason-question", - "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is pcs-which-instruction-stalls in extended_evals.json.", - "prompt": "which instruction in my AMD GPU kernel stalls the most, and what is the stall reason?" - }, { "id": "pcs-rank-kernels-by-time", "skill_should_trigger": false, - "note": "Hotspot wording, but ranking whole kernels is counter work, not instruction sampling.", "prompt": "which of my GPU kernels accounts for the most total time?" }, { diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json deleted file mode 100644 index ff20f340bd48..000000000000 --- a/projects/rocprofiler-compute/skills/rocprof-compute-pc-sampling/evals/extended_evals.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", - "evaluations": [ - { - "id": "pcs-which-instruction-stalls", - "skill_should_trigger": true, - "prompt": "which instruction in my MI300 kernel stalls the most?\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "queries the device for supported PC sampling configurations before choosing an interval", - "collects with --experimental --pc-sampling", - "sorts the analysis by sample count and reads the stall reason column" - ], - "unexpected_behavior": [ - "hardcodes a sampling interval without querying the device" - ], - "logs_contain": [ - "--pc-sampling", - "rocprofv3-avail" - ] - }, - { - "id": "pcs-source-line-attribution", - "skill_should_trigger": true, - "prompt": "map the hot GPU instructions back to my HIP source lines\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "states that source line mapping requires the application to be built with debug info", - "notes that samples map to assembly even without debug info" - ], - "unexpected_behavior": [ - "promises source line attribution for a build with no debug info" - ] - }, - { - "id": "pcs-export-disassembly", - "skill_should_trigger": true, - "prompt": "export the annotated disassembly with sample counts from my pc sampling workload\n\nThe application is ./vmem.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "uses --output-format csv or db", - "notes that the per-kernel disassembly is written under per_kernel_pc_sampling" - ], - "logs_contain": [ - "--output-format" - ] - } - ] -} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json index 9f59aabca83b..60a72d51135f 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/evals.json @@ -1,29 +1,57 @@ { - "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ { - "id": "roof-missing-data", + "id": "roof-compute-or-memory-bound", "skill_should_trigger": true, - "prompt": "my rocprof-compute report has no roofline section, what went wrong?", + "prompt": "is this rocprof-compute kernel compute bound or memory bound?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", "expected_behavior": [ - "checks whether the profile used --no-roof", - "checks whether the architecture supports the roofline microbenchmarks before advising a re-profile" + "analyzes the roof block for a single kernel id", + "compares arithmetic intensity against the empirical peaks printed in the table", + "says when the kernel is below both ceilings instead of forcing a bound" ], "unexpected_behavior": [ - "tells the user to re-profile without --no-roof when the architecture is the actual reason" - ] + "opens the generated roofline HTML chart instead of reading the table", + "calculates a peak instead of using the measured one" + ], + "logs_contain": ["-b roof"] }, { - "id": "roof-which-bound", + "id": "roof-arithmetic-intensity", "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is roof-compute-or-memory-bound in extended_evals.json.", - "prompt": "is my AMD GPU kernel compute-bound or memory-bound on the rocprof-compute roofline?" + "prompt": "what is the arithmetic intensity of my hottest kernel and how much performance am I leaving on the table?\n\nThe application is ./mat_mul_max.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "reads FLOPs per byte and the peak values from the roofline table", + "narrows to one kernel id first" + ], + "logs_contain": ["roof"] }, { - "id": "roof-left-on-table", + "id": "roof-precision-selection", "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is roof-arithmetic-intensity in extended_evals.json.", - "prompt": "how much performance is my kernel leaving on the table according to the rocprof-compute roofline?" + "prompt": "my matrix multiply sits far below the top ceiling on the roofline, is it slow?\n\nThe application is ./mat_mul_max.hip.", + "workspace": "evals/samples", + "expected_behavior": [ + "reads the precision from the instruction mix rather than assuming it", + "compares against the FP32 peak, not the highest matrix or FP16 peak" + ], + "unexpected_behavior": [ + "calls the kernel slow because it is below the highest ceiling in the table" + ], + "logs_contain": ["cu_ins"] + }, + { + "id": "roof-missing-data", + "skill_should_trigger": true, + "prompt": "my rocprof-compute report has no roofline section, what went wrong?", + "expected_behavior": [ + "checks whether the profile used --no-roof", + "checks whether the architecture supports the roofline microbenchmarks before advising a re-profile" + ], + "unexpected_behavior": [ + "tells the user to re-profile without --no-roof when the architecture is the actual reason" + ] }, { "id": "roof-generic-optimization", @@ -38,7 +66,6 @@ { "id": "roof-concept-question", "skill_should_trigger": false, - "note": "Asks what a roofline is in the abstract, with no workload to analyze.", "prompt": "explain the roofline performance model to me" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json deleted file mode 100644 index 363380eb1b32..000000000000 --- a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/extended_evals.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", - "evaluations": [ - { - "id": "roof-compute-or-memory-bound", - "skill_should_trigger": true, - "prompt": "is this rocprof-compute kernel compute bound or memory bound?\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "analyzes the roof block for a single kernel id", - "compares arithmetic intensity against the empirical peaks printed in the table", - "says when the kernel is below both ceilings instead of forcing a bound" - ], - "unexpected_behavior": [ - "opens the generated roofline HTML chart instead of reading the table", - "calculates a peak instead of using the measured one" - ], - "logs_contain": [ - "-b roof" - ] - }, - { - "id": "roof-arithmetic-intensity", - "skill_should_trigger": true, - "prompt": "what is the arithmetic intensity of my hottest kernel and how much performance am I leaving on the table?\n\nThe application is ./mat_mul_max.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "reads FLOPs per byte and the peak values from the roofline table", - "narrows to one kernel id first" - ], - "logs_contain": [ - "roof" - ] - }, - { - "id": "roof-precision-selection", - "skill_should_trigger": true, - "prompt": "my matrix multiply sits far below the top ceiling on the roofline, is it slow?\n\nThe application is ./mat_mul_max.hip.", - "workspace": "evals/samples", - "expected_behavior": [ - "reads the precision from the instruction mix rather than assuming it", - "compares against the FP32 peak, not the highest matrix or FP16 peak" - ], - "unexpected_behavior": [ - "calls the kernel slow because it is below the highest ceiling in the table" - ], - "logs_contain": [ - "cu_ins" - ] - } - ] -} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml index 4d7efd456e93..78cd19714df9 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml +++ b/projects/rocprofiler-compute/skills/rocprof-compute-roofline/evals/machine.yml @@ -1,2 +1 @@ os: [Linux] -labels: [mi300x, gpu, rocm] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json index f3f27bddec8a..628f8c8ddfc4 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/evals.json @@ -1,6 +1,34 @@ { - "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ + { + "id": "sol-percent-of-peak", + "skill_should_trigger": true, + "prompt": "how close to peak is this kernel running in my rocprof-compute workload?\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "analyzes the sol block for a single kernel id", + "reads the Percent of Peak column from the report", + "compares against the Peak value the report prints rather than a remembered threshold" + ], + "unexpected_behavior": [ + "calculates a theoretical peak instead of using the one in the report" + ], + "logs_contain": ["-b sol"] + }, + { + "id": "sol-overall-efficiency", + "skill_should_trigger": true, + "prompt": "give me an overall efficiency summary of this rocprofiler-compute capture\n\nThe application is ./vcopy.cpp.", + "workspace": "evals/samples", + "expected_behavior": [ + "selects one kernel with -k before reading Speed-of-Light", + "summarizes using the printed peak and percent of peak values" + ], + "unexpected_behavior": [ + "reports a Speed-of-Light table averaged across unrelated kernels" + ], + "logs_contain": ["-b sol"] + }, { "id": "sol-mfma-zero", "skill_should_trigger": true, @@ -25,16 +53,9 @@ "states a universal threshold such as 70% as if the tool defined one" ] }, - { - "id": "sol-how-close-to-peak", - "skill_should_trigger": true, - "note": "Catalog trigger check. The profiled version is sol-percent-of-peak in extended_evals.json.", - "prompt": "how close to hardware peak is this AMD GPU kernel in rocprof-compute?" - }, { "id": "sol-instruction-level", "skill_should_trigger": false, - "note": "Peak comparison is not the question; this is per-instruction work.", "prompt": "which individual instruction in this kernel stalls the most?" }, { @@ -45,7 +66,6 @@ { "id": "sol-hardware-spec-lookup", "skill_should_trigger": false, - "note": "Asks for a datasheet number, not a reading from a profile.", "prompt": "what is the theoretical peak FP64 throughput of an MI300X?" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json deleted file mode 100644 index fa4d6250470e..000000000000 --- a/projects/rocprofiler-compute/skills/rocprof-compute-speed-of-light/evals/extended_evals.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", - "evaluations": [ - { - "id": "sol-percent-of-peak", - "skill_should_trigger": true, - "prompt": "how close to peak is this kernel running in my rocprof-compute workload?\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "analyzes the sol block for a single kernel id", - "reads the Percent of Peak column from the report", - "compares against the Peak value the report prints rather than a remembered threshold" - ], - "unexpected_behavior": [ - "calculates a theoretical peak instead of using the one in the report" - ], - "logs_contain": [ - "-b sol" - ] - }, - { - "id": "sol-overall-efficiency", - "skill_should_trigger": true, - "prompt": "give me an overall efficiency summary of this rocprofiler-compute capture\n\nThe application is ./vcopy.cpp.", - "workspace": "evals/samples", - "expected_behavior": [ - "selects one kernel with -k before reading Speed-of-Light", - "summarizes using the printed peak and percent of peak values" - ], - "unexpected_behavior": [ - "reports a Speed-of-Light table averaged across unrelated kernels" - ], - "logs_contain": [ - "-b sol" - ] - } - ] -} diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json index 23ff37726945..82c6975fddd6 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/evals.json @@ -1,5 +1,4 @@ { - "comment": "Minimum dataset graded by the AMD skills catalog. Cases that profile a sample and read the report live in extended_evals.json.", "evaluations": [ { "id": "tt-triton-inductor-kernels", @@ -11,9 +10,7 @@ "unexpected_behavior": [ "relies on --triton-trace alone to capture torch.compile kernels" ], - "logs_contain": [ - "--torch-trace" - ] + "logs_contain": ["--torch-trace"] }, { "id": "tt-filter-by-operator", @@ -23,9 +20,7 @@ "filters with --torch-operator", "treats the pattern as a shell-style glob rather than a regex" ], - "logs_contain": [ - "--torch-operator" - ] + "logs_contain": ["--torch-operator"] }, { "id": "tt-custom-roctx-unsupported", @@ -59,7 +54,6 @@ { "id": "tt-model-accuracy", "skill_should_trigger": false, - "note": "PyTorch question with no performance profiling in it.", "prompt": "why is my PyTorch model's validation accuracy dropping?" } ] diff --git a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json index 681a3fe9f15d..21a5c654e905 100644 --- a/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json +++ b/projects/rocprofiler-compute/skills/rocprof-compute-torch-trace/evals/extended_evals.json @@ -1,5 +1,4 @@ { - "comment": "Profile-and-read cases. The AMD skills catalog does not grade this file. Run it here with ./skills/run_evals.py --extended.", "evaluations": [ { "id": "tt-which-operator-is-slow", @@ -14,10 +13,7 @@ "unexpected_behavior": [ "guesses an operator pattern without listing the captured operators first" ], - "logs_contain": [ - "--torch-trace", - "--list-torch-operators" - ] + "logs_contain": ["--torch-trace", "--list-torch-operators"] }, { "id": "tt-map-kernels-to-model-code", @@ -28,9 +24,7 @@ "profiles the Python script with torch tracing enabled", "advises profiling a few steady-state iterations rather than a full run" ], - "logs_contain": [ - "--torch-trace" - ] + "logs_contain": ["--torch-trace"] } ] } diff --git a/projects/rocprofiler-compute/skills/run_evals.py b/projects/rocprofiler-compute/skills/run_evals.py index b2da14ecadbc..9688d2ed77a0 100755 --- a/projects/rocprofiler-compute/skills/run_evals.py +++ b/projects/rocprofiler-compute/skills/run_evals.py @@ -12,7 +12,6 @@ Usage: ./skills/run_evals.py # routing only, no tools ./skills/run_evals.py --mode behavioral # also grade what the agent did - ./skills/run_evals.py --extended --mode behavioral # include the long profile cases ./skills/run_evals.py --skill rocprof-compute-memory # one skill ./skills/run_evals.py --only kb-kernel-name-filter # one case ./skills/run_evals.py --list # validate datasets, run nothing @@ -78,38 +77,28 @@ def discover_skills(only: Optional[str]) -> list[Path]: return found -def load_cases(skill: Path, *, extended: bool) -> tuple[list[dict[str, Any]], int]: - """Parse and validate one skill's datasets. +def load_cases(skill: Path) -> list[dict[str, Any]]: + """Return the cases from both of a skill's datasets, erroring on bad ones. - ``evals/evals.json`` is the catalog minimum and always runs. The optional - ``evals/extended_evals.json`` is validated whenever it exists and is run - only when `extended` is set. Returns the cases to run and how many of - them came from the extended file. + The minimum applies to evals.json only, the one file amd/skills runs. """ - required = _read_cases(skill, "evals.json") + required = _read_evaluations(skill / "evals" / "evals.json") + extended = _read_evaluations(skill / "evals" / "extended_evals.json") + cases = required + extended + _check_cases(skill, cases) _require_minimum(skill, required) - extra_path = skill / "evals" / "extended_evals.json" - extra = _read_cases(skill, "extended_evals.json") if extra_path.is_file() else [] - seen = {case["id"] for case in required} - for case in extra: - if case["id"] in seen: - sys.exit(f"{skill.name}: duplicate case id {case['id']!r}") - seen.add(case["id"]) - if extended: - return required + extra, len(extra) - return required, len(extra) + return cases + +def _read_evaluations(path: Path) -> list[dict[str, Any]]: + """Return the evaluations in one dataset file, or none when it is absent.""" + if not path.is_file(): + return [] + return json.loads(path.read_text(encoding="utf-8"))["evaluations"] -def _read_cases(skill: Path, filename: str) -> list[dict[str, Any]]: - """Parse one dataset file, erroring on anything malformed.""" - dataset = json.loads((skill / "evals" / filename).read_text()) - unknown_top = set(dataset) - {"evaluations", "comment"} - if unknown_top: - sys.exit(f"{skill.name}: unknown key(s) {sorted(unknown_top)} in {filename}") - cases = dataset.get("evaluations") - if not isinstance(cases, list): - sys.exit(f"{skill.name}: {filename} needs an evaluations array") +def _check_cases(skill: Path, cases: list[dict[str, Any]]) -> None: + """Exit on the first malformed case.""" seen: set[str] = set() for case in cases: unknown = set(case) - KNOWN_KEYS @@ -126,26 +115,19 @@ def _read_cases(skill: Path, filename: str) -> list[dict[str, Any]]: seen.add(case["id"]) if "workspace" in case and not workspace_dir(skill, case["workspace"]).is_dir(): sys.exit(f"{skill.name}: {case['id']} names a missing workspace") - if not case["skill_should_trigger"]: - extra = set(case) - {"id", "prompt", "skill_should_trigger", "note"} - if extra: - sys.exit( - f"{skill.name}: {case['id']} is negative but sets {sorted(extra)}" - ) - return cases + if case["skill_should_trigger"]: + continue + extra = set(case) - {"id", "prompt", "skill_should_trigger", "note"} + if extra: + sys.exit(f"{skill.name}: {case['id']} is negative but sets {sorted(extra)}") def _require_minimum(skill: Path, cases: list[dict[str, Any]]) -> None: - """The catalog bar applies to evals.json only, not the extended file.""" - positive = sum(case["skill_should_trigger"] for case in cases) - negative = len(cases) - positive - judged = sum( - case["skill_should_trigger"] and any(case.get(key) for key in JUDGED_KEYS) - for case in cases - ) - if positive < 3 or negative < 2: + """Exit unless the cases meet the minimum amd/skills requires.""" + positive = [case for case in cases if case["skill_should_trigger"]] + if len(positive) < 3 or len(cases) - len(positive) < 2: sys.exit(f"{skill.name}: needs at least 3 positive and 2 negative cases") - if not judged: + if not any(case.get(key) for case in positive for key in JUDGED_KEYS): sys.exit(f"{skill.name}: needs at least one positive case with judged behavior") @@ -368,14 +350,6 @@ def main() -> int: parser.add_argument("--skill", help="Run one skill instead of all of them.") parser.add_argument("--only", help="Comma-separated case ids to run.") parser.add_argument("--list", action="store_true", help="Validate datasets only.") - parser.add_argument( - "--extended", - action="store_true", - help=( - "Also run evals/extended_evals.json. Those cases profile a sample " - "and are not graded by the AMD skills catalog." - ), - ) parser.add_argument( "--jobs", type=int, @@ -385,24 +359,12 @@ def main() -> int: args = parser.parse_args() skills = discover_skills(args.skill) - datasets = {} - extended_counts = {} - for skill in skills: - cases, extended_count = load_cases(skill, extended=args.extended) - datasets[skill.name] = cases - extended_counts[skill.name] = extended_count + datasets = {skill.name: load_cases(skill) for skill in skills} if args.list: for name, cases in datasets.items(): positive = sum(case["skill_should_trigger"] for case in cases) - extra = extended_counts[name] - if args.extended and extra: - detail = f"{positive} positive, including {extra} extended" - elif extra: - detail = f"{positive} positive; {extra} extended not selected" - else: - detail = f"{positive} positive" - print(f"{name}: {len(cases)} cases ({detail})") + print(f"{name}: {len(cases)} cases ({positive} positive)") return 0 if not shutil.which("claude"):