diff --git a/61-rate-models.sh b/61-rate-models.sh new file mode 100755 index 0000000..cfe5238 --- /dev/null +++ b/61-rate-models.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Rate the models this machine actually serves, on the tasks a coding agent +# actually performs, and write the evidence to a file. +# +# This is what turns catalog_ratings() from a column of `unknown` into +# something with a method, a date and an artifact behind it. It does NOT edit +# the catalog: the table is curated by hand, and a script that rewrites its own +# evidence base is not evidence. It prints the rows to paste, and prints the +# artifact they cite. +# +# Read lib/rate.sh before trusting the number. In particular: nothing the model +# produces is executed, the suite is twelve text-graded tasks, and the +# confidence ceiling is `medium` on purpose. +# +# Usage: +# ./61-rate-models.sh # every served model, one pass each +# ./61-rate-models.sh --repeats 3 # three passes; disagreement -> low +# ./61-rate-models.sh --model qwen3-4b # one served model, by name +# ./61-rate-models.sh --dry-run # print the plan, call nothing +# +# Output -> ~/llm-rating-.txt +set -uo pipefail +RIG_SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$RIG_SRC_DIR/lib/detect.sh" +source "$RIG_SRC_DIR/lib/catalog.sh" +source "$RIG_SRC_DIR/lib/preflight.sh" +source "$RIG_SRC_DIR/lib/rate.sh" + +ONLY="" +DRY=0 + +while (( $# )); do + case "$1" in + --repeats) shift; RATE_REPEATS="${1:-1}" ;; + --repeats=*) RATE_REPEATS="${1#--repeats=}" ;; + --model) shift; ONLY="${1:-}" ;; + --model=*) ONLY="${1#--model=}" ;; + --dry-run) DRY=1 ;; + -h|--help) sed -n '2,21p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +[[ "$RATE_REPEATS" =~ ^[1-9][0-9]*$ ]] || die "--repeats must be a positive integer, got '$RATE_REPEATS'" +command -v jq >/dev/null 2>&1 || die "jq is required: sudo apt-get install -y jq" + +STAMP="$(date +%Y%m%d-%H%M)" +TODAY="$(date +%F)" +ARTIFACT="$HOME/llm-rating-$STAMP.txt" +ERRFILE="$(mktemp)" +trap 'rm -f "$ERRFILE"' EXIT + +BASE="$(preflight_endpoint)" +c_info "Rating against $BASE" + +AVAILABLE="$(preflight_check_models "$BASE")" || die \ + "no models served at $BASE -- start the stack first (./40-serve.sh), then re-run" + +if [[ -n "$ONLY" ]]; then + printf '%s\n' "$AVAILABLE" | grep -qxF "$ONLY" \ + || die "$BASE does not serve '$ONLY'. It serves: $(printf '%s' "$AVAILABLE" | tr '\n' ' ')" + AVAILABLE="$ONLY" +fi + +TOTAL_W="$(rate_total_weight)" +NTASKS="$(rate_task_count)" + +c_info "suite v$RATE_SUITE_VERSION -- $NTASKS tasks, total weight $TOTAL_W, $RATE_REPEATS repeat(s)" + +if (( DRY )); then + printf '%s\n' "$AVAILABLE" | while IFS= read -r served; do + [[ -n "$served" ]] || continue + if id="$(rate_catalog_id "$served")"; then + printf ' %-40s -> %s\n' "$served" "$id" + else + printf ' %-40s -> (not in the catalog; will be measured, not recorded)\n' "$served" + fi + done + exit 0 +fi + +# --- run -------------------------------------------------------------------- + +CFG="${LLAMA_SWAP_CFG:-$RIG_DIR/etc/llama-swap.yaml}" + +{ + printf 'llm-rig local coding rating %s\n' "$STAMP" + printf 'suite: v%s (%s tasks, total weight %s)\n' "$RATE_SUITE_VERSION" "$NTASKS" "$TOTAL_W" + printf 'endpoint: %s\n' "$BASE" + printf 'sampling: temperature=%s seed=%s max_tokens=%s repeats=%s\n' \ + "$RATE_TEMPERATURE" "$RATE_SEED" "$RATE_MAX_TOKENS" "$RATE_REPEATS" + printf 'grading: text only. No model output is executed.\n' + # Runtime identity. A rating is only reproducible if you know what was + # running: the served alias does not say which quant, which llama.cpp build + # or which context produced it. Anything that cannot be read is recorded as + # `unavailable` rather than guessed. + printf 'llama.cpp revision: %s\n' "$(rate_llamacpp_rev "$RIG_DIR")" + printf 'llama-swap config: %s\n' "$( [[ -f "$CFG" ]] && printf '%s' "$CFG" || printf '%s' "$RATE_UNAVAILABLE" )" + printf '\n' +} > "$ARTIFACT" + +ROWS="" + +while IFS= read -r served; do + [[ -n "$served" ]] || continue + + cid="" + if ! cid="$(rate_catalog_id "$served")"; then + c_warn "$served is not in the catalog -- measuring it, but no rating row can be written" + cid="" + fi + + # Read before the suite runs, except the live context: that needs the model + # loaded, which the first task does. + gguf="$(rate_swap_gguf "$CFG" "$served")" + quant="$(rate_quant_of "$gguf")" + flags="$(rate_swap_flags "$CFG" "$served")" + + c_info "$served${cid:+ (catalog: $cid)} quant=$quant" + { + printf 'model: %s\n' "$served" + printf 'catalog-id: %s\n' "${cid:-none}" + printf ' weights: %s\n' "$( [[ "$gguf" == "$RATE_UNAVAILABLE" ]] && printf '%s' "$gguf" || printf '%s' "${gguf##*/}" )" + printf ' quant: %s\n' "$quant" + printf ' serving flags: %s\n' "$flags" + } >> "$ARTIFACT" + + got_w=0; answered=0; flips=0 + + while IFS= read -r task; do + [[ -n "$task" ]] || continue + tid="${task%%;*}" + weight="$(rate_task_get "$tid" weight)" + + # Each repeat is graded on its own; the task counts as passed only if every + # repeat passed, and a disagreement is recorded rather than averaged away. + passes=0; errors=0 + for (( r = 1; r <= RATE_REPEATS; r++ )); do + # The reason goes through a file rather than RATE_LAST_ERROR: the + # response arrives on stdout, so the call is a command substitution, and + # a variable set inside one does not survive it. + if resp="$(rate_call "$BASE" "$served" "$tid" 2>"$ERRFILE")"; then + if rate_grade "$tid" "$resp"; then + passes=$(( passes + 1 )) + fi + else + errors=$(( errors + 1 )) + printf ' task %-20s error: %s\n' "$tid" \ + "$(head -1 "$ERRFILE" 2>/dev/null || printf 'unknown')" >> "$ARTIFACT" + fi + done + + if (( errors )); then + verdict="error" + elif (( passes == RATE_REPEATS )); then + verdict="pass"; got_w=$(( got_w + weight )); answered=$(( answered + 1 )) + elif (( passes == 0 )); then + verdict="fail"; answered=$(( answered + 1 )) + else + verdict="unstable"; answered=$(( answered + 1 )); flips=$(( flips + 1 )) + fi + + printf ' task %-20s kind=%-6s weight=%s %s (%s/%s)\n' \ + "$tid" "$(rate_task_get "$tid" kind)" "$weight" "$verdict" "$passes" "$RATE_REPEATS" \ + >> "$ARTIFACT" + printf ' %-20s %s\n' "$tid" "$verdict" >&2 + done < <(rate_tasks) + + value="$(rate_value "$got_w" "$TOTAL_W")" + conf="$(rate_confidence "$answered" "$NTASKS" "$RATE_REPEATS" "$flips")" + + # Now, while the model is still loaded, ask the server what it actually has. + # /props is the only source for this; the config says what was asked for. + ctx="$(rate_live_ctx "$CFG" "$gguf")" + printf ' live n_ctx: %s\n' "$ctx" >> "$ARTIFACT" + + printf 'RESULT %s value=%s quant=%s weight=%s/%s answered=%s/%s flips=%s confidence=%s\n\n' \ + "${cid:-$served}" "$value" "$quant" "$got_w" "$TOTAL_W" "$answered" "$NTASKS" "$flips" "$conf" \ + >> "$ARTIFACT" + + c_ok "$served value=$value weight=$got_w/$TOTAL_W confidence=$conf" + + # A model that errored on any task is measured but not offered as a rating: + # a partial run is a report, not evidence. + if [[ -n "$cid" ]] && (( answered == NTASKS )); then + ROWS+="$(rate_row "$cid" "$value" "$TODAY" "$ARTIFACT" "$conf")"$'\n' + fi +done < <(printf '%s\n' "$AVAILABLE") + +c_info "artifact: $ARTIFACT" + +if [[ -n "$ROWS" ]]; then + printf '\n' + c_info "Paste these into catalog_ratings() in lib/catalog.sh, replacing the matching ids:" + printf '%s' "$ROWS" + printf '\n' + c_info "Then re-validate: bash -c 'source lib/models.sh; catalog_validate || printf \"%%s\" \"\$CATALOG_ERRORS\"'" +else + c_warn "No complete result for any catalogued model -- nothing to record" +fi diff --git a/README.md b/README.md index a74e763..e48b9d1 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ claude | Script | Purpose | |---|---| | `./71-verify-runtime.sh` | Query the **running** server's `/props` — confirms live `n_ctx`, flash-attn, KV cache types. Trust this over the config file. Grades its evidence; see below. | +| `./61-rate-models.sh` | Rate the models you serve on a fixed coding suite, and print rows for `catalog_ratings()`. Nothing it grades is executed; see [Rating the models you serve](#rating-the-models-you-serve). | | `./70-thermal-sweep.sh` | Re-derive the best power limit for your chassis under a heat-soaked load. | | `./80-try-bigger.sh [quant]` | Assess, download, auto-tune `--n-cpu-moe` and benchmark a model **larger than VRAM**. Empirically finds the lowest working offload level. `--list` sizes it without downloading. | | `./19-os-revert.sh` | Undo `10-os-tune.sh`, restoring the values captured before it ran — not assumed defaults. See [Rollback](#what-reversible-means). | @@ -104,8 +105,12 @@ not reproducible here, and sorting one vendor's SWE-bench figure against another's HumanEval figure produces an ordering that means nothing. The validator enforces the consequence in both directions: a value cannot be recorded without a method and a source behind it, and a method claiming -evidence cannot be recorded without a value. Filling these in is the deferred -local-benchmark work. +evidence cannot be recorded without a value. + +Filling them in is what [`./61-rate-models.sh`](61-rate-models.sh) is for — it +measures the models **you** serve and prints rows you can paste in. The shipped +table stays `unknown` because the shipped table cannot contain your +measurements. See [Rating the models you serve](#rating-the-models-you-serve). This replaced a single `coding_score` column carrying values from 42 to 88 with no source, no date and no method — weighted at 25% of the recommendation. @@ -178,7 +183,7 @@ One consequence to be aware of: with every coding rating still `unknown`, the quality term does no discriminating work, so the ranking runs on freshness, hardware fit, speed and features. On a 31 GB machine that makes Laguna XS 2.1 the top `medium` pick ahead of `qwen3-coder-30b`, on metadata alone. That is -the existing design behaving as designed, and it is an argument for finishing +the existing design behaving as designed, and it is an argument for running the local benchmark, not for hand-weighting the table. Running them here: XS 2.1 at `Q4_K_M` is 18.9 GiB and needs both cards @@ -286,8 +291,14 @@ publisher, the rating behind a quarter of the weight is not. Confidence counts three independent kinds of evidence — verified facts, a sourced rating, and current live data. Three of three is `high`, two is -`medium`, fewer is `low`. **Nothing reaches `high` today**, because no model has -a rating; that ceiling lifts on its own once local benchmarking lands. +`medium`, fewer is `low`. **Nothing reaches `high` on the shipped table**, +because no model has a rating; that ceiling lifts on its own once you record +one with [`./61-rate-models.sh`](61-rate-models.sh). + +The rating counts only when the rating itself is `medium` or better, and that +qualifier is load-bearing: a single unrepeated benchmark pass is recorded as +`low`, and a hurried run must not be able to raise the confidence of the +ranking it feeds. Note what the neutral rating does to the ranking: with the quality term equal for every model, the total is driven by fit and speed, so the smallest model @@ -331,6 +342,105 @@ whose next step downloads tens of gigabytes. afterwards so a run can be reproduced exactly. Duplicate, out-of-range and non-numeric answers are refused by name rather than with "invalid input". +### Rating the models you serve + +`./61-rate-models.sh` is the answer to "a quarter of the score is a neutral +placeholder". It runs a fixed suite against the models **this machine actually +serves**, at the quant they are actually served at, and writes the evidence to +`~/llm-rating-.txt`. + +```bash +./61-rate-models.sh # every served model, one pass each +./61-rate-models.sh --repeats 3 # three passes; any disagreement -> low +./61-rate-models.sh --model qwen3-4b # one model +./61-rate-models.sh --dry-run # show the plan, call nothing +``` + +Read [`lib/rate.sh`](lib/rate.sh) before trusting a number out of it. Four +things about it are deliberate and constrain what it can claim: + +- **Nothing the model produces is executed.** The obvious way to grade + generated code is to run it; that means running text from a model on your + machine, as you, for a score. Every task is graded by reading the response — + an exact answer, a `tool_use` block, a parse. The task set is written around + that constraint rather than pretending it is not there. +- **It measures agent-shaped competence, not SWE-bench.** Read a snippet and + say what it does, pick the right tool with the right arguments, obey an + output format. Those are the failures that make a local model useless as a + Claude Code backend. Tool tasks carry double weight, and all three tools are + offered on every one of them, so a model that always calls the first tool + fails two of the three. +- **Format tasks are graded strictly; comprehension tasks are not.** The + distinction is deliberate and is what makes "obey an output format" a claim + the score actually measures — see the table below. +- **It is comparable across models on your machine and nowhere else.** That is + the comparison the ranking needs, and it is why the method is called + `local-benchmark` rather than `benchmark`. +- **The confidence ceiling is `medium`.** Twelve text-graded tasks at one quant + on one machine does not settle how good a model is at coding. A single pass + is `low`; two clean repeats are `medium`; `high` is not reachable from here. + +#### Two grading regimes, on purpose + +| Kind | Tasks | Graded on | +| --- | --- | --- | +| `answer` | 6 | The last non-empty line, normalised. Fences, quotes, trailing punctuation and preceding prose are stripped: the question is whether the model knows the answer. | +| `tool` | 3 | A `tool_use` block satisfying a jq filter. Wrapping text is irrelevant — the block either exists with the right arguments or it does not. | +| `oneword` | 1 | **The whole response.** Trimmed of surrounding whitespace, it must be exactly the word. A fence, a full stop or a sentence around it is the failure being measured. | +| `json-only` | 1 | **The whole response.** It must parse as a JSON object on its own — no fence, no prose — and satisfy a jq filter. | +| `diff-only` | 1 | **The whole response.** At least one `@@` hunk header, every non-empty line valid diff syntax, and the required line present. A preamble fails. | + +The lenient normaliser would otherwise turn `Let me think.\n\nbash.` into +`bash` and pass a task whose entire subject is formatting. A strict kind may +not call it at all, and a test enforces that structurally, so the bug cannot +come back quietly. A model that cannot suppress its preamble cannot be trusted +to emit a patch a tool will apply — that is the thing `diff-only` measures. + +Sampling is pinned — temperature 0, fixed seed — and `--repeats` is what checks +that the pinning held. A task that does not give the same verdict every time +counts as unstable, and one unstable task drops the whole run to `low`: a model +that answers differently at temperature 0 is telling you the measurement is +not stable. + +#### The artifact records what was running + +A served alias does not identify a runtime. `qwen3-4b` fronts whatever GGUF +`30-models.sh` last downloaded, built by whatever `20-build-llamacpp.sh` last +compiled, at whatever context `40-serve.sh` last configured — so two runs that +disagree could otherwise produce the same catalog row. Each run therefore +records: + +| Field | Read from | +| --- | --- | +| llama.cpp revision | `.llamacpp-rev`, written by the build | +| weights | the `-m` path in the generated `llama-swap.yaml`, per model | +| quant | the filename **on disk**, not the catalog's preference — the reason to record it is that the two can differ | +| serving flags | the per-model flags in the same config, including `CUDA_VISIBLE_DEVICES` | +| live `n_ctx` | `/props` on the upstream serving *those* weights | + +Anything that cannot be read is written as `unavailable`. The live context is +matched by model path rather than taken from the first server that answers: +with several models loaded, the first answer is some other model's context, +and recording that is worse than recording nothing. The quant also appears on +the `RESULT` line, so two ratings taken at different quants cannot be compared +by accident. + +The script **does not edit the catalog**. It prints rows to paste: + +``` +qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-20260811-1930.txt;medium +``` + +The source is the artifact's **basename**, not a URL. There is no https address +for a file in your `$HOME`, and the validator refuses an invented one — +`local-benchmark` must cite `file:.txt`, `vendor-benchmark` must cite +https. A table that is curated by hand is not improved by a script that +rewrites its own evidence base, so the paste is manual and the run that +produced it is a file you can open. + +A run where any task errored is written down but **not** offered as a row: a +partial run is a report, not evidence. + ### The llama-swap binary is pinned and verified `40-serve.sh` installs executable code into `/usr/local/bin` as root, so it diff --git a/lib/catalog.sh b/lib/catalog.sh index 3e4a894..eed9c5d 100644 --- a/lib/catalog.sh +++ b/lib/catalog.sh @@ -109,7 +109,11 @@ CATALOG_FACT_METHODS=(hf-api card-stated vendor-blog derived) # # none no comparable evidence; rating_value must be "unknown" # vendor-benchmark a number the publisher reports for its own model -# local-benchmark measured on this machine by lib/bench.sh +# local-benchmark measured on this machine by 61-rate-models.sh, against +# the suite in lib/rate.sh, at the quant this rig serves. +# Comparable across models HERE and nowhere else, which is +# the comparison the ranking needs. Its source is the +# artifact basename, not a URL -- see the validator. CATALOG_RATING_METHODS=(none vendor-benchmark local-benchmark) CATALOG_RATING_CONFIDENCE=(none low medium high) @@ -715,8 +719,26 @@ catalog_validate_ratings_into() { || errs+="rating row $n ($id): method '$method' claims evidence but value is unknown"$'\n' [[ "$conf" != "none" ]] \ || errs+="rating row $n ($id): method '$method' claims evidence but confidence is 'none'"$'\n' - [[ "$source" == https://* ]] \ - || errs+="rating row $n ($id): rating_source '$source' must be an https URL"$'\n' + # What counts as a source depends on the method, because the two kinds of + # evidence live in different places. A vendor benchmark is published and + # must be linkable. A local benchmark is a file in the runner's own + # $HOME -- there is no URL for it, and inventing an https address so the + # field validates would be a fabrication in the one column whose whole + # job is to say where a number came from. + # + # `file:`, not an absolute path: the artifact is in the $HOME + # of whoever ran it, and a path from someone else's machine would not + # resolve on yours. The basename is what you look for in your own. + case "$method" in + local-benchmark) + [[ "$source" == file:*.txt && "$source" != *"/"* ]] \ + || errs+="rating row $n ($id): rating_source '$source' must be 'file:.txt' -- a basename under \$HOME, produced by 61-rate-models.sh"$'\n' + ;; + *) + [[ "$source" == https://* ]] \ + || errs+="rating row $n ($id): rating_source '$source' must be an https URL"$'\n' + ;; + esac if [[ ! "$rdate" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || ! date -d "$rdate" +%Y-%m-%d >/dev/null 2>&1; then errs+="rating row $n ($id): rating_date '$rdate' is not a real ISO date"$'\n' fi diff --git a/lib/rate.sh b/lib/rate.sh new file mode 100644 index 0000000..258e40e --- /dev/null +++ b/lib/rate.sh @@ -0,0 +1,566 @@ +#!/usr/bin/env bash +# Local coding benchmark -- the evidence behind a `local-benchmark` rating. +# +# Every row in catalog_ratings() reads `unknown`, and the README says filling +# them in is deferred work. This is that work: a fixed, versioned suite run +# against the models actually served on this machine, producing a number with +# an artifact behind it. +# +# WHAT THIS MEASURES, AND WHAT IT DOES NOT +# +# It is not SWE-bench and does not claim to be. It measures whether a model, +# at the quant and context this rig serves it at, can do the small things a +# coding agent does constantly: read a snippet and say what it does, pick the +# right tool with the right arguments, and obey an output format. Those are +# the failures that make a local model useless as a Claude Code backend, and +# they are the ones that are cheap to check without a judge model. +# +# It is therefore comparable ACROSS MODELS ON THIS MACHINE and nowhere else. +# That is exactly the comparison the ranking needs, and it is why the rating +# method is called `local-benchmark` rather than `benchmark`. +# +# NOTHING THE MODEL PRODUCES IS EVER EXECUTED. +# +# The obvious way to grade generated code is to run it. This suite does not, +# and will not: it would mean executing text from a model on the developer's +# own machine, as their own user, for a score. Every task is graded by reading +# the response -- an exact answer, a tool_use block, a parse. That constrains +# what can be asked (no "write a function and let's see if it works"), and the +# task set is written around the constraint rather than pretending it is not +# there. +# +# shellcheck shell=bash + +[[ -z "${_LLMRIG_RATE_SH:-}" ]] || return 0 +_LLMRIG_RATE_SH=1 + +# rate_catalog_id joins a served model back to a catalog row. +# shellcheck source=lib/catalog.sh +source "$(dirname "${BASH_SOURCE[0]}")/catalog.sh" +# swap_upstream_ports: which ports llama-swap may have opened, derived from the +# generated config rather than assumed to be 9100. +# shellcheck source=lib/bench.sh +source "$(dirname "${BASH_SOURCE[0]}")/bench.sh" + +# Bump when a task is added, removed or reworded. Two ratings from different +# suite versions are not comparable, and the artifact records which one ran. +# shellcheck disable=SC2034 # documented return channel, read by callers +RATE_SUITE_VERSION=1 + +# Sampling. Zero temperature and a fixed seed because a rating that changes +# between runs is not a rating; `--repeats` exists to check that it doesn't. +RATE_TEMPERATURE="${RATE_TEMPERATURE:-0}" +RATE_SEED="${RATE_SEED:-42}" +RATE_MAX_TOKENS="${RATE_MAX_TOKENS:-256}" +# shellcheck disable=SC2034 # the driver reads and overrides this one +RATE_REPEATS="${RATE_REPEATS:-1}" +RATE_TIMEOUT="${RATE_TIMEOUT:-600}" # first call to a cold model loads it +RATE_PROBE_TIMEOUT="${RATE_PROBE_TIMEOUT:-5}" # /props on an already-running server + +# --- the task suite --------------------------------------------------------- +# id;weight;kind;expect;prompt +# +# Fields are SEMICOLON-delimited, matching lib/catalog.sh, and for the same +# reason: the data contains pipes. +# +# kind decides how the response is graded, and every kind is a pure text +# check. The kinds divide into two groups, and the division is the whole +# reason the format tasks mean anything: +# +# LENIENT -- grades the ANSWER, ignoring how it was wrapped: +# +# answer the last non-empty line, normalised, must equal `expect`. For +# questions with exactly one right answer. Fences, quotes, trailing +# punctuation and preceding prose are stripped first, because the +# question is whether the model knows the answer. +# tool the response must contain a tool_use block satisfying the jq +# filter in `expect`. Wrapping text is irrelevant: what is graded is +# a structured block that either exists with the right arguments or +# does not. +# +# STRICT -- grades the WHOLE response, because the task IS the format: +# +# oneword the entire response, trimmed of surrounding whitespace, must be +# exactly `expect`. One word means one word: a fence, a full stop +# or a sentence around it is the failure being measured. +# json-only the entire trimmed response must parse as JSON on its own -- no +# fence, no prose -- be an object, and satisfy the jq filter in +# `expect`. +# diff-only the entire response must be a unified diff and nothing else: +# at least one @@ hunk header, every non-empty line valid diff +# syntax, and `expect` present as a literal line (so the diff +# also has to make the requested change). +# +# A strict kind may not use rate_normalise_answer, and a test enforces that. +# Grading a format task through the lenient normaliser is exactly the bug this +# split fixes: it made "obey an output format" a claim the score did not +# measure, since prose, fences and punctuation all passed. +# +# Tool tasks carry double weight. A model that cannot call a tool correctly is +# unusable as a Claude Code backend no matter how well it reads code, and a +# flat weighting hides that behind eleven other tasks. +rate_tasks() { + sed -e 's/[[:space:]]*#.*$//' -e '/^[[:space:]]*$/d' <<'TASKS' +comprehension-loop;1;answer;6;What does this print? Reply with only the number.\n\nx = 0\nfor i in range(4):\n x += i\nprint(x) +comprehension-slice;1;answer;cd;What does this print? Reply with only the value, no quotes.\n\ns = "abcdef"\nprint(s[2:4]) +comprehension-shell;1;answer;1;In bash, after running `false; echo $?` what number is printed? Reply with only the number. +bug-lineno;1;answer;3;Which line number has the off-by-one bug? Reply with only the number.\n\n1: def last(xs):\n2: if not xs:\n3: return xs[1]\n4: return xs[-1] +bug-name;1;answer;total;This function raises NameError. Which name is undefined? Reply with only the name.\n\ndef f(items):\n for i in items:\n total += i\n return total +regex-match;1;answer;no;Does the regex ^a.c$ match the string "abcd"? Reply with only yes or no. +format-oneword;1;oneword;bash;Which language is this? Reply with exactly one word, and nothing else -- no punctuation, no code fence, no explanation.\n\nfor f in *.txt; do mv -- "$f" "${f%.txt}.md"; done +format-json;1;json-only;.language == "python" and (.functions | index("parse")) != null;Reply with only a JSON object, no prose and no code fence, with keys "language" (string) and "functions" (array of function names) for this file.\n\ndef parse(s):\n return s.split(",") +format-diff;1;diff-only;+bar;Produce a unified diff that changes the string "foo" to "bar" in file a.txt. Output only the diff -- no explanation before or after it. +tool-read;2;tool;.name == "read_file" and (.input.path == "src/main.py");Read the file src/main.py. +tool-args;2;tool;.name == "write_file" and (.input.path == "notes.md") and (.input.content | type == "string");Create notes.md containing the single word hello. +tool-choose;2;tool;.name == "list_dir" and (.input.path == "src");What files are in the src directory? +TASKS +} + +RATE_TASK_FIELDS=(id weight kind expect prompt) + +# The tools offered on every `tool` task. All three are offered every time, so +# a tool task also tests CHOOSING -- a model that always calls the first tool +# passes tool-read and fails the other two. +rate_tool_schema() { + jq -nc '[ + {name: "read_file", + description: "Read the contents of a file", + input_schema: {type: "object", + properties: {path: {type: "string", description: "Path to the file"}}, + required: ["path"]}}, + {name: "write_file", + description: "Create a file with the given contents", + input_schema: {type: "object", + properties: {path: {type: "string"}, content: {type: "string"}}, + required: ["path", "content"]}}, + {name: "list_dir", + description: "List the files in a directory", + input_schema: {type: "object", + properties: {path: {type: "string"}}, + required: ["path"]}} + ]' +} + +rate_task_count() { rate_tasks | awk 'END { print NR }'; } +rate_task_ids() { rate_tasks | cut -d';' -f1; } + +# Total weight of the suite, so a partial run can report against the whole. +rate_total_weight() { rate_tasks | awk -F';' '{ w += $2 } END { print w + 0 }'; } + +# rate_task_get +rate_task_get() { + local id="$1" field="$2" idx=0 i row + for i in "${!RATE_TASK_FIELDS[@]}"; do + [[ "${RATE_TASK_FIELDS[$i]}" == "$field" ]] && { idx=$(( i + 1 )); break; } + done + (( idx )) || return 1 + row="$(rate_tasks | awk -F';' -v id="$id" '$1 == id { print; exit }')" + [[ -n "$row" ]] || return 1 + # The prompt is the last field and contains no semicolons by construction, + # but cut -f5 would truncate at one if a future task did. Take the remainder. + if (( idx == ${#RATE_TASK_FIELDS[@]} )); then + printf '%s' "$row" | cut -d';' -f"$idx"- + else + printf '%s' "$row" | cut -d';' -f"$idx" + fi +} + +# --- request construction --------------------------------------------------- + +# rate_payload -- the /v1/messages body for one task. +# +# `\n` in the task table becomes a real newline here rather than in the table, +# so the table stays one line per task and greppable. +rate_payload() { + local model="$1" id="$2" kind prompt + kind="$(rate_task_get "$id" kind)" || return 1 + prompt="$(rate_task_get "$id" prompt)" || return 1 + prompt="$(printf '%b' "$prompt")" + + local base + base=$(jq -nc \ + --arg m "$model" --arg p "$prompt" \ + --argjson mt "$RATE_MAX_TOKENS" --argjson t "$RATE_TEMPERATURE" \ + --argjson s "$RATE_SEED" ' + {model: $m, max_tokens: $mt, temperature: $t, seed: $s, + messages: [{role: "user", content: $p}]}') + + if [[ "$kind" == "tool" ]]; then + jq -nc --argjson b "$base" --argjson tools "$(rate_tool_schema)" '$b + {tools: $tools}' + else + printf '%s' "$base" + fi +} + +# _rate_http [payload] -- the ONLY function here that touches the +# network. Prints the body on stdout and the status on stderr's last line via +# the caller's own check; returns 1 on anything but 200. +# +# One function, two callers (rate_call and rate_props), so every other +# function in this file is pure and testable without a server -- the same rule +# lib/hfmeta.sh follows, enforced by a test. +# The status comes back as the FIRST LINE of stdout, not in a variable. Both +# callers want the body, so both invoke this in a command substitution -- and +# an assignment made inside one is discarded when the subshell ends. Returning +# it in-band is the only channel that survives. +_rate_http() { + local url="$1" payload="${2:-}" out code + out="$(mktemp)" + if [[ -n "$payload" ]]; then + code=$(curl -s -o "$out" -w '%{http_code}' --max-time "$RATE_TIMEOUT" \ + -X POST "$url" \ + -H 'content-type: application/json' \ + -H 'anthropic-version: 2023-06-01' \ + -H 'x-api-key: local' \ + -d "$payload" 2>/dev/null) + else + code=$(curl -s -o "$out" -w '%{http_code}' --max-time "$RATE_PROBE_TIMEOUT" \ + "$url" 2>/dev/null) + fi + printf '%s\n' "${code:-000}" + cat "$out" + rm -f "$out" +} + +# rate_call -- POST it, print the raw response. +rate_call() { + local base="$1" model="$2" id="$3" payload raw code body + payload="$(rate_payload "$model" "$id")" || return 1 + raw="$(_rate_http "${base%/}/v1/messages" "$payload")" + code="${raw%%$'\n'*}" + body="${raw#*$'\n'}" + if [[ "$code" != 200 ]]; then + # Reported twice, on purpose. The variable is for a caller that invokes + # this directly; stderr is for the one that does not, because the response + # comes back on stdout and therefore through a command substitution -- + # which runs in a subshell, where an assignment to RATE_LAST_ERROR is + # discarded the moment the function returns. The driver reads stderr. + # shellcheck disable=SC2034 # documented return channel, read by callers + RATE_LAST_ERROR="http $code: $(printf '%s' "$body" | head -c 200)" + printf '%s\n' "$RATE_LAST_ERROR" >&2 + return 1 + fi + printf '%s' "$body" + return 0 +} + +# --- runtime identity ------------------------------------------------------- +# A rating is only reproducible if you know what was actually running. The +# served alias does not tell you: llama-swap names a model after its GGUF +# directory, and the same alias can front a different quant, a different +# llama.cpp build, or a different context after any re-run of 30-models.sh or +# 20-build-llamacpp.sh. Two runs that disagree would then produce the same +# catalog row. +# +# Every field here is read from something that exists, and is the string +# `unavailable` when it does not. None of them is inferred from another. + +RATE_UNAVAILABLE='unavailable' + +# rate_swap_gguf -- the -m path for that model, or +# `unavailable`. +# +# The config is generated by 40-serve.sh: a quoted model key, then an indented +# cmd block containing `-m `. Read the first -m after the key and stop at +# the next key, so two models cannot borrow each other's weights. +rate_swap_gguf() { + local cfg="$1" want="$2" path + [[ -f "$cfg" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + path="$(awk -v want="$want" ' + /^ "[^"]+":[[:space:]]*$/ { + key = $0 + sub(/^ "/, "", key); sub(/":[[:space:]]*$/, "", key) + in_model = (key == want) + next + } + in_model && /(^|[[:space:]])-m[[:space:]]+/ { + line = $0 + sub(/.*(^|[[:space:]])-m[[:space:]]+/, "", line) + sub(/[[:space:]].*$/, "", line) + print line + exit + } + ' "$cfg")" + [[ -n "$path" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + printf '%s' "$path" +} + +# rate_quant_of -- the quant token in the filename, or +# `unavailable`. +# +# Derived from the filename and nowhere else. A quant read off the catalog's +# preference would be what was ASKED for; this has to be what is on disk, +# because the whole point of recording it is that they can differ. +rate_quant_of() { + local name="${1##*/}" q + q="$(grep -oiE '(IQ[0-9]+_[A-Z]+|Q[0-9]+_[0-9]+|Q[0-9]+_K(_[A-Z]+)?|BF16|F16|F32)' <<<"$name" | head -1)" + [[ -n "$q" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + printf '%s' "$q" +} + +# rate_swap_flags -- the serving flags for that model, +# on one line. The generated `${base}` flags are shared by every model and are +# recorded once in the artifact header; these are the per-model ones that +# differ, which is where two runs of the "same" model diverge. +rate_swap_flags() { + local cfg="$1" want="$2" flags + [[ -f "$cfg" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + flags="$(awk -v want="$want" ' + /^ "[^"]+":[[:space:]]*$/ { + key = $0 + sub(/^ "/, "", key); sub(/":[[:space:]]*$/, "", key) + in_model = (key == want) + next + } + in_model && /^[[:space:]]*(ttl|aliases|env|proxy):/ { in_model = 0 } + in_model && /(--[a-z-]+|CUDA_VISIBLE_DEVICES=)/ { + line = $0 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + # The -m line names the weights, which are recorded separately. + if (line !~ /^-m[[:space:]]/) printf "%s ", line + } + ' "$cfg" | sed -e 's/[[:space:]]\+/ /g' -e 's/[[:space:]]*$//')" + [[ -n "$flags" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + printf '%s' "$flags" +} + +# rate_llamacpp_rev [dir] -- the commit 20-build-llamacpp.sh recorded, or +# `unavailable`. Never "master": a branch name is not a revision. +rate_llamacpp_rev() { + local dir="${1:-${RIG_DIR:-$HOME/llm-rig}}" rev + rev="$(head -1 "$dir/.llamacpp-rev" 2>/dev/null | tr -d '[:space:]')" + [[ -n "$rev" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + printf '%s' "$rev" +} + +# rate_props -- the /props body from an upstream llama-server, or +# nothing. llama-swap fronts one server per model on its own port; /props is +# the only source that reports what the server ACTUALLY has, as opposed to +# what the config asked for. +rate_props() { + local port="$1" raw code + raw="$(_rate_http "http://127.0.0.1:$port/props")" + code="${raw%%$'\n'*}" + [[ "$code" == 200 ]] || return 1 + printf '%s' "${raw#*$'\n'}" +} + +# rate_live_ctx -- the n_ctx of the upstream actually +# serving those weights, or `unavailable`. +# +# Matched by model path rather than by taking the first server that answers: +# with several models loaded, the first answer is some other model's context, +# and recording it would be worse than recording nothing. +rate_live_ctx() { + local cfg="$1" gguf="$2" port props served ctx + [[ "$gguf" != "$RATE_UNAVAILABLE" ]] || { printf '%s' "$RATE_UNAVAILABLE"; return 1; } + for port in $(swap_upstream_ports "$cfg" 2>/dev/null); do + props="$(rate_props "$port")" || continue + served="$(jq -r '.model_path // .default_generation_settings.model // ""' <<<"$props" 2>/dev/null)" + [[ "${served##*/}" == "${gguf##*/}" ]] || continue + ctx="$(jq -r '.default_generation_settings.n_ctx // .n_ctx // ""' <<<"$props" 2>/dev/null)" + [[ -n "$ctx" && "$ctx" != "null" ]] || break + printf '%s' "$ctx" + return 0 + done + printf '%s' "$RATE_UNAVAILABLE" + return 1 +} + +# --- grading ---------------------------------------------------------------- + +# The concatenated text blocks of a Messages response, or empty. +rate_response_text() { + jq -r '[.content[]? | select(.type == "text") | .text] | join("")' 2>/dev/null <<<"$1" +} + +# Normalisation for `answer` tasks. Models wrap a one-word answer in prose, +# punctuation, quotes, backticks and code fences no matter how the prompt is +# worded, and refusing all of that would grade formatting rather than the +# answer -- `format-oneword` is the task that grades formatting, deliberately +# and on its own. +# +# So: take the last non-empty line, strip fences, quotes and trailing +# punctuation, lowercase. What survives must equal the expected answer exactly; +# a substring test would pass "the answer is not 6" for expecting 6. +rate_normalise_answer() { + local text="$1" line + line="$(printf '%s\n' "$text" \ + | sed -e 's/^[[:space:]]*```[a-zA-Z0-9]*[[:space:]]*$//' \ + | grep -v '^[[:space:]]*$' \ + | tail -1)" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + line="$(printf '%s' "$line" | sed -e 's/^[`"'"'"']*//' -e 's/[`"'"'"'.,!]*$//')" + printf '%s' "${line,,}" +} + +# Surrounding whitespace only. This is the ONLY thing a strict kind forgives, +# and it is a transport detail rather than a formatting choice: a trailing +# newline is not a model deciding to add prose. +rate_trim() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +# Is every non-empty line of this text valid unified-diff syntax? +# +# The check is on EVERY line rather than on the presence of a diff, because +# "here is the diff you asked for:" followed by a correct diff is the failure +# format-diff exists to catch. A model that cannot suppress its preamble +# cannot be trusted to emit a patch a tool will apply. +rate_is_diff_only() { + local text="$1" + # A hunk header is the one part prose never produces by accident. + printf '%s\n' "$text" | grep -qE '^@@' || return 1 + ! printf '%s\n' "$text" \ + | grep -v '^[[:space:]]*$' \ + | grep -qvE '^(---|\+\+\+|@@|diff |index |new file|deleted file|similarity index|rename |[-+ ]|\\ No newline)' +} + +# rate_grade -- status 0 for a pass. +# +# Takes the whole response rather than the extracted text, because tool tasks +# are graded on structure the text does not contain. +rate_grade() { + local id="$1" response="$2" kind expect text + kind="$(rate_task_get "$id" kind)" || return 2 + expect="$(rate_task_get "$id" expect)" || return 2 + + case "$kind" in + tool) + # The filter is interpolated rather than passed as data because it is a + # jq expression from our own table, not input. jq has no other way to + # apply a filter held in a string. + jq -e ' + [.content[]? | select(.type == "tool_use")] as $t + | ($t | length) > 0 and ([$t[] | select('"$expect"')] | length) > 0 + ' >/dev/null 2>&1 <<<"$response" + return $? + ;; + answer) + text="$(rate_response_text "$response")" + [[ "$(rate_normalise_answer "$text")" == "${expect,,}" ]] + return $? + ;; + oneword) + # No normalisation. The entire response, minus surrounding whitespace, + # has to be the word -- which is what the prompt asked for. + text="$(rate_trim "$(rate_response_text "$response")")" + [[ "${text,,}" == "${expect,,}" ]] + return $? + ;; + json-only) + text="$(rate_trim "$(rate_response_text "$response")")" + # Parsed as-is: a ```json fence does not parse, and neither does an + # object with a sentence in front of it. Both are the failure. + jq -e 'type == "object"' >/dev/null 2>&1 <<<"$text" || return 1 + jq -e "$expect" >/dev/null 2>&1 <<<"$text" + return $? + ;; + diff-only) + text="$(rate_response_text "$response")" + rate_is_diff_only "$text" || return 1 + # And it must actually make the change: `expect` as a whole line. + grep -qxF -- "$expect" <<<"$text" + return $? + ;; + esac + return 2 +} + +# --- aggregation ------------------------------------------------------------ + +# rate_value -- the 0-100 rating. +# +# A straight weighted pass rate. No curve: a curve would make the number +# unexplainable, and catalog_ratings has to survive someone asking where it +# came from. +rate_value() { + local got="$1" total="$2" + (( total > 0 )) || return 1 + printf '%d' $(( got * 100 / total )) +} + +# rate_confidence +# +# `flips` is the number of tasks that did not give the same verdict on every +# repeat. One is enough to drop to low: a model that answers differently at +# temperature 0 is telling you the measurement is not stable. +# +# The ceiling is `medium`, always. `high` would claim this suite settles the +# question of how good a model is at coding, and twelve text-graded tasks on +# one machine at one quant does not. Raising it needs a bigger suite and more +# than one machine, which is a different piece of work. +rate_confidence() { + local answered="$1" total="$2" repeats="$3" flips="${4:-0}" + (( answered == total )) || { printf 'low'; return 0; } + (( flips == 0 )) || { printf 'low'; return 0; } + (( repeats >= 2 )) || { printf 'low'; return 0; } + printf 'medium' +} + +# --- catalog wiring --------------------------------------------------------- + +# rate_catalog_id -- the catalog id for a model llama-swap +# serves, or status 1. +# +# 40-serve.sh names a model after its GGUF directory, lowercased and with +# -GGUF stripped: `Qwen3-Coder-30B-A3B-Instruct-GGUF` becomes +# `qwen3-coder-30b-a3b-instruct`. The catalog knows the canonical repo, whose +# basename lowercases to the same string. +# +# An unmapped model is an error and not a guess. Writing a rating against the +# wrong id would put a number on a model nobody measured, which is the exact +# failure this table was split out to prevent. +rate_catalog_id() { + local served="${1,,}" row id repo + served="${served%-local}" # the alias 40-serve.sh adds + while IFS=';' read -r id repo _; do + [[ -n "$id" ]] || continue + local base="${repo##*/}" + [[ "${base,,}" == "$served" ]] && { printf '%s' "$id"; return 0; } + done < <(catalog_rows) + return 1 +} + +# rate_row +# +# The line to paste into catalog_ratings(). The source is the artifact's +# basename, not a URL: the evidence is a file in the runner's $HOME, and +# citing an https address for it would be a fabrication. catalog_validate +# enforces that distinction rather than trusting this function. +rate_row() { + local id="$1" value="$2" date="$3" artifact="$4" conf="$5" + printf '%s;%s;%s;local-benchmark;file:%s;%s' \ + "$id" "$value" "$date" "${artifact##*/}" "$conf" +} + +# --- artifacts -------------------------------------------------------------- +# The machine-readable line is `RESULT value= ...`. Everything else in +# the file is for a human, and the parser ignores it -- so the report can be +# reworded without breaking the reader. + +# rate_artifact_result -- one field of a RESULT line. +rate_artifact_result() { + local file="$1" id="$2" key="$3" + [[ -f "$file" ]] || return 1 + awk -v id="$id" -v key="$key" ' + $1 == "RESULT" && $2 == id { + for (i = 3; i <= NF; i++) { + split($i, kv, "=") + if (kv[1] == key) { print kv[2]; found = 1 } + } + exit + } + END { if (!found) exit 1 } + ' "$file" +} + +# The newest rating artifact, or nothing. +rate_latest_artifact() { + local dir="${1:-$HOME}" + ls -1t "$dir"/llm-rating-*.txt 2>/dev/null | head -1 +} diff --git a/lib/score.sh b/lib/score.sh index ec720b4..ff416fc 100644 --- a/lib/score.sh +++ b/lib/score.sh @@ -209,20 +209,29 @@ score_trust() { # than collapsed, because they fail independently: # # facts the catalog row was confirmed against the publisher -# rating a coding rating exists, with a method and a source behind it +# rating a coding rating exists, and the rating itself is not `low` # live download counts and file listings are current, not stale or absent # # 3 of 3 high 2 medium 0 or 1 low # +# The rating counts only at `medium` or better, and that qualifier is +# load-bearing. 61-rate-models.sh returns `low` for a single unrepeated pass, +# or when any task errored -- exactly the runs that establish least. Counting +# those the same as a repeated, complete measurement would let a hurried run +# raise the confidence of the ranking it feeds. +# # Today every row has verified facts and no rating, so a model with current # live data reaches medium and no further. That ceiling is deliberate: nothing # should report high confidence while a quarter of the weight rests on a # neutral placeholder. score_confidence() { - local id="$1" live_source="${2:-missing}" value evidence=0 + local id="$1" live_source="${2:-missing}" value rconf evidence=0 catalog_facts_verified "$id" && evidence=$(( evidence + 1 )) value="$(catalog_rating_get "$id" rating_value)" || return 1 - [[ "$value" != "unknown" ]] && evidence=$(( evidence + 1 )) + rconf="$(catalog_rating_get "$id" rating_confidence)" || return 1 + if [[ "$value" != "unknown" && ( "$rconf" == "medium" || "$rconf" == "high" ) ]]; then + evidence=$(( evidence + 1 )) + fi case "$live_source" in fresh|cached) evidence=$(( evidence + 1 )) ;; esac if (( evidence >= 3 )); then printf 'high' diff --git a/tests/cases/catalog_test.sh b/tests/cases/catalog_test.sh index d35f916..9abc6fa 100755 --- a/tests/cases/catalog_test.sh +++ b/tests/cases/catalog_test.sh @@ -336,10 +336,19 @@ test_an_unknown_rating_method_is_rejected() { test_a_well_formed_measured_rating_is_accepted() { # The schema must not be so strict that a real measurement cannot be # recorded -- that is the route by which these become permanently unknown. - validate_table "$GOOD_ROW" "ok-model;72;2026-01-01;local-benchmark;https://example.com/run;medium" + # + # A local measurement cites the artifact 61-rate-models.sh wrote, by + # basename. There is no URL for a file in the runner's own $HOME, and the + # validator refuses an invented one; see tests/cases/rating_test.sh. + validate_table "$GOOD_ROW" "ok-model;72;2026-01-01;local-benchmark;file:llm-rating-20260101-0900.txt;medium" assert_eq "$VALIDATE_STATUS" 0 "a sourced, dated measurement must validate: $VALIDATE_ERRORS" } +test_a_published_vendor_rating_must_still_be_linkable() { + validate_table "$GOOD_ROW" "ok-model;72;2026-01-01;vendor-benchmark;https://example.com/run;medium" + assert_eq "$VALIDATE_STATUS" 0 "a vendor figure with a URL must validate: $VALIDATE_ERRORS" +} + test_no_row_currently_claims_a_coding_rating() { # Documents the shipped state: every rating is unknown, and deliberately so. # If a future edit adds one, this test fails and forces the author to look at diff --git a/tests/cases/rating_test.sh b/tests/cases/rating_test.sh new file mode 100644 index 0000000..c518a99 --- /dev/null +++ b/tests/cases/rating_test.sh @@ -0,0 +1,878 @@ +#!/usr/bin/env bash +# The local coding benchmark: what it asks, how it grades, what it refuses to +# claim, and how a result reaches catalog_ratings(). +# +# No test here contacts a server. rate_call is the only function that touches +# the network, and every grading test feeds it a canned response instead -- +# which is the point of keeping the grading pure. The two tests that do +# exercise rate_call go through the curl mock and its routes table. +set -uo pipefail + +TEST_ROOT="${TEST_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +REPO_ROOT="${REPO_ROOT:-$(cd "$TEST_ROOT/.." && pwd)}" +source "$TEST_ROOT/lib/harness.sh" +source "$TEST_ROOT/lib/mockenv.sh" + +# shellcheck disable=SC2034 +SUITE_NAME="local coding rating" + +# shellcheck source=lib/rate.sh +source "$REPO_ROOT/lib/rate.sh" +# shellcheck source=lib/catalog.sh +source "$REPO_ROOT/lib/catalog.sh" +# shellcheck source=lib/score.sh +source "$REPO_ROOT/lib/score.sh" + +setup_test() { + mock_init + RATE_REPEATS=1 +} + +# A Messages response carrying text. +text_response() { jq -nc --arg t "$1" '{content: [{type: "text", text: $t}]}'; } + +# A Messages response carrying one tool_use block. +tool_response() { + jq -nc --arg n "$1" --argjson i "$2" \ + '{content: [{type: "tool_use", id: "t1", name: $n, input: $i}]}' +} + +# --- the suite itself ------------------------------------------------------- + +test_every_task_has_at_least_the_five_fields() { + # At least, not exactly: the prompt is the last field and carries code, and + # code contains semicolons. rate_task_get reads it as the remainder of the + # line for that reason -- a strict five would ban `false; echo $?` from a + # benchmark whose whole subject is reading code. + local row n + while IFS= read -r row; do + [[ -n "$row" ]] || continue + n="$(awk -F';' '{print NF}' <<<"$row")" + assert_gt "$n" "4" "task row must have at least 5 fields: $row" || return 1 + done < <(rate_tasks) +} + +test_a_prompt_containing_a_semicolon_survives_intact() { + local content + content="$(rate_payload m comprehension-shell | jq -r '.messages[0].content')" + assert_contains "$content" 'false; echo $?' "the snippet must not be truncated at its semicolon" +} + +test_the_first_four_fields_never_contain_a_separator() { + # Only the prompt may. If `expect` grew a semicolon, cut -f4 would silently + # truncate the jq filter and the task would grade against half a condition. + local id weight kind expect + while IFS=';' read -r id weight kind expect _; do + [[ -n "$id" ]] || continue + assert_matches "$weight" '^[0-9]+$' "$id weight" || return 1 + assert_matches "$kind" '^[a-z-]+$' "$id kind" || return 1 + assert_ne "$expect" "" "$id expect" || return 1 + done < <(rate_tasks) +} + +test_task_ids_are_unique() { + local total uniq + total="$(rate_task_ids | wc -l)" + uniq="$(rate_task_ids | sort -u | wc -l)" + assert_eq "$uniq" "$total" "duplicate task id" +} + +RATE_KINDS=(answer tool oneword json-only diff-only) + +test_every_task_kind_is_one_the_grader_implements() { + local kind k ok + while IFS= read -r kind; do + [[ -n "$kind" ]] || continue + ok=0 + for k in "${RATE_KINDS[@]}"; do [[ "$kind" == "$k" ]] && { ok=1; break; }; done + (( ok )) || { _fail "unknown task kind '$kind' -- rate_grade would return 2"; return 1; } + done < <(rate_tasks | cut -d';' -f3) +} + +test_every_kind_the_grader_implements_is_used_by_a_task() { + # The other direction. An unused kind is untested code in the one place + # where untested code decides whether a model passed. + local k used + used="$(rate_tasks | cut -d';' -f3 | sort -u)" + for k in "${RATE_KINDS[@]}"; do + printf '%s\n' "$used" | grep -qxF "$k" || { _fail "kind '$k' is implemented but unused"; return 1; } + done +} + +test_tool_tasks_are_weighted_double() { + local id kind weight + while IFS=';' read -r id weight kind _; do + [[ -n "$id" ]] || continue + if [[ "$kind" == "tool" ]]; then + assert_eq "$weight" "2" "$id is a tool task and must carry double weight" || return 1 + else + assert_eq "$weight" "1" "$id is not a tool task" || return 1 + fi + done < <(rate_tasks) +} + +test_total_weight_is_the_sum_of_the_rows() { + local expected + expected="$(rate_tasks | awk -F';' '{ w += $2 } END { print w }')" + assert_eq "$(rate_total_weight)" "$expected" "total weight" +} + +test_task_get_reads_a_field() { + assert_eq "$(rate_task_get comprehension-loop kind)" "answer" "kind" || return 1 + assert_eq "$(rate_task_get tool-read weight)" "2" "weight" +} + +test_task_get_rejects_an_unknown_task() { + run rate_task_get no-such-task kind + assert_fails "unknown task id" +} + +# --- request construction --------------------------------------------------- + +test_payload_pins_the_sampling_parameters() { + local p + p="$(rate_payload m comprehension-loop)" + assert_eq "$(jq -r '.temperature' <<<"$p")" "0" "temperature must be 0" || return 1 + assert_eq "$(jq -r '.seed' <<<"$p")" "42" "seed must be fixed" || return 1 + assert_eq "$(jq -r '.model' <<<"$p")" "m" "model" +} + +test_payload_turns_the_escaped_newlines_into_real_ones() { + local content + content="$(rate_payload m comprehension-loop | jq -r '.messages[0].content')" + assert_contains "$content" "for i in range(4):" "the snippet must survive" || return 1 + assert_not_contains "$content" '\n' "no literal backslash-n reaches the model" +} + +test_only_tool_tasks_carry_tools() { + assert_eq "$(rate_payload m comprehension-loop | jq -r '.tools // "none"')" "none" \ + "a text task must not offer tools" || return 1 + assert_eq "$(rate_payload m tool-read | jq -r '.tools | length')" "3" \ + "a tool task offers all three tools, so it tests choosing" +} + +# --- grading: answers ------------------------------------------------------- + +test_a_bare_correct_answer_passes() { + run rate_grade comprehension-loop "$(text_response "6")" + assert_ok "6 is correct" +} + +test_a_wrong_answer_fails() { + run rate_grade comprehension-loop "$(text_response "10")" + assert_fails "10 is wrong" +} + +test_the_answer_may_be_wrapped_in_prose_and_punctuation() { + # Grading formatting here would duplicate format-oneword and make every + # other task partly a formatting test. + run rate_grade comprehension-loop "$(text_response $'Let me work through it.\n\n6.')" + assert_ok "trailing prose line with a full stop" +} + +test_a_fenced_answer_passes() { + run rate_grade comprehension-slice "$(text_response $'```\ncd\n```')" + assert_ok "code fence around the answer" +} + +test_a_negated_answer_does_not_pass() { + # The reason grading is an equality test on the last line and not a + # substring search: "not 6" contains "6". + run rate_grade comprehension-loop "$(text_response "The answer is not 6")" + assert_fails "a sentence containing the right token is not the right answer" +} + +test_answers_are_case_insensitive() { + run rate_grade regex-match "$(text_response "No")" + assert_ok "No matches no" +} + +# --- grading: the strict format kinds --------------------------------------- +# These three tasks ARE the format check. Everything the lenient normaliser +# forgives -- prose, fences, quotes, trailing punctuation -- must fail here, +# or the suite claims to measure output-format compliance while measuring +# nothing of the sort. + +test_oneword_accepts_exactly_the_word() { + run rate_grade format-oneword "$(text_response "bash")" + assert_ok "the bare word" || return 1 + run rate_grade format-oneword "$(text_response "Bash")" + assert_ok "case is not a formatting failure" || return 1 + run rate_grade format-oneword "$(text_response $' bash\n')" + assert_ok "surrounding whitespace is transport, not prose" +} + +test_oneword_rejects_a_sentence_a_fence_or_punctuation() { + local bad + for bad in \ + "It is bash." \ + "bash." \ + 'The answer is bash' \ + '`bash`' \ + '"bash"' \ + $'```\nbash\n```' \ + $'bash\n\nIt renames the files.' \ + "shell script" + do + run rate_grade format-oneword "$(text_response "$bad")" + assert_fails "must reject [$bad]" || return 1 + done +} + +test_oneword_does_not_go_through_the_lenient_normaliser() { + # The regression the review caught: the last-line normaliser turns + # "Let me think.\n\nbash." into "bash", so a task whose entire subject is + # formatting passed on a response that ignored the format. + run rate_grade format-oneword "$(text_response $'Let me think.\n\nbash.')" + assert_fails "prose plus a punctuated answer" || return 1 + # ... and the same response would still pass a lenient answer task, which is + # what makes the two kinds different rather than one of them wrong. + assert_eq "$(rate_normalise_answer $'Let me think.\n\nbash.')" "bash" \ + "the normaliser itself is unchanged, and still lenient by design" +} + +test_json_only_requires_a_bare_object() { + run rate_grade format-json "$(text_response '{"language":"python","functions":["parse"]}')" + assert_ok "a bare object" || return 1 + run rate_grade format-json "$(text_response $'\n {"language":"python","functions":["parse"]} \n')" + assert_ok "surrounding whitespace only" +} + +test_json_only_rejects_a_fence_or_any_prose() { + local bad + for bad in \ + $'```json\n{"language":"python","functions":["parse"]}\n```' \ + $'```\n{"language":"python","functions":["parse"]}\n```' \ + 'Here it is: {"language":"python","functions":["parse"]}' \ + $'{"language":"python","functions":["parse"]}\n\nLet me know if you need more.' \ + '[{"language":"python","functions":["parse"]}]' + do + run rate_grade format-json "$(text_response "$bad")" + assert_fails "must reject [$bad]" || return 1 + done +} + +test_json_only_still_checks_the_content() { + run rate_grade format-json "$(text_response '{"language":"ruby","functions":["parse"]}')" + assert_fails "well-formatted and wrong is still wrong" +} + +test_diff_only_accepts_a_diff_and_nothing_else() { + run rate_grade format-diff "$(text_response $'--- a.txt\n+++ b.txt\n@@ -1 +1 @@\n-foo\n+bar')" + assert_ok "a clean unified diff" || return 1 + run rate_grade format-diff \ + "$(text_response $'diff --git a/a.txt b/a.txt\nindex 1234567..89abcde 100644\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-foo\n+bar')" + assert_ok "git-style headers are still only a diff" +} + +test_diff_only_rejects_a_preamble_or_a_trailing_explanation() { + # A model that cannot suppress its preamble cannot be trusted to emit a + # patch a tool will apply. + run rate_grade format-diff \ + "$(text_response $'Here is the diff you asked for:\n\n--- a.txt\n+++ b.txt\n@@ -1 +1 @@\n-foo\n+bar')" + assert_fails "preamble" || return 1 + run rate_grade format-diff \ + "$(text_response $'--- a.txt\n+++ b.txt\n@@ -1 +1 @@\n-foo\n+bar\n\nThat replaces foo with bar.')" + assert_fails "trailing explanation" || return 1 + run rate_grade format-diff \ + "$(text_response $'```diff\n--- a.txt\n+++ b.txt\n@@ -1 +1 @@\n-foo\n+bar\n```')" + assert_fails "a fence is not diff syntax" +} + +test_diff_only_requires_the_change_to_be_made() { + # Structure alone is not enough: a well-formed diff that does not add the + # line asked for did not do the task. + run rate_grade format-diff "$(text_response $'--- a.txt\n+++ b.txt\n@@ -1 +1 @@\n-foo\n+baz')" + assert_fails "the wrong replacement" || return 1 + run rate_grade format-diff "$(text_response $'--- a.txt\n+++ b.txt\n@@ -1 +1 @@\n-foo\n+ bar')" + assert_fails "an added line that is not exactly +bar" +} + +test_no_strict_kind_calls_the_lenient_normaliser() { + # Structural, so a future edit cannot quietly reintroduce the bug: the + # normaliser may be referenced only by the `answer` branch. + local body + body="$(sed -n '/^rate_grade()/,/^}/p' "$REPO_ROOT/lib/rate.sh")" + assert_eq "$(grep -c 'rate_normalise_answer' <<<"$body")" "1" \ + "exactly one call, in the answer branch" +} + +# --- grading: tools --------------------------------------------------------- + +test_the_right_tool_with_the_right_argument_passes() { + run rate_grade tool-read "$(tool_response read_file '{"path":"src/main.py"}')" + assert_ok "read_file with the right path" +} + +test_the_right_tool_with_the_wrong_argument_fails() { + run rate_grade tool-read "$(tool_response read_file '{"path":"main.py"}')" + assert_fails "wrong path" +} + +test_the_wrong_tool_fails() { + run rate_grade tool-read "$(tool_response list_dir '{"path":"src/main.py"}')" + assert_fails "list_dir is not read_file" +} + +test_describing_a_tool_call_in_prose_is_not_a_tool_call() { + # This is the failure mode that makes a model useless to Claude Code, and + # the one a text-only grader would score as a pass. + run rate_grade tool-read "$(text_response 'I would call read_file with path src/main.py')" + assert_fails "prose about a tool call" +} + +test_a_tool_call_among_several_blocks_passes() { + local resp + resp="$(jq -nc '{content: [ + {type: "text", text: "Sure."}, + {type: "tool_use", id: "t1", name: "list_dir", input: {path: "src"}} + ]}')" + run rate_grade tool-choose "$resp" + assert_ok "a tool_use block alongside text" +} + +# --- grading: json and shape ------------------------------------------------ + +test_json_task_fails_on_unparseable_output() { + run rate_grade format-json "$(text_response 'It is python, with one function called parse.')" + assert_fails "prose is not JSON" +} + +test_the_diff_task_requires_a_hunk_header() { + # The hunk header is the one part prose never produces by accident. File + # headers alone are not a diff -- "--- a.txt" is a sentence a model writes + # while explaining what it is about to do. + run rate_grade format-diff "$(text_response $'--- a.txt\n+++ b.txt')" + assert_fails "headers without a hunk" +} + +test_an_empty_response_fails_every_kind() { + local id + while IFS= read -r id; do + [[ -n "$id" ]] || continue + run rate_grade "$id" '{"content":[]}' + assert_fails "$id must fail on an empty response" || return 1 + done < <(rate_task_ids) +} + +test_grading_an_unknown_task_is_an_error_not_a_failure() { + # Status 2, distinct from 1: a task that does not exist is a bug in the + # runner, and must not be recorded as a model getting something wrong. + run rate_grade no-such-task "$(text_response "6")" + assert_status 2 "unknown task" +} + +# --- aggregation ------------------------------------------------------------ + +test_the_value_is_a_straight_weighted_percentage() { + assert_eq "$(rate_value 15 15)" "100" "everything passed" || return 1 + assert_eq "$(rate_value 0 15)" "0" "nothing passed" || return 1 + assert_eq "$(rate_value 9 15)" "60" "9 of 15" +} + +test_the_value_is_in_range_for_a_full_suite() { + local v + v="$(rate_value "$(rate_total_weight)" "$(rate_total_weight)")" + assert_eq "$v" "100" "a clean sweep is exactly 100" +} + +test_a_single_pass_is_never_better_than_low_confidence() { + # One sample says nothing about stability, so it cannot support a rating + # that the ranking will count as evidence. + assert_eq "$(rate_confidence 12 12 1 0)" "low" "one repeat" +} + +test_repeated_and_complete_reaches_medium() { + assert_eq "$(rate_confidence 12 12 2 0)" "medium" "two clean repeats" +} + +test_one_unstable_task_drops_confidence_to_low() { + assert_eq "$(rate_confidence 12 12 3 1)" "low" "a single flip" +} + +test_an_incomplete_run_is_low_however_many_repeats() { + assert_eq "$(rate_confidence 11 12 5 0)" "low" "a task errored" +} + +test_a_local_benchmark_never_reports_high() { + # The ceiling is a property of the suite, not of a run. Twelve text-graded + # tasks on one machine at one quant does not settle the question. + local a r f conf + for a in 0 6 12; do for r in 1 2 9; do for f in 0 1 4; do + conf="$(rate_confidence "$a" 12 "$r" "$f")" + assert_ne "$conf" "high" "answered=$a repeats=$r flips=$f" || return 1 + done; done; done +} + +# --- catalog wiring --------------------------------------------------------- + +test_a_served_name_maps_back_to_its_catalog_id() { + assert_eq "$(rate_catalog_id qwen3-coder-30b-a3b-instruct)" "qwen3-coder-30b" "moe" || return 1 + assert_eq "$(rate_catalog_id devstral-small-2507)" "devstral-small" "dense" || return 1 + assert_eq "$(rate_catalog_id phi-4)" "phi-4" "id equal to the repo name" +} + +test_the_alias_suffix_maps_too() { + # 40-serve.sh registers "-local" as an alias, and a request may arrive + # under either. + assert_eq "$(rate_catalog_id qwen3-4b-local)" "qwen3-4b" "alias" +} + +test_an_unknown_served_model_is_refused_not_guessed() { + run rate_catalog_id some-finetune-nobody-catalogued + assert_fails "an unmapped model must not resolve to a near neighbour" +} + +test_every_catalogued_model_is_reachable_from_its_served_name() { + # The mapping is derived from canonical_repo, so a catalog row whose repo + # basename is not what 40-serve.sh would name it can never be rated. + local id repo served back + while IFS=';' read -r id repo _; do + [[ -n "$id" ]] || continue + served="${repo##*/}" + back="$(rate_catalog_id "${served,,}")" \ + || { _fail "$id: served name '${served,,}' does not map back"; return 1; } + assert_eq "$back" "$id" "round trip for $id" || return 1 + done < <(catalog_rows) +} + +test_the_row_cites_the_artifact_and_not_a_url() { + local row + row="$(rate_row qwen3-4b 67 2026-08-11 "$HOME/llm-rating-20260811-1930.txt" medium)" + assert_eq "$row" "qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-20260811-1930.txt;medium" \ + "the row pastes straight into catalog_ratings" +} + +test_the_row_drops_the_directory() { + # An absolute path from someone else's $HOME would not resolve on yours. + local row + row="$(rate_row qwen3-4b 67 2026-08-11 /home/someone/llm-rating-1.txt medium)" + assert_not_contains "$row" "/home/someone" "no foreign path in the table" +} + +# --- what the validator will accept ----------------------------------------- + +# Swap in a ratings table for the duration of one test: the given row, plus an +# `unknown` row for every other model so the table still validates as a whole. +# catalog_ratings is a function, so overriding it is enough -- no file is +# written, and the next test's subshell gets the shipped table back. +with_rating() { + # Two statements, not one: bash expands the whole `local` word list before it + # assigns any of it, so `local row="$1" want="${row%%;*}"` reads an unset row. + local row="$1" id + local want="${row%%;*}" + RATINGS_OVERRIDE="$row" + for id in $(catalog_ids); do + [[ "$id" == "$want" ]] || RATINGS_OVERRIDE+=$'\n'"$id;unknown;-;none;-;none" + done + catalog_ratings() { printf '%s\n' "$RATINGS_OVERRIDE"; } +} + +test_a_local_benchmark_row_validates() { + with_rating "qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-20260811-1930.txt;medium" + run catalog_validate + assert_ok "a well-formed local-benchmark rating: ${CATALOG_ERRORS:-}" +} + +test_a_local_benchmark_row_citing_a_url_is_refused() { + # The whole point of the method: there is no URL for a file on this machine. + with_rating "qwen3-4b;67;2026-08-11;local-benchmark;https://example.com/run;medium" + catalog_validate + assert_contains "${CATALOG_ERRORS:-}" "must be 'file:" "https is not evidence of a local run" +} + +test_a_local_benchmark_row_citing_a_path_is_refused() { + with_rating "qwen3-4b;67;2026-08-11;local-benchmark;file:/home/kiwi/llm-rating-1.txt;medium" + catalog_validate + assert_contains "${CATALOG_ERRORS:-}" "must be 'file:" "a path from another machine" +} + +test_a_vendor_benchmark_still_requires_an_https_source() { + with_rating "qwen3-4b;67;2026-08-11;vendor-benchmark;file:llm-rating-1.txt;medium" + catalog_validate + assert_contains "${CATALOG_ERRORS:-}" "must be an https URL" "a published claim must be linkable" +} + +test_a_rating_still_cannot_be_recorded_without_a_value() { + with_rating "qwen3-4b;unknown;2026-08-11;local-benchmark;file:llm-rating-1.txt;medium" + catalog_validate + assert_contains "${CATALOG_ERRORS:-}" "claims evidence but value is unknown" "method without a number" +} + +# --- what a rating does to the ranking -------------------------------------- + +test_a_medium_rating_lifts_confidence_but_a_low_one_does_not() { + # 61-rate-models.sh returns low for a single unrepeated pass. Counting that + # the same as a repeated measurement would let a hurried run raise the + # confidence of the ranking it feeds. + with_rating "qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-1.txt;low" + assert_eq "$(score_confidence qwen3-4b fresh)" "medium" "facts + live, rating too weak to count" || return 1 + + with_rating "qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-1.txt;medium" + assert_eq "$(score_confidence qwen3-4b fresh)" "high" "facts + live + a rating that counts" +} + +test_the_rating_feeds_the_coding_component() { + with_rating "qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-1.txt;medium" + SCORE_CODING_KNOWN=0 + local v + v="$(score_coding qwen3-4b)" + assert_eq "$v" "67" "the measured value, not the neutral 50" +} + +test_an_unrated_model_still_scores_at_the_neutral_50() { + assert_eq "$(score_coding qwen3-4b)" "50" "unknown means neutral, not zero" +} + +# --- runtime identity ------------------------------------------------------- +# A rating is only reproducible if you know what was running. The served alias +# does not say which quant, which llama.cpp build, or which context produced +# it -- the same alias fronts different weights after any re-run of +# 30-models.sh. Every field is read from something that exists, or is the +# string `unavailable`; none is inferred from another. + +# A config in the shape 40-serve.sh generates. +write_cfg() { + mkdir -p "$RIG_DIR/etc" + cat >"$RIG_DIR/etc/llama-swap.yaml" <<'CFG' +startPort: 9100 +models: + "qwen3-4b": + # Qwen3-4B-Q5_K_M.gguf (~2835 MB) -- pinned to GPU0 + cmd: | + CUDA_VISIBLE_DEVICES=0 llama-server ${base} + -m /models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf + ttl: 900 + aliases: ["qwen3-4b-local"] + + "qwen3-coder-30b-a3b-instruct": + cmd: | + llama-server ${base} + -m /models/Qwen3-Coder-30B-A3B-Instruct-GGUF/Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf + --tensor-split 0.5,0.5 --n-cpu-moe 7 + ttl: 900 + aliases: ["qwen3-coder-30b-a3b-instruct-local"] +CFG + printf '%s' "$RIG_DIR/etc/llama-swap.yaml" +} + +test_the_weights_come_from_the_config_not_the_catalog() { + local cfg; cfg="$(write_cfg)" + assert_eq "$(rate_swap_gguf "$cfg" qwen3-4b)" \ + "/models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf" "the -m path" +} + +test_one_model_cannot_borrow_another_models_weights() { + # The parse has to stop at the next model key. If it runs on, every model + # after the first reports the first one's file. + local cfg; cfg="$(write_cfg)" + assert_eq "$(rate_swap_gguf "$cfg" qwen3-coder-30b-a3b-instruct)" \ + "/models/Qwen3-Coder-30B-A3B-Instruct-GGUF/Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf" \ + "the second model's own weights" +} + +test_an_unknown_model_or_missing_config_is_unavailable_not_a_guess() { + local cfg; cfg="$(write_cfg)" + assert_eq "$(rate_swap_gguf "$cfg" not-served)" "unavailable" "unknown model" || return 1 + assert_eq "$(rate_swap_gguf "$SANDBOX/nope.yaml" qwen3-4b)" "unavailable" "no config" +} + +test_the_quant_is_read_off_the_file_on_disk() { + # Not from the catalog's preference: that is what was ASKED for, and the + # reason to record it at all is that the two can differ. + assert_eq "$(rate_quant_of /models/x/Qwen3-4B-Q5_K_M.gguf)" "Q5_K_M" "K quant" || return 1 + assert_eq "$(rate_quant_of Devstral-Small-2507-IQ4_XS.gguf)" "IQ4_XS" "I quant" || return 1 + assert_eq "$(rate_quant_of model-Q8_0.gguf)" "Q8_0" "legacy quant" || return 1 + assert_eq "$(rate_quant_of some-model-BF16.gguf)" "BF16" "unquantised" +} + +test_a_filename_with_no_quant_is_unavailable() { + assert_eq "$(rate_quant_of /models/mystery.gguf)" "unavailable" "no quant in the name" || return 1 + assert_eq "$(rate_quant_of unavailable)" "unavailable" "an unavailable path stays unavailable" +} + +test_the_per_model_serving_flags_are_recorded() { + local cfg flags; cfg="$(write_cfg)" + flags="$(rate_swap_flags "$cfg" qwen3-coder-30b-a3b-instruct)" + assert_contains "$flags" "--n-cpu-moe 7" "offload level" || return 1 + assert_contains "$flags" "--tensor-split 0.5,0.5" "split" || return 1 + assert_not_contains "$flags" "-m /models" "the weights are recorded on their own line" +} + +test_a_pinned_gpu_counts_as_a_serving_flag() { + # CUDA_VISIBLE_DEVICES changes what the measurement measures as surely as + # any --flag does. + local cfg; cfg="$(write_cfg)" + assert_contains "$(rate_swap_flags "$cfg" qwen3-4b)" "CUDA_VISIBLE_DEVICES=0" "pinning" +} + +test_the_llamacpp_revision_is_read_from_the_build_record() { + printf 'abc1234def\n' >"$RIG_DIR/.llamacpp-rev" + assert_eq "$(rate_llamacpp_rev "$RIG_DIR")" "abc1234def" "the recorded commit" || return 1 + rm -f "$RIG_DIR/.llamacpp-rev" + assert_eq "$(rate_llamacpp_rev "$RIG_DIR")" "unavailable" "no build record" +} + +test_the_live_context_is_matched_to_the_model_being_rated() { + # With several models loaded, taking the first server that answers records + # some other model's context. That is worse than recording nothing. + local cfg; cfg="$(write_cfg)" + { + printf '*\t127.0.0.1:9100/props\t200\t{"model_path":"/models/other/Other-Q4_K_M.gguf","default_generation_settings":{"n_ctx":8192}}\n' + printf '*\t127.0.0.1:9101/props\t200\t{"model_path":"/models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf","default_generation_settings":{"n_ctx":65536}}\n' + } >"$MOCK_ROUTES" + assert_eq "$(rate_live_ctx "$cfg" /models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf)" "65536" \ + "the context of the server serving these weights" +} + +test_no_matching_upstream_means_unavailable() { + local cfg; cfg="$(write_cfg)" + printf '*\t/props\t200\t{"model_path":"/models/other/Other-Q4_K_M.gguf","default_generation_settings":{"n_ctx":8192}}\n' >"$MOCK_ROUTES" + assert_eq "$(rate_live_ctx "$cfg" /models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf)" "unavailable" \ + "another model's 8192 must not be recorded as this one's" +} + +test_no_props_at_all_means_unavailable() { + local cfg; cfg="$(write_cfg)" + : >"$MOCK_ROUTES" + assert_eq "$(rate_live_ctx "$cfg" /models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf)" "unavailable" \ + "nothing answered" +} + +test_unavailable_weights_cannot_produce_a_live_context() { + local cfg; cfg="$(write_cfg)" + printf '*\t/props\t200\t{"model_path":"/models/x.gguf","default_generation_settings":{"n_ctx":8192}}\n' >"$MOCK_ROUTES" + assert_eq "$(rate_live_ctx "$cfg" unavailable)" "unavailable" "nothing to match against" +} + +# --- the artifact ----------------------------------------------------------- + +write_artifact() { + cat >"$SANDBOX/llm-rating-20260811-1930.txt" <<'ART' +llm-rig local coding rating 20260811-1930 +suite: v1 (12 tasks, total weight 15) + +model: qwen3-4b + task comprehension-loop kind=answer weight=1 pass (2/2) +RESULT qwen3-4b value=67 weight=10/15 answered=12/12 flips=0 confidence=medium + +model: phi-4 +RESULT phi-4 value=40 weight=6/15 answered=12/12 flips=0 confidence=medium +ART + printf '%s' "$SANDBOX/llm-rating-20260811-1930.txt" +} + +test_a_result_line_is_machine_readable() { + local f; f="$(write_artifact)" + assert_eq "$(rate_artifact_result "$f" qwen3-4b value)" "67" "value" || return 1 + assert_eq "$(rate_artifact_result "$f" qwen3-4b confidence)" "medium" "confidence" || return 1 + assert_eq "$(rate_artifact_result "$f" phi-4 value)" "40" "the second model" +} + +test_a_missing_model_in_the_artifact_is_an_error() { + local f; f="$(write_artifact)" + run rate_artifact_result "$f" qwen3-32b value + assert_fails "a model that was not rated" +} + +test_a_missing_artifact_is_an_error_not_an_empty_value() { + run rate_artifact_result "$SANDBOX/nope.txt" qwen3-4b value + assert_fails "no artifact" +} + +test_the_newest_artifact_wins() { + mkdir -p "$SANDBOX/arts" + : >"$SANDBOX/arts/llm-rating-20260101-0000.txt" + : >"$SANDBOX/arts/llm-rating-20260811-1930.txt" + touch -d '2026-01-01' "$SANDBOX/arts/llm-rating-20260101-0000.txt" + touch -d '2026-08-11' "$SANDBOX/arts/llm-rating-20260811-1930.txt" + assert_eq "$(basename "$(rate_latest_artifact "$SANDBOX/arts")")" \ + "llm-rating-20260811-1930.txt" "newest" +} + +# --- the one function that touches the network ------------------------------ + +test_curl_appears_in_exactly_one_function() { + # Same rule as lib/hfmeta.sh: if the network call is in one place, every + # other function can be tested without a server, and the suite behaves + # identically inside tests/isolated.sh. + # + # Counted by function rather than by line: _rate_http invokes curl twice, + # once per method, and that is still one place. What must not happen is a + # second function growing its own call. + local everywhere inside + everywhere="$(grep -c '^[[:space:]]*code=$(curl' "$REPO_ROOT/lib/rate.sh")" + inside="$(sed -n '/^_rate_http()/,/^}/p' "$REPO_ROOT/lib/rate.sh" \ + | grep -c '^[[:space:]]*code=$(curl')" + assert_eq "$everywhere" "$inside" "every curl call must be inside _rate_http" || return 1 + assert_gt "$inside" "0" "and _rate_http must actually make one" +} + +test_a_call_posts_to_v1_messages_and_returns_the_body() { + printf 'POST\t/v1/messages\t200\t{"content":[{"type":"text","text":"6"}]}\n' >"$MOCK_ROUTES" + local resp + resp="$(rate_call http://127.0.0.1:8081 qwen3-4b comprehension-loop)" + assert_eq "$(rate_response_text "$resp")" "6" "the body comes back intact" || return 1 + assert_contains "$(cat "$MOCK_CALLS")" "/v1/messages" "posted to the Messages endpoint" +} + +test_a_non_200_is_a_failure_with_the_reason_kept() { + printf 'POST\t/v1/messages\t500\tmodel failed to load\n' >"$MOCK_ROUTES" + run rate_call http://127.0.0.1:8081 qwen3-4b comprehension-loop + assert_fails "http 500" || return 1 + rate_call http://127.0.0.1:8081 qwen3-4b comprehension-loop >/dev/null 2>&1 + assert_contains "$RATE_LAST_ERROR" "500" "the status is reported, not swallowed" +} + +test_a_trailing_slash_on_the_endpoint_does_not_double_up() { + printf 'POST\t/v1/messages\t200\t{"content":[]}\n' >"$MOCK_ROUTES" + rate_call http://127.0.0.1:8081/ qwen3-4b comprehension-loop >/dev/null 2>&1 + assert_not_contains "$(cat "$MOCK_CALLS")" "//v1/messages" "no doubled slash" +} + +# --- 61-rate-models.sh, driven for real ------------------------------------- +# These run the script end to end against the curl mock. No server, no model, +# no network -- the routes table answers /v1/models and /v1/messages, so the +# whole path from endpoint resolution to the pasteable row is exercised. + +# Serve `qwen3-4b` and answer every task with the given text. +serve_model() { + local answer="${1:-6}" + { + printf 'GET\t/v1/models\t200\t{"data":[{"id":"qwen3-4b"}]}\n' + printf 'POST\t/v1/messages\t200\t{"content":[{"type":"text","text":"%s"}]}\n' "$answer" + } >"$MOCK_ROUTES" +} + +drive() { run bash -c "cd '$REPO_ROOT' && HOME='$HOME' PATH='$PATH' bash ./61-rate-models.sh $1"; } + +test_the_script_writes_an_artifact_and_a_pasteable_row() { + serve_model 6 + drive "" + assert_ok "the run must complete: $RUN_OUTPUT" || return 1 + + local artifact + artifact="$(rate_latest_artifact "$HOME")" + assert_ne "$artifact" "" "an artifact must be written" || return 1 + assert_contains "$(cat "$artifact")" "RESULT qwen3-4b" "with a machine-readable result" || return 1 + assert_contains "$(cat "$artifact")" "No model output is executed" \ + "and a standing statement of what the grading does" || return 1 + + # The row it prints must be one catalog_validate will accept. + local row + row="$(printf '%s\n' "$RUN_OUTPUT" | grep '^qwen3-4b;')" + with_rating "$row" + run catalog_validate + assert_ok "the printed row must validate: ${CATALOG_ERRORS:-}" +} + +test_the_artifact_records_what_was_actually_running() { + write_cfg >/dev/null + printf 'abc1234def\n' >"$RIG_DIR/.llamacpp-rev" + serve_model 6 + # /props answers for the weights the config names, so the live context is + # establishable rather than unavailable. + printf '*\t127.0.0.1:9100/props\t200\t{"model_path":"/models/Qwen3-4B-GGUF/Qwen3-4B-Q5_K_M.gguf","default_generation_settings":{"n_ctx":65536}}\n' \ + >>"$MOCK_ROUTES" + drive "" + assert_ok "the run must complete: $RUN_OUTPUT" || return 1 + + local art + art="$(cat "$(rate_latest_artifact "$HOME")")" + assert_contains "$art" "llama.cpp revision: abc1234def" "the build" || return 1 + assert_contains "$art" "weights: Qwen3-4B-Q5_K_M.gguf" "the file on disk" || return 1 + assert_contains "$art" "quant: Q5_K_M" "the quant it was served at" || return 1 + assert_contains "$art" "CUDA_VISIBLE_DEVICES=0" "the serving flags" || return 1 + assert_contains "$art" "live n_ctx: 65536" "what the server reported" || return 1 + assert_contains "$art" "quant=Q5_K_M" "and the RESULT line carries the quant" +} + +test_missing_runtime_identity_is_marked_not_omitted() { + # No config, no build record, no /props. Every field must say so: a blank + # is indistinguishable from a field nobody thought to record. + serve_model 6 + drive "" + assert_ok "the run must still complete" || return 1 + local art + art="$(cat "$(rate_latest_artifact "$HOME")")" + assert_contains "$art" "llama.cpp revision: unavailable" "no build record" || return 1 + assert_contains "$art" "weights: unavailable" "no config" || return 1 + assert_contains "$art" "quant: unavailable" "nothing to derive it from" || return 1 + assert_contains "$art" "live n_ctx: unavailable" "nothing answered /props" +} + +test_a_model_that_answers_everything_wrong_still_produces_a_row() { + # A rating of 0 is a result. Refusing to record it would leave the worst + # model looking unmeasured, which reads as "no evidence" rather than "bad". + serve_model "definitely not the answer" + drive "" + assert_ok "the run must complete" || return 1 + assert_contains "$RUN_OUTPUT" "value=0" "zero is a legitimate rating" +} + +test_a_single_pass_is_recorded_as_low_confidence() { + serve_model 6 + drive "" + assert_contains "$RUN_OUTPUT" "confidence=low" "one repeat cannot support more" || return 1 + local row + row="$(printf '%s\n' "$RUN_OUTPUT" | grep '^qwen3-4b;')" + assert_contains "$row" ";low" "and the row says so" +} + +test_the_script_refuses_when_nothing_is_served() { + : >"$MOCK_ROUTES" # every request is a connection failure + drive "" + assert_fails "no endpoint, no rating" || return 1 + assert_contains "$RUN_OUTPUT" "no models served" "with the reason named" || return 1 + assert_eq "$(rate_latest_artifact "$HOME")" "" "and no artifact is left behind" +} + +test_dry_run_maps_names_and_calls_nothing() { + serve_model 6 + drive "--dry-run" + assert_ok "dry run" || return 1 + assert_contains "$RUN_OUTPUT" "qwen3-4b" "the served model is listed" || return 1 + assert_not_contains "$(cat "$MOCK_CALLS")" "/v1/messages" "no task may be sent" || return 1 + assert_eq "$(rate_latest_artifact "$HOME")" "" "and nothing is written" +} + +test_an_unserved_model_is_refused_by_name() { + serve_model 6 + drive "--model qwen3-32b" + assert_fails "a model the endpoint does not serve" || return 1 + assert_contains "$RUN_OUTPUT" "does not serve" "named, not 'invalid input'" || return 1 + assert_contains "$RUN_OUTPUT" "qwen3-4b" "and what it does serve is listed" +} + +test_repeats_must_be_a_positive_integer() { + serve_model 6 + drive "--repeats 0" + assert_fails "zero repeats" || return 1 + drive "--repeats two" + assert_fails "non-numeric repeats" || return 1 + assert_contains "$RUN_OUTPUT" "positive integer" "with the reason stated" +} + +test_two_clean_repeats_reach_medium_confidence() { + serve_model 6 + drive "--repeats 2" + assert_ok "two repeats" || return 1 + assert_contains "$RUN_OUTPUT" "confidence=medium" "stable across repeats" +} + +test_a_task_erroring_blocks_the_row_but_not_the_report() { + # A partial run is a report, not evidence. It must still be written down. + { + printf 'GET\t/v1/models\t200\t{"data":[{"id":"qwen3-4b"}]}\n' + printf 'POST\t/v1/messages\t500\tmodel failed to load\n' + } >"$MOCK_ROUTES" + drive "" + assert_contains "$RUN_OUTPUT" "nothing to record" "no row may be offered" || return 1 + local artifact + artifact="$(rate_latest_artifact "$HOME")" + assert_ne "$artifact" "" "the failed run is still written down" || return 1 + assert_contains "$(cat "$artifact")" "error: http 500" "with what went wrong" +} + +run_suite +suite_exit