Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 179 additions & 0 deletions 61-rate-models.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
#!/usr/bin/env bash
# Rate the models this machine actually serves, on the tasks a coding agent
# actually performs, and write the evidence to a file.
#
# This is what turns catalog_ratings() from a column of `unknown` into
# something with a method, a date and an artifact behind it. It does NOT edit
# the catalog: the table is curated by hand, and a script that rewrites its own
# evidence base is not evidence. It prints the rows to paste, and prints the
# artifact they cite.
#
# Read lib/rate.sh before trusting the number. In particular: nothing the model
# produces is executed, the suite is twelve text-graded tasks, and the
# confidence ceiling is `medium` on purpose.
#
# Usage:
# ./61-rate-models.sh # every served model, one pass each
# ./61-rate-models.sh --repeats 3 # three passes; disagreement -> low
# ./61-rate-models.sh --model qwen3-4b # one served model, by name
# ./61-rate-models.sh --dry-run # print the plan, call nothing
#
# Output -> ~/llm-rating-<date>.txt
set -uo pipefail
RIG_SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$RIG_SRC_DIR/lib/detect.sh"
source "$RIG_SRC_DIR/lib/catalog.sh"
source "$RIG_SRC_DIR/lib/preflight.sh"
source "$RIG_SRC_DIR/lib/rate.sh"

ONLY=""
DRY=0

while (( $# )); do
case "$1" in
--repeats) shift; RATE_REPEATS="${1:-1}" ;;
--repeats=*) RATE_REPEATS="${1#--repeats=}" ;;
--model) shift; ONLY="${1:-}" ;;
--model=*) ONLY="${1#--model=}" ;;
--dry-run) DRY=1 ;;
-h|--help) sed -n '2,21p' "${BASH_SOURCE[0]}"; exit 0 ;;
*) die "unknown argument: $1" ;;
esac
shift
done

[[ "$RATE_REPEATS" =~ ^[1-9][0-9]*$ ]] || die "--repeats must be a positive integer, got '$RATE_REPEATS'"
command -v jq >/dev/null 2>&1 || die "jq is required: sudo apt-get install -y jq"

STAMP="$(date +%Y%m%d-%H%M)"
TODAY="$(date +%F)"
ARTIFACT="$HOME/llm-rating-$STAMP.txt"
ERRFILE="$(mktemp)"
trap 'rm -f "$ERRFILE"' EXIT

BASE="$(preflight_endpoint)"
c_info "Rating against $BASE"

AVAILABLE="$(preflight_check_models "$BASE")" || die \
"no models served at $BASE -- start the stack first (./40-serve.sh), then re-run"

if [[ -n "$ONLY" ]]; then
printf '%s\n' "$AVAILABLE" | grep -qxF "$ONLY" \
|| die "$BASE does not serve '$ONLY'. It serves: $(printf '%s' "$AVAILABLE" | tr '\n' ' ')"
AVAILABLE="$ONLY"
fi

TOTAL_W="$(rate_total_weight)"
NTASKS="$(rate_task_count)"

c_info "suite v$RATE_SUITE_VERSION -- $NTASKS tasks, total weight $TOTAL_W, $RATE_REPEATS repeat(s)"

if (( DRY )); then
printf '%s\n' "$AVAILABLE" | while IFS= read -r served; do
[[ -n "$served" ]] || continue
if id="$(rate_catalog_id "$served")"; then
printf ' %-40s -> %s\n' "$served" "$id"
else
printf ' %-40s -> (not in the catalog; will be measured, not recorded)\n' "$served"
fi
done
exit 0
fi

# --- run --------------------------------------------------------------------

{
printf 'llm-rig local coding rating %s\n' "$STAMP"
printf 'suite: v%s (%s tasks, total weight %s)\n' "$RATE_SUITE_VERSION" "$NTASKS" "$TOTAL_W"
printf 'endpoint: %s\n' "$BASE"
printf 'sampling: temperature=%s seed=%s max_tokens=%s repeats=%s\n' \
"$RATE_TEMPERATURE" "$RATE_SEED" "$RATE_MAX_TOKENS" "$RATE_REPEATS"
printf 'grading: text only. No model output is executed.\n'
printf '\n'
} > "$ARTIFACT"

ROWS=""

while IFS= read -r served; do
[[ -n "$served" ]] || continue

cid=""
if ! cid="$(rate_catalog_id "$served")"; then
c_warn "$served is not in the catalog -- measuring it, but no rating row can be written"
cid=""
fi

c_info "$served${cid:+ (catalog: $cid)}"
{
printf 'model: %s\n' "$served"
printf 'catalog-id: %s\n' "${cid:-none}"
} >> "$ARTIFACT"

got_w=0; answered=0; flips=0

while IFS= read -r task; do
[[ -n "$task" ]] || continue
tid="${task%%;*}"
weight="$(rate_task_get "$tid" weight)"

# Each repeat is graded on its own; the task counts as passed only if every
# repeat passed, and a disagreement is recorded rather than averaged away.
passes=0; errors=0
for (( r = 1; r <= RATE_REPEATS; r++ )); do
# The reason goes through a file rather than RATE_LAST_ERROR: the
# response arrives on stdout, so the call is a command substitution, and
# a variable set inside one does not survive it.
if resp="$(rate_call "$BASE" "$served" "$tid" 2>"$ERRFILE")"; then
if rate_grade "$tid" "$resp"; then
passes=$(( passes + 1 ))
fi
else
errors=$(( errors + 1 ))
printf ' task %-20s error: %s\n' "$tid" \
"$(head -1 "$ERRFILE" 2>/dev/null || printf 'unknown')" >> "$ARTIFACT"
fi
done

if (( errors )); then
verdict="error"
elif (( passes == RATE_REPEATS )); then
verdict="pass"; got_w=$(( got_w + weight )); answered=$(( answered + 1 ))
elif (( passes == 0 )); then
verdict="fail"; answered=$(( answered + 1 ))
else
verdict="unstable"; answered=$(( answered + 1 )); flips=$(( flips + 1 ))
fi

printf ' task %-20s kind=%-6s weight=%s %s (%s/%s)\n' \
"$tid" "$(rate_task_get "$tid" kind)" "$weight" "$verdict" "$passes" "$RATE_REPEATS" \
>> "$ARTIFACT"
printf ' %-20s %s\n' "$tid" "$verdict" >&2
done < <(rate_tasks)

value="$(rate_value "$got_w" "$TOTAL_W")"
conf="$(rate_confidence "$answered" "$NTASKS" "$RATE_REPEATS" "$flips")"

printf 'RESULT %s value=%s weight=%s/%s answered=%s/%s flips=%s confidence=%s\n\n' \
"${cid:-$served}" "$value" "$got_w" "$TOTAL_W" "$answered" "$NTASKS" "$flips" "$conf" \
>> "$ARTIFACT"

c_ok "$served value=$value weight=$got_w/$TOTAL_W confidence=$conf"

# A model that errored on any task is measured but not offered as a rating:
# a partial run is a report, not evidence.
if [[ -n "$cid" ]] && (( answered == NTASKS )); then
ROWS+="$(rate_row "$cid" "$value" "$TODAY" "$ARTIFACT" "$conf")"$'\n'
fi
done < <(printf '%s\n' "$AVAILABLE")

c_info "artifact: $ARTIFACT"

if [[ -n "$ROWS" ]]; then
printf '\n'
c_info "Paste these into catalog_ratings() in lib/catalog.sh, replacing the matching ids:"
printf '%s' "$ROWS"
printf '\n'
c_info "Then re-validate: bash -c 'source lib/models.sh; catalog_validate || printf \"%%s\" \"\$CATALOG_ERRORS\"'"
else
c_warn "No complete result for any catalogued model -- nothing to record"
fi
76 changes: 72 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ claude
| Script | Purpose |
|---|---|
| `./71-verify-runtime.sh` | Query the **running** server's `/props` — confirms live `n_ctx`, flash-attn, KV cache types. Trust this over the config file. Grades its evidence; see below. |
| `./61-rate-models.sh` | Rate the models you serve on a fixed coding suite, and print rows for `catalog_ratings()`. Nothing it grades is executed; see [Rating the models you serve](#rating-the-models-you-serve). |
| `./70-thermal-sweep.sh` | Re-derive the best power limit for your chassis under a heat-soaked load. |
| `./80-try-bigger.sh <hf-repo> [quant]` | Assess, download, auto-tune `--n-cpu-moe` and benchmark a model **larger than VRAM**. Empirically finds the lowest working offload level. `--list` sizes it without downloading. |
| `./19-os-revert.sh` | Undo `10-os-tune.sh`. |
Expand Down Expand Up @@ -92,8 +93,12 @@ not reproducible here, and sorting one vendor's SWE-bench figure against
another's HumanEval figure produces an ordering that means nothing. The
validator enforces the consequence in both directions: a value cannot be
recorded without a method and a source behind it, and a method claiming
evidence cannot be recorded without a value. Filling these in is the deferred
local-benchmark work.
evidence cannot be recorded without a value.

Filling them in is what [`./61-rate-models.sh`](61-rate-models.sh) is for — it
measures the models **you** serve and prints rows you can paste in. The shipped
table stays `unknown` because the shipped table cannot contain your
measurements. See [Rating the models you serve](#rating-the-models-you-serve).

This replaced a single `coding_score` column carrying values from 42 to 88 with
no source, no date and no method — weighted at 25% of the recommendation.
Expand Down Expand Up @@ -227,8 +232,14 @@ publisher, the rating behind a quarter of the weight is not.

Confidence counts three independent kinds of evidence — verified facts, a
sourced rating, and current live data. Three of three is `high`, two is
`medium`, fewer is `low`. **Nothing reaches `high` today**, because no model has
a rating; that ceiling lifts on its own once local benchmarking lands.
`medium`, fewer is `low`. **Nothing reaches `high` on the shipped table**,
because no model has a rating; that ceiling lifts on its own once you record
one with [`./61-rate-models.sh`](61-rate-models.sh).

The rating counts only when the rating itself is `medium` or better, and that
qualifier is load-bearing: a single unrepeated benchmark pass is recorded as
`low`, and a hurried run must not be able to raise the confidence of the
ranking it feeds.

Note what the neutral rating does to the ranking: with the quality term equal
for every model, the total is driven by fit and speed, so the smallest model
Expand Down Expand Up @@ -272,6 +283,63 @@ whose next step downloads tens of gigabytes.
afterwards so a run can be reproduced exactly. Duplicate, out-of-range and
non-numeric answers are refused by name rather than with "invalid input".

### Rating the models you serve

`./61-rate-models.sh` is the answer to "a quarter of the score is a neutral
placeholder". It runs a fixed suite against the models **this machine actually
serves**, at the quant they are actually served at, and writes the evidence to
`~/llm-rating-<date>.txt`.

```bash
./61-rate-models.sh # every served model, one pass each
./61-rate-models.sh --repeats 3 # three passes; any disagreement -> low
./61-rate-models.sh --model qwen3-4b # one model
./61-rate-models.sh --dry-run # show the plan, call nothing
```

Read [`lib/rate.sh`](lib/rate.sh) before trusting a number out of it. Four
things about it are deliberate and constrain what it can claim:

- **Nothing the model produces is executed.** The obvious way to grade
generated code is to run it; that means running text from a model on your
machine, as you, for a score. Every task is graded by reading the response —
an exact answer, a `tool_use` block, a parse. The task set is written around
that constraint rather than pretending it is not there.
- **It measures agent-shaped competence, not SWE-bench.** Read a snippet and
say what it does, pick the right tool with the right arguments, obey an
output format. Those are the failures that make a local model useless as a
Claude Code backend. Tool tasks carry double weight, and all three tools are
offered on every one of them, so a model that always calls the first tool
fails two of the three.
- **It is comparable across models on your machine and nowhere else.** That is
the comparison the ranking needs, and it is why the method is called
`local-benchmark` rather than `benchmark`.
- **The confidence ceiling is `medium`.** Twelve text-graded tasks at one quant
on one machine does not settle how good a model is at coding. A single pass
is `low`; two clean repeats are `medium`; `high` is not reachable from here.

Sampling is pinned — temperature 0, fixed seed — and `--repeats` is what checks
that the pinning held. A task that does not give the same verdict every time
counts as unstable, and one unstable task drops the whole run to `low`: a model
that answers differently at temperature 0 is telling you the measurement is
not stable.

The script **does not edit the catalog**. It prints rows to paste:

```
qwen3-4b;67;2026-08-11;local-benchmark;file:llm-rating-20260811-1930.txt;medium
```

The source is the artifact's **basename**, not a URL. There is no https address
for a file in your `$HOME`, and the validator refuses an invented one —
`local-benchmark` must cite `file:<artifact>.txt`, `vendor-benchmark` must cite
https. A table that is curated by hand is not improved by a script that
rewrites its own evidence base, so the paste is manual and the run that
produced it is a file you can open.

A run where any task errored is written down but **not** offered as a row: a
partial run is a report, not evidence.

## Configuration

Everything is derived from detected hardware, and everything is overridable by
Expand Down
28 changes: 25 additions & 3 deletions lib/catalog.sh
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,11 @@ CATALOG_FACT_METHODS=(hf-api card-stated vendor-blog derived)
#
# none no comparable evidence; rating_value must be "unknown"
# vendor-benchmark a number the publisher reports for its own model
# local-benchmark measured on this machine by lib/bench.sh
# local-benchmark measured on this machine by 61-rate-models.sh, against
# the suite in lib/rate.sh, at the quant this rig serves.
# Comparable across models HERE and nowhere else, which is
# the comparison the ranking needs. Its source is the
# artifact basename, not a URL -- see the validator.
CATALOG_RATING_METHODS=(none vendor-benchmark local-benchmark)

CATALOG_RATING_CONFIDENCE=(none low medium high)
Expand Down Expand Up @@ -650,8 +654,26 @@ catalog_validate_ratings_into() {
|| errs+="rating row $n ($id): method '$method' claims evidence but value is unknown"$'\n'
[[ "$conf" != "none" ]] \
|| errs+="rating row $n ($id): method '$method' claims evidence but confidence is 'none'"$'\n'
[[ "$source" == https://* ]] \
|| errs+="rating row $n ($id): rating_source '$source' must be an https URL"$'\n'
# What counts as a source depends on the method, because the two kinds of
# evidence live in different places. A vendor benchmark is published and
# must be linkable. A local benchmark is a file in the runner's own
# $HOME -- there is no URL for it, and inventing an https address so the
# field validates would be a fabrication in the one column whose whole
# job is to say where a number came from.
#
# `file:<basename>`, not an absolute path: the artifact is in the $HOME
# of whoever ran it, and a path from someone else's machine would not
# resolve on yours. The basename is what you look for in your own.
case "$method" in
local-benchmark)
[[ "$source" == file:*.txt && "$source" != *"/"* ]] \
|| errs+="rating row $n ($id): rating_source '$source' must be 'file:<artifact>.txt' -- a basename under \$HOME, produced by 61-rate-models.sh"$'\n'
;;
*)
[[ "$source" == https://* ]] \
|| errs+="rating row $n ($id): rating_source '$source' must be an https URL"$'\n'
;;
esac
if [[ ! "$rdate" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] || ! date -d "$rdate" +%Y-%m-%d >/dev/null 2>&1; then
errs+="rating row $n ($id): rating_date '$rdate' is not a real ISO date"$'\n'
fi
Expand Down
Loading