Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,16 @@ This is the `.github-private` org infrastructure repo for `petry-projects`. It c

- Scripts must be POSIX-compatible shell (`#!/usr/bin/env bash` with `set -euo pipefail`).
- No hardcoded tokens or secrets — use `$GITHUB_TOKEN` from the environment.

### Cost reporting

- **All surfaced USD amounts are rounded to 2 decimals (cents).** Render every
dollar figure through a single formatter (`_fmt_usd` in `scripts/token_report.sh`,
`printf "$%.2f"`) so reports, issue comments, and step summaries stay consistent.
Sub-cent values round to `$0.00`; use Effective Tokens (ET, `_fmt_int`) as the
fine-grained comparator when cent precision is too coarse.
- Prices are data, not code: keep per-model rates in `scripts/lib/model-pricing.tsv`
(effective-dated) — never hardcode dollar rates in scripts.
- **Promotion:** this is currently a repo-local standard. To make it org-wide,
lift it into [`petry-projects/.github`](https://github.com/petry-projects/.github/blob/main/AGENTS.md)
(the org AGENTS.md / `standards/`) and have repos defer to it.
16 changes: 14 additions & 2 deletions docs/token-report.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,20 @@ scans all non-archived repos.
## Cost (USD)

The report shows **estimated USD cost** alongside tokens, everywhere tokens are
surfaced (totals, by workflow/tier/model, by repository, and a most-expensive-PRs
rollup). Cost is the headline metric; ET is kept as a normalised comparator.
surfaced: totals, by workflow/tier/model, by repository, a **cost-per-day stacked-by-repo
chart**, and a most-expensive-PRs rollup (with the first 35 chars of each PR title). Cost
is the headline metric; ET is kept as a normalised comparator.

**Formatting (org standard):** all surfaced dollar amounts are rounded to **2 decimals
(cents)** via a single formatter (`_fmt_usd`, `printf "$%.2f"`) — see
[AGENTS.md → Cost reporting](../AGENTS.md). Sub-cent values render as `$0.00`; use ET for
finer granularity.

The **cost-per-day chart** is an ASCII stacked bar (one bar per day, length scaled to the
priciest day, segments lettered per top repo with the rest bucketed as `.`) — GitHub
Markdown / Mermaid has no native stacked-bar type, so an ASCII bar inside a fenced block
renders reliably everywhere. PR titles are fetched once (top-10 PRs) by `main()` and passed
to the pure renderer via `PR_TITLE_FILE`.

### Single source of truth: `scripts/lib/model-pricing.tsv`

Expand Down
112 changes: 105 additions & 7 deletions scripts/token_report.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,19 @@ _fmt_int() {
}

# _fmt_usd <dollars>
# Renders a USD amount to 4 decimals (small per-window costs stay legible), e.g. $1.0548.
# Renders a USD amount rounded to 2 decimals (cents), e.g. $1.05. Org standard for
# all surfaced dollar amounts — see AGENTS.md "Cost reporting". Sub-cent values
# round to $0.00; ET (see _fmt_int) remains the fine-grained comparator.
_fmt_usd() {
awk -v v="${1:-0}" 'BEGIN { printf "$%.4f", v }'
awk -v v="${1:-0}" 'BEGIN { printf "$%.2f", v }'
}

# annotate_records <jsonl_dir>
# Joins every record against the dated price table and emits enriched TSV (one row per
# call), pricing each call at the rate in effect on its OWN timestamp. Columns:
# 1 repo 2 workflow 3 tier 4 model 5 input 6 cache 7 output
# 8 cost_usd (-1 when the model price is unknown) 9 et 10 known(1/0) 11 context
# 12 cache_write 13 date (YYYY-MM-DD)
# ET is recomputed here from the table (m = input(model)/input(haiku)), so it can never
# drift from the dollar figure.
annotate_records() {
Expand Down Expand Up @@ -115,14 +118,83 @@ annotate_records() {
m = (bpin > 0) ? tin[mi] / bpin : 1.0
} else { known = 0; cost = -1; m = 1.0 }
et = m * (1.0 * inp + 0.1 * ca + 4.0 * out)
# Enriched cols 1-11 unchanged for downstream; cache_write appended as col 12.
printf "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%.6f\t%.4f\t%d\t%s\t%d\n",
repo, wf, tier, model, inp, ca, out, cost, et, known, ctx, cw
# Enriched cols 1-11 unchanged for downstream; cache_write=12, date=13 appended.
printf "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%.6f\t%.4f\t%d\t%s\t%d\t%s\n",
repo, wf, tier, model, inp, ca, out, cost, et, known, ctx, cw, d
}'
}

# top_pr_urls <jsonl_dir> [n]
# Prints the top-N PR URLs by total cost (one per line). Pure: reuses annotate_records.
# Used by main() to resolve PR titles for the cost-per-PR table.
top_pr_urls() {
local dir="$1" n="${2:-10}"
annotate_records "$dir" \
| awk -F'\t' '$11 ~ /\/pull\// { c[$11] += ($10 == 1 ? $8 : 0) }
END { for (k in c) printf "%.6f\t%s\n", c[k], k }' \
| sort -t$'\t' -k1,1rn | awk -v n="$n" 'NR <= n { print $2 }'
}

# render_cost_per_day <enriched_file>
# Emits an ASCII stacked-bar chart of daily cost composed by repository: one bar
# per day, length scaled to the priciest day, each segment a top-cost repo (others
# bucketed as "."). Pure: reads only the enriched TSV (date=col13, repo=col1,
# cost=col8, known=col10). No-op when there are no priced, dated rows.
render_cost_per_day() {
local enriched="$1"
awk -F'\t' '
function fmt_usd(v) { return sprintf("$%.2f", v) }
$10 == 1 && $13 ~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ {
d = $13; repo = $1
daycost[d] += $8; cell[d SUBSEP repo] += $8; repotot[repo] += $8
days[d] = 1; repos[repo] = 1
}
END {
nd = 0; for (d in days) dl[nd++] = d
for (i = 0; i < nd; i++) for (j = i+1; j < nd; j++) if (dl[j] < dl[i]) { t = dl[i]; dl[i] = dl[j]; dl[j] = t }
if (nd == 0) exit 0
nr = 0; for (r in repos) rl[nr++] = r
for (i = 0; i < nr; i++) for (j = i+1; j < nr; j++) if (repotot[rl[j]] > repotot[rl[i]]) { t = rl[i]; rl[i] = rl[j]; rl[j] = t }
K = 6; nk = (nr < K ? nr : K)
for (x = 1; x <= 8; x++) L[x] = substr("ABCDEFGH", x, 1)
maxday = 0; for (i = 0; i < nd; i++) if (daycost[dl[i]] > maxday) maxday = daycost[dl[i]]
MAXW = 50

printf "## Cost per day (stacked by repo)\n\n"
# Legend: letter → repo (org prefix stripped) with its window total.
printf "Legend: "
for (k = 0; k < nk; k++) {
r = rl[k]; short = r; sub(/^[^/]+\//, "", short)
printf "`%s` %s (%s) ", L[k+1], short, fmt_usd(repotot[r])
}
if (nr > nk) {
oc = 0; for (k = nk; k < nr; k++) oc += repotot[rl[k]]
printf "`.` other (%s)", fmt_usd(oc)
}
printf "\n\n```\n"
for (i = 0; i < nd; i++) {
d = dl[i]; tot = daycost[d]
barlen = (maxday > 0) ? int(tot / maxday * MAXW + 0.5) : 0
line = ""; used = 0
for (k = 0; k < nk; k++) {
c = cell[d SUBSEP rl[k]] + 0
seg = (tot > 0) ? int(c / tot * barlen + 0.5) : 0
if (used + seg > barlen) seg = barlen - used
used += seg
for (s = 0; s < seg; s++) line = line L[k+1]
}
oc = 0; for (k = nk; k < nr; k++) oc += cell[d SUBSEP rl[k]] + 0
if (oc > 0) { seg = barlen - used; for (s = 0; s < seg; s++) line = line "." }
printf "%s %8s %s\n", d, fmt_usd(tot), line
}
printf "```\n\n"
}
' "$enriched"
}

# render_token_report <jsonl_dir> <lookback_days> <repo_count> <artifact_count> [generated_at]
# Writes the full Markdown report (with USD cost) to stdout. Pure: no network.
# Optional: PR_TITLE_FILE (TSV "url<TAB>title") adds PR titles to the cost-per-PR table.
render_token_report() {
local dir="$1" lookback="$2" repo_count="$3" artifact_count="$4" generated_at="${5:-}"

Expand Down Expand Up @@ -207,21 +279,34 @@ render_token_report() {
done
printf '\n'

render_cost_per_day "$enriched"

# Cost-per-PR — each record carries its PR URL in context; surface the priciest PRs.
# Limit with `awk 'NR<=10'` rather than `head -10`: awk consumes the whole stream, so
# `sort` never receives SIGPIPE and the command substitution can't fail under pipefail
# (which would otherwise abort the report — see PR #456 review).
# When PR_TITLE_FILE (TSV "url<TAB>title") is provided by main(), the first 35 chars
# of the title are shown; otherwise the Title cell is blank (keeps render network-free).
local pr_rows
pr_rows="$(awk -F'\t' '$11 ~ /\/pull\// { k = $11
calls[k]++; if ($10 == 1) cost[k] += $8 }
END { for (k in calls) printf "%.6f\t%d\t%s\n", cost[k], calls[k], k }' "$enriched" \
| sort -t$'\t' -k1,1rn | awk 'NR <= 10')"
if [ -n "$pr_rows" ]; then
printf '## Most expensive PRs (top 10)\n\n'
printf '| PR | Calls | Cost |\n|---|---:|---:|\n'
printf '| PR | Title | Calls | Cost |\n|---|---|---:|---:|\n'
while IFS=$'\t' read -r cost calls pr; do
[ -n "$pr" ] || continue
printf '| %s | %s | %s |\n' "$pr" "$(_fmt_int "$calls")" "$(_fmt_usd "$cost")"
local title=""
if [ -n "${PR_TITLE_FILE:-}" ] && [ -f "$PR_TITLE_FILE" ]; then
# Look up title by URL, strip pipes/newlines, truncate to 35 chars (+…).
title="$(awk -F'\t' -v u="$pr" '$1 == u {
t = $2; gsub(/\r/, "", t); gsub(/\|/, "/", t)
if (length(t) > 35) t = substr(t, 1, 35) "…"
print t; exit
}' "$PR_TITLE_FILE")"
fi
printf '| %s | %s | %s | %s |\n' "$pr" "$title" "$(_fmt_int "$calls")" "$(_fmt_usd "$cost")"
done <<< "$pr_rows"
printf '\n'
fi
Expand Down Expand Up @@ -335,6 +420,19 @@ main() {
artifact_count="${counts##* }"
generated_at="$(date -u +%Y-%m-%d 2>/dev/null || echo '')"

# Resolve PR titles for the priciest PRs so render can show them (network step;
# render itself stays pure and just reads PR_TITLE_FILE).
local titles_file purl ptitle
titles_file="$jsonl_dir/pr_titles.tsv"
if command -v gh >/dev/null 2>&1; then
while IFS= read -r purl; do
[ -n "$purl" ] || continue
ptitle="$(gh pr view "$purl" --json title -q '.title' 2>/dev/null || true)"
[ -n "$ptitle" ] && printf '%s\t%s\n' "$purl" "$ptitle" >> "$titles_file"
Comment thread
don-petry marked this conversation as resolved.
Resolved
done < <(top_pr_urls "$jsonl_dir" 10)
fi
export PR_TITLE_FILE="$titles_file"

report="$(render_token_report "$jsonl_dir" "$LOOKBACK_DAYS" \
"$repo_count" "$artifact_count" "$generated_at")"

Expand Down
71 changes: 66 additions & 5 deletions tests/token_report.bats
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,14 @@ setup() {
[ "$output" = "0" ]
}

@test "_fmt_usd: renders dollars to 4 decimals" {
@test "_fmt_usd: renders dollars rounded to 2 decimals (cents)" {
run _fmt_usd 1.0548
[ "$output" = "\$1.0548" ]
[ "$output" = "\$1.05" ]
}

@test "_fmt_usd: sub-cent amounts render as \$0.00" {
run _fmt_usd 0.0042
[ "$output" = "\$0.00" ]
}

# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -62,11 +67,11 @@ setup() {
# render_token_report
# ---------------------------------------------------------------------------

@test "render_token_report: total cost sums priced calls" {
# 0.010250 + 0.010500 + 0.004500 + 0.000850 = 0.026100
@test "render_token_report: total cost sums priced calls (rounded to cents)" {
# 0.010250 + 0.010500 + 0.004500 + 0.000850 = 0.026100 → $0.03 at 2 decimals
run render_token_report "$FIXTURES" 7 2 2 2026-06-07
[ "$status" -eq 0 ]
[[ "$output" == *"Total cost:** \$0.0261"* ]]
[[ "$output" == *"Total cost:** \$0.03"* ]]
}

@test "render_token_report: has cost columns and a cost-per-PR section" {
Expand Down Expand Up @@ -115,6 +120,62 @@ setup() {
[[ "$output" == *"/pull/25 |"* ]]
}

# ---------------------------------------------------------------------------
# cost-per-day stacked chart
# ---------------------------------------------------------------------------

@test "render_token_report: includes a cost-per-day stacked-by-repo chart" {
run render_token_report "$FIXTURES" 7 2 2 2026-06-07
[ "$status" -eq 0 ]
[[ "$output" == *"Cost per day (stacked by repo)"* ]]
[[ "$output" == *"Legend:"* ]]
# One dated bar row per day present in the fixtures (2026-06-01 and 2026-06-02).
[[ "$output" == *"2026-06-01"* ]]
[[ "$output" == *"2026-06-02"* ]]
}

@test "render_cost_per_day: bar length scales with the daily total" {
# Day with the higher total gets the longer (50-char max) bar; the lighter day shorter.
run render_cost_per_day <(annotate_records "$FIXTURES")
[ "$status" -eq 0 ]
d1="$(printf '%s\n' "$output" | awk '/^2026-06-01/ { n=gsub(/[A-H.]/,""); print n }')"
d2="$(printf '%s\n' "$output" | awk '/^2026-06-02/ { n=gsub(/[A-H.]/,""); print n }')"
[ "$d1" -gt "$d2" ]
}

# ---------------------------------------------------------------------------
# cost-per-PR — PR title column (PR_TITLE_FILE)
# ---------------------------------------------------------------------------

@test "render_token_report: shows first 35 chars of PR title from PR_TITLE_FILE" {
local tf; tf="$(mktemp)"
printf 'https://github.com/petry-projects/markets/pull/1\tFix the widget alignment bug in the dashboard header\n' > "$tf"
export PR_TITLE_FILE="$tf"
run render_token_report "$FIXTURES" 7 2 2 2026-06-07
unset PR_TITLE_FILE; rm -f "$tf"
[ "$status" -eq 0 ]
[[ "$output" == *"| Title |"* ]]
# First 35 chars + ellipsis.
[[ "$output" == *"Fix the widget alignment bug in the…"* ]]
}

@test "render_token_report: PR title pipes are sanitized so the table doesn't break" {
local tf; tf="$(mktemp)"
printf 'https://github.com/petry-projects/markets/pull/1\tfix a|b parser\n' > "$tf"
export PR_TITLE_FILE="$tf"
run render_token_report "$FIXTURES" 7 2 2 2026-06-07
unset PR_TITLE_FILE; rm -f "$tf"
[[ "$output" == *"fix a/b parser"* ]]
}

@test "render_token_report: Title column is blank when no PR_TITLE_FILE" {
unset PR_TITLE_FILE
run render_token_report "$FIXTURES" 7 2 2 2026-06-07
[[ "$output" == *"| Title |"* ]]
# No stray title text leaks in — the PR rows still render (URL + empty title cell).
[[ "$output" == *"/pull/1 |"* ]]
}

# ---------------------------------------------------------------------------
# collect_org_jsonl — error-handling (gh is stubbed; no network)
# ---------------------------------------------------------------------------
Expand Down
Loading