From 027109fea17eb1537cf4f3e37eebd8a610828a47 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Sat, 30 May 2026 01:11:49 +0200 Subject: [PATCH 1/2] feat(bench): wire telemetrygen OTLP capture into bench.yml (PR-N1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `capture` input to bench.yml. Default `telemetrygen` installs otelcol-contrib + telemetrygen at a pinned version, boots the collector with an OTLP-gRPC receiver + fileexporter (config in bench-collector.yaml next to the workflow), generates N synthetic log records via telemetrygen, snapshots the captured OTLP/JSON file into bench-corpus/. `capture: none` preserves the existing manual-corpus path. Defaults updated: corpus_dir to bench-corpus/telemetrygen- synthetic-v1, hardware_kind from the unknown `github-ubuntu-4vcpu` to the RFC-blessed `ci-runner`. Job summary explicitly tags the run as synthetic-on-content (real OTLP wire envelope, uniform body shape) — A1/C1 numbers from this corpus are diagnostic, not canonical. An OTel Demo capture is the natural follow-up corpus tag. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/bench-collector.yaml | 44 +++++++ .github/workflows/bench.yml | 152 ++++++++++++++++++++----- 2 files changed, 168 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/bench-collector.yaml diff --git a/.github/workflows/bench-collector.yaml b/.github/workflows/bench-collector.yaml new file mode 100644 index 00000000..64332eff --- /dev/null +++ b/.github/workflows/bench-collector.yaml @@ -0,0 +1,44 @@ +# OTel Collector config used by `bench.yml`'s `capture: +# telemetrygen` path. Minimal pipeline — OTLP-gRPC receiver → +# fileexporter writing OTLP/JSON Lines, which is the format +# `ourios-bench`'s `*.jsonl` loader (RFC 0006 §3.1) consumes. +# +# Lives next to the workflow that loads it so the `(workflow, +# collector config)` pair is reviewed as one unit. Not used by +# any other crate — this is purely the bench's capture path. + +receivers: + otlp: + protocols: + grpc: + endpoint: 127.0.0.1:4317 + http: + endpoint: 127.0.0.1:4318 + +exporters: + # `file/corpus` matches the named-exporter convention in the + # fileexporter docs. `format: json` emits one OTLP `LogsData` + # JSON object per line — the wire shape the `*.jsonl` loader + # reads. `flush_interval` keeps the trailing bytes from sitting + # in the in-process buffer when the collector is SIGTERM'd at + # the end of the capture. + file/corpus: + path: /tmp/otel-capture/logs.jsonl + format: json + flush_interval: 1s + +service: + pipelines: + logs: + receivers: [otlp] + exporters: [file/corpus] + telemetry: + # Silence collector-self telemetry — the bench is measuring + # the captured corpus, not the collector. Prevents the + # collector's own scrape endpoint from binding (which would + # need its own port and complicate the GH Actions runner + # surface). + metrics: + level: none + logs: + level: warn diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8212bc15..159b77f8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -5,42 +5,66 @@ name: Bench # for now — the bench is too slow / hardware-dependent for the # per-PR loop, so it isn't wired to `push` / `pull_request`. # -# The `corpus_dir` input selects which directory the loader -# walks. It defaults to the committed seed (`testdata/corpus/`, -# ~tens of lines, plain text); operators can point it at the -# committed OTLP fixture (`crates/ourios-bench/tests/data/otlp`, -# exercising the PR-K2 loader path) or a future representative -# corpus. Whichever corpus a run picks, the numbers are -# **indicative only**: a hosted runner is not the -# `docs/benchmarks.md` §1 hardware baseline, so results aren't -# authoritative regardless of corpus size. The default seed is -# additionally far too small to be representative (A1 is -# *expected* to FAIL on it — Parquet footer / dictionary overhead -# dwarfs a few KB of data). The workflow uploads the results -# JSON as an artifact and never commits `docs/benchmarks.md` §9 — -# landing authoritative numbers stays a deliberate act. +# Two phases: +# +# 1. **Corpus capture** — driven by the `capture` input. Default +# `telemetrygen` boots an OTel collector with a fileexporter +# (config: `bench-collector.yaml` next to this file), runs +# `telemetrygen logs` against it for `telemetrygen_logs` +# records, then snapshots the captured OTLP/JSON file into +# `bench-corpus/telemetrygen-synthetic-v1/`. `capture: none` +# skips this and uses `corpus_dir` as supplied (lets an +# operator pre-stage a corpus the workflow doesn't know how +# to fetch). +# +# 2. **Bench** — runs `ourios-bench` against whichever corpus +# the capture phase produced (or was pre-staged). Numbers +# land in `bench-results/*.json` as an artifact. +# +# Whichever corpus a run picks, the numbers are **indicative +# only**: a hosted runner is not the `docs/benchmarks.md` §1 +# hardware baseline (`baseline-8vcpu-32gib`), so results aren't +# authoritative regardless of corpus. The `telemetrygen` capture +# is additionally synthetic on content (real OTLP wire envelope, +# uniform body shape) — `corpus: telemetrygen-synthetic-v1` in +# the JSON is the warning label. The workflow uploads the +# results JSON as an artifact and never commits +# `docs/benchmarks.md` §9 — landing authoritative numbers stays +# a deliberate human act. on: workflow_dispatch: inputs: + capture: + description: Corpus capture strategy + type: choice + default: telemetrygen + options: + - telemetrygen + - none + required: false + telemetrygen_logs: + description: Number of synthetic log records to generate (capture=telemetrygen) + default: '1000000' + required: false corpus_dir: - # Default matches the historical workflow behaviour - # (the committed seed text corpus). The PR-K2 OTLP - # loader added `*.jsonl` / `*.json` dispatch in - # `corpus::walk`, so an operator can point this at - # e.g. `crates/ourios-bench/tests/data/otlp` to - # exercise the OTLP path in CI without any external - # tooling, or at any future representative corpus. - description: Corpus directory (defaults to the committed seed) - default: testdata/corpus + # When `capture=telemetrygen`, this is **overwritten** by + # the captured directory. When `capture=none`, this is + # the path the bench reads as-is. + description: Corpus directory (used as-is when capture=none) + default: bench-corpus/telemetrygen-synthetic-v1 required: false gates: description: Comma-separated gates to run (a1,c1,c2) default: a1,c1,c2 required: false hardware_kind: + # `ci-runner` is the RFC 0006 §3.5 tag for the GH Actions + # hosted runner. The §1 baseline `baseline-8vcpu-32gib` + # is a different (larger) tag — pass it explicitly when + # dispatching against an actual baseline-class VM. description: Hardware tag recorded in the results - default: github-ubuntu-4vcpu + default: ci-runner required: false permissions: @@ -50,6 +74,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +# Pin the OTel toolchain version across the workflow so the +# downloaded collector binary and the `go install` of +# telemetrygen agree. Bump deliberately; reproducibility of +# captured corpora across re-dispatches depends on this pin. +env: + OTEL_VER: v0.115.0 + jobs: bench: name: thesis-gate bench @@ -62,6 +93,67 @@ jobs: - name: Build (release — §3.7 pins release mode for A1) run: cargo build -p ourios-bench --release + - name: Install OTel collector contrib + telemetrygen + if: inputs.capture == 'telemetrygen' + # fileexporter is contrib-only, so we use the + # collector-contrib distribution. telemetrygen lives in + # the contrib repo under `cmd/telemetrygen` — `go + # install` pins the same version as the collector for + # API agreement. + run: | + set -euxo pipefail + curl -fL -o /tmp/otelcol.tar.gz \ + "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/${OTEL_VER}/otelcol-contrib_${OTEL_VER#v}_linux_amd64.tar.gz" + sudo tar -xzf /tmp/otelcol.tar.gz -C /usr/local/bin otelcol-contrib + otelcol-contrib --version + go install "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen@${OTEL_VER}" + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + + - name: Capture corpus (telemetrygen → fileexporter) + if: inputs.capture == 'telemetrygen' + env: + BENCH_LOG_COUNT: ${{ inputs.telemetrygen_logs }} + # `BENCH_LOG_COUNT` is the only operator-controllable + # input here — passed via `env:` (not `${{ ... }}` + # splicing) per the same template-injection-avoidance + # rule the bench invocation below documents. + run: | + set -euxo pipefail + mkdir -p bench-corpus/telemetrygen-synthetic-v1 + mkdir -p /tmp/otel-capture + # Start collector in background. The trap on EXIT + # makes a flaky telemetrygen failure clean up the + # collector instead of leaving an orphaned process + # blocking the runner's exit. + otelcol-contrib --config .github/workflows/bench-collector.yaml >/tmp/collector.log 2>&1 & + COLLECTOR_PID=$! + trap 'kill -TERM $COLLECTOR_PID 2>/dev/null || true' EXIT + # Wait for OTLP gRPC port. `bash`'s built-in + # `/dev/tcp` saves installing `netcat`. 30s is generous + # for a cold collector binary on a hosted runner. + for _ in $(seq 1 30); do + if (echo > /dev/tcp/127.0.0.1/4317) 2>/dev/null; then + break + fi + sleep 1 + done + telemetrygen logs \ + --otlp-insecure \ + --otlp-endpoint=127.0.0.1:4317 \ + --logs "$BENCH_LOG_COUNT" + # Let the fileexporter's `flush_interval: 1s` finish + # writing the trailing batch before we SIGTERM. + sleep 3 + kill -TERM $COLLECTOR_PID + wait $COLLECTOR_PID 2>/dev/null || true + trap - EXIT + # Move the captured file into the bench corpus dir. + # The bench loader dispatches on the `.jsonl` + # extension (RFC 0006 §3.1 OTLP/JSON path). + mv /tmp/otel-capture/logs.jsonl bench-corpus/telemetrygen-synthetic-v1/logs.jsonl + echo "captured $(wc -l < bench-corpus/telemetrygen-synthetic-v1/logs.jsonl) records, \ + $(stat -c%s bench-corpus/telemetrygen-synthetic-v1/logs.jsonl) bytes" + - name: Run thesis-gate bench # `pipefail` so a C1 reconstruction failure (the bench's # only non-zero exit, §3.4.2) fails the job rather than @@ -94,10 +186,12 @@ jobs: shell: bash env: BENCH_CORPUS: ${{ inputs.corpus_dir }} + BENCH_CAPTURE: ${{ inputs.capture }} run: | { echo '## ourios-bench — thesis-gate results' echo + echo "Capture: \`${BENCH_CAPTURE}\`" echo "Corpus: \`${BENCH_CORPUS}\`" echo echo '```' @@ -105,10 +199,12 @@ jobs: echo '```' echo echo '> **Indicative only.** A hosted runner is not the `docs/benchmarks.md` §1' - echo '> hardware baseline; results are not authoritative regardless of corpus size.' - echo '> The default seed corpus (`testdata/corpus/`) is also far too small to be' - echo '> representative — A1 is expected to FAIL there (Parquet overhead dominates a' - echo '> few KB of data). Real numbers need a representative corpus + baseline hardware.' + echo '> hardware baseline (`baseline-8vcpu-32gib`); results are not authoritative' + echo '> regardless of corpus. The `telemetrygen` capture path is additionally' + echo '> synthetic on content (real OTLP wire envelope, uniform body shape) — the' + echo '> `corpus` field in the results JSON carries `telemetrygen-synthetic-v1` as' + echo '> the warning label. A representative-corpus capture (OTel Demo) is a' + echo '> follow-up `capture` option.' } >> "$GITHUB_STEP_SUMMARY" - name: Upload results JSON From 906394d68958ceb5942bbb234d003f266a81c54d Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Sat, 30 May 2026 01:21:38 +0200 Subject: [PATCH 2/2] fixup! feat(bench): wire telemetrygen OTLP capture into bench.yml (PR-N1) --- .github/workflows/bench.yml | 39 +++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 159b77f8..c6cd7829 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -110,6 +110,7 @@ jobs: echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" - name: Capture corpus (telemetrygen → fileexporter) + id: capture if: inputs.capture == 'telemetrygen' env: BENCH_LOG_COUNT: ${{ inputs.telemetrygen_logs }} @@ -117,9 +118,16 @@ jobs: # input here — passed via `env:` (not `${{ ... }}` # splicing) per the same template-injection-avoidance # rule the bench invocation below documents. + # + # Emits `corpus_dir` as a step output so the downstream + # bench + summary steps consume the *captured* path, + # not the (now meaningless when capture=telemetrygen) + # `inputs.corpus_dir`. The capture-strategy → corpus-dir + # binding lives in one place this way. run: | set -euxo pipefail - mkdir -p bench-corpus/telemetrygen-synthetic-v1 + CAPTURED_DIR="bench-corpus/telemetrygen-synthetic-v1" + mkdir -p "$CAPTURED_DIR" mkdir -p /tmp/otel-capture # Start collector in background. The trap on EXIT # makes a flaky telemetrygen failure clean up the @@ -131,12 +139,24 @@ jobs: # Wait for OTLP gRPC port. `bash`'s built-in # `/dev/tcp` saves installing `netcat`. 30s is generous # for a cold collector binary on a hosted runner. + # Explicit fail-after-timeout — the loop falling + # through silently used to make telemetrygen the + # error-surface, which logged a connection refusal + # with no pointer at the collector itself. + READY=false for _ in $(seq 1 30); do if (echo > /dev/tcp/127.0.0.1/4317) 2>/dev/null; then + READY=true break fi sleep 1 done + if [ "$READY" != "true" ]; then + echo "::error::collector did not open OTLP gRPC port 4317 within 30s" + echo "--- /tmp/collector.log ---" + cat /tmp/collector.log || true + exit 1 + fi telemetrygen logs \ --otlp-insecure \ --otlp-endpoint=127.0.0.1:4317 \ @@ -150,9 +170,10 @@ jobs: # Move the captured file into the bench corpus dir. # The bench loader dispatches on the `.jsonl` # extension (RFC 0006 §3.1 OTLP/JSON path). - mv /tmp/otel-capture/logs.jsonl bench-corpus/telemetrygen-synthetic-v1/logs.jsonl - echo "captured $(wc -l < bench-corpus/telemetrygen-synthetic-v1/logs.jsonl) records, \ - $(stat -c%s bench-corpus/telemetrygen-synthetic-v1/logs.jsonl) bytes" + mv /tmp/otel-capture/logs.jsonl "$CAPTURED_DIR/logs.jsonl" + echo "captured $(wc -l < "$CAPTURED_DIR/logs.jsonl") records, \ + $(stat -c%s "$CAPTURED_DIR/logs.jsonl") bytes" + echo "corpus_dir=$CAPTURED_DIR" >> "$GITHUB_OUTPUT" - name: Run thesis-gate bench # `pipefail` so a C1 reconstruction failure (the bench's @@ -171,7 +192,10 @@ jobs: env: BENCH_GATES: ${{ inputs.gates }} BENCH_HARDWARE: ${{ inputs.hardware_kind }} - BENCH_CORPUS: ${{ inputs.corpus_dir }} + # Prefer the captured-path step output (set only when + # `capture=telemetrygen` ran). Falls through to the + # operator-supplied `corpus_dir` when capture=none. + BENCH_CORPUS: ${{ steps.capture.outputs.corpus_dir || inputs.corpus_dir }} run: | set -euo pipefail ./target/release/ourios-bench \ @@ -185,7 +209,10 @@ jobs: if: always() shell: bash env: - BENCH_CORPUS: ${{ inputs.corpus_dir }} + # Same effective-path rule the bench step uses, so the + # summary's `Corpus:` line matches what was actually + # measured (not the now-overridden input). + BENCH_CORPUS: ${{ steps.capture.outputs.corpus_dir || inputs.corpus_dir }} BENCH_CAPTURE: ${{ inputs.capture }} run: | {