diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 5488e9274..206452ac5 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -38,6 +38,31 @@ on: type: boolean default: false description: "Run the SDK performance regression benchmarks (adds ~45m per SDK). Needs two builds per SDK: set the *-ref inputs to 'main latest', since a bare 'main' installs no release to use as a baseline and every cell will skip." + bench-baseline-ref: + required: false + type: string + default: "" + description: "Benchmark the ref named here against bench-candidate-ref, instead of the default newest-release-vs-branch-head comparison. Any ref otdf-sdk-mgr resolves: 'main', a branch, a tag, a SHA, 'refs/pull/N/head'. Requires bench-candidate-ref and a focus-sdk naming one SDK; ignores the *-ref inputs, which drive the functional matrix rather than this." + bench-candidate-ref: + required: false + type: string + default: "" + description: "The build under suspicion, measured against bench-baseline-ref. e.g. 'feat/DSPX-2604-createtdf-chunked'." + bench-payloads: + required: false + type: string + default: "" + description: "Payload sizes to benchmark, comma-separated, e.g. '1KiB,1MiB,32MiB,1GiB'. Default is 1KiB,1MiB,32MiB, at which ~86% of a go encrypt is fixed startup cost -- so the 1.15x gate is wider than the whole payload-dependent part and no throughput change can fail a cell. Add 1GiB to actually gate throughput; it needs a bench-budget-seconds to match and ~5 GiB of runner disk." + bench-budget-seconds: + required: false + type: string + default: "" + description: "Wall-clock allowance shared by every benchmark cell (default 1500). Manual runs are not on the nightly's schedule, so this is the knob to raise when adding payload sizes -- each one adds an encrypt and a decrypt cell, and the budget is divided evenly as cells start." + bench-max-rounds: + required: false + type: string + default: "" + description: "Hard cap on paired rounds per cell (default 60). Raise it together with the budget: at the default, cells routinely stop on max_rounds with budget left over, and every unspent round is interval width that could have been bought." workflow_call: inputs: platform-ref: @@ -68,6 +93,26 @@ on: required: false type: boolean default: false + bench-baseline-ref: + required: false + type: string + default: "" + bench-candidate-ref: + required: false + type: string + default: "" + bench-payloads: + required: false + type: string + default: "" + bench-budget-seconds: + required: false + type: string + default: "" + bench-max-rounds: + required: false + type: string + default: "" schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -88,6 +133,7 @@ jobs: platform-tag-list: ${{ steps.version-info.outputs.platform-tag-list }} heads: ${{ steps.version-info.outputs.platform-heads }} default-tags: ${{ steps.version-info.outputs.default-tags }} + bench-sdks: ${{ steps.bench-inputs.outputs.sdks }} go: ${{ steps.version-info.outputs.go-version-info }} java: ${{ steps.version-info.outputs.java-version-info }} js: ${{ steps.version-info.outputs.js-version-info }} @@ -107,6 +153,45 @@ jobs: echo "Invalid focus-sdk input: ${FOCUS_SDK_INPUT}. Must be one of: all, go, java, js." >> "$GITHUB_STEP_SUMMARY" exit 1 fi + # Decided here rather than in the bench job because a matrix cannot be + # narrowed from inside the job it belongs to: a bad combination would + # already have spun up three runners for 45 minutes each. + - name: Validate benchmark inputs and pick the bench matrix + id: bench-inputs + env: + FOCUS_SDK: ${{ inputs.focus-sdk || 'all' }} + BASELINE_REF: ${{ inputs.bench-baseline-ref }} + CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} + BUDGET_SECONDS: ${{ inputs.bench-budget-seconds }} + MAX_ROUNDS: ${{ inputs.bench-max-rounds }} + run: |- + # Only the numeric inputs are checked here. bench-payloads has a + # grammar, and a second copy of it in bash would drift from the one + # pytest enforces and start rejecting runs that would have worked; + # the bench job validates it with the real parser instead. + for pair in "bench-budget-seconds:$BUDGET_SECONDS" "bench-max-rounds:$MAX_ROUNDS"; do + name=${pair%%:*} + value=${pair#*:} + if [[ -n "$value" && ! "$value" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::${name} must be a positive whole number, got '${value}'." + exit 1 + fi + done + if [[ -n "$BASELINE_REF" && -z "$CANDIDATE_REF" ]] \ + || [[ -z "$BASELINE_REF" && -n "$CANDIDATE_REF" ]]; then + echo "::error::bench-baseline-ref and bench-candidate-ref must be set together; a comparison needs both arms named." + exit 1 + fi + if [[ -n "$CANDIDATE_REF" && "$FOCUS_SDK" == "all" ]]; then + echo "::error::bench-baseline-ref/bench-candidate-ref name refs of one SDK, so focus-sdk must be go, java, or js -- not 'all'." + exit 1 + fi + if [[ "$FOCUS_SDK" == "all" ]]; then + echo 'sdks=["go","java","js"]' >> "$GITHUB_OUTPUT" + else + echo "sdks=[\"${FOCUS_SDK}\"]" >> "$GITHUB_OUTPUT" + fi + - name: Default Versions depend on context id: default-tags run: |- @@ -769,7 +854,14 @@ jobs: # Never runs on pull requests: 30 minutes of serial measurement is too slow # for a PR gate, and a PR runner is the noisiest place to measure. bench: - timeout-minutes: 45 + # Has to cover setup plus the whole of bench-budget-seconds, and setup is + # not a small constant: a warm Go module cache builds both arms in ~3 + # minutes, a cold one took 19. At 45 this job could not even finish its + # own default 1500s budget after a cold start -- it would be killed + # mid-measurement, which loses the report entirely rather than reporting + # fewer rounds. The budget is the knob that bounds the run; this is only + # the backstop for a hung one. + timeout-minutes: 90 runs-on: ubuntu-latest needs: resolve-versions # Nightly cron only, not the Mon/Wed or weekly ones: three runs a week of @@ -783,10 +875,11 @@ jobs: packages: read strategy: # One runner per SDK. Two SDKs on one runner would contend for the very - # CPU being measured. + # CPU being measured. Narrowed by focus-sdk, so investigating one SDK + # does not spend 45 minutes measuring the two nobody asked about. fail-fast: false matrix: - sdk: [go, java, js] + sdk: ${{ fromJSON(needs.resolve-versions.outputs.bench-sdks) }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -794,6 +887,30 @@ jobs: path: otdftests persist-credentials: false + # Before the platform, which takes ~15 minutes to come up: a typo'd size + # is worth catching in the first thirty seconds. This calls the harness's + # own parser rather than reimplementing the grammar in bash -- perf.cells + # imports nothing outside the standard library, so a bare python3 can + # read it, and a bash copy that drifted would start refusing specs the + # run itself would have accepted. + - name: Validate benchmark payload sizes + if: inputs.bench-payloads != '' + working-directory: otdftests/xtest + env: + BENCH_PAYLOADS: ${{ inputs.bench-payloads }} + run: |- + python3 - "$BENCH_PAYLOADS" <<'PY' + import sys + + from perf.cells import parse_payloads + + try: + sizes = parse_payloads(sys.argv[1]) + except ValueError as e: + raise SystemExit(f"::error::invalid bench-payloads: {e}") + print("payload sizes:", ", ".join(p.label for p in sizes)) + PY + - name: load extra keys from file id: load-extra-keys run: |- @@ -859,16 +976,57 @@ jobs: PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} ######## INSTALL BOTH ARMS OF THE COMPARISON ############# + # Two named refs instead of the default release-vs-branch pair. Resolved + # here rather than in resolve-versions because the *-ref inputs there + # drive the functional matrix, and a benchmark wants to name its two + # arms without also changing what the rest of the workflow tests. + # + # Baseline first: the tag order becomes configure-sdk's `heads` output, + # and conftest.py takes heads[0] as the otdfctl that provisions + # attributes and the KAS registry. That provisioning is not measured, + # and it should be the same build for both arms. + - name: Resolve the two benchmark arms + id: bench-arms + if: inputs.bench-candidate-ref != '' + working-directory: otdftests/otdf-sdk-mgr + env: + SDK: ${{ matrix.sdk }} + BASELINE_REF: ${{ inputs.bench-baseline-ref }} + CANDIDATE_REF: ${{ inputs.bench-candidate-ref }} + run: |- + info=$(uv run --project . otdf-sdk-mgr versions resolve \ + "$SDK" "$BASELINE_REF" "$CANDIDATE_REF") + jq . <<<"$info" + err=$(jq -r '[.[] | select(.err != null) | .err] | join("; ")' <<<"$info") + if [[ -n "$err" ]]; then + echo "::error::Could not resolve benchmark arms: $err" + exit 1 + fi + # `versions resolve` drops a ref whose SHA it has already seen, so + # two names for one commit come back as a single entry. Left alone + # that installs one build, fails arm selection in every cell, and + # spends the runner's 45 minutes arriving at NOTHING MEASURED. + if [[ "$(jq 'length' <<<"$info")" -ne 2 ]]; then + echo "::error::${BASELINE_REF} and ${CANDIDATE_REF} resolve to the same commit -- nothing to compare." + exit 1 + fi + { + echo "version-info=$(jq -c . <<<"$info")" + echo "baseline-spec=${SDK}@$(jq -r '.[0].tag' <<<"$info")" + echo "candidate-spec=${SDK}@$(jq -r '.[1].tag' <<<"$info")" + } >> "$GITHUB_OUTPUT" + # The whole design rests on this step laying down two builds side by - # side under sdk//dist/: the branch head (candidate) and the - # newest release (baseline). Arm selection picks them up from there. + # side under sdk//dist/: by default the branch head (candidate) and + # the newest release (baseline), or the two refs resolved above. Arm + # selection picks them up from there. - name: Configure ${{ matrix.sdk }} sdk id: configure-sdk uses: ./otdftests/xtest/setup-cli-tool with: path: otdftests/xtest/sdk sdk: ${{ matrix.sdk }} - version-info: "${{ needs.resolve-versions.outputs[matrix.sdk] }}" + version-info: "${{ steps.bench-arms.outputs.version-info || needs.resolve-versions.outputs[matrix.sdk] }}" platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} @@ -929,7 +1087,7 @@ jobs: fi done env: - java_version_info: ${{ needs.resolve-versions.outputs.java }} + java_version_info: ${{ steps.bench-arms.outputs.version-info || needs.resolve-versions.outputs.java }} platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)['main'] }} - name: Build the ${{ matrix.sdk }} cli @@ -962,10 +1120,22 @@ jobs: - name: Run performance benchmarks id: bench run: |- + # Empty unless the two arms were named explicitly, in which case + # arm selection must not fall back to "newest release vs branch + # head": neither named ref need be a release, and with two branch + # builds installed the default would pick the wrong pair or none. + arms=() + if [[ -n "$BENCH_BASELINE_SPEC" ]]; then + arms=(--bench-baseline "$BENCH_BASELINE_SPEC" + --bench-candidate "$BENCH_CANDIDATE_SPEC") + fi uv run --frozen --no-build pytest -ra -v \ --bench \ --sdks "$BENCH_SDK" \ - --bench-budget-seconds 1500 \ + "${arms[@]}" \ + --bench-payloads "$BENCH_PAYLOADS" \ + --bench-budget-seconds "$BENCH_BUDGET_SECONDS" \ + --bench-max-rounds "$BENCH_MAX_ROUNDS" \ --bench-out test-results/benchmarks \ --html "test-results/bench-${BENCH_SDK}.html" \ --self-contained-html \ @@ -973,6 +1143,14 @@ jobs: working-directory: otdftests/xtest env: BENCH_SDK: ${{ matrix.sdk }} + BENCH_BASELINE_SPEC: ${{ steps.bench-arms.outputs.baseline-spec }} + BENCH_CANDIDATE_SPEC: ${{ steps.bench-arms.outputs.candidate-spec }} + # Fallbacks rather than input defaults: the scheduled nightly + # supplies no inputs at all, so `inputs.*` is empty there and these + # are what keeps its matrix and budget where they have always been. + BENCH_PAYLOADS: ${{ inputs.bench-payloads || '1KiB,1MiB,32MiB' }} + BENCH_BUDGET_SECONDS: ${{ inputs.bench-budget-seconds || '1500' }} + BENCH_MAX_ROUNDS: ${{ inputs.bench-max-rounds || '60' }} PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" PLATFORM_TAG: main diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 9c50d2ce1..7ff3b6811 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -300,7 +300,14 @@ def _resolve_against( "alias": version, "head": True, "sha": sha, - "tag": version, + # Flattened the same way _classify_sha_match flattens a branch + # it reached by SHA: the tag becomes a single dist// and + # src// path component. A slash here nests those + # directories, and every consumer walks them one level deep -- + # xtest's all_versions_of() lists dist/*/ and the go Makefile + # finds src/*/, so "feat/x" is discovered as a "feat" build + # with no cli.sh in it. + "tag": version.replace("/", "--"), } if infix and version.startswith(f"{infix}/"): diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index a7c30b057..29efa6f6f 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -76,7 +76,24 @@ def test_refs_heads_non_main_branch(self): result = resolve("js", "refs/heads/release/sdk-v0.17", None) assert is_resolve_success(result) assert "head" in result and result["head"] is True - assert result["tag"] == "release/sdk-v0.17" + assert result["tag"] == "release--sdk-v0.17" + assert result["sha"] == SHA40 + + def test_branch_by_name_flattens_slashes(self): + # Same flattening the SHA path applies, and for the same reason: the + # tag is one path component under dist/ and src/. Reached by name + # rather than by SHA, which is the shape a workflow_dispatch input + # arrives in. + ls = make_ls_remote( + (SHA40, "refs/heads/feat/DSPX-2604-createtdf-chunked"), + ("d" * 40, "refs/heads/main"), + ) + with patch_git(ls): + result = resolve("go", "feat/DSPX-2604-createtdf-chunked", None) + assert is_resolve_success(result) + assert result.get("head") is True + assert result["tag"] == "feat--DSPX-2604-createtdf-chunked" + assert result["alias"] == "feat/DSPX-2604-createtdf-chunked" assert result["sha"] == SHA40 diff --git a/xtest/conftest.py b/xtest/conftest.py index 5604bf0c1..da3949447 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -23,9 +23,10 @@ import pytest import tdfs +from fixtures.bench import payloads_from_options from otdfctl import OpentdfCommandLineTool from perf import report, stats -from perf.cells import cells_for +from perf.cells import DEFAULT_PAYLOAD_SPEC, cells_for logging.basicConfig(level=os.environ.get("LOGLEVEL", "DEBUG")) @@ -181,6 +182,16 @@ def _add_benchmark_options(parser: pytest.Parser): help="build under test, e.g. go@main; defaults to the installed " "unreleased build of each sdk", ) + group.addoption( + "--bench-payloads", + default=DEFAULT_PAYLOAD_SPEC, + help="comma-separated payload sizes to measure, e.g. " + "'1KiB,1MiB,32MiB,1GiB' (default: %(default)s). Sizes above the " + "default are opt-in because they are what a throughput gate actually " + "needs and what a nightly cannot afford: each one adds two cells, and " + "a run holds roughly twice the total plus the largest twice over on " + "disk", + ) group.addoption( "--bench-threshold", type=float, @@ -342,7 +353,7 @@ def _parametrize_bench_cells(metafunc: pytest.Metafunc): typing.get_args(tdfs.sdk_type) ) names = list(dict.fromkeys(s.split("@", 1)[0] for s in str(specs).split())) - cells = cells_for(names) + cells = cells_for(names, payloads_from_options(metafunc.config)) metafunc.config.stash[report.CELLS_KEY] = cells metafunc.parametrize("bench_cell", cells, ids=[c.id for c in cells]) diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py index d5194b380..e5a1b08d1 100644 --- a/xtest/fixtures/bench.py +++ b/xtest/fixtures/bench.py @@ -12,6 +12,8 @@ import os import platform import random +import shutil +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from typing import cast @@ -21,7 +23,7 @@ import abac import tdfs from perf import report -from perf.cells import PAYLOADS, BenchCell +from perf.cells import BenchCell, Payload, parse_payloads, payloads_to_generate from perf.runner import Arm, BenchConfig, Budget, Invocation @@ -133,8 +135,77 @@ def bench_config(request: pytest.FixtureRequest) -> BenchConfig: return config_from_options(request.config) +def payloads_from_options(config: pytest.Config) -> tuple[Payload, ...]: + """The run's payload set, from ``--bench-payloads``.""" + spec = cast(str, config.getoption("--bench-payloads")) + try: + return parse_payloads(spec) + except ValueError as e: + raise pytest.UsageError(f"invalid --bench-payloads: {e}") from e + + +@pytest.fixture(scope="session") +def bench_payload_set(request: pytest.FixtureRequest) -> tuple[Payload, ...]: + """Payload sizes this run measures, from --bench-payloads.""" + return payloads_from_options(request.config) + + +#: Bytes generated per ``randbytes`` call. Must stay a multiple of 4: CPython +#: draws a 32-bit word at a time, so chunking on a 4-byte boundary yields the +#: same stream as one call for the whole payload, and the promise below -- +#: that a given seed and label always produce the same bytes -- survives both +#: this constant changing and a payload growing past it. +_CHUNK_BYTES = 8 * 2**20 + +#: Free space a run keeps in hand beyond its payload arithmetic, for the +#: platform's own logs and database growth over a long benchmark. +_DISK_HEADROOM_BYTES = 2**30 + + +def write_payload(path: Path, payload: Payload, seed: int) -> None: + """Write one payload file, in chunks so a 1 GiB file is not built in RAM.""" + rng = random.Random(f"{seed}:{payload.label}") + remaining = payload.n_bytes + with path.open("wb") as f: + while remaining > 0: + n = min(remaining, _CHUNK_BYTES) + f.write(rng.randbytes(n)) + remaining -= n + + +def disk_shortfall(tmp_dir: Path, payloads: Sequence[Payload]) -> str | None: + """Return why ``payloads`` will not fit in ``tmp_dir``, or None. + + Checked up front because the alternative is finding out mid-run: ENOSPC + reaches the harness as a non-zero exit from the CLI under measurement, + which is reported as a failed measurement of that build. A run can lose + an hour before anyone notices the disk was the problem, and the report + points at the wrong thing while they look. + + The estimate is the plaintexts, plus a cached ciphertext for each (the + decrypt cells share one per size), plus the two output files the largest + cell holds while it runs. Outputs are deleted as each cell finishes, so + only one cell's worth is ever live. + """ + total = sum(p.n_bytes for p in payloads) + largest = max(p.n_bytes for p in payloads) + need = 2 * total + 2 * largest + _DISK_HEADROOM_BYTES + free = shutil.disk_usage(tmp_dir).free + if free >= need: + return None + gib = 2**30 + sizes = ", ".join(p.label for p in payloads) + return ( + f"payloads {sizes} need about {need / gib:.1f} GiB of scratch space " + f"in {tmp_dir} but only {free / gib:.1f} GiB is free; drop the largest " + "size from --bench-payloads or run somewhere with more disk" + ) + + @pytest.fixture(scope="session") -def bench_payloads(tmp_dir: Path, bench_config: BenchConfig) -> dict[str, Path]: +def bench_payloads( + tmp_dir: Path, bench_config: BenchConfig, bench_payload_set: tuple[Payload, ...] +) -> dict[str, Path]: """Generate one plaintext file per payload size, shared by both arms. Content is pseudo-random but seeded, so a rerun measures byte-identical @@ -147,13 +218,19 @@ def bench_payloads(tmp_dir: Path, bench_config: BenchConfig) -> dict[str, Path]: calls and shifts the stream for every payload after it. Deriving each payload's bytes from the seed *and* its label keeps the promise above true whether the cache is empty, full, or half there. + + The control's payload is generated whether or not it was selected -- see + :func:`perf.cells.payloads_to_generate`. """ + wanted = payloads_to_generate(bench_payload_set) + shortfall = disk_shortfall(tmp_dir, wanted) + if shortfall: + raise pytest.UsageError(shortfall) out: dict[str, Path] = {} - for payload in PAYLOADS: + for payload in wanted: path = tmp_dir / f"bench-plain-{payload.label}.bin" if not path.is_file() or path.stat().st_size != payload.n_bytes: - rng = random.Random(f"{bench_config.seed}:{payload.label}") - path.write_bytes(rng.randbytes(payload.n_bytes)) + write_payload(path, payload, bench_config.seed) out[payload.label] = path return out diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 49d933d90..fd109e433 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -19,6 +19,10 @@ measure. > baseline — and every cell skips. The run fails rather than passing empty > (see [NOTHING MEASURED](#the-verdicts)), but it will have wasted 45 minutes > to tell you that. +> +> To compare **two named refs** instead — a branch against `main`, say — use +> `bench-baseline-ref` / `bench-candidate-ref` and skip all of the above; see +> [Benchmarking one branch against another](#benchmarking-one-branch-against-another). - **Section 1 — [Reading a result](#1-reading-a-result)** is for developers on the SDKs and the platform: your build got flagged, what does that mean. @@ -49,7 +53,9 @@ a 30-minute job to look at the same numbers again. ``` - **cell** — `--`, plus `-control` for the A/A cell. - Payload sizes are 1 KiB, 1 MiB, and 32 MiB. + Payload sizes default to 1 KiB, 1 MiB, and 32 MiB; `--bench-payloads` selects + others. See [Payload sizes and what they can gate](#payload-sizes-and-what-they-can-gate) + before reading a throughput result — at the default sizes there is not one. - **ratio** — candidate ÷ baseline. `1.208x` means the candidate took 20.8% longer. Below 1.0 means faster. - **95% CI** — the bootstrap interval on that ratio. Its *width* is how precisely @@ -141,8 +147,10 @@ Censored cells report inconclusive with the floor named in the note. 2. **Check the control row.** If the A/A cell for your SDK also looks strange, suspect the runner before your code. 3. **Look at which cells fired.** Only the 1 KiB cells means startup cost — - process boot, package resolution, TLS handshake, token fetch. Only 32 MiB - means throughput — the crypto and IO path. Both means something structural. + process boot, package resolution, TLS handshake, token fetch. The largest + cell firing on its own points at throughput — the crypto and IO path — but + only if that cell is large enough for throughput to be most of it, which at + 32 MiB it is not. Both means something structural. 4. **Reproduce locally.** The comparison is self-contained; it does not need CI. ```bash @@ -160,6 +168,7 @@ Useful knobs while investigating: | Option | Default | Use | | --- | --- | --- | | `--bench-threshold` | `1.15` | Smallest slowdown worth failing on | +| `--bench-payloads` | `1KiB,1MiB,32MiB` | Sizes to measure, e.g. `1KiB,1GiB` | | `--bench-min-rounds` / `--bench-max-rounds` | `20` / `60` | Rounds per cell | | `--bench-warmup` | `5` | Discarded rounds paying one-time costs | | `--bench-budget-seconds` | `1500` | Wall-clock allowance shared by all cells | @@ -170,6 +179,102 @@ Useful knobs while investigating: A local run is noisier than CI unless the machine is otherwise idle. Close things; the noise floor will tell you whether you succeeded. +### Payload sizes and what they can gate + +**At the default sizes this harness cannot fail a build on throughput.** Not +"is unlikely to" — cannot. On a 4-core Linux runner a go encrypt costs about +450 ms before it touches the payload: runtime start, config load, TLS +handshake, token fetch, KAS key fetch. Going from 1 KiB to 32 MiB — a 32,000x +increase in bytes — adds about 72 ms on top of that. + +| operation | 1 KiB | 1 MiB | 32 MiB | payload-dependent | +| --- | --- | --- | --- | --- | +| encrypt | 455.0 ms | 447.6 ms | 526.7 ms | ~72 ms (13.6%) | +| decrypt | 533.6 ms | 513.1 ms | 600.7 ms | ~67 ms (11.2%) | + +The gate is 1.15x of the *whole cell*, which at 32 MiB encrypt is +79 ms — +more than the entire payload-dependent portion. A candidate that doubled every +per-segment cost would come in at 1.136x and report **PASS**. The 1 MiB cells +are worse: indistinguishable from 1 KiB, so they measure startup twice. + +This is not a statistics problem. The intervals are tight and the control is +clean; the matrix is simply asking the wrong sizes. To gate throughput the +payload term has to dominate, which means going much larger: + +```bash +uv run pytest --bench --sdks go \ + --bench-payloads 1KiB,1GiB \ + --bench-budget-seconds 5400 --bench-max-rounds 200 \ + -v test_benchmarks.py +``` + +At 1 GiB the payload term is ~2.3 s against the same ~450 ms fixed cost, so it +is ~84% of the cell and a 15% gate lands inside the part being tested. + +Three things to know before adding a large size: + +- **Budget.** Each size adds an encrypt and a decrypt cell, and the budget is + divided evenly as cells start. A 1 GiB round costs ~6 s against ~1 s at 32 + MiB, so the default 1500 s will not reach `min_rounds` on both new cells. +- **Disk.** A run holds roughly twice the payload total plus the largest size + twice over. 1 GiB needs ~5 GiB free. This is checked before the first + measurement, because running out mid-run arrives as a non-zero exit from the + CLI under test and reads as "this build is broken". +- **`max_rounds` binds before the budget does.** In the run these numbers come + from, 3 of 7 cells stopped on `max_rounds` while only 418 s of 1500 s was + spent. Raising the budget alone buys nothing; raise both. + +The control stays at 1 MiB whatever you select. Its CI width is the run's noise +floor and every other cell is judged against it, so it must not move with the +matrix — otherwise two runs of the same comparison can disagree about which +cells were trustworthy for a reason unrelated to either build. + +### Benchmarking one branch against another + +The nightly comparison is newest-release vs branch head, which is the right +question to ask every night and the wrong one to ask about a specific change: +the baseline carries every other commit that landed since the release. To +point the harness at two refs you name, dispatch X-Test with: + +| Input | Example | Meaning | +| --- | --- | --- | +| `run-benchmarks` | ✅ | Required; the bench job is off otherwise | +| `focus-sdk` | `go` | Must name one SDK — the matrix runs only this one | +| `bench-baseline-ref` | `main` | The build you are comparing *against* | +| `bench-candidate-ref` | `feat/DSPX-2604-createtdf-chunked` | The build under suspicion | +| `bench-payloads` | `1KiB,1GiB` | Sizes to measure; default `1KiB,1MiB,32MiB` | +| `bench-budget-seconds` | `5400` | Shared allowance; default `1500` | +| `bench-max-rounds` | `200` | Cap per cell; default `60` | + +The last three are why a dispatch can answer a question the nightly cannot. A +nightly runs unattended every day and has to stay inside a sensible cost; a +dispatch is asked for, once, about one thing. If the change is a throughput +claim, spend the budget — see [Payload sizes and what they can +gate](#payload-sizes-and-what-they-can-gate), because at the defaults the +answer will be **PASS** whatever the change did. + +Either ref can be anything `otdf-sdk-mgr versions resolve` accepts: a branch, a +tag, a full or short SHA, or `refs/pull/N/head`. Both are built from source and +installed side by side, and arm selection is told which is which explicitly — +so neither has to be a release, which is the whole point. + +The `*-ref` inputs are ignored by the bench job in this mode. They still drive +the functional test matrix, so a dispatch can answer "is it slower?" without +also changing what the rest of the run tests. + +Two things this mode does **not** change, both of which bound what a result +means: + +- **The server stays on `main`.** The bench job pins the platform and runs a + single KAS, whatever the refs say. A candidate whose speed depends on a + matching server change will not show it here. +- **The baseline is whatever you named.** For a stacked branch, `main` as the + baseline measures the whole stack. Name the parent branch instead to isolate + the top commit. + +It fails fast, before spending a runner, when the two refs resolve to the same +commit or when `focus-sdk` is `all`. + ### What this benchmark cannot tell you - **Anything about absolute speed.** A number from a GitHub-hosted runner is not @@ -327,6 +432,18 @@ and the directory listing breaks the tie. That is a baseline nobody chose, and i differs run to run. Baseline selection uses `is_final_release()`, which matches only a plain `vX.Y.Z`. +#### A dist tag is one path component + +`otdf-sdk-mgr` flattens `/` to `--` when it resolves a ref, so +`feat/DSPX-2604-createtdf-chunked` installs as +`dist/feat--DSPX-2604-createtdf-chunked/`. Everything downstream walks those +directories exactly one level deep — `tdfs.all_versions_of()` lists `dist/*/`, +the go `Makefile` finds `src/*/` — so a slash that survives resolution is +discovered as a build named `feat` with no `cli.sh` in it, which +`all_versions_of()` raises on before any cell runs. Branch-vs-branch dispatch +is the first thing to routinely feed it a slashed ref, and the `--bench-*` +specs name the flattened tag: `go@feat--DSPX-2604-createtdf-chunked`. + #### Payloads are seeded per payload, not per run `tmp_dir` persists between runs. With one RNG stream shared across the payloads, a @@ -337,6 +454,12 @@ be comparable with. Each payload derives from `f"{seed}:{label}"` instead. Content is random rather than repetitive because compressible input would let an SDK that happens to compress look faster for reasons unrelated to crypto. +Large payloads are written in chunks so a 1 GiB file is not first built as a +1 GiB `bytes` in RAM. The chunk size must stay a multiple of 4: CPython's +`randbytes` draws a 32-bit word at a time, so a 4-byte-aligned split produces +the same stream as one call would, and the seed-to-bytes promise survives both +the constant changing and a payload growing past it. + #### Cells record; the session gates The verdict cannot be reached cell by cell — the multiplicity correction spans @@ -369,11 +492,13 @@ CPU under measurement. ### Adding to it -**A new payload size** — add a `Payload` to `PAYLOADS` in `cells.py`. Note that -`CONTROL_PAYLOAD = PAYLOADS[1]`, so inserting at the front moves the control. -Cell count per SDK is `1 + 2 × len(PAYLOADS)`; the 1500s budget is divided -across all of them, so adding sizes makes every cell poorer unless the budget -grows too. +**A new payload size** — no code change: `--bench-payloads 1KiB,1GiB` (or the +`bench-payloads` dispatch input). Sizes parse as a count and a binary unit — +`B`, `KiB`, `MiB`, `GiB` — and the list is sorted ascending and deduplicated by +byte count, so `1KiB,1024B` is one cell rather than two identical ones. Changing +`DEFAULT_PAYLOAD_SPEC` in `cells.py` changes what the nightly measures; think +about the budget first. Cell count per SDK is `1 + 2 × len(payloads)`, and the +budget is divided evenly across all of them. **A new metric** — add it to `METRICS` and `METRIC_LABELS` in `measure.py`, teach `Sample.metric()` and `format_metric()` about it, and decide whether it belongs diff --git a/xtest/perf/cells.py b/xtest/perf/cells.py index 4a1d4ba47..f8a34413f 100644 --- a/xtest/perf/cells.py +++ b/xtest/perf/cells.py @@ -6,6 +6,8 @@ from __future__ import annotations +import re +from collections.abc import Sequence from dataclasses import dataclass from typing import Literal @@ -22,22 +24,100 @@ class Payload: 32 MiB the crypto and IO dominate and a startup regression is invisible. A benchmark at one size only will miss half the regressions it claims to cover. + + "Dominate" is relative, and at 32 MiB it is not yet true. On a 4-core + Linux runner a go encrypt costs ~450 ms of fixed startup against ~72 ms + that scales with the payload, so payload work is ~14% of the cell and the + default 1.15x gate is wider than the whole of it -- a candidate that + doubled every per-segment cost would still report PASS. Gating throughput + needs a size where the ratio inverts, which is what ``--bench-payloads`` + is for: at 1 GiB the payload term is ~2.3 s against the same ~450 ms. """ label: str n_bytes: int -PAYLOADS: tuple[Payload, ...] = ( - Payload("1KiB", 1024), - Payload("1MiB", 2**20), - Payload("32MiB", 32 * 2**20), +#: Binary units only. A label is a filename and a cell id, and "1MB" sitting +#: next to "1MiB" in a report is a misreading waiting to happen. +_UNITS: tuple[tuple[str, int], ...] = ( + ("B", 1), + ("KiB", 2**10), + ("MiB", 2**20), + ("GiB", 2**30), ) +_SIZE_RE = re.compile(r"^\s*(\d+)\s*([a-z]+)\s*$", re.IGNORECASE) + + +def parse_payload(spec: str) -> Payload: + """Parse one size spec, e.g. ``"32MiB"``, into a :class:`Payload`. + + The unit is matched case-insensitively but the label is rebuilt from the + canonical spelling, so ``"1gib"`` and ``"1GiB"`` name the same cell rather + than two cells that measure the same thing under different ids. + """ + match = _SIZE_RE.match(spec) + units = {name.lower(): (name, mult) for name, mult in _UNITS} + if match is None or match.group(2).lower() not in units: + raise ValueError( + f"{spec!r} is not a payload size; expected a count and one of " + f"{', '.join(name for name, _ in _UNITS)}, e.g. '32MiB'" + ) + count = int(match.group(1)) + name, multiplier = units[match.group(2).lower()] + if count <= 0: + raise ValueError(f"{spec!r} is not a payload size; it must be above zero") + return Payload(f"{count}{name}", count * multiplier) + + +def parse_payloads(spec: str) -> tuple[Payload, ...]: + """Parse a comma-separated size list into the run's payload set. + + Sorted ascending and deduplicated *by byte count*, not by label: ``1024B`` + and ``1KiB`` are one size written two ways, and admitting both would run + two identically-sized cells whose only difference is the id in the report. + """ + by_size: dict[int, Payload] = {} + for part in spec.split(","): + if not part.strip(): + continue + payload = parse_payload(part) + by_size.setdefault(payload.n_bytes, payload) + if not by_size: + raise ValueError("no payload sizes given") + return tuple(by_size[n] for n in sorted(by_size)) + + +#: What a run measures unless ``--bench-payloads`` says otherwise. Anything +#: larger is opt-in: a 1 GiB cell costs minutes of budget and gigabytes of +#: scratch disk, which a nightly should not spend without being asked. +DEFAULT_PAYLOAD_SPEC = "1KiB,1MiB,32MiB" + +PAYLOADS: tuple[Payload, ...] = parse_payloads(DEFAULT_PAYLOAD_SPEC) + #: Payload used for the A/A control. Mid-size: large enough that startup noise #: does not dominate it, small enough that the control is not a big slice of #: the budget. -CONTROL_PAYLOAD = PAYLOADS[1] +#: +#: Fixed rather than picked out of the selected set, because the control's +#: reported width *is* the run's noise floor and every other cell is judged +#: against it. Letting it follow ``--bench-payloads`` would move the floor +#: whenever the matrix changed, so two runs of the same comparison could +#: disagree about which cells were trustworthy for a reason that has nothing +#: to do with either build. +CONTROL_PAYLOAD = Payload("1MiB", 2**20) + + +def payloads_to_generate(payloads: Sequence[Payload]) -> tuple[Payload, ...]: + """Every payload file a run needs, including the control's. + + The control's size need not be in the selected set -- ``--bench-payloads + 1GiB`` is a legitimate ask -- but its file is still required, and a + missing one surfaces as a KeyError deep in arm construction. + """ + by_label = {p.label: p for p in (*payloads, CONTROL_PAYLOAD)} + return tuple(sorted(by_label.values(), key=lambda p: p.n_bytes)) @dataclass(frozen=True, slots=True) @@ -61,7 +141,9 @@ def __str__(self) -> str: return self.id -def cells_for(sdks: list[str]) -> list[BenchCell]: +def cells_for( + sdks: list[str], payloads: Sequence[Payload] = PAYLOADS +) -> list[BenchCell]: """Build the full cell list for a run, each SDK's control cell first. One control per SDK rather than one per run: a control measures a @@ -72,13 +154,17 @@ def cells_for(sdks: list[str]) -> list[BenchCell]: whatever is at the end. Losing one comparison leaves the rest trustworthy; losing the control leaves nothing trustworthy at all, since without a noise floor no cell may report PASS. + + Within an SDK the payloads run smallest first, so that when the budget + does run out it is the most expensive cell that is lost rather than an + arbitrary one. """ cells: list[BenchCell] = [] for sdk in sdks: cells.append(BenchCell(sdk, "encrypt", CONTROL_PAYLOAD, control=True)) cells += [ BenchCell(sdk, op, payload) + for payload in payloads for op in ("encrypt", "decrypt") - for payload in PAYLOADS ] return cells diff --git a/xtest/perf/runner.py b/xtest/perf/runner.py index af69ff68b..ba8f34a00 100644 --- a/xtest/perf/runner.py +++ b/xtest/perf/runner.py @@ -261,66 +261,76 @@ def one_round(into: dict[str, dict[str, list[float]]]) -> None: for metric in METRICS: into[arm.name][metric].append(sample.metric(metric)) - for i in range(config.warmup): - # Warm-up rounds pay the one-time costs -- page cache, `go build` - # cache, npx package resolution, JIT warm-up -- that would otherwise - # land unevenly and show up as a difference between builds. Their - # samples are collected into a throwaway dict and dropped. - # - # The deadline is checked here too, and not only in the measured loop - # below. The budget's end is absolute, so warm-ups that overrun it - # spend the *following* cells' time and then reach the measured loop - # with nothing left -- paying the full cost of the cell and producing - # no data. Better to give up here and say why. - if deadline is not None and clock() >= deadline: - raise BudgetExhausted( - f"{cell_id}: budget ran out after {i} of {config.warmup} " - f"warm-up rounds ({clock() - started:.0f}s), " - "before any measurement began" - ) - one_round(_empty_samples()) - - stopped_because = "max_rounds" - for _ in range(config.max_rounds): - round_start = clock() - if deadline is not None and round_start >= deadline: - stopped_because = "budget" - break - if deadline is not None and round_durations: - # Do not start a round we cannot finish: a half-measured round is - # unpaired data, and unpaired data is exactly what this design - # exists to avoid. - expected = float(np.median(round_durations)) - if round_start + expected > deadline: + try: + for i in range(config.warmup): + # Warm-up rounds pay the one-time costs -- page cache, `go build` + # cache, npx package resolution, JIT warm-up -- that would + # otherwise land unevenly and show up as a difference between + # builds. Their samples are collected into a throwaway dict and + # dropped. + # + # The deadline is checked here too, and not only in the measured + # loop below. The budget's end is absolute, so warm-ups that + # overrun it spend the *following* cells' time and then reach the + # measured loop with nothing left -- paying the full cost of the + # cell and producing no data. Better to give up here and say why. + if deadline is not None and clock() >= deadline: + raise BudgetExhausted( + f"{cell_id}: budget ran out after {i} of {config.warmup} " + f"warm-up rounds ({clock() - started:.0f}s), " + "before any measurement began" + ) + one_round(_empty_samples()) + + stopped_because = "max_rounds" + for _ in range(config.max_rounds): + round_start = clock() + if deadline is not None and round_start >= deadline: stopped_because = "budget" break - one_round(samples) - round_durations.append(clock() - round_start) + if deadline is not None and round_durations: + # Do not start a round we cannot finish: a half-measured round + # is unpaired data, and unpaired data is exactly what this + # design exists to avoid. + expected = float(np.median(round_durations)) + if round_start + expected > deadline: + stopped_because = "budget" + break + one_round(samples) + round_durations.append(clock() - round_start) + + n = len(samples["baseline"]["wall"]) + if n >= config.min_rounds and _precise_enough(samples, config): + stopped_because = "precision" + break + elapsed = clock() - started n = len(samples["baseline"]["wall"]) - if n >= config.min_rounds and _precise_enough(samples, config): - stopped_because = "precision" - break - - elapsed = clock() - started - n = len(samples["baseline"]["wall"]) - if n < stats.MIN_USABLE_ROUNDS: - raise BudgetExhausted( - f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " - f"below the {stats.MIN_USABLE_ROUNDS} needed for any verdict" + if n < stats.MIN_USABLE_ROUNDS: + raise BudgetExhausted( + f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " + f"below the {stats.MIN_USABLE_ROUNDS} needed for any verdict" + ) + return CellResult( + cell_id=cell_id, + baseline_label=baseline.label, + candidate_label=candidate.label, + samples=samples, + n_warmup=config.warmup, + elapsed_s=elapsed, + stopped_because=stopped_because, + control=control, + sdk=sdk, + rss_floor_bytes=rss_floor, ) - return CellResult( - cell_id=cell_id, - baseline_label=baseline.label, - candidate_label=candidate.label, - samples=samples, - n_warmup=config.warmup, - elapsed_s=elapsed, - stopped_because=stopped_because, - control=control, - sdk=sdk, - rss_floor_bytes=rss_floor, - ) + finally: + # Each arm leaves behind an output the size of the payload, and + # nothing reads it once the cell is done. Keeping them costs 2 GiB per + # 1 GiB cell, which every later cell then has to fit around -- so the + # cell that fails on disk is not the one that filled it. + for arm in arms: + if arm.invocation.output is not None: + arm.invocation.output.unlink(missing_ok=True) def _precise_enough( diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py index bf7a3194e..97f654350 100644 --- a/xtest/test_bench_arms.py +++ b/xtest/test_bench_arms.py @@ -10,13 +10,22 @@ ``tmp_path``. """ +from collections.abc import Sequence from pathlib import Path import pytest import tdfs from fixtures import bench -from perf.cells import PAYLOADS +from perf.cells import ( + CONTROL_PAYLOAD, + DEFAULT_PAYLOAD_SPEC, + PAYLOADS, + Payload, + cells_for, + parse_payload, + parse_payloads, +) from perf.runner import BenchConfig @@ -86,10 +95,121 @@ def test_refuses_to_compare_a_build_against_itself(self, cwd: Path): with pytest.raises(bench.ArmSelectionError, match="nothing to compare"): bench.select_arms("go", baseline_spec="go@main", candidate_spec="go@main") + def test_two_branch_builds_need_explicit_specs(self, cwd: Path): + # What a branch-vs-branch dispatch installs: two heads and no release + # at all. Named explicitly it is a fine comparison; left to the default + # there is no baseline, and "newest final release" cannot invent one. + install(cwd, "go", "main", "feat--DSPX-2604-createtdf-chunked") + baseline, candidate = bench.select_arms( + "go", + baseline_spec="go@main", + candidate_spec="go@feat--DSPX-2604-createtdf-chunked", + ) + assert baseline.version == "main" + assert candidate.version == "feat--DSPX-2604-createtdf-chunked" + with pytest.raises(bench.ArmSelectionError, match="no final go release"): + bench.select_arms("go") + + +class TestDistTagShape: + def test_a_slashed_tag_breaks_discovery(self, cwd: Path): + # Why otdf-sdk-mgr flattens '/' to '--' in a resolved ref. A branch + # installed as dist/feat/x/ is listed as a build named "feat", which + # has no cli.sh -- and this raises during collection, before any cell + # has a chance to report why. + install(cwd, "go", "feat/DSPX-2604-createtdf-chunked") + with pytest.raises(FileNotFoundError): + tdfs.all_versions_of("go") + + def test_a_flattened_tag_is_discovered(self, cwd: Path): + install(cwd, "go", "feat--DSPX-2604-createtdf-chunked") + assert [s.version for s in tdfs.all_versions_of("go")] == [ + "feat--DSPX-2604-createtdf-chunked" + ] + + +class TestPayloadSpec: + @pytest.mark.parametrize( + ("spec", "n_bytes"), + [ + ("512B", 512), + ("1KiB", 1024), + ("32MiB", 32 * 2**20), + ("1GiB", 2**30), + ("4GiB", 4 * 2**30), + ], + ) + def test_sizes_parse(self, spec: str, n_bytes: int): + assert parse_payload(spec).n_bytes == n_bytes + + def test_the_label_is_canonical_regardless_of_case(self): + # The label is a filename and a cell id. '1gib' and '1GiB' naming two + # cells would measure one size twice and report it as two results. + assert parse_payload("1gib").label == "1GiB" + assert parse_payload(" 1 GIB ").label == "1GiB" + + @pytest.mark.parametrize( + "spec", ["", "1", "MiB", "1MB", "1.5GiB", "-1GiB", "0GiB", "1GiB extra"] + ) + def test_junk_is_refused(self, spec: str): + with pytest.raises(ValueError): + parse_payload(spec) + + def test_a_list_is_sorted_ascending(self): + labels = [p.label for p in parse_payloads("1GiB,1KiB,32MiB")] + assert labels == ["1KiB", "32MiB", "1GiB"] + + def test_one_size_written_two_ways_is_one_payload(self): + # Otherwise the run pays for two identical cells and reports them as + # independent results, which the multiplicity correction then treats + # as two tests. + assert [p.label for p in parse_payloads("1KiB,1024B")] == ["1KiB"] + + def test_an_empty_list_is_refused(self): + with pytest.raises(ValueError, match="no payload sizes"): + parse_payloads(" , ") + + +class TestCellMatrix: + def test_the_default_matrix_is_unchanged(self): + ids = [c.id for c in cells_for(["go"], parse_payloads(DEFAULT_PAYLOAD_SPEC))] + assert ids == [ + "go-encrypt-1MiB-control", + "go-encrypt-1KiB", + "go-decrypt-1KiB", + "go-encrypt-1MiB", + "go-decrypt-1MiB", + "go-encrypt-32MiB", + "go-decrypt-32MiB", + ] + + def test_the_control_comes_first_and_the_biggest_pair_last(self): + # The budget is spent in cell order, so whatever is last is what a + # short run loses. Losing the control invalidates every other cell; + # losing the largest pair costs the most expensive measurement but + # leaves the rest readable. + cells = cells_for(["go"], parse_payloads("1KiB,1GiB")) + assert cells[0].control + assert [c.id for c in cells[-2:]] == ["go-encrypt-1GiB", "go-decrypt-1GiB"] + + def test_the_control_size_does_not_follow_the_selection(self): + # The control's CI width is the run's noise floor and every cell is + # judged against it. If it moved with --bench-payloads, two runs of + # the same comparison could disagree on which cells are trustworthy. + for spec in ("1KiB", "1GiB", DEFAULT_PAYLOAD_SPEC): + control = next(c for c in cells_for(["go"], parse_payloads(spec))) + assert control.payload == CONTROL_PAYLOAD + #: The fixture body, called directly: these tests are about the bytes it #: writes, not about pytest's fixture wiring. -make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] +_make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] + + +def make_payloads( + tmp_path: Path, config: BenchConfig, payloads: Sequence[Payload] = PAYLOADS +) -> dict[str, Path]: + return _make_payloads(tmp_path, config, tuple(payloads)) class TestPayloads: @@ -124,6 +244,37 @@ def test_a_truncated_cache_entry_is_regenerated(self, tmp_path: Path): second = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) assert first == second + def test_the_controls_payload_is_generated_even_when_not_selected( + self, tmp_path: Path + ): + # --bench-payloads 1GiB is a legitimate ask, and the A/A control still + # needs its own file. Without it the control cell dies on a KeyError + # in arm construction -- and a run with no control can pass nothing. + out = make_payloads(tmp_path, BenchConfig(seed=1), [Payload("4KiB", 4096)]) + assert out[CONTROL_PAYLOAD.label].stat().st_size == CONTROL_PAYLOAD.n_bytes + + def test_chunking_does_not_change_the_bytes( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ): + # A 1 GiB payload is written in chunks rather than built in RAM. The + # chunk size must not be part of the seed contract: a run compared + # against an earlier one has to measure the same bytes, and the check + # is cheap next to the cost of discovering otherwise. + payload = Payload("40KiB", 40 * 1024) + whole = subdir(tmp_path, "whole") + bench.write_payload(whole / "p.bin", payload, seed=7) + monkeypatch.setattr(bench, "_CHUNK_BYTES", 4096) + chunked = subdir(tmp_path, "chunked") + bench.write_payload(chunked / "p.bin", payload, seed=7) + assert (whole / "p.bin").read_bytes() == (chunked / "p.bin").read_bytes() + + def test_a_payload_too_big_for_the_disk_is_refused_up_front(self, tmp_path: Path): + # Running out of disk mid-benchmark surfaces as a non-zero exit from + # the CLI under measurement, which reads as "this build is broken". + huge = Payload("1024GiB", 1024 * 2**30) + assert bench.disk_shortfall(tmp_path, [huge]) is not None + assert bench.disk_shortfall(tmp_path, [Payload("1KiB", 1024)]) is None + def read_all(paths: dict[str, Path]) -> dict[str, bytes]: return {label: p.read_bytes() for label, p in paths.items()}