From 130152c0eb2a69b5cad50d87ef92ed65e2c9cc9d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 13 Aug 2026 10:45:18 -0400 Subject: [PATCH 1/4] spec: scaffold for DSPX-4372 --- spec/DSPX-4372.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 spec/DSPX-4372.md diff --git a/spec/DSPX-4372.md b/spec/DSPX-4372.md new file mode 100644 index 000000000..b76bd3865 --- /dev/null +++ b/spec/DSPX-4372.md @@ -0,0 +1,34 @@ +--- +ticket: DSPX-4372 +title: +status: draft +authors: [dmihalcik@virtru.com] +branches: [opentdf/tests:DSPX-4372] +prs: [] +created: 2026-08-13 +updated: 2026-08-13 +--- + +# DSPX-4372 + +## Summary + + +## Problem / Motivation +_Why does this work need to happen? What is the user/business pain?_ + +## Proposed Solution +_What will you build, at a functional level? Sketch the approach._ + +## Inputs / Outputs / Contracts +_Function signatures, data shapes, API contracts, CLI flags._ + +## Edge Cases & Constraints +_Boundary conditions, error states, performance limits, security considerations._ + +## Out of Scope +_What this work item explicitly does not cover._ + +## Acceptance Criteria +- [ ] _Clear, testable condition_ +- [ ] _…_ From 27d7ea19fd2f0176cace357f96de7756fdb2e24d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 13 Aug 2026 11:14:37 -0400 Subject: [PATCH 2/4] feat(xtest): paired A/B SDK performance regression benchmarks Nothing in this repo measured the cost of an SDK operation, so a performance regression in any SDK shipped silently. The obvious design -- record timings, store them, compare to last week -- does not work on GitHub-hosted runners. CPU models vary, tenancy is shared, and steal time is unbounded, so run-to-run variation on identical code exceeds any regression worth catching. A historical gate would produce false alarms until people muted it. So no history is stored and no absolute number is ever compared. Each cell runs the newest installed release and the branch build on the *same runner*, paired within randomized interleaved rounds. Runner speed is a shared term that cancels in the per-round ratio. Verdicts come from the median log-ratio with a BCa bootstrap CI, a one-sided Wilcoxon signed-rank test, and Benjamini-Hochberg control across the run. A cell fails only when the CI lower bound clears 1.15x *and* the adjusted p clears 0.05: the interval clause cannot fire on noise, and the p clause cannot fire on a trivial effect that got lucky across ~14 comparisons. Two guards make the verdict honest rather than merely computed: - An A/A control per SDK compares the baseline against itself through the identical pipeline, so its true ratio is 1.0 and anything it reports is the harness's own error. If it trips, the run reports but does not fail. Its interval width is the empirical noise floor; if that is wider than the threshold the run had no power, and no cell may report PASS. "We could not tell" must never be reported as "no regression". - Rounds stop on attained CI precision, never on significance. Stopping when p drops below alpha is optional stopping and inflates the false positive rate well past nominal. This is easy to "optimize" away -- significance-stopping finishes sooner -- and doing so silently invalidates every number the job produces. Confounders are pinned rather than hoped away: same plaintext, same RSA attribute, one container and target mode for both arms, and for decrypt both arms read one baseline-produced ciphertext, since letting each arm read its own output would measure two different files. A cell skips with a stated reason when the arms disagree on a feature in the measured path. Wall clock and peak RSS gate the build; CPU time is reported but never fails. Measurement uses Popen + os.wait4 rather than getrusage(CHILDREN), whose ru_maxrss is a process-lifetime high-water mark and so has meaningless deltas. CI runs nightly and on manual dispatch, one runner per SDK, serial -- the xdist guard is a hard error because parallel workers contending for the CPU under measurement would invalidate everything. Never on PRs. The harness tests demonstrate the gate catching a planted 25% slowdown and ignoring a planted 3% one; a gate never shown to do both is not yet known to work. --- .github/workflows/check.yml | 13 + .github/workflows/pr-lint.yaml | 1 + .github/workflows/xtest.yml | 227 ++++++++++++++ .gitignore | 1 + spec/DSPX-4372.md | 240 ++++++++++++++- xtest/conftest.py | 199 +++++++++++- xtest/fixtures/bench.py | 407 ++++++++++++++++++++++++ xtest/perf/__init__.py | 12 + xtest/perf/_launcher.py | 146 +++++++++ xtest/perf/cells.py | 84 +++++ xtest/perf/measure.py | 254 +++++++++++++++ xtest/perf/report.py | 263 ++++++++++++++++ xtest/perf/runner.py | 431 ++++++++++++++++++++++++++ xtest/perf/stats.py | 544 +++++++++++++++++++++++++++++++++ xtest/pyproject.toml | 8 +- xtest/tdfs.py | 91 +++++- xtest/test_bench_arms.py | 135 ++++++++ xtest/test_bench_measure.py | 233 ++++++++++++++ xtest/test_bench_runner.py | 447 +++++++++++++++++++++++++++ xtest/test_bench_stats.py | 344 +++++++++++++++++++++ xtest/test_benchmarks.py | 91 ++++++ xtest/test_sdk_commands.py | 149 +++++++++ xtest/uv.lock | 86 ++++++ 23 files changed, 4390 insertions(+), 16 deletions(-) create mode 100644 xtest/fixtures/bench.py create mode 100644 xtest/perf/__init__.py create mode 100644 xtest/perf/_launcher.py create mode 100644 xtest/perf/cells.py create mode 100644 xtest/perf/measure.py create mode 100644 xtest/perf/report.py create mode 100644 xtest/perf/runner.py create mode 100644 xtest/perf/stats.py create mode 100644 xtest/test_bench_arms.py create mode 100644 xtest/test_bench_measure.py create mode 100644 xtest/test_bench_runner.py create mode 100644 xtest/test_bench_stats.py create mode 100644 xtest/test_benchmarks.py create mode 100644 xtest/test_sdk_commands.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5d52208aa..7e42f1435 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,6 +34,19 @@ jobs: uv run ruff format --check . uv run pyright working-directory: xtest + # The benchmark harness's own tests: statistics, measurement, and the + # CLI command builders. No platform and no SDK builds required, so the + # part of the gate that has to be *correct* is checked on every PR + # rather than only when the nightly benchmark runs. + # --frozen --no-build: resolve nothing and build nothing, so a + # dependency cannot slip in an unlocked version or a setup script on a + # runner that already has everything installed from the step above. + - name: Test xtest benchmark harness + run: >- + uv run --frozen --no-build pytest --no-header -q + test_bench_stats.py test_bench_measure.py test_bench_runner.py + test_bench_arms.py test_sdk_commands.py + working-directory: xtest - name: Lint and test otdf-local run: | uv sync diff --git a/.github/workflows/pr-lint.yaml b/.github/workflows/pr-lint.yaml index 6da092109..7ade36efc 100644 --- a/.github/workflows/pr-lint.yaml +++ b/.github/workflows/pr-lint.yaml @@ -29,6 +29,7 @@ jobs: java web xtest + perf ci dependabot env: diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 0f8e2f439..740a3648d 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -33,6 +33,11 @@ on: type: boolean default: false description: "Enable DPoP nonce challenge on KAS instances" + run-benchmarks: + required: false + type: boolean + default: false + description: "Run the SDK performance regression benchmarks (adds ~45m per SDK)" workflow_call: inputs: platform-ref: @@ -59,6 +64,10 @@ on: required: false type: boolean default: false + run-benchmarks: + required: false + type: boolean + default: false schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -748,6 +757,224 @@ jobs: ${{ steps.kas-km2.outputs.log-file }} if-no-files-found: ignore + # Paired A/B performance regression benchmark. + # + # Absolute timings from a GitHub-hosted runner are not comparable to + # timings from any other runner -- CPU model, tenancy, and steal time all + # vary more than any regression worth catching. So nothing is compared to + # history. Instead both builds under comparison run on *this* runner, in + # the same interleaved round, and only their ratio is reported. Runner + # speed divides out. + # + # Never runs on pull requests: 30 minutes of serial measurement is too slow + # for a PR gate, and a PR runner is the noisiest place to measure. + bench: + timeout-minutes: 45 + runs-on: ubuntu-latest + needs: resolve-versions + # Nightly cron only, not the Mon/Wed or weekly ones: three runs a week of + # the same comparison would tell us nothing the first one did not. + if: >- + github.event.schedule == '30 6 * * *' || + ((github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') + && inputs.run-benchmarks) + permissions: + contents: read + packages: read + strategy: + # One runner per SDK. Two SDKs on one runner would contend for the very + # CPU being measured. + fail-fast: false + matrix: + sdk: [go, java, js] + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: opentdf/tests + path: otdftests + persist-credentials: false + + - name: load extra keys from file + id: load-extra-keys + run: |- + echo "EXTRA_KEYS=$(jq -c > "${GITHUB_OUTPUT}" + + ######## SPIN UP PLATFORM BACKEND ############# + # Pinned to main, and to the default KAS only. We are measuring SDK + # regressions, so the server is held constant; and the six extra KAS + # instances the ABAC tests need would draw background CPU on the runner + # doing the measuring, which is noise rather than merely waste. + - name: Check out and start up platform with deps/containers + id: run-platform + uses: opentdf/platform/test/start-up-with-containers@18b8070f7ae1e3547234342f42d0d686dc77788f # keycloak-26.4 (opentdf/platform#3792) + with: + platform-ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)['main'] }} + bootstrap-ref: main + ec-tdf-enabled: true + extra-keys: ${{ steps.load-extra-keys.outputs.EXTRA_KEYS }} + log-type: json + pqc-enabled: true + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + - uses: bufbuild/buf-action@fd21066df7214747548607aaa45548ba2b9bc1ff # v1.4.0 + if: matrix.sdk == 'java' + with: + setup_only: true + token: ${{ secrets.BUF_TOKEN }} + version: "1.56.0" + + - name: Set up JDK + if: matrix.sdk == 'java' + uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 + with: + java-version: "11" + distribution: "adopt" + server-id: github + + - name: Set up Node 22 + if: matrix.sdk == 'js' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: "22.x" + + - name: Capture platform otdfctl location + if: matrix.sdk == 'go' + id: platform-otdfctl + run: |- + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD) || { + echo "::error::Failed to get SHA from platform checkout at $PLATFORM_DIR" + exit 1 + } + echo "sha=$sha" >> "$GITHUB_OUTPUT" + else + echo "dir=" >> "$GITHUB_OUTPUT" + echo "sha=" >> "$GITHUB_OUTPUT" + fi + env: + PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} + + ######## INSTALL BOTH ARMS OF THE COMPARISON ############# + # The whole design rests on this step laying down two builds side by + # side under sdk//dist/: the branch head (candidate) and the + # newest release (baseline). Arm selection picks them up from there. + - name: Configure ${{ matrix.sdk }} sdk + id: configure-sdk + uses: ./otdftests/xtest/setup-cli-tool + with: + path: otdftests/xtest/sdk + sdk: ${{ matrix.sdk }} + version-info: "${{ needs.resolve-versions.outputs[matrix.sdk] }}" + platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} + platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} + + - name: Cache Go modules + if: matrix.sdk == 'go' + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }} + restore-keys: | + go-${{ runner.os }}- + + - name: Cache npm + if: matrix.sdk == 'js' + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }} + restore-keys: | + npm-${{ runner.os }}- + + - name: Cache Maven repository + if: matrix.sdk == 'java' + uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }} + restore-keys: | + maven-${{ runner.os }}- + + - name: point java heads at the platform under test + if: matrix.sdk == 'java' && fromJson(steps.configure-sdk.outputs.heads)[0] != null + run: |- + for row in $(echo "$java_version_info" | jq -c '.[]'); do + TAG=$(echo "$row" | jq -r '.tag') + HEAD=$(echo "$row" | jq -r '.head') + if [[ "$HEAD" == "true" ]]; then + echo "PLATFORM_BRANCH=$platform_ref" > "otdftests/xtest/sdk/java/${TAG}.env" + fi + done + env: + java_version_info: ${{ needs.resolve-versions.outputs.java }} + platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)['main'] }} + + - name: Build the ${{ matrix.sdk }} cli + if: fromJson(steps.configure-sdk.outputs.heads)[0] != null + run: make + working-directory: otdftests/xtest/sdk/${{ matrix.sdk }} + env: + BUF_INPUT_HTTPS_USERNAME: opentdf-bot + BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + + ######## MEASURE ############# + # --locked --no-build: install exactly what uv.lock pins, and run no + # setup scripts doing it. A benchmark that measured a differently + # resolved dependency set would be measuring the wrong thing anyway. + - name: Install test dependencies + run: uv sync --locked --no-build + working-directory: otdftests/xtest + + # Deliberately serial: no -n / --dist. Parallel pytest workers contend + # for the CPU under measurement and would invalidate every number here. + # conftest.py refuses to run --bench under xdist for the same reason. + - name: Run performance benchmarks + id: bench + run: |- + uv run --frozen --no-build pytest -ra -v \ + --bench \ + --sdks "$BENCH_SDK" \ + --bench-budget-seconds 1500 \ + --bench-out test-results/benchmarks \ + --html "test-results/bench-${BENCH_SDK}.html" \ + --self-contained-html \ + test_benchmarks.py + working-directory: otdftests/xtest + env: + BENCH_SDK: ${{ matrix.sdk }} + PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" + SCHEMA_FILE: "manifest.schema.json" + PLATFORM_TAG: main + OTDFCTL_HEADS: ${{ steps.configure-sdk.outputs.heads }} + # The benchmark never touches the audit-log fixture; asserting on + # logs would also add file IO to the measured path. + DISABLE_AUDIT_ASSERTIONS: "1" + + # Raw per-round samples, not just the verdict. Re-analysing a + # surprising result offline beats re-running a 30-minute job to look at + # the same numbers again. + - name: Upload benchmark results + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: success() || failure() + with: + name: ${{ job.status == 'success' && '✅' || '❌' }} bench-${{ matrix.sdk }} + path: | + otdftests/xtest/test-results/benchmarks/*.json + otdftests/xtest/test-results/*.html + if-no-files-found: warn + + - name: Upload server logs on failure + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: failure() + with: + name: bench-server-logs-${{ matrix.sdk }} + path: ${{ steps.run-platform.outputs.platform-log-file }} + if-no-files-found: ignore + publish-results: runs-on: ubuntu-latest needs: xct diff --git a/.gitignore b/.gitignore index a1bb32382..8eab78bda 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ vulnerability/tilt_modules/ /xtest/node_modules/ /xtest/tilt_modules/ /xtest/tmp/ +/xtest/test-results/ /xtest/sdk/js/web/dist/ /xtest/.helm diff --git a/spec/DSPX-4372.md b/spec/DSPX-4372.md index b76bd3865..dcd1740b2 100644 --- a/spec/DSPX-4372.md +++ b/spec/DSPX-4372.md @@ -1,6 +1,6 @@ --- ticket: DSPX-4372 -title: +title: Statistically valid SDK performance regression benchmarks status: draft authors: [dmihalcik@virtru.com] branches: [opentdf/tests:DSPX-4372] @@ -9,26 +9,248 @@ created: 2026-08-13 updated: 2026-08-13 --- -# DSPX-4372 +# DSPX-4372 — Statistically valid SDK performance regression benchmarks ## Summary +A nightly, per-SDK CI job measures the branch build against the newest release +**on the same runner, in the same interleaved round**, and fails the job on a +confirmed wall-clock or peak-RSS regression. CPU time is measured and reported +but never gates. + +Nothing is compared to history. Absolute timings from a GitHub-hosted runner +are not comparable to timings from any other runner, so the only quantity the +job reports is a *ratio between two builds measured under identical +conditions*. ## Problem / Motivation -_Why does this work need to happen? What is the user/business pain?_ + +Nothing in this repo measures the cost of an SDK operation. `tdfs.SDK.encrypt` +and `decrypt` return `None`, there is no timing code, and CI produces no +machine-readable durations. A performance regression in any SDK ships +silently, and is found by a customer rather than by us. + +The ticket asks for "performance metric tests ... (memory usage, wall clock +time, cpu usage)" and already points at the design that makes them work: +*"comparing main to release of the SDKs ... separate jobs from the existing job +matrix so we can have encrypt from different versions running on the same +instance."* + +The naive alternative — record timings, store them, compare to last week — does +not work here. GitHub-hosted runners vary in CPU model, are shared tenancy, and +suffer unbounded steal time. Run-to-run variation on identical code exceeds any +regression worth catching, so a historical gate produces false alarms until +people mute it, at which point it is worse than nothing. ## Proposed Solution -_What will you build, at a functional level? Sketch the approach._ + +### Paired A/B on one runner + +A **cell** is one operation at one payload size for one SDK, e.g. +`go-encrypt-1MiB`. Each cell runs *rounds*; each round runs both arms once: + +- **baseline** — the newest installed release (`go@v0.36.0`) +- **candidate** — the branch build (`go@main`) + +Runner speed, thermal state, and noisy neighbours are shared within a round and +cancel in the per-round ratio. Order within the round is randomized from a +seeded RNG so neither arm systematically inherits the other's cache state. + +### Statistics + +Per round, on the log scale: `d_i = ln(candidate_i) - ln(baseline_i)`. + +| Quantity | Method | +|---|---| +| Point estimate | median of `d_i`, exponentiated | +| Interval | BCa percentile bootstrap, 95%, 10 000 resamples | +| Test | one-sided Wilcoxon signed-rank (`alternative="greater"`) | +| Multiplicity | Benjamini–Hochberg across the run's gated cells | + +**Decision rule: regression iff `ci_low > threshold` AND BH-adjusted +`p < 0.05`.** Default threshold 1.15 (+15%). The conjunction is deliberate and +neither clause is redundant: the interval clause cannot fire on pure noise — +that would require excluding an effect that is not there — and the p clause +cannot fire on a real-but-trivial effect surviving by luck across ~14 cells. +Symmetrically, `ci_high < 1/threshold` reports an improvement, which is +informational and never fails. + +### The A/A control + +Each SDK runs one extra cell comparing the **baseline against itself** through +the identical pipeline. Its true ratio is 1.0 by construction, so whatever it +reports is the harness's own error. Two things come out of it: + +- If it *trips* — reports an effect past the threshold — the runner is too noisy + or the harness is biased. The whole run is downgraded: verdicts are reported, + the build is not failed. +- Its interval width is the run's empirical **noise floor**. If that is not + tighter than the threshold, the run had no power to detect the effect being + gated on, and no cell may report `PASS` — only `INCONCLUSIVE`. "We could not + tell" must never be reported as "no regression". + +Controls run **first** in each SDK's cell list. A run that overruns its budget +loses whatever is at the end; losing one comparison leaves the rest +trustworthy, losing the control leaves nothing trustworthy at all. + +### Stopping rule + +Rounds continue until the bootstrap CI half-width falls below +`ln(threshold)/3`, bounded below by `--bench-min-rounds` (20), above by +`--bench-max-rounds` (60), and by a shared wall-clock budget. + +**Stopping is on precision, never on significance.** Peeking at the p-value and +stopping when it drops below alpha is optional stopping: it inflates the +false-positive rate well past nominal, because each round is a fresh chance to +cross the line and the loop only ever stops on the lucky side. Attained CI width +is driven by dispersion rather than location, so it is approximately ancillary +to the effect being tested. This is easy to "optimize away" — stopping on +significance finishes sooner — and doing so silently invalidates every number +the job produces. ## Inputs / Outputs / Contracts -_Function signatures, data shapes, API contracts, CLI flags._ + +### New modules + +| Path | Responsibility | +|---|---| +| `xtest/perf/measure.py` | one invocation → wall ns, CPU s, peak RSS bytes | +| `xtest/perf/_launcher.py` | forks the measured command from an empty process, so its RSS is its own | +| `xtest/perf/stats.py` | log-ratios, BCa CI, Wilcoxon, BH, decision rule | +| `xtest/perf/runner.py` | paired round loop, warm-up, stopping rule, budget | +| `xtest/perf/cells.py` | the experiment matrix | +| `xtest/perf/report.py` | JSON artifact + `$GITHUB_STEP_SUMMARY` markdown | +| `xtest/fixtures/bench.py` | arm selection, payloads, ciphertexts, comparability guards | +| `xtest/test_benchmarks.py` | the cells (needs a platform) | +| `xtest/test_bench_stats.py` | statistics, offline | +| `xtest/test_bench_measure.py` | measurement primitive, offline | +| `xtest/test_bench_runner.py` | round loop and gate, offline | +| `xtest/test_sdk_commands.py` | the `XT_WITH_*` CLI contract, offline | + +`xtest/tdfs.py` gains `SDK.encrypt_command` / `SDK.decrypt_command` — argv+env +builders extracted from the existing `encrypt`/`decrypt`, whose behaviour is +unchanged — and `SDK.semver()`. The benchmark drives the CLI through the same +builders the functional tests use, so the `XT_WITH_*` contract cannot drift +between them. + +### Measurement primitive + +```python +@dataclass(frozen=True, slots=True) +class Sample: + wall_ns: int # perf_counter_ns around the call + cpu_s: float # ru_utime + ru_stime + max_rss_bytes: int # ru_maxrss, unit-normalized + exit_code: int + rss_floor_bytes: int # RSS of the process that forked it +``` + +`os.wait4`, not `resource.getrusage(RUSAGE_CHILDREN)`: the latter's +`ru_maxrss` is a process-lifetime high-water mark, so deltas are meaningless. +rusage folds in reaped descendants, so the `java`/`node` process behind each +`cli.sh` shim is counted. `ru_maxrss` is KiB on Linux and bytes on macOS; +normalized on `sys.platform`. + +The command is **not** forked from the pytest process. On Linux a child +inherits the parent's resident-set accounting and `execve` does not clear it, +so `ru_maxrss` comes back as `max(the child's true peak, the parent's RSS at +fork time)`. Measured from a pytest process holding numpy, scipy and a +session's worth of samples, every SDK invocation reported *pytest's* footprint +— about 165 MiB on a CI runner — instead of its own. That does not look +broken; it looks like a stable ratio of 1.000, which reads as "no regression" +forever. `posix_spawn` and `sh -c 'exec …'` were measured and are equally +contaminated: an exec is too late, the accounting is already latched. So +`perf/_launcher.py` runs as a small `python -I -S` process holding nothing and +forks the real command itself, reporting its own RSS as the floor under the +reading. It also puts the command in its own process group, so a timeout kills +the whole tree rather than just the shim. + +### CLI options + +`--bench` (opt-in; without it `test_benchmarks.py` collects nothing), +`--bench-baseline`, `--bench-candidate`, `--bench-threshold` (1.15), +`--bench-min-rounds` (20), `--bench-max-rounds` (60), `--bench-warmup` (5), +`--bench-budget-seconds` (1500), `--bench-seed` (0), `--bench-out` +(`test-results/benchmarks`), `--bench-no-gate`. + +### Outputs + +- `test-results/benchmarks/.json` — **every raw per-round sample** + alongside the derived statistics, runner metadata, seed, and thresholds. + Re-analysing a surprising result offline is the difference between + understanding a red build and re-running a 30-minute job to look at the same + numbers again. +- `$GITHUB_STEP_SUMMARY` — one row per (cell, metric): baseline median, + candidate median, ratio with CI, adjusted p, verdict, plus the noise floor. +- Exit status: `pytest_sessionfinish` fails the session on a confirmed + regression. + +### CI + +New `bench` job in `.github/workflows/xtest.yml`: matrix over +`sdk: [go, java, js]`, one runner each, `timeout-minutes: 45`, platform pinned +to the `main` SHA, default KAS only, **serial** (no `-n`). Triggers on the +nightly cron and on `workflow_dispatch`/`workflow_call` with +`run-benchmarks: true`. Never on pull requests. ## Edge Cases & Constraints -_Boundary conditions, error states, performance limits, security considerations._ + +| Threat to validity | Handling | +|---|---| +| Runner CPU heterogeneity | Both arms on one runner; ratios, not absolutes | +| Noisy neighbours, steal time | Paired interleaved rounds; median + Wilcoxon; A/A gate | +| Thermal and slow drift | Randomized within-round order; pairing differences it out | +| Page cache, first `go build`, npx resolve | Warm-up rounds discarded | +| JVM/npx startup dominating small payloads | Reported separately per payload size; 1 KiB *is* the startup cell | +| Platform/KAS latency in the decrypt path | Shared by both arms in a round; cancels | +| Arms differing in function, not speed | Container, target mode, and attribute pinned; cell skipped if the arms disagree on `SDK.supports()` for anything in the measured path | +| Decrypt arms reading different ciphertexts | Both arms decrypt one baseline-produced file | +| ~14 simultaneous comparisons | BH correction plus an effect threshold | +| Optional-stopping bias | Stop on precision, never significance; min-round floor | +| xdist contention | `--bench` under xdist is a hard `UsageError`; CI runs serial | +| Peak RSS inheriting the measuring process's memory | The command is forked from an empty launcher, not from pytest | +| Peak RSS pinned at the measurement floor | Both arms clip to the same number, so the ratio is 1.000 with a tight interval — the most convincing PASS the harness can emit, carrying no information. A floored cell is forced `INCONCLUSIVE` and excluded from the BH correction, like the control | +| A failing operation | Any non-zero exit aborts the cell with captured stderr; a benchmark over the error path is worse than no benchmark | + +Payload sizes are 1 KiB / 1 MiB / 32 MiB because they separate two regimes that +fail independently: at 1 KiB nearly all cost is process startup, so a throughput +regression is invisible; at 32 MiB crypto and IO dominate, so a startup +regression is invisible. + +Budget: java is the worst case at roughly 1.0 s/op small and 2.5 s at 32 MiB, +giving ~19 s per round across both operations and both arms; 45 rounds plus the +control lands around 16 minutes, inside the 30-minute target and the 45-minute +job timeout. Go and JS are substantially cheaper. ## Out of Scope -_What this work item explicitly does not cover._ + +Historical trend storage; gh-pages dashboards or `github-action-benchmark`; +flamegraphs and profiling artifacts; cross-SDK comparison (go vs java is not a +regression signal); cross-platform-version performance comparison; perf gating +on pull requests; nano and other container types; in-process microbenchmarks or +`pytest-benchmark` semantics. ## Acceptance Criteria -- [ ] _Clear, testable condition_ -- [ ] _…_ + +- [x] Wall clock, CPU time, and peak RSS are measured per invocation, with + descendant processes folded in and RSS units normalized across platforms. +- [x] Both arms are measured on one runner, paired within randomized + interleaved rounds, with warm-up rounds discarded. +- [x] Decrypt cells compare two arms reading the *same* baseline-produced + ciphertext; encrypt cells pin container, target mode, and attribute. +- [x] A cell is skipped with a stated reason when a build is missing or the two + arms disagree on a feature in the measured path. +- [x] The verdict uses a robust CI, a one-sided signed-rank test, a minimum + effect threshold, and BH multiplicity control across the run. +- [x] An A/A control runs per SDK; if it trips, the run reports but does not + fail; if its interval is wider than the threshold, no cell reports PASS. +- [x] The round loop stops on attained precision, never on significance, and + refuses a verdict below the minimum usable round count. +- [x] Raw per-round samples are written to JSON and a summary table to + `$GITHUB_STEP_SUMMARY`. +- [x] A confirmed wall-clock or peak-RSS regression fails the job; CPU time + never does. +- [x] The offline harness tests demonstrate the gate catching a planted 25% + slowdown and *ignoring* a planted 3% one. +- [ ] A `workflow_dispatch` run with `run-benchmarks: true` produces step + summaries and artifacts for all three SDKs inside the 45-minute timeout. diff --git a/xtest/conftest.py b/xtest/conftest.py index 4f39176be..903c54724 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -24,6 +24,8 @@ import tdfs from otdfctl import OpentdfCommandLineTool +from perf import report, stats +from perf.cells import cells_for logging.basicConfig(level=os.environ.get("LOGLEVEL", "DEBUG")) @@ -35,8 +37,16 @@ def pytest_report_header() -> list[str]: and pytest does not show captured output for skipped tests. Echoing the detected version and feature set into the report header makes it visible in CI even when every gated test skips. + + Detection probes the platform over HTTP, which fails when there is no + platform -- running only the offline unit tests, for instance. That is a + header, not a test result, so report the failure and carry on rather than + breaking collection for tests that never needed a platform. """ - pfs = tdfs.get_platform_features() + try: + pfs = tdfs.get_platform_features() + except Exception as e: # a header must never break collection + return [f"platform features unavailable: {e}"] return [ f"platform version: {pfs.version} (semver={pfs.semver})", f"detected features: {', '.join(sorted(pfs.features))}", @@ -52,6 +62,7 @@ def pytest_report_header() -> list[str]: "fixtures.keys", "fixtures.audit", "fixtures.encryption", + "fixtures.bench", ] @@ -144,6 +155,83 @@ def pytest_addoption(parser: pytest.Parser): help="select which sdks to run for encrypt only; accepts same format as --sdks", type=sdk_spec_type, ) + _add_benchmark_options(parser) + + +def _add_benchmark_options(parser: pytest.Parser): + """Options for the SDK performance regression benchmarks. + + Grouped separately because they configure an experiment rather than + selecting tests, and because none of them do anything without --bench. + """ + group = parser.getgroup("benchmarks", "SDK performance regression benchmarks") + group.addoption( + "--bench", + action="store_true", + help="run the performance regression benchmarks (they are long, so they " + "are opt-in and collect nothing otherwise)", + ) + group.addoption( + "--bench-baseline", + help="build to compare against, e.g. go@v0.29.0; defaults to the newest " + "installed release of each sdk", + ) + group.addoption( + "--bench-candidate", + help="build under test, e.g. go@main; defaults to the installed " + "unreleased build of each sdk", + ) + group.addoption( + "--bench-threshold", + type=float, + default=stats.DEFAULT_THRESHOLD, + help="smallest slowdown ratio worth failing on (default: %(default)s, " + "i.e. 15%% slower)", + ) + group.addoption( + "--bench-min-rounds", + type=int, + default=20, + help="paired rounds to run before the stopping rule may fire " + "(default: %(default)s)", + ) + group.addoption( + "--bench-max-rounds", + type=int, + default=60, + help="hard cap on paired rounds per cell (default: %(default)s)", + ) + group.addoption( + "--bench-warmup", + type=int, + default=5, + help="paired rounds discarded before measuring, to pay one-time costs " + "like page cache and package resolution (default: %(default)s)", + ) + group.addoption( + "--bench-budget-seconds", + type=float, + default=1500.0, + help="wall-clock allowance shared by every cell (default: %(default)s)", + ) + group.addoption( + "--bench-seed", + type=int, + default=0, + help="seed for payload generation, round ordering, and the bootstrap; " + "fixing it makes a run reproducible (default: %(default)s)", + ) + group.addoption( + "--bench-out", + type=Path, + default=Path("test-results/benchmarks"), + help="directory for the JSON result artifact (default: %(default)s)", + ) + group.addoption( + "--bench-no-gate", + action="store_true", + help="measure and report, but never fail the run on a regression", + ) def pytest_generate_tests(metafunc: pytest.Metafunc): @@ -231,6 +319,115 @@ def sdk_specs_opt(names: list[str]) -> list[str]: containers = list(typing.get_args(tdfs.container_type)) metafunc.parametrize("container", containers) + if "bench_cell" in metafunc.fixturenames: + _parametrize_bench_cells(metafunc) + + +def _parametrize_bench_cells(metafunc: pytest.Metafunc): + """Fan the benchmark module out over its cells. + + Without --bench there is nothing to fan out over, and the items are + dropped wholesale in :func:`pytest_collection_modifyitems` rather than + parametrized here. Parametrizing over an empty list would *not* collect + zero items: pytest's default ``empty_parameter_set_mark`` turns an empty + set into one skipped item per test, so every ordinary run would carry + benchmark skips it never asked for. + """ + if not metafunc.config.getoption("--bench"): + return + + # --sdks may be version-qualified (go@main); benchmark arms come from + # --bench-baseline/--bench-candidate instead, so only the name matters. + specs = metafunc.config.getoption("--sdks") or " ".join( + typing.get_args(tdfs.sdk_type) + ) + names = list(dict.fromkeys(s.split("@", 1)[0] for s in str(specs).split())) + cells = cells_for(names) + metafunc.config.stash[report.CELLS_KEY] = cells + metafunc.parametrize("bench_cell", cells, ids=[c.id for c in cells]) + + +def pytest_configure(config: pytest.Config): + if not config.getoption("--bench", default=False): + return + # Parallel workers contend for the CPU the benchmark is measuring, which + # turns every number into noise. The CI step also omits -n; this guard is + # what stops a later edit from silently invalidating the whole job. + distributed = getattr(config, "workerinput", None) is not None or bool( + config.getoption("numprocesses", default=None) + ) + if distributed: + raise pytest.UsageError( + "--bench cannot run under pytest-xdist: parallel workers compete " + "for the CPU being measured. Drop -n / --dist." + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + """Drop the benchmark cells entirely unless --bench asked for them. + + Deselected rather than skipped: a 20-minute cell has no business in the + regular integration matrix, and a skip would report it as a test that + exists and was declined rather than one that was never in scope. + """ + if config.getoption("--bench", default=False): + return + keep, drop = [], [] + for item in items: + (drop if item.get_closest_marker("benchmark") else keep).append(item) + if drop: + config.hook.pytest_deselected(items=drop) + items[:] = keep + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int): + """Analyse every recorded cell, write the artifacts, and gate the run. + + The gate lives here rather than in the cells because it is a run-level + decision: the multiplicity correction spans all cells, and the A/A control + can invalidate the lot. Artifacts are written first and unconditionally -- + a run that is about to fail is exactly the run whose raw numbers someone + will want to read. + """ + del exitstatus # the benchmark's own verdict is independent of test outcomes + config = session.config + if not config.getoption("--bench", default=False): + return + recorder = config.stash.get(report.RECORDER_KEY, None) + if recorder is None or not (recorder.results or recorder.skipped): + return + + # Imported here, not at module scope: importing a pytest plugin from a + # conftest before pytest registers it costs the plugin its assertion + # rewriting, which the fixture module's own asserts rely on. + from fixtures import bench + + bench_config = bench.config_from_options(config) + recorder.metadata = bench.runner_metadata(config) + gate = recorder.gate(bench_config) + + cells = config.stash.get(report.CELLS_KEY, []) + name = "-".join(dict.fromkeys(c.sdk for c in cells)) or "benchmarks" + out_dir = cast(Path, config.getoption("--bench-out")) + json_path = report.write_json( + out_dir / f"{name}.json", recorder, bench_config, gate + ) + + summary = report.markdown(recorder, bench_config, gate) + report.append_step_summary(summary) + reporter = config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_sep("=", "benchmark results") + reporter.write_line(gate.summary) + reporter.write_line(f"raw samples and statistics: {json_path}") + + if config.getoption("--bench-no-gate", default=False): + return + if gate.should_fail: + session.exitstatus = pytest.ExitCode.TESTS_FAILED + def pytest_runtest_setup(item: pytest.Item): if not item.config.getoption("--skip-released-pairs", default=False): diff --git a/xtest/fixtures/bench.py b/xtest/fixtures/bench.py new file mode 100644 index 000000000..d5194b380 --- /dev/null +++ b/xtest/fixtures/bench.py @@ -0,0 +1,407 @@ +"""Fixtures for the SDK performance regression benchmarks. + +The experiment matrix, the payload files, the arm selection, and the shared +time budget all live here. The measurement loop itself is in ``perf/runner.py`` +and the statistics in ``perf/stats.py``; this module is the glue that turns +pytest's world (options, fixtures, SDK discovery) into the runner's world +(two arms and a config). +""" + +from __future__ import annotations + +import os +import platform +import random +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +import pytest + +import abac +import tdfs +from perf import report +from perf.cells import PAYLOADS, BenchCell +from perf.runner import Arm, BenchConfig, Budget, Invocation + + +class ArmSelectionError(Exception): + """The two builds a comparison needs are not both installed.""" + + +def select_arms( + sdk: str, + *, + baseline_spec: str | None = None, + candidate_spec: str | None = None, +) -> tuple[tdfs.SDK, tdfs.SDK]: + """Pick (baseline, candidate) builds for one SDK. + + By default the candidate is the branch build (``main``) and the baseline + is the newest installed release, which is exactly what the CI setup action + lays down side by side. Explicit specs override either side, for + reproducing a comparison or for pinning a specific release. + + Raises: + ArmSelectionError: if either side is missing or the two resolve to the + same build (a comparison of a build against itself is only + meaningful as the explicit A/A control). + """ + installed = tdfs.all_versions_of(sdk) # pyright: ignore[reportArgumentType] + if not installed: + raise ArmSelectionError(f"no {sdk} builds installed under sdk/{sdk}/dist/") + + def resolve(spec: str, role: str) -> tdfs.SDK: + try: + matches = tdfs.parse_sdk_spec(spec) + except (FileNotFoundError, ValueError) as e: + raise ArmSelectionError(f"{role} {spec!r}: {e}") from e + if len(matches) != 1: + raise ArmSelectionError( + f"{role} {spec!r} resolved to {len(matches)} builds; " + "name one exactly, e.g. go@v0.29.0" + ) + return matches[0] + + if candidate_spec: + candidate = resolve(candidate_spec, "candidate") + else: + heads = [s for s in installed if not s.is_released()] + if not heads: + raise ArmSelectionError( + f"no unreleased {sdk} build to test; installed: " + f"{', '.join(sorted(s.version for s in installed))}" + ) + # Prefer 'main' when several branch builds are present. + candidate = next((s for s in heads if s.version == "main"), heads[0]) + + if baseline_spec: + baseline = resolve(baseline_spec, "baseline") + else: + # Final releases only. A release candidate parses to the same semver + # as its final release, so including them leaves `max` breaking a tie + # on whatever order the directory listing happened to produce -- and a + # baseline that is silently an rc is a baseline nobody chose. + releases = [s for s in installed if s.is_final_release()] + if not releases: + raise ArmSelectionError( + f"no final {sdk} release to compare against (prereleases do " + f"not count); installed: " + f"{', '.join(sorted(s.version for s in installed))}" + ) + baseline = max(releases, key=lambda s: s.semver() or (0, 0, 0)) + + if baseline == candidate: + raise ArmSelectionError( + f"baseline and candidate are both {baseline}; nothing to compare" + ) + return baseline, candidate + + +# --- Session-scoped configuration ------------------------------------------- + + +def config_from_options(config: pytest.Config) -> BenchConfig: + """Build a :class:`BenchConfig` from the ``--bench-*`` options. + + Every option has a default, so ``getoption`` never returns None here; the + casts are for the type checker, which cannot see the parser setup. + """ + + def as_int(name: str) -> int: + return int(cast(int, config.getoption(name))) + + def as_float(name: str) -> float: + return float(cast(float, config.getoption(name))) + + try: + return BenchConfig( + min_rounds=as_int("--bench-min-rounds"), + max_rounds=as_int("--bench-max-rounds"), + warmup=as_int("--bench-warmup"), + budget_seconds=as_float("--bench-budget-seconds"), + seed=as_int("--bench-seed"), + threshold=as_float("--bench-threshold"), + ) + except ValueError as e: + raise pytest.UsageError(f"invalid benchmark options: {e}") from e + + +@pytest.fixture(scope="session") +def bench_config(request: pytest.FixtureRequest) -> BenchConfig: + """Round-loop and analysis settings, from the --bench-* options.""" + return config_from_options(request.config) + + +@pytest.fixture(scope="session") +def bench_payloads(tmp_dir: Path, bench_config: BenchConfig) -> dict[str, Path]: + """Generate one plaintext file per payload size, shared by both arms. + + Content is pseudo-random but seeded, so a rerun measures byte-identical + input. Random rather than repetitive because compressible input would let + an SDK that happens to compress look faster for reasons unrelated to the + crypto path. + + One RNG per payload rather than one stream shared across them: ``tmp_dir`` + persists between runs, so a partially cached set skips some ``randbytes`` + calls and shifts the stream for every payload after it. Deriving each + payload's bytes from the seed *and* its label keeps the promise above true + whether the cache is empty, full, or half there. + """ + out: dict[str, Path] = {} + for payload in PAYLOADS: + path = tmp_dir / f"bench-plain-{payload.label}.bin" + if not path.is_file() or path.stat().st_size != payload.n_bytes: + rng = random.Random(f"{bench_config.seed}:{payload.label}") + path.write_bytes(rng.randbytes(payload.n_bytes)) + out[payload.label] = path + return out + + +@pytest.fixture(scope="session") +def bench_budget(request: pytest.FixtureRequest, bench_config: BenchConfig) -> Budget: + """One wall-clock allowance shared by every cell in the session. + + Divided evenly as cells start, so a cell that stops early on precision + donates its unused time to the ones after it instead of leaving the last + cell starved by whatever the first ones happened to spend. + """ + n_cells = max(1, len(_selected_cells(request.config))) + return Budget(bench_config.budget_seconds, n_cells) + + +@pytest.fixture(scope="session") +def bench_recorder(request: pytest.FixtureRequest) -> report.BenchmarkRecorder: + """The session-wide collector that the end-of-run gate reads.""" + return report.recorder_for(request.config) + + +def _selected_cells(config: pytest.Config) -> list[BenchCell]: + """Cells this session will run, cached on the config by the parametrizer.""" + return config.stash.get(report.CELLS_KEY, []) + + +# --- Module-scoped experiment inputs ---------------------------------------- + + +@dataclass(frozen=True, slots=True) +class BenchArms: + baseline: tdfs.SDK + candidate: tdfs.SDK + + +class ArmResolver: + """Resolves and memoizes the two builds to compare, per SDK. + + Resolution is lazy so that a missing build skips one SDK's cells with a + readable reason instead of erroring out every cell in the module. + """ + + def __init__(self, baseline_spec: str | None, candidate_spec: str | None) -> None: + self._baseline_spec = baseline_spec + self._candidate_spec = candidate_spec + self._cache: dict[str, BenchArms] = {} + + def __call__(self, sdk: str) -> BenchArms: + cached = self._cache.get(sdk) + if cached is None: + baseline, candidate = select_arms( + sdk, + baseline_spec=_spec_for(self._baseline_spec, sdk), + candidate_spec=_spec_for(self._candidate_spec, sdk), + ) + cached = self._cache[sdk] = BenchArms(baseline, candidate) + return cached + + +@pytest.fixture(scope="module") +def bench_arms(request: pytest.FixtureRequest) -> ArmResolver: + """Resolver for the (baseline, candidate) pair of any SDK in the run.""" + return ArmResolver( + cast(str | None, request.config.getoption("--bench-baseline")), + cast(str | None, request.config.getoption("--bench-candidate")), + ) + + +def _spec_for(spec: str | None, sdk: str) -> str | None: + """Return ``spec`` only if it names this SDK, so one flag can cover a run.""" + if not spec: + return None + return spec if spec.split("@", 1)[0] == sdk else None + + +#: Features whose presence changes what an encrypt or decrypt actually *does*. +#: If the two arms disagree on one of these they are not performing the same +#: operation, and a timing difference between them is a difference in work, +#: not in speed. +_COMPARABILITY_FEATURES: tuple[tdfs.feature_type, ...] = ( + "hexless", + "hexaflexible", + "autoconfigure", +) + + +def comparability_problem(arms: BenchArms) -> str | None: + """Return why these two builds cannot be fairly compared, or None.""" + for feature in _COMPARABILITY_FEATURES: + if arms.baseline.supports(feature) != arms.candidate.supports(feature): + supporter, other = ( + (arms.baseline, arms.candidate) + if arms.baseline.supports(feature) + else (arms.candidate, arms.baseline) + ) + return ( + f"{supporter} supports [{feature}] and {other} does not, so the " + "two arms would not be doing the same work" + ) + return None + + +def pinned_target_mode(arms: BenchArms) -> tdfs.container_version | None: + """Pick one container version both arms emit, or None for their default. + + Letting each arm choose its own target would compare two output formats. + ``None`` is only returned when neither arm can be told which to use, in + which case :func:`comparability_problem` has already established that they + agree on the relevant features and will pick the same one. + """ + if not ( + arms.baseline.supports("hexaflexible") + and arms.candidate.supports("hexaflexible") + ): + return None + if arms.baseline.supports("hexless") and arms.candidate.supports("hexless"): + return "4.3.0" + return "4.2.2" + + +class CiphertextFactory: + """Baseline-produced ciphertexts for the decrypt cells, made on demand. + + Both arms of a decrypt comparison must read the *same* file. If each arm + decrypted its own output, a difference in how the two builds *write* a TDF + would show up as a difference in how fast they read one. + """ + + def __init__( + self, + payloads: dict[str, Path], + tmp_dir: Path, + attr_values: list[str], + ) -> None: + self._payloads = payloads + self._tmp_dir = tmp_dir + self._attr_values = attr_values + self._cache: dict[tuple[str, str], Path] = {} + + def __call__(self, arms: BenchArms, payload_label: str) -> Path: + key = (str(arms.baseline), payload_label) + cached = self._cache.get(key) + if cached is not None: + return cached + ct_file = self._tmp_dir / f"bench-ct-{arms.baseline}-{payload_label}.tdf" + arms.baseline.encrypt( + self._payloads[payload_label], + ct_file, + container="ztdf", + attr_values=self._attr_values, + target_mode=pinned_target_mode(arms), + ) + assert ct_file.is_file() + self._cache[key] = ct_file + return ct_file + + +@pytest.fixture(scope="module") +def bench_ciphertexts( + bench_payloads: dict[str, Path], + tmp_dir: Path, + attribute_default_rsa: abac.Attribute, +) -> CiphertextFactory: + """Ciphertext source for decrypt cells. + + Pinned to the explicit RSA attribute so both arms wrap with RSA regardless + of what base key the platform happens to have configured -- an arm that + silently switched to EC would look slower for reasons that have nothing to + do with a regression. + """ + return CiphertextFactory(bench_payloads, tmp_dir, attribute_default_rsa.value_fqns) + + +def build_arms( + cell: BenchCell, + arms: BenchArms, + *, + pt_file: Path, + ct_file: Path | None, + tmp_dir: Path, + attr_values: list[str], +) -> tuple[Arm, Arm]: + """Turn a cell plus its two builds into two ready-to-run invocations. + + Everything that is not the build under test is pinned identically across + the arms: same plaintext, same attribute (so both wrap with RSA), same + container, same target mode. A functional difference between the builds + that changed any of these would otherwise show up as a speed difference. + + For decrypt, both arms read the *same* ``ct_file``, produced once by the + baseline. Letting each arm decrypt its own output would compare the cost + of reading two different files. + + In a control cell both arms are the baseline build, so the pair differs + only in the output path -- exactly the harness overhead the A/A cell + exists to measure. + """ + baseline_sdk = arms.baseline + candidate_sdk = arms.baseline if cell.control else arms.candidate + target_mode = pinned_target_mode(arms) + + def invocation(sdk: tdfs.SDK, role: str) -> Invocation: + out = tmp_dir / f"bench-{cell.id}-{role}" + if cell.operation == "encrypt": + out = out.with_suffix(".tdf") + argv, env = sdk.encrypt_command( + pt_file, + out, + container="ztdf", + attr_values=attr_values, + target_mode=target_mode, + ) + else: + if ct_file is None: + raise ValueError(f"{cell.id} is a decrypt cell but has no ciphertext") + out = out.with_suffix(".untdf") + argv, env = sdk.decrypt_command(ct_file, out, container="ztdf") + return Invocation(argv, env, out) + + return ( + Arm("baseline", str(baseline_sdk), invocation(baseline_sdk, "baseline")), + Arm("candidate", str(candidate_sdk), invocation(candidate_sdk, "candidate")), + ) + + +def runner_metadata(config: pytest.Config) -> dict[str, object]: + """Machine facts worth keeping alongside the numbers. + + Absolute timings are not comparable across runners, which is why nothing + here feeds the decision rule. It is recorded so that a human reading an + old artifact can tell what they are looking at. + """ + return { + "python": platform.python_version(), + "platform": platform.platform(), + "processor": platform.processor() or "unknown", + "cpu_count": os.cpu_count(), + "runner_os": os.environ.get("RUNNER_OS", ""), + "runner_arch": os.environ.get("RUNNER_ARCH", ""), + "github_run_id": os.environ.get("GITHUB_RUN_ID", ""), + "platform_version": _platform_version(), + "seed": config.getoption("--bench-seed"), + } + + +def _platform_version() -> str: + try: + return tdfs.get_platform_features().version or "unknown" + except Exception: # pragma: no cover - reporting must not break the run + return "unknown" diff --git a/xtest/perf/__init__.py b/xtest/perf/__init__.py new file mode 100644 index 000000000..5ff6e1479 --- /dev/null +++ b/xtest/perf/__init__.py @@ -0,0 +1,12 @@ +"""Performance regression benchmarking for the OpenTDF SDK CLIs. + +The suite compares two SDK builds -- typically the latest release against +``main`` -- by running them against each other on the same machine at the same +time. See ``perf/stats.py`` for why the comparison is structured that way. + +Modules: +- ``measure``: wall-clock / CPU / peak-RSS for a single CLI invocation. +- ``stats``: the paired statistical comparison and its decision rule. +- ``runner``: the round loop that produces paired samples. +- ``report``: JSON artifacts and GitHub step-summary markdown. +""" diff --git a/xtest/perf/_launcher.py b/xtest/perf/_launcher.py new file mode 100644 index 000000000..aab0326ce --- /dev/null +++ b/xtest/perf/_launcher.py @@ -0,0 +1,146 @@ +"""Run one command and report its resource usage, isolated from the caller. + +Why this extra process exists +----------------------------- +On Linux a forked child inherits the parent's resident-set accounting, and +``execve`` does not clear it. ``ru_maxrss`` from ``wait4`` therefore comes back +as ``max(the child's true peak, the parent's RSS at fork time)``. + +Measured straight from a pytest process holding numpy, scipy and a session's +worth of samples, every SDK invocation reports *pytest's* footprint -- about +165 MiB on a CI runner -- instead of its own. Every cell cheaper than that +reports the same number, so a peak-RSS comparison between two builds becomes a +comparison between two readings of the harness. It does not look broken: it +looks like a stable ratio of 1.000, which reads as "no regression". + +``posix_spawn`` and ``sh -c 'exec ...'`` do not help. Both were measured on +Linux and both inherit the same floor; an exec is too late, the accounting is +already latched. The only fix is to fork the measured command from a process +that is holding nothing, which is what this one is for. + +Its own RSS is the floor under every reading it produces, so it reports that +alongside them: a measurement sitting at the floor is censored, not small. + +Private protocol -- :mod:`perf.measure` is the only caller:: + + -I -S _launcher.py [args...] + +One line of space-separated integers is written to ````:: + + + + +``maxrss_raw`` is passed through in whatever unit the platform uses; the caller +normalizes it. ``floor_bytes`` is already bytes. +""" + +import os +import signal +import sys +import time + + +class _Timeout(Exception): + """Raised in the alarm handler to interrupt a blocking wait.""" + + +def _on_alarm(_signum: int, _frame: object) -> None: + raise _Timeout + + +def _current_rss_bytes() -> int: + """This process's resident size right now -- the floor for its children.""" + try: + with open("/proc/self/statm", "rb") as f: + pages = int(f.read().split()[1]) + except OSError, IndexError, ValueError: + # macOS has no /proc. Its ru_maxrss is already bytes, and it does not + # show the inheritance above, so a high-water reading is close enough. + import resource + + return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + return pages * os.sysconf("SC_PAGE_SIZE") + + +def _spawn(command: list[str], err_w: int) -> int: + """Fork and exec ``command``, reporting a failed exec down ``err_w``.""" + pid = os.fork() + if pid != 0: + return pid + try: + # Its own process group, so a timeout kills the whole tree instead of + # just the shim -- leaving a wedged JVM behind would hold the runner + # until the job timeout. + os.setpgid(0, 0) + os.execvp(command[0], command) + # BaseException, not Exception: this is a forked child, and letting a + # SystemExit or a KeyboardInterrupt unwind past here would run the + # *parent's* cleanup -- atexit handlers, buffered output -- a second time, + # from a process that only exists to exec. NOSONAR + except BaseException as e: # noqa: BLE001 - nothing may escape into a fork + try: + os.write(err_w, str(getattr(e, "errno", 0) or 0).encode()) + except OSError: + pass + os._exit(127) + + +def _kill_tree(pid: int) -> None: + try: + os.killpg(pid, signal.SIGKILL) + except OSError: + # setpgid may not have run yet; the bare process is all there is. + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + + +def main(argv: list[str]) -> int: + result_path, timeout_arg = argv[1], argv[2] + command = argv[3:] + timeout = None if timeout_arg == "-" else float(timeout_arg) + + # Closed by a successful exec; carries an errno if the exec never happened. + err_r, err_w = os.pipe() + + floor = _current_rss_bytes() + started = time.perf_counter_ns() + pid = _spawn(command, err_w) + os.close(err_w) + + timed_out = False + if timeout is not None: + signal.signal(signal.SIGALRM, _on_alarm) + signal.setitimer(signal.ITIMER_REAL, timeout) + try: + _, status, ru = os.wait4(pid, 0) + except _Timeout: + timed_out = True + _kill_tree(pid) + _, status, ru = os.wait4(pid, 0) + finally: + if timeout is not None: + signal.setitimer(signal.ITIMER_REAL, 0) + elapsed = time.perf_counter_ns() - started + + exec_errno = os.read(err_r, 32) or b"0" + os.close(err_r) + + fields = ( + status, + elapsed, + int(ru.ru_utime * 1e6), + int(ru.ru_stime * 1e6), + int(ru.ru_maxrss), + floor, + int(timed_out), + int(exec_errno), + ) + with open(result_path, "w") as f: + f.write(" ".join(str(v) for v in fields)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/xtest/perf/cells.py b/xtest/perf/cells.py new file mode 100644 index 000000000..4a1d4ba47 --- /dev/null +++ b/xtest/perf/cells.py @@ -0,0 +1,84 @@ +"""The benchmark's experiment matrix. + +Kept free of pytest and of ``tdfs`` so that both the conftest parametrizer and +the reporting layer can name a cell without importing each other. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal + +operation_type = Literal["encrypt", "decrypt"] + + +@dataclass(frozen=True, slots=True) +class Payload: + """One payload size regime. + + The sizes separate two failure modes that hide each other. At 1 KiB + essentially all the cost is process startup -- JVM boot, npx resolution, + TLS handshake, token fetch -- so a throughput regression is invisible. At + 32 MiB the crypto and IO dominate and a startup regression is invisible. + A benchmark at one size only will miss half the regressions it claims to + cover. + """ + + label: str + n_bytes: int + + +PAYLOADS: tuple[Payload, ...] = ( + Payload("1KiB", 1024), + Payload("1MiB", 2**20), + Payload("32MiB", 32 * 2**20), +) + +#: Payload used for the A/A control. Mid-size: large enough that startup noise +#: does not dominate it, small enough that the control is not a big slice of +#: the budget. +CONTROL_PAYLOAD = PAYLOADS[1] + + +@dataclass(frozen=True, slots=True) +class BenchCell: + """One comparison: an operation at a payload size, for one SDK.""" + + sdk: str + operation: operation_type + payload: Payload + #: A/A control -- the same build in both arms, through the same pipeline. + #: Its true ratio is 1.0 by construction, so whatever it reports is the + #: harness's own error. + control: bool = False + + @property + def id(self) -> str: + suffix = "-control" if self.control else "" + return f"{self.sdk}-{self.operation}-{self.payload.label}{suffix}" + + def __str__(self) -> str: + return self.id + + +def cells_for(sdks: list[str]) -> list[BenchCell]: + """Build the full cell list for a run, each SDK's control cell first. + + One control per SDK rather than one per run: a control measures a + particular SDK's harness path, and go's noise floor says nothing about + java's. + + Controls run first because a run that overruns its time budget loses + whatever is at the end. Losing one comparison leaves the rest + trustworthy; losing the control leaves nothing trustworthy at all, since + without a noise floor no cell may report PASS. + """ + cells: list[BenchCell] = [] + for sdk in sdks: + cells.append(BenchCell(sdk, "encrypt", CONTROL_PAYLOAD, control=True)) + cells += [ + BenchCell(sdk, op, payload) + for op in ("encrypt", "decrypt") + for payload in PAYLOADS + ] + return cells diff --git a/xtest/perf/measure.py b/xtest/perf/measure.py new file mode 100644 index 000000000..0385a9895 --- /dev/null +++ b/xtest/perf/measure.py @@ -0,0 +1,254 @@ +"""Resource measurement for a single SDK CLI invocation. + +Measures wall-clock time, CPU time, and peak resident memory for one child +process and everything it spawns. The SDK CLIs are bash shims that exec a Go +binary, a JVM, or node, so "everything it spawns" is the interesting part. + +Why ``os.wait4`` rather than ``resource.getrusage`` +--------------------------------------------------- +``resource.getrusage(RUSAGE_CHILDREN)`` reports a *process-lifetime* high-water +mark for ``ru_maxrss``. Subtracting successive readings does not give the peak +of the most recent child -- once one big child has run, every later delta reads +zero. ``os.wait4`` returns rusage for the specific child being reaped, which is +what we actually want. + +The kernel folds a child's reaped descendants into its rusage, so the shim's +``java``/``node``/``otdfctl`` grandchild is included: CPU times sum and +``ru_maxrss`` takes the maximum. Both are the right aggregation here. + +Why the command is not spawned from this process +------------------------------------------------ +A forked child inherits the parent's resident-set accounting on Linux, and +exec does not clear it, so ``ru_maxrss`` would report the *measuring* process's +memory whenever the measured command uses less. Everything therefore goes +through :mod:`perf._launcher`, which holds nothing and forks the real command +itself. That module's docstring has the measurements behind this. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import threading +from dataclasses import dataclass, fields +from pathlib import Path +from typing import IO + +#: ``ru_maxrss`` is kilobytes on Linux and bytes on macOS. There is no portable +#: way to ask, so branch on the platform. +_RSS_SCALE = 1 if sys.platform == "darwin" else 1024 + +#: How much of a failing child's stderr to keep for the error message. +_STDERR_TAIL_BYTES = 4000 + +_LAUNCHER = Path(__file__).with_name("_launcher.py") + +#: Grace period on top of the launcher's own timeout, before this process +#: gives up on it. Only reachable if the launcher itself wedges. +_LAUNCHER_GRACE_S = 30.0 + + +class MeasurementError(RuntimeError): + """A measured invocation failed, timed out, or could not be measured. + + Benchmarking an operation that does not succeed is worse than not + benchmarking it: a build that errors out early looks fast. + """ + + +@dataclass(frozen=True, slots=True) +class Sample: + """Resource usage of one CLI invocation.""" + + #: Wall-clock duration including fork/exec, which is part of the real cost. + wall_ns: int + #: User + system CPU seconds, summed over the process tree. + cpu_s: float + #: Peak resident set size in bytes, maximum over the process tree. + max_rss_bytes: int + exit_code: int + #: RSS of the launcher at fork time. A child cannot be measured below the + #: memory of whatever forked it, so a reading at this value is censored + #: rather than small. Reported so that fact is visible. + rss_floor_bytes: int = 0 + + @property + def rss_is_floored(self) -> bool: + """Whether peak RSS is indistinguishable from the measurement floor.""" + return self.rss_floor_bytes > 0 and self.max_rss_bytes <= self.rss_floor_bytes + + @property + def wall_s(self) -> float: + return self.wall_ns / 1e9 + + def metric(self, name: str) -> float: + """Return a metric by name, for generic iteration over metric sets.""" + match name: + case "wall": + return float(self.wall_ns) + case "cpu": + return self.cpu_s + case "rss": + return float(self.max_rss_bytes) + case _: + raise KeyError(f"unknown metric {name!r}") + + +#: Metrics a :class:`Sample` can report, in display order. +METRICS: tuple[str, ...] = ("wall", "cpu", "rss") + +#: Human-facing labels and units, keyed by metric name. +METRIC_LABELS: dict[str, tuple[str, str]] = { + "wall": ("wall clock", "ms"), + "cpu": ("cpu time", "s"), + "rss": ("peak rss", "MiB"), +} + + +def format_metric(name: str, value: float) -> str: + """Render a raw metric value in its display unit.""" + match name: + case "wall": + return f"{value / 1e6:.1f} ms" + case "cpu": + return f"{value:.3f} s" + case "rss": + return f"{value / 2**20:.1f} MiB" + case _: + raise KeyError(f"unknown metric {name!r}") + + +def measure( + argv: list[str], + env: dict[str, str] | None = None, + cwd: Path | None = None, + *, + timeout_s: float | None = 600.0, + check: bool = True, +) -> Sample: + """Run ``argv`` once and return its resource usage. + + Args: + argv: command to run, already fully constructed. + env: complete environment for the child (not merged with the parent's). + cwd: working directory for the child. + timeout_s: kill the child after this long. ``None`` waits forever, + which will hang a CI job on a wedged CLI. + check: raise :class:`MeasurementError` on a non-zero exit. + + Raises: + MeasurementError: on non-zero exit (when ``check``), on timeout, or if + the process could not be started. + """ + if not hasattr(os, "wait4"): # pragma: no cover - Unix-only test suite + raise MeasurementError( + "per-process rusage requires os.wait4, which this platform lacks" + ) + + # Redirect to real files rather than pipes: nothing here drains a pipe, so + # a child that fills its stdout buffer would deadlock against us. + with ( + tempfile.TemporaryFile() as out, + tempfile.TemporaryFile() as err, + tempfile.TemporaryDirectory() as scratch, + ): + result_path = Path(scratch) / "rusage" + launcher_argv = [ + sys.executable, + "-I", + "-S", + str(_LAUNCHER), + str(result_path), + "-" if timeout_s is None else repr(float(timeout_s)), + *argv, + ] + try: + proc = subprocess.Popen( + launcher_argv, stdout=out, stderr=err, env=env, cwd=cwd + ) + except OSError as e: # pragma: no cover - our own interpreter is missing + raise MeasurementError( + f"could not start the measurement launcher: {e}" + ) from e + + # The launcher enforces the real timeout and kills the measured process + # group. This only covers the launcher itself wedging. + backstop: threading.Timer | None = None + if timeout_s is not None: + backstop = threading.Timer(timeout_s + _LAUNCHER_GRACE_S, proc.kill) + backstop.daemon = True + backstop.start() + try: + proc.wait() + finally: + if backstop is not None: + backstop.cancel() + + reading = _read_result(result_path, argv) + if reading.exec_errno: + raise MeasurementError( + f"could not start {argv[0]!r}: {os.strerror(reading.exec_errno)}" + ) + if reading.timed_out: + raise MeasurementError( + f"{argv[0]!r} exceeded the {timeout_s:g}s measurement timeout" + ) + + sample = Sample( + wall_ns=reading.wall_ns, + cpu_s=(reading.utime_us + reading.stime_us) / 1e6, + max_rss_bytes=reading.maxrss_raw * _RSS_SCALE, + exit_code=os.waitstatus_to_exitcode(reading.status), + rss_floor_bytes=reading.floor_bytes, + ) + if check and sample.exit_code != 0: + raise MeasurementError( + f"{' '.join(argv)} exited {sample.exit_code}\nstderr: {_tail(err)}" + ) + return sample + + +@dataclass(frozen=True, slots=True) +class _Reading: + """The launcher's raw report, before it is turned into a :class:`Sample`.""" + + status: int + wall_ns: int + utime_us: int + stime_us: int + maxrss_raw: int + floor_bytes: int + timed_out: int + exec_errno: int + + +#: Integers the launcher writes, one per :class:`_Reading` field. +_RESULT_FIELD_COUNT = len(fields(_Reading)) + + +def _read_result(path: Path, argv: list[str]) -> _Reading: + """Parse the launcher's report, or say clearly that it never made one.""" + try: + fields = [int(v) for v in path.read_text().split()] + except (OSError, ValueError) as e: + raise MeasurementError( + f"the measurement launcher did not report on {argv[0]!r}: {e}" + ) from e + if len(fields) != _RESULT_FIELD_COUNT: + raise MeasurementError( + f"the measurement launcher reported {len(fields)} fields for " + f"{argv[0]!r}, expected {_RESULT_FIELD_COUNT}" + ) + return _Reading(*fields) + + +def _tail(fh: IO[bytes]) -> str: + """Read the last few KiB of a temp file, for error messages.""" + try: + fh.seek(0, os.SEEK_END) + fh.seek(max(0, fh.tell() - _STDERR_TAIL_BYTES)) + return fh.read().decode(errors="replace").strip() + except OSError: # pragma: no cover - defensive + return "" diff --git a/xtest/perf/report.py b/xtest/perf/report.py new file mode 100644 index 000000000..a100044ae --- /dev/null +++ b/xtest/perf/report.py @@ -0,0 +1,263 @@ +"""Collecting benchmark results and turning them into artifacts. + +Cells do not assert. Each one records its raw samples here and moves on, +because the decision rule needs every cell before it can decide anything: +the multiplicity correction is computed across the run, and the A/A control +can invalidate the whole thing. The gate therefore runs once, at session +finish, from :meth:`BenchmarkRecorder.gate`. + +Two artifacts come out: + +- A JSON file per run, holding **every raw per-round sample** alongside the + derived statistics. Re-analysing a surprising result offline is the + difference between understanding a red build and re-running a 30-minute job + to look at the same numbers again. +- A markdown table for ``$GITHUB_STEP_SUMMARY``, so the answer is on the job + page rather than buried in log output. +""" + +from __future__ import annotations + +import json +import math +import os +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from perf import stats +from perf.cells import BenchCell +from perf.measure import METRIC_LABELS, METRICS, format_metric +from perf.runner import BenchConfig, CellResult, analyze + +#: Cells the session intends to run. Set by the conftest parametrizer, read by +#: the budget and arm-resolution fixtures. +CELLS_KEY: pytest.StashKey[list[BenchCell]] = pytest.StashKey() + +#: The session's recorder, reachable from both fixtures and session hooks. +RECORDER_KEY: pytest.StashKey[BenchmarkRecorder] = pytest.StashKey() + + +def recorder_for(config: pytest.Config) -> BenchmarkRecorder: + """Return the session's recorder, creating it on first use.""" + existing = config.stash.get(RECORDER_KEY, None) + if existing is not None: + return existing + recorder = BenchmarkRecorder() + config.stash[RECORDER_KEY] = recorder + return recorder + + +@dataclass(slots=True) +class BenchmarkRecorder: + """Session-wide collector for cell results, skips, and failures.""" + + results: list[CellResult] = field(default_factory=list) + #: cell id -> why it did not run. Reported so a quiet run is visibly + #: quiet rather than indistinguishable from a clean one. + skipped: dict[str, str] = field(default_factory=dict) + metadata: dict[str, object] = field(default_factory=dict) + + def record(self, result: CellResult) -> None: + self.results.append(result) + + def skip(self, cell_id: str, reason: str) -> None: + self.skipped[cell_id] = reason + + def gate(self, config: BenchConfig) -> stats.GateResult: + return analyze(self.results, config) + + +def to_dict( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> dict[str, object]: + """Serialize a whole run, raw samples included.""" + return { + "schema": 1, + "metadata": recorder.metadata, + "config": { + "min_rounds": config.min_rounds, + "max_rounds": config.max_rounds, + "warmup": config.warmup, + "budget_seconds": config.budget_seconds, + "seed": config.seed, + "threshold": config.threshold, + "confidence": config.confidence, + "n_resamples": config.n_resamples, + "gated_metrics": list(config.gated_metrics), + }, + "noise_floor": _noise_dict(gate.noise), + # Per-control floors as well as the run-level worst case: with several + # SDKs in one run, "which one was noisy" is the first question a + # surprising verdict raises. + "noise_floor_by_control": { + k: _noise_dict(n) for k, n in gate.noise_by_control.items() + }, + "trustworthy": gate.trustworthy, + "regressions": gate.regressions, + "improvements": gate.improvements, + "summary": gate.summary, + "skipped": recorder.skipped, + "cells": [ + { + "id": r.cell_id, + "baseline": r.baseline_label, + "candidate": r.candidate_label, + "control": r.control, + "n_rounds": r.n_rounds, + "n_warmup": r.n_warmup, + "elapsed_s": round(r.elapsed_s, 3), + "stopped_because": r.stopped_because, + "rss_floor_bytes": r.rss_floor_bytes, + "samples": r.samples, + "metrics": { + m: _comparison_dict(gate.comparisons[f"{r.cell_id}/{m}"]) + for m in METRICS + if f"{r.cell_id}/{m}" in gate.comparisons + }, + } + for r in recorder.results + ], + } + + +def _noise_dict(noise: stats.NoiseFloor | None) -> dict[str, object]: + if noise is None: + return {"assessed": False} + return { + "assessed": True, + "tripped": noise.tripped, + "underpowered": noise.underpowered, + "width_ratio": _jsonable(noise.width_ratio), + "detail": noise.detail, + } + + +def _comparison_dict(c: stats.PairedComparison) -> dict[str, object]: + return { + "n_rounds": c.n_rounds, + "baseline_median": _jsonable(c.baseline_median), + "candidate_median": _jsonable(c.candidate_median), + "ratio": _jsonable(c.ratio), + "ci_low": _jsonable(c.ci_low), + "ci_high": _jsonable(c.ci_high), + "p_value": _jsonable(c.p_value), + "p_adjusted": _jsonable(c.p_adjusted), + "verdict": str(c.verdict), + "note": c.note, + } + + +def _jsonable(v: float | None) -> float | None: + """JSON has no NaN or infinity; emit null rather than invalid JSON.""" + if v is None or not math.isfinite(v): + return None + return v + + +def write_json( + path: Path, + recorder: BenchmarkRecorder, + config: BenchConfig, + gate: stats.GateResult, +) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(to_dict(recorder, config, gate), indent=2)) + return path + + +def markdown( + recorder: BenchmarkRecorder, config: BenchConfig, gate: stats.GateResult +) -> str: + """Render the run as a GitHub step summary.""" + threshold_pct = (config.threshold - 1) * 100 + lines = [ + "## SDK performance regression benchmark", + "", + f"Paired A/B on one runner. A cell fails only if the 95% CI lower " + f"bound exceeds **{config.threshold:.2f}x** (+{threshold_pct:.0f}%) " + f"*and* the BH-adjusted p < {stats.DEFAULT_ALPHA}.", + "", + f"**{gate.summary}**", + "", + ] + + noise = gate.noise + if noise is not None and noise.detail: + lines += [f"> {noise.detail}", ""] + elif noise is not None and math.isfinite(noise.width_ratio): + lines += [ + f"A/A noise floor: +/-{(noise.width_ratio - 1) * 100:.1f}% " + f"(the smallest effect this run could resolve).", + "", + ] + + lines += [ + "| cell | metric | baseline | candidate | ratio (95% CI) | p (BH) | n | verdict |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for result in recorder.results: + for metric in METRICS: + key = f"{result.cell_id}/{metric}" + c = gate.comparisons.get(key) + if c is None: + continue + gated = metric in config.gated_metrics and not result.control + label = METRIC_LABELS[metric][0] + ("" if gated else " (ungated)") + lines.append( + f"| {result.cell_id} | {label} " + f"| {format_metric(metric, c.baseline_median)} " + f"| {format_metric(metric, c.candidate_median)} " + f"| {_ratio_cell(c)} | {_p_cell(c)} | {c.n_rounds} " + f"| {_verdict_cell(c)} |" + ) + + if recorder.skipped: + lines += ["", "### Not measured", ""] + lines += [f"- `{cid}`: {why}" for cid, why in sorted(recorder.skipped.items())] + + lines += [ + "", + f"seed {config.seed}; warm-up {config.warmup} rounds; " + f"{config.min_rounds}-{config.max_rounds} measured rounds per cell; " + "stopping on attained CI width, never on significance.", + ] + return "\n".join(lines) + "\n" + + +def _ratio_cell(c: stats.PairedComparison) -> str: + if not math.isfinite(c.ratio): + return "-" + if not (math.isfinite(c.ci_low) and math.isfinite(c.ci_high)): + return f"{c.ratio:.3f}x" + return f"{c.ratio:.3f}x [{c.ci_low:.3f}, {c.ci_high:.3f}]" + + +def _p_cell(c: stats.PairedComparison) -> str: + p = c.p_adjusted if c.p_adjusted is not None else c.p_value + if p is None or not math.isfinite(p): + return "-" + return f"{p:.3f}" if p >= 0.001 else "<0.001" + + +_VERDICT_ICONS = { + stats.Verdict.PASS: "PASS", + stats.Verdict.REGRESSION: "**REGRESSION**", + stats.Verdict.IMPROVED: "IMPROVED", + stats.Verdict.INCONCLUSIVE: "inconclusive", +} + + +def _verdict_cell(c: stats.PairedComparison) -> str: + text = _VERDICT_ICONS[c.verdict] + return f"{text} ({c.note})" if c.note else text + + +def append_step_summary(text: str) -> None: + """Append to ``$GITHUB_STEP_SUMMARY`` when running in Actions.""" + target = os.environ.get("GITHUB_STEP_SUMMARY") + if not target: + return + with open(target, "a", encoding="utf-8") as f: + f.write(text) diff --git a/xtest/perf/runner.py b/xtest/perf/runner.py new file mode 100644 index 000000000..af69ff68b --- /dev/null +++ b/xtest/perf/runner.py @@ -0,0 +1,431 @@ +"""The paired round loop that produces comparable samples for one cell. + +A *cell* is one operation at one payload size, measured for two SDK builds. +The loop runs both arms once per round, in a randomized order, until it has +either enough precision or no more time. + +Why rounds rather than "run A 30 times, then B 30 times" +-------------------------------------------------------- +A shared runner drifts: a noisy neighbour arrives, the CPU thermally throttles, +the page cache warms. Run all of A and then all of B and every one of those +effects lands entirely on one arm and shows up as a difference between builds. +Interleaving means both arms see the same conditions within a round, and the +per-round ratio differences it out. + +The order *within* a round is randomized because a fixed order is itself a +confounder -- whichever arm runs second inherits the first one's cache state. + +Why stopping on precision and not on significance +------------------------------------------------- +The loop stops when the confidence interval is narrow enough, never when the +p-value gets small. Peeking at the p-value and stopping the moment it drops +below alpha is optional stopping: it inflates the false-positive rate well +past the nominal level, because you get a fresh chance to cross the line every +round and only ever stop on the lucky side. Attained CI width, in contrast, is +driven by the dispersion of the differences rather than their location, so it +is approximately ancillary to the effect being tested and stopping on it does +not bias the verdict. + +This distinction is easy to "optimize away" -- stopping on significance would +finish sooner -- and doing so silently invalidates every result the job +produces. Do not. +""" + +from __future__ import annotations + +import os +import random +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np + +from perf import stats +from perf.measure import METRICS, MeasurementError, Sample, measure + +#: Target CI half-width on the log scale, as a fraction of the log threshold. +#: At 1/3, an interval centred on "no change" is comfortably clear of the +#: threshold, so a PASS is a real statement about precision rather than a +#: shrug. Tighter costs rounds superlinearly; looser makes PASS meaningless. +PRECISION_FRACTION = 1 / 3 + +#: Bootstrap resamples for the between-round precision check. Far fewer than +#: the final analysis uses: this only needs to answer "is the interval roughly +#: narrow enough yet", and it runs after every round. +_INTERIM_RESAMPLES = 1000 + +#: The A/A control metric that assesses an SDK's noise floor. Wall clock is +#: the most sensitive of the gated metrics to runner noise, which makes it the +#: honest choice of canary. +_CONTROL_METRIC = "wall" + + +class BudgetExhausted(RuntimeError): + """The time budget ran out before the cell could collect usable rounds.""" + + +@dataclass(frozen=True, slots=True) +class BenchConfig: + """Knobs for the round loop and the analysis that follows it.""" + + min_rounds: int = 20 + max_rounds: int = 60 + warmup: int = 5 + budget_seconds: float = 1500.0 + seed: int = 0 + threshold: float = stats.DEFAULT_THRESHOLD + confidence: float = 0.95 + n_resamples: int = stats.DEFAULT_BOOTSTRAP_RESAMPLES + #: Per-invocation timeout. A wedged CLI must not eat the whole job. + timeout_s: float = 600.0 + #: Metrics whose verdict can fail the build. CPU time is measured and + #: reported but excluded: it is the noisiest of the three on a shared + #: runner, and a real CPU regression shows up in wall clock anyway. + gated_metrics: tuple[str, ...] = ("wall", "rss") + + def __post_init__(self) -> None: + if self.min_rounds < stats.MIN_USABLE_ROUNDS: + raise ValueError( + f"min_rounds must be at least {stats.MIN_USABLE_ROUNDS}, " + f"below which no verdict is possible" + ) + if self.max_rounds < self.min_rounds: + raise ValueError("max_rounds must not be below min_rounds") + if self.warmup < 0: + raise ValueError("warmup must not be negative") + if self.threshold <= 1.0: + raise ValueError("threshold is a ratio above 1.0, e.g. 1.15 for 15%") + unknown = set(self.gated_metrics) - set(METRICS) + if unknown: + raise ValueError(f"unknown gated metrics: {sorted(unknown)}") + + @property + def target_half_width_log(self) -> float: + """CI half-width, on the log scale, that ends the round loop.""" + return float(np.log(self.threshold)) * PRECISION_FRACTION + + +@dataclass(frozen=True, slots=True) +class Invocation: + """One fully-built CLI call, ready to run repeatedly.""" + + argv: list[str] + #: CLI-specific overrides, merged over ``os.environ`` at run time. + env: dict[str, str] = field(default_factory=dict) + #: Removed before each measured run, so every round starts from the same + #: state rather than measuring an overwrite in round 2 onwards. + output: Path | None = None + + def child_env(self) -> dict[str, str]: + return dict(os.environ) | self.env + + +@dataclass(frozen=True, slots=True) +class Arm: + """One side of a comparison.""" + + #: ``"baseline"`` or ``"candidate"``; identifies the role, not the build. + name: str + #: The build under this role, e.g. ``"go@v0.29.0"``. + label: str + invocation: Invocation + + +@dataclass(slots=True) +class CellResult: + """Everything one cell measured, before any verdict is assigned. + + Raw per-round vectors are kept in full. Re-analysing a surprising result + offline is the difference between understanding a red build and re-running + a 30-minute job to look at it again. + """ + + cell_id: str + baseline_label: str + candidate_label: str + #: ``samples[arm_name][metric]`` is the per-round vector, warm-up excluded. + samples: dict[str, dict[str, list[float]]] + n_warmup: int + elapsed_s: float + stopped_because: str + #: True for the A/A control, where both arms are the same build. + control: bool = False + #: Which SDK's control cell assesses this cell's noise floor. A run may + #: measure several SDKs, and each has its own harness path and its own + #: floor; go's says nothing about java's. + sdk: str = "" + #: Highest measurement floor seen in this cell -- the RSS of the process + #: that forked each invocation. See :mod:`perf._launcher`. + rss_floor_bytes: int = 0 + + @property + def rss_censored_reason(self) -> str | None: + """Why this cell's peak RSS cannot be compared, or None if it can. + + A command whose peak sits at the floor was not measured, it was + clipped, and both arms clip to the same value. The resulting ratio is + 1.000 with a tight interval, which is the most convincing-looking + PASS the harness can emit and means nothing at all. + """ + if self.rss_floor_bytes <= 0: + return None + rss = [v for arm in self.samples.values() for v in arm.get("rss", [])] + if not rss or min(rss) > self.rss_floor_bytes: + return None + return ( + f"peak rss reaches the {self.rss_floor_bytes / 2**20:.0f} MiB " + "measurement floor, so the two arms are not distinguishable" + ) + + @property + def n_rounds(self) -> int: + first = next(iter(self.samples.values()), {}) + return len(next(iter(first.values()), [])) + + def metric_pair(self, metric: str) -> tuple[list[float], list[float]]: + """Return ``(baseline, candidate)`` vectors for one metric.""" + return self.samples["baseline"][metric], self.samples["candidate"][metric] + + def compare(self, metric: str, config: BenchConfig) -> stats.PairedComparison: + baseline, candidate = self.metric_pair(metric) + return stats.compare( + baseline, + candidate, + confidence=config.confidence, + seed=config.seed, + n_resamples=config.n_resamples, + ) + + +def _empty_samples() -> dict[str, dict[str, list[float]]]: + return {arm: {m: [] for m in METRICS} for arm in ("baseline", "candidate")} + + +def run_cell( + cell_id: str, + baseline: Arm, + candidate: Arm, + config: BenchConfig, + *, + deadline: float | None = None, + control: bool = False, + sdk: str = "", + clock: Callable[[], float] = time.monotonic, + run: Callable[..., Sample] = measure, +) -> CellResult: + """Run one cell's paired rounds and return its raw samples. + + Args: + cell_id: stable identifier, also the per-cell RNG seed material so + that two cells do not share an interleaving order. + baseline: the arm the candidate is compared against. + candidate: the arm under test. For an A/A control this is the same + build as ``baseline``, running through the identical path. + deadline: absolute ``clock()`` value past which no new round starts. + control: records that this is the A/A cell; does not change the loop. + sdk: which SDK this cell belongs to, so that the analysis can pair it + with the right control. Only matters in a multi-SDK run. + clock: injectable monotonic clock. + run: injectable measurement function, for testing the loop itself. + + Raises: + MeasurementError: if any invocation fails. A benchmark over an + operation that errors out is measuring the error path. + BudgetExhausted: if the deadline passed before ``min_rounds``, or + during warm-up. + """ + arms = (baseline, candidate) + # Seeded per cell so a rerun reproduces the interleaving exactly, but the + # cells do not all share one order (which would correlate their noise). + rng = random.Random(f"{config.seed}:{cell_id}") + samples = _empty_samples() + round_durations: list[float] = [] + rss_floor = 0 + started = clock() + + def one_round(into: dict[str, dict[str, list[float]]]) -> None: + order = list(arms) + rng.shuffle(order) + for arm in order: + inv = arm.invocation + if inv.output is not None: + inv.output.unlink(missing_ok=True) + try: + sample = run(inv.argv, inv.child_env(), timeout_s=config.timeout_s) + except MeasurementError as e: + raise MeasurementError(f"{cell_id}: {arm.label} failed: {e}") from e + nonlocal rss_floor + rss_floor = max(rss_floor, sample.rss_floor_bytes) + for metric in METRICS: + into[arm.name][metric].append(sample.metric(metric)) + + for i in range(config.warmup): + # Warm-up rounds pay the one-time costs -- page cache, `go build` + # cache, npx package resolution, JIT warm-up -- that would otherwise + # land unevenly and show up as a difference between builds. Their + # samples are collected into a throwaway dict and dropped. + # + # The deadline is checked here too, and not only in the measured loop + # below. The budget's end is absolute, so warm-ups that overrun it + # spend the *following* cells' time and then reach the measured loop + # with nothing left -- paying the full cost of the cell and producing + # no data. Better to give up here and say why. + if deadline is not None and clock() >= deadline: + raise BudgetExhausted( + f"{cell_id}: budget ran out after {i} of {config.warmup} " + f"warm-up rounds ({clock() - started:.0f}s), " + "before any measurement began" + ) + one_round(_empty_samples()) + + stopped_because = "max_rounds" + for _ in range(config.max_rounds): + round_start = clock() + if deadline is not None and round_start >= deadline: + stopped_because = "budget" + break + if deadline is not None and round_durations: + # Do not start a round we cannot finish: a half-measured round is + # unpaired data, and unpaired data is exactly what this design + # exists to avoid. + expected = float(np.median(round_durations)) + if round_start + expected > deadline: + stopped_because = "budget" + break + one_round(samples) + round_durations.append(clock() - round_start) + + n = len(samples["baseline"]["wall"]) + if n >= config.min_rounds and _precise_enough(samples, config): + stopped_because = "precision" + break + + elapsed = clock() - started + n = len(samples["baseline"]["wall"]) + if n < stats.MIN_USABLE_ROUNDS: + raise BudgetExhausted( + f"{cell_id}: only {n} rounds completed in {elapsed:.0f}s, " + f"below the {stats.MIN_USABLE_ROUNDS} needed for any verdict" + ) + return CellResult( + cell_id=cell_id, + baseline_label=baseline.label, + candidate_label=candidate.label, + samples=samples, + n_warmup=config.warmup, + elapsed_s=elapsed, + stopped_because=stopped_because, + control=control, + sdk=sdk, + rss_floor_bytes=rss_floor, + ) + + +def _precise_enough( + samples: dict[str, dict[str, list[float]]], config: BenchConfig +) -> bool: + """True once every gated metric's CI is narrow enough to decide on. + + Deliberately looks only at interval *width*, never at where the interval + sits or at any p-value -- see the module docstring. + """ + for metric in config.gated_metrics: + c = stats.compare( + samples["baseline"][metric], + samples["candidate"][metric], + confidence=config.confidence, + seed=config.seed, + n_resamples=_INTERIM_RESAMPLES, + ) + # `not (a <= b)` rather than `a > b`, which is not the same thing when + # a is NaN: an unusable interval must read as "keep going", and + # `NaN > b` is False, which would end the loop and call it precise. + if not c.ci_half_width_log <= config.target_half_width_log: # NOSONAR + return False + return True + + +class Budget: + """Shares one wall-clock allowance across a run's cells. + + Cells are measured one at a time, so a cell that stops early on precision + should hand its unused time to the cells after it rather than letting the + last cell get squeezed by whatever the first ones happened to use. + """ + + def __init__( + self, + total_seconds: float, + n_cells: int, + *, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if n_cells <= 0: + raise ValueError("a budget needs at least one cell to divide across") + self._clock = clock + self._end = clock() + total_seconds + self._cells_left = n_cells + + def next_deadline(self) -> float: + """Absolute deadline for the next cell: an even share of what is left.""" + now = self._clock() + share = max(0.0, self._end - now) / max(1, self._cells_left) + self._cells_left = max(0, self._cells_left - 1) + return now + share + + @property + def remaining_s(self) -> float: + return max(0.0, self._end - self._clock()) + + +def analyze(results: Sequence[CellResult], config: BenchConfig) -> stats.GateResult: + """Turn every cell's raw samples into one gate decision. + + ``GateResult.comparisons`` is keyed by ``"/"`` and holds + the *finalized* comparisons -- the ones carrying adjusted p-values and + verdicts. Ungated metrics are included so they appear in the report, but + they cannot fail the build, and they are corrected separately from the + gated ones: adjusting across tests nobody gates on only makes a real + regression harder to confirm. + + Each SDK's cells are paired with *that SDK's* control. A run measuring go + and java has two harness paths and two noise floors, and judging java's + cells against go's control would be judging them against a floor that was + never measured for them. + """ + comparisons: dict[str, stats.PairedComparison] = {} + gated: set[str] = set() + censored: dict[str, str] = {} + control_keys: set[str] = set() + controls: dict[str, str] = {} + control_for_sdk = { + r.sdk: f"{r.cell_id}/{_CONTROL_METRIC}" for r in results if r.control + } + + for result in results: + floored = result.rss_censored_reason + for metric in METRICS: + key = f"{result.cell_id}/{metric}" + comparisons[key] = result.compare(metric, config) + control_key = control_for_sdk.get(result.sdk) + if control_key is not None: + controls[key] = control_key + if metric == "rss" and floored is not None: + censored[key] = floored + continue + if result.control: + control_keys.add(key) + elif metric in config.gated_metrics: + gated.add(key) + + return stats.apply_multiplicity_control( + comparisons, + gated=gated, + controls=controls, + control_keys=control_keys, + censored=censored, + threshold=config.threshold, + alpha=stats.DEFAULT_ALPHA, + ) diff --git a/xtest/perf/stats.py b/xtest/perf/stats.py new file mode 100644 index 000000000..2f3e49255 --- /dev/null +++ b/xtest/perf/stats.py @@ -0,0 +1,544 @@ +"""Paired statistical comparison of two SDK builds. + +Why this shape +-------------- +Absolute timings from a GitHub-hosted runner are not comparable across runs: +CPU models vary, tenancy is shared, and steal time is unbounded. Storing a +baseline and diffing against it produces false alarms until people mute the +job. So we never compare across runs. Both builds are measured on the *same* +runner, interleaved in time, and the statistic is the within-round *ratio*. +Runner speed is then a shared factor that divides out. + +Everything here is a pure function over sample vectors, so the decision logic +is testable without a platform, an SDK, or a subprocess. + +The scale +--------- +Comparisons use the log-ratio ``d_i = ln(candidate_i) - ln(baseline_i)`` of the +i-th paired round. Logs make ratios symmetric (a 2x slowdown and a 2x speedup +are equal and opposite) and additive, which is what the median and the +bootstrap want. Results are exponentiated back to ratios for reporting. + +The decision rule +----------------- +A cell is a regression iff **both**: + +1. the lower bound of the 95% CI on the ratio exceeds ``threshold``, and +2. the Benjamini-Hochberg adjusted one-sided p-value is below ``alpha``. + +Requiring both is deliberate, and neither clause is redundant: + +- Clause 1 alone would fire on a real-but-trivial effect measured precisely + enough -- a reproducible 0.5% slowdown is not worth a red build. + It cannot fire on pure noise, since that would require the interval to + exclude an effect that is not there. +- Clause 2 alone would fire on noise roughly ``alpha`` of the time per cell, + and a run has enough cells that "roughly alpha" becomes "most nights". + BH adjustment across cells controls the false discovery rate. + +Together they answer the only question worth gating on: is the slowdown both +real and large enough to care about? +""" + +from __future__ import annotations + +import math +import warnings +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from enum import StrEnum +from typing import cast + +import numpy as np +from scipy import stats as _scipy_stats + +# Rounds below this cannot support a meaningful interval. The one-sided +# signed-rank test cannot reach p < 0.05 at all below n=5, so anything less is +# reported as INCONCLUSIVE rather than given a verdict. +MIN_USABLE_ROUNDS = 5 + +#: A vector of per-round measurements for one arm of one cell. Accepts a plain +#: list from the runner or an array from a test's synthetic data generator. +type Samples = Sequence[float] | np.ndarray + +DEFAULT_ALPHA = 0.05 +DEFAULT_THRESHOLD = 1.15 +DEFAULT_BOOTSTRAP_RESAMPLES = 10000 + + +class Verdict(StrEnum): + """Outcome for a single comparison cell.""" + + PASS = "PASS" + REGRESSION = "REGRESSION" + IMPROVED = "IMPROVED" + INCONCLUSIVE = "INCONCLUSIVE" + + +@dataclass(frozen=True, slots=True) +class PairedComparison: + """The statistical summary of one (cell, metric) comparison. + + Ratios are candidate-over-baseline: 1.20 means the candidate took 20% + longer (or used 20% more memory) than the baseline. + """ + + n_rounds: int + baseline_median: float + candidate_median: float + ratio: float + ci_low: float + ci_high: float + p_value: float + #: Set by :func:`apply_multiplicity_control` once every cell is known. + p_adjusted: float | None = None + verdict: Verdict = Verdict.INCONCLUSIVE + note: str = "" + + @property + def ci_half_width_log(self) -> float: + """Half-width of the CI on the log scale, the run's attained precision.""" + if not (math.isfinite(self.ci_low) and math.isfinite(self.ci_high)): + return math.inf + if self.ci_low <= 0 or self.ci_high <= 0: + return math.inf + return (math.log(self.ci_high) - math.log(self.ci_low)) / 2 + + +def log_ratios(baseline: Samples, candidate: Samples) -> np.ndarray: + """Return per-round log-ratios ``ln(candidate) - ln(baseline)``. + + The two vectors must be the same length: entry i of each comes from the + same round, which is what makes the comparison paired. Non-positive + measurements cannot be log-transformed and indicate a broken measurement + rather than a fast one, so they are rejected outright. + """ + b = np.asarray(baseline, dtype=float) + c = np.asarray(candidate, dtype=float) + if b.shape != c.shape: + raise ValueError( + f"paired vectors must be the same length, got {b.shape} and {c.shape}" + ) + if b.size == 0: + return np.empty(0, dtype=float) + if not (np.all(np.isfinite(b)) and np.all(np.isfinite(c))): + raise ValueError("measurements must all be finite") + if np.any(b <= 0) or np.any(c <= 0): + raise ValueError("measurements must all be positive to take a log-ratio") + return np.log(c) - np.log(b) + + +def _bootstrap_ci( + d: np.ndarray, *, confidence: float, seed: int, n_resamples: int +) -> tuple[float, float]: + """Percentile-bootstrap CI on the median log-ratio. + + Returns log-scale bounds. BCa is preferred but degenerates when the + jackknife acceleration is undefined (every value identical), so fall back + to the basic percentile method there. + """ + if np.allclose(d, d[0]): + # A perfectly constant difference has no sampling variability to + # estimate; the interval is the point itself. + return float(d[0]), float(d[0]) + try: + with warnings.catch_warnings(): + # SciPy announces a degenerate BCa interval (DegenerateDataWarning, + # a RuntimeWarning) through the warnings machinery and returns NaN + # bounds rather than raising. Promote it, or the fallback below is + # only reachable via the isfinite check and every other degenerate + # case prints a warning nobody reads. + warnings.simplefilter("error", RuntimeWarning) + res = _scipy_stats.bootstrap( + (d,), + np.median, + confidence_level=confidence, + method="BCa", + n_resamples=n_resamples, + rng=np.random.default_rng(seed), + ) + low = float(res.confidence_interval.low) + high = float(res.confidence_interval.high) + if math.isfinite(low) and math.isfinite(high): + return low, high + except ValueError, RuntimeWarning: + pass + res = _scipy_stats.bootstrap( + (d,), + np.median, + confidence_level=confidence, + method="percentile", + n_resamples=n_resamples, + rng=np.random.default_rng(seed), + ) + return float(res.confidence_interval.low), float(res.confidence_interval.high) + + +def _one_sided_p(d: np.ndarray) -> float: + """One-sided Wilcoxon signed-rank p-value for "candidate is slower". + + Signed-rank rather than a t-test because latency distributions are + skewed and occasionally have a stray outlier round; we do not want a + single stalled invocation to drive the verdict. + """ + if np.all(d == 0): + # No difference whatsoever. Wilcoxon rejects an all-zero input. + return 1.0 + # scipy's stubs type the result as an opaque tuple-like; index and cast. + return cast(float, _scipy_stats.wilcoxon(d, alternative="greater")[1]) + + +def compare( + baseline: Samples, + candidate: Samples, + *, + confidence: float = 0.95, + seed: int = 0, + n_resamples: int = DEFAULT_BOOTSTRAP_RESAMPLES, +) -> PairedComparison: + """Compute the paired comparison for one metric of one cell. + + The returned comparison has no final verdict yet: ``p_adjusted`` is unset + and ``verdict`` is INCONCLUSIVE until :func:`apply_multiplicity_control` + has seen every cell in the run. + """ + d = log_ratios(baseline, candidate) + n = int(d.size) + b_med = float(np.median(baseline)) if n else math.nan + c_med = float(np.median(candidate)) if n else math.nan + + if n < MIN_USABLE_ROUNDS: + return PairedComparison( + n_rounds=n, + baseline_median=b_med, + candidate_median=c_med, + ratio=math.exp(float(np.median(d))) if n else math.nan, + ci_low=math.nan, + ci_high=math.nan, + p_value=math.nan, + note=f"only {n} usable rounds; need at least {MIN_USABLE_ROUNDS}", + ) + + lo_log, hi_log = _bootstrap_ci( + d, confidence=confidence, seed=seed, n_resamples=n_resamples + ) + return PairedComparison( + n_rounds=n, + baseline_median=b_med, + candidate_median=c_med, + ratio=math.exp(float(np.median(d))), + ci_low=math.exp(lo_log), + ci_high=math.exp(hi_log), + p_value=_one_sided_p(d), + ) + + +@dataclass(frozen=True, slots=True) +class NoiseFloor: + """What the A/A control says about this runner's measurement noise. + + The A/A control compares the baseline build against *itself* through the + identical pipeline, so its true ratio is exactly 1.0 by construction. Any + apparent effect it reports is pure measurement noise, which makes it a + direct, run-specific check on whether the verdicts can be believed. + """ + + #: True if the A/A comparison itself looked like a regression. The gate + #: is then unreliable and the run must not fail the build on its findings. + tripped: bool + #: CI half-width on the log scale, as an equivalent ratio (e.g. 1.04). + width_ratio: float + #: True if the noise floor is wider than the effect we claim to detect. + underpowered: bool + detail: str = "" + + +def assess_noise_floor( + control: PairedComparison | None, *, threshold: float = DEFAULT_THRESHOLD +) -> NoiseFloor: + """Judge whether this runner was quiet enough to trust the verdicts. + + Two independent failure modes: + + - The control *tripped*: A/A produced an apparent effect past the + threshold. Something is systematically biased (ordering, caching, + thermal drift) and every verdict in the run is suspect. + - The run is *underpowered*: the control's interval is wider than the + effect size we are gating on, so a real regression of that size could + not have been distinguished from noise. A PASS here means "we could not + tell", which must not be reported as "no regression". + """ + if control is None: + return NoiseFloor( + tripped=False, + width_ratio=math.nan, + underpowered=True, + detail="no A/A control cell was run", + ) + if control.n_rounds < MIN_USABLE_ROUNDS or not math.isfinite( + control.ci_half_width_log + ): + return NoiseFloor( + tripped=False, + width_ratio=math.nan, + underpowered=True, + detail=f"A/A control did not produce a usable interval ({control.note})", + ) + + width_ratio = math.exp(control.ci_half_width_log) + # The control's true ratio is 1.0. If its interval excludes the threshold + # in either direction, the pipeline is measuring a difference that cannot + # exist. + tripped = control.ci_low > threshold or control.ci_high < 1 / threshold + underpowered = control.ci_half_width_log >= math.log(threshold) + + detail = "" + if tripped: + detail = ( + f"A/A control reported ratio {control.ratio:.3f} " + f"[{control.ci_low:.3f}, {control.ci_high:.3f}] against a true 1.000; " + "runner is too noisy or the harness is biased" + ) + elif underpowered: + detail = ( + f"A/A noise floor +/-{(width_ratio - 1) * 100:.1f}% is not tighter than " + f"the {(threshold - 1) * 100:.0f}% detection threshold" + ) + return NoiseFloor( + tripped=tripped, + width_ratio=width_ratio, + underpowered=underpowered, + detail=detail, + ) + + +def _worst_noise(noises: Iterable[NoiseFloor]) -> NoiseFloor | None: + """The least reassuring control in the run, or None if there were none. + + Worst case rather than average: a single tripped control means the + harness may be biased on this runner, and averaging that away with two + quiet ones is exactly the reassurance the control exists to withhold. + """ + + def rank(n: NoiseFloor) -> tuple[bool, bool, float]: + # A NaN width is an unusable interval, which is worse than any real one. + width = n.width_ratio if math.isfinite(n.width_ratio) else math.inf + return (n.tripped, n.underpowered, width) + + return max(noises, key=rank, default=None) + + +def benjamini_hochberg(p_values: Sequence[float]) -> list[float]: + """BH-adjusted p-values, controlling the false discovery rate. + + NaNs (cells with too few rounds to test) pass through untouched and are + excluded from the adjustment, so an unmeasurable cell neither gains nor + confers significance. + """ + p = np.asarray(p_values, dtype=float) + out = p.copy() + testable = np.isfinite(p) + if not testable.any(): + return out.tolist() + out[testable] = _scipy_stats.false_discovery_control(p[testable], method="bh") + return out.tolist() + + +@dataclass(slots=True) +class GateResult: + """The run-level outcome after every cell has been compared.""" + + comparisons: dict[str, PairedComparison] = field(default_factory=dict) + #: The run-level noise floor: the *worst* of the per-control assessments, + #: since one biased control means the harness may be biased everywhere. + noise: NoiseFloor | None = None + #: Every control's own assessment, keyed by its comparison key. A run with + #: several SDKs has one control each, and go's noise floor says nothing + #: about java's. + noise_by_control: dict[str, NoiseFloor] = field(default_factory=dict) + #: Keys of cells that are confirmed regressions on a gated metric. + regressions: list[str] = field(default_factory=list) + improvements: list[str] = field(default_factory=list) + #: True if the run may fail the build. False when the A/A control tripped: + #: we still report, but a gate we cannot trust must not turn the build red. + trustworthy: bool = True + summary: str = "" + + @property + def should_fail(self) -> bool: + return self.trustworthy and bool(self.regressions) + + +def apply_multiplicity_control( + comparisons: dict[str, PairedComparison], + *, + gated: set[str] | None = None, + controls: Mapping[str, str] | None = None, + control_keys: set[str] | None = None, + censored: dict[str, str] | None = None, + threshold: float = DEFAULT_THRESHOLD, + alpha: float = DEFAULT_ALPHA, +) -> GateResult: + """Assign final verdicts to every cell and decide the run's outcome. + + Args: + comparisons: cell key -> comparison, for every measured (cell, metric). + gated: keys allowed to fail the build. Keys outside this set are still + given a verdict and reported, but never counted as a regression. + ``None`` means every key is gated. + controls: comparison key -> the A/A control key that assesses *its* + noise floor. A run measuring several SDKs has one control each, + and a cell judged against another SDK's control is judged against + a noise floor that was never measured for it. A key absent from + this mapping has no control, which is treated as underpowered. + control_keys: every key belonging to a control cell. Kept out of the + multiplicity correction and out of the regression and improvement + tallies: an A/A cell is not a hypothesis about the candidate. + censored: keys whose measurement is known to be invalid, mapped to why. + Reported as INCONCLUSIVE and never counted as a regression or an + improvement. A censored reading that happens to land inside the + threshold is otherwise indistinguishable from a real PASS, which + is the more dangerous of the two ways to be wrong. + threshold: minimum ratio worth calling a regression, e.g. 1.15. + alpha: false discovery rate for the BH adjustment. + + Returns: + A :class:`GateResult` whose ``comparisons`` hold the finalized + verdicts. The input mapping is not mutated. + """ + keys = list(comparisons) + controls = controls or {} + # The keys actually doing the assessing: one metric of one control cell + # per SDK. A control cell's other metrics are still control keys -- kept + # out of the gate -- but they are not anybody's noise floor. + assessors = set(controls.values()) + control_keys = control_keys or assessors + censored = censored or {} + + noise_by_control = { + ck: assess_noise_floor(comparisons.get(ck), threshold=threshold) + for ck in assessors + } + #: What a cell with no control of its own is judged against: nothing, which + #: `assess_noise_floor` calls underpowered, so it may report at worst + #: INCONCLUSIVE rather than a PASS nobody measured the power for. + uncontrolled = assess_noise_floor(None, threshold=threshold) + noise = _worst_noise(noise_by_control.values()) or uncontrolled + + # Neither a control nor a censored reading is a hypothesis about the + # candidate, so neither may dilute the correction applied to the cells + # that are. The gated keys are corrected as their own family for the same + # reason: adjusting them against metrics nobody gates on only makes a real + # regression harder to confirm. Ungated metrics still get a family of + # their own so that they carry a reportable verdict. + adjustable = [k for k in keys if k not in control_keys and k not in censored] + gated_family = [k for k in adjustable if gated is None or k in gated] + rest = [k for k in adjustable if k not in set(gated_family)] + p_adj: dict[str, float] = {} + for family in (gated_family, rest): + p_adj.update( + zip( + family, + benjamini_hochberg([comparisons[k].p_value for k in family]), + strict=True, + ) + ) + + result = GateResult(noise=noise, noise_by_control=noise_by_control) + for key in keys: + c = comparisons[key] + pa = p_adj.get(key) + if key in censored: + verdict, note = Verdict.INCONCLUSIVE, censored[key] + else: + verdict, note = _verdict_for( + c, + pa, + threshold=threshold, + alpha=alpha, + noise=noise_by_control.get(controls.get(key, ""), uncontrolled), + is_control=key in control_keys, + ) + result.comparisons[key] = PairedComparison( + n_rounds=c.n_rounds, + baseline_median=c.baseline_median, + candidate_median=c.candidate_median, + ratio=c.ratio, + ci_low=c.ci_low, + ci_high=c.ci_high, + p_value=c.p_value, + p_adjusted=pa, + verdict=verdict, + note=note or c.note, + ) + if key in control_keys: + continue + if verdict is Verdict.REGRESSION and (gated is None or key in gated): + result.regressions.append(key) + elif verdict is Verdict.IMPROVED: + result.improvements.append(key) + + result.trustworthy = not noise.tripped + result.summary = _summarize(result, noise, threshold) + return result + + +def _verdict_for( + c: PairedComparison, + p_adjusted: float | None, + *, + threshold: float, + alpha: float, + noise: NoiseFloor, + is_control: bool, +) -> tuple[Verdict, str]: + if c.n_rounds < MIN_USABLE_ROUNDS or not math.isfinite(c.ci_low): + return Verdict.INCONCLUSIVE, c.note or "no usable interval" + + p = c.p_value if is_control else p_adjusted + if p is None or not math.isfinite(p): + return Verdict.INCONCLUSIVE, "no p-value" + + if c.ci_low > threshold and p < alpha: + return Verdict.REGRESSION, "" + if c.ci_high < 1 / threshold and p > 1 - alpha: + return Verdict.IMPROVED, "" + + # Not a regression. But "we looked and found nothing" only counts as PASS + # if we could have found something. Without the power to resolve an effect + # of `threshold`, the honest answer is that we do not know. + if not is_control and noise.underpowered: + return Verdict.INCONCLUSIVE, noise.detail + if c.ci_half_width_log >= math.log(threshold): + return ( + Verdict.INCONCLUSIVE, + f"interval +/-{(math.exp(c.ci_half_width_log) - 1) * 100:.1f}% is wider " + f"than the {(threshold - 1) * 100:.0f}% threshold", + ) + return Verdict.PASS, "" + + +def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: + if noise.tripped: + return ( + f"INCONCLUSIVE: the A/A control failed its own comparison. {noise.detail}. " + "Verdicts below are reported but not gated." + ) + if result.regressions: + return ( + f"{len(result.regressions)} confirmed regression(s) past the " + f"{(threshold - 1) * 100:.0f}% threshold: {', '.join(result.regressions)}" + ) + inconclusive = [ + k for k, c in result.comparisons.items() if c.verdict is Verdict.INCONCLUSIVE + ] + if inconclusive: + return ( + f"No confirmed regressions. {len(inconclusive)} cell(s) INCONCLUSIVE " + f"(runner noise floor +/-{(noise.width_ratio - 1) * 100:.1f}%)." + ) + return ( + f"No regressions. All cells resolved within the " + f"{(threshold - 1) * 100:.0f}% threshold " + f"(runner noise floor +/-{(noise.width_ratio - 1) * 100:.1f}%)." + ) diff --git a/xtest/pyproject.toml b/xtest/pyproject.toml index a21816fd8..182e5e55c 100644 --- a/xtest/pyproject.toml +++ b/xtest/pyproject.toml @@ -46,6 +46,8 @@ dependencies = [ "smmap>=5.0.3", "typing_extensions>=4.15.0", "urllib3>=2.7.0", + "numpy>=2.5.2", + "scipy>=1.18.0", ] [project.optional-dependencies] @@ -79,7 +81,7 @@ ignore = [ ] [tool.ruff.lint.isort] -known-first-party = ["abac", "tdfs", "otdfctl", "assertions", "fixtures"] +known-first-party = ["abac", "tdfs", "otdfctl", "assertions", "fixtures", "perf"] [tool.ruff.format] quote-style = "double" @@ -100,3 +102,7 @@ testpaths = ["."] python_files = ["test_*.py"] python_functions = ["test_*"] addopts = "-ra -v" +markers = [ + "benchmark: paired A/B performance cell; only collected under --bench", + "no_audit_logs: opt this test out of the default audit-log assertions", +] diff --git a/xtest/tdfs.py b/xtest/tdfs.py index d09ca470d..b73bc508c 100644 --- a/xtest/tdfs.py +++ b/xtest/tdfs.py @@ -505,7 +505,26 @@ def is_released(self) -> bool: ) ) - def encrypt( + def is_final_release(self) -> bool: + """True only for a plain ``vX.Y.Z`` tag -- no prerelease, no build metadata. + + :meth:`is_released` accepts ``v0.29.0-rc.1``, and :meth:`semver` + parses it to the same ``(0, 29, 0)`` as the final release, so ordering + by semver alone leaves the two tied and directory-listing order breaks + the tie. Callers that must not pick a release candidate by accident -- + choosing a benchmark baseline, for one -- want this instead. + """ + return bool(re.fullmatch(r"(?:sdk/)?v?\d+\.\d+\.\d+", self.version)) + + def semver(self) -> tuple[int, int, int] | None: + """Parsed (major, minor, patch), or None for branch builds like 'main'. + + Lets callers order the installed versions -- picking the newest + release as a benchmark baseline, for instance. + """ + return _parse_semver(self.version.removeprefix("sdk/")) + + def encrypt_command( self, pt_file: Path, ct_file: Path, @@ -515,7 +534,17 @@ def encrypt( assert_value: str = "", policy_mode: str = "encrypted", target_mode: container_version | None = None, - ): + ) -> tuple[list[str], dict[str, str]]: + """Build the argv and CLI-specific env vars for an encrypt invocation. + + Split out from :meth:`encrypt` so that callers which need to run the + command themselves -- the benchmark harness measures resource usage + around it -- share this one definition of the `XT_WITH_*` contract + instead of keeping a second copy that drifts. + + The returned env holds only the CLI-specific overrides; merge it over + ``os.environ`` before handing it to a subprocess. + """ use_ecwrap = container == "ztdf-ecwrap" fmt = simple_container(container) c = [ @@ -541,6 +570,29 @@ def encrypt( if use_ecwrap: local_env |= {"XT_WITH_ECWRAP": "true"} + return c, local_env + + def encrypt( + self, + pt_file: Path, + ct_file: Path, + mime_type: str = "application/octet-stream", + container: container_type = "ztdf", + attr_values: list[str] | None = None, + assert_value: str = "", + policy_mode: str = "encrypted", + target_mode: container_version | None = None, + ): + c, local_env = self.encrypt_command( + pt_file, + ct_file, + mime_type=mime_type, + container=container, + attr_values=attr_values, + assert_value=assert_value, + policy_mode=policy_mode, + target_mode=target_mode, + ) logger.debug(f"enc [{' '.join([fmt_env(local_env)] + c)}]") env = dict(os.environ) env |= local_env @@ -554,7 +606,7 @@ def encrypt( result.returncode, c, output=result.stdout, stderr=result.stderr ) - def decrypt( + def decrypt_command( self, ct_file: Path, rt_file: Path, @@ -562,10 +614,15 @@ def decrypt( assert_keys: str = "", verify_assertions: bool = True, ecwrap: bool = False, - expect_error: bool = False, kasallowlist: str = "", ignore_kas_allowlist: bool = False, - ): + ) -> tuple[list[str], dict[str, str]]: + """Build the argv and CLI-specific env vars for a decrypt invocation. + + See :meth:`encrypt_command` for why this is separate. ``expect_error`` + has no counterpart here: it selects how the caller runs the command, + not what the command is. + """ fmt = simple_container(container) c = [ @@ -587,6 +644,30 @@ def decrypt( local_env |= {"XT_WITH_KAS_ALLOWLIST": kasallowlist} if ignore_kas_allowlist: local_env |= {"XT_WITH_IGNORE_KAS_ALLOWLIST": "true"} + return c, local_env + + def decrypt( + self, + ct_file: Path, + rt_file: Path, + container: container_type = "ztdf", + assert_keys: str = "", + verify_assertions: bool = True, + ecwrap: bool = False, + expect_error: bool = False, + kasallowlist: str = "", + ignore_kas_allowlist: bool = False, + ): + c, local_env = self.decrypt_command( + ct_file, + rt_file, + container=container, + assert_keys=assert_keys, + verify_assertions=verify_assertions, + ecwrap=ecwrap, + kasallowlist=kasallowlist, + ignore_kas_allowlist=ignore_kas_allowlist, + ) logger.info(f"dec [{' '.join([fmt_env(local_env)] + c)}]") env = dict(os.environ) env |= local_env diff --git a/xtest/test_bench_arms.py b/xtest/test_bench_arms.py new file mode 100644 index 000000000..bf7a3194e --- /dev/null +++ b/xtest/test_bench_arms.py @@ -0,0 +1,135 @@ +"""Unit tests for benchmark arm selection and payload generation. + +Both decide *what* gets measured, before any measuring happens, and both fail +quietly when they get it wrong: a baseline that is silently a release +candidate, or a payload whose bytes changed between two runs that claim to be +comparable. Neither shows up as an error -- only as numbers that mean +something other than what the report says they mean. + +No platform and no real SDK; the builds are stub ``cli.sh`` trees in +``tmp_path``. +""" + +from pathlib import Path + +import pytest + +import tdfs +from fixtures import bench +from perf.cells import PAYLOADS +from perf.runner import BenchConfig + + +def install(root: Path, sdk: str, *versions: str) -> None: + """Lay down a stub build tree, as ``otdf-sdk-mgr install`` would.""" + for version in versions: + cli = root / "sdk" / sdk / "dist" / version / "cli.sh" + cli.parent.mkdir(parents=True) + cli.write_text("#!/bin/sh\nexit 0\n") + + +@pytest.fixture +def cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """`SDK.__init__` resolves `cli.sh` relative to the cwd, so move there.""" + monkeypatch.chdir(tmp_path) + return tmp_path + + +class TestFinalRelease: + @pytest.mark.parametrize("version", ["v0.29.0", "0.29.0"]) + def test_accepts_a_plain_tag(self, cwd: Path, version: str): + install(cwd, "go", version) + assert tdfs.SDK("go", version).is_final_release() + + @pytest.mark.parametrize( + "version", ["v0.29.0-rc.1", "v0.29.0+build.5", "main", "DSPX-1234"] + ) + def test_rejects_anything_else(self, cwd: Path, version: str): + install(cwd, "go", version) + assert not tdfs.SDK("go", version).is_final_release() + + +class TestBaselineSelection: + def test_picks_the_newest_final_release(self, cwd: Path): + install(cwd, "go", "main", "v0.28.0", "v0.29.0") + baseline, candidate = bench.select_arms("go") + assert baseline.version == "v0.29.0" + assert candidate.version == "main" + + def test_a_release_candidate_never_becomes_the_baseline(self, cwd: Path): + # An rc parses to the same semver as its final release, so ordering by + # semver alone leaves the two tied and the directory listing breaks + # the tie -- a baseline nobody chose, differing run to run. + install(cwd, "go", "main", "v0.29.0", "v0.29.0-rc.1", "v0.30.0-rc.1") + baseline, _ = bench.select_arms("go") + assert baseline.version == "v0.29.0" + + def test_no_final_release_is_a_clear_refusal(self, cwd: Path): + install(cwd, "go", "main", "v0.30.0-rc.1") + with pytest.raises(bench.ArmSelectionError, match="no final go release"): + bench.select_arms("go") + + def test_no_branch_build_is_a_clear_refusal(self, cwd: Path): + install(cwd, "go", "v0.29.0") + with pytest.raises(bench.ArmSelectionError, match="no unreleased go build"): + bench.select_arms("go") + + def test_explicit_specs_win(self, cwd: Path): + install(cwd, "go", "main", "v0.28.0", "v0.29.0") + baseline, candidate = bench.select_arms( + "go", baseline_spec="go@v0.28.0", candidate_spec="go@v0.29.0" + ) + assert (baseline.version, candidate.version) == ("v0.28.0", "v0.29.0") + + def test_refuses_to_compare_a_build_against_itself(self, cwd: Path): + install(cwd, "go", "main", "v0.29.0") + with pytest.raises(bench.ArmSelectionError, match="nothing to compare"): + bench.select_arms("go", baseline_spec="go@main", candidate_spec="go@main") + + +#: The fixture body, called directly: these tests are about the bytes it +#: writes, not about pytest's fixture wiring. +make_payloads = bench.bench_payloads.__wrapped__ # pyright: ignore[reportAttributeAccessIssue] + + +class TestPayloads: + def test_every_size_is_generated(self, tmp_path: Path): + out = make_payloads(tmp_path, BenchConfig(seed=1)) + for payload in PAYLOADS: + assert out[payload.label].stat().st_size == payload.n_bytes + + def test_a_seed_reproduces_the_bytes(self, tmp_path: Path): + a = read_all(make_payloads(subdir(tmp_path, "a"), BenchConfig(seed=1))) + b = read_all(make_payloads(subdir(tmp_path, "b"), BenchConfig(seed=1))) + assert a == b + + def test_a_different_seed_changes_them(self, tmp_path: Path): + a = read_all(make_payloads(subdir(tmp_path, "a"), BenchConfig(seed=1))) + b = read_all(make_payloads(subdir(tmp_path, "b"), BenchConfig(seed=2))) + assert a != b + + def test_a_partial_cache_still_reproduces_the_bytes(self, tmp_path: Path): + # tmp_dir persists between runs. With one RNG stream shared across the + # payloads, skipping a cached file shifts every payload after it, so a + # rerun measures different input than the run it is compared against. + first = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + (tmp_path / f"bench-plain-{PAYLOADS[0].label}.bin").unlink() + second = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + assert first == second + + def test_a_truncated_cache_entry_is_regenerated(self, tmp_path: Path): + first = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + path = tmp_path / f"bench-plain-{PAYLOADS[1].label}.bin" + path.write_bytes(b"truncated") + second = read_all(make_payloads(tmp_path, BenchConfig(seed=1))) + assert first == second + + +def read_all(paths: dict[str, Path]) -> dict[str, bytes]: + return {label: p.read_bytes() for label, p in paths.items()} + + +def subdir(root: Path, name: str) -> Path: + out = root / name + out.mkdir() + return out diff --git a/xtest/test_bench_measure.py b/xtest/test_bench_measure.py new file mode 100644 index 000000000..d7990df80 --- /dev/null +++ b/xtest/test_bench_measure.py @@ -0,0 +1,233 @@ +"""Unit tests for ``perf/measure.py``. + +No platform and no SDK: these drive small Python and shell children with known +resource profiles, so they run in ``check.yml`` next to the stats tests. + +Tolerances are deliberately loose. The point is to catch a primitive that is +plain wrong -- reporting kilobytes as bytes, missing a grandchild's CPU, +returning the parent's memory instead of the child's -- not to assert precise +timings on a shared runner. +""" + +import os +import subprocess +import sys +import textwrap + +import pytest + +from perf import measure +from perf.measure import MeasurementError, Sample + +pytestmark = pytest.mark.skipif( + not hasattr(os, "wait4"), + reason="per-process rusage requires os.wait4", +) + + +def python_child(body: str) -> list[str]: + """Build an argv running a short Python snippet as a child process.""" + return [sys.executable, "-c", textwrap.dedent(body)] + + +class TestWallClock: + def test_tracks_sleep_duration(self): + s = measure.measure(python_child("import time; time.sleep(0.25)")) + assert 0.2 < s.wall_s < 1.5 + + def test_wall_s_matches_wall_ns(self): + s = measure.measure(python_child("pass")) + assert s.wall_s == pytest.approx(s.wall_ns / 1e9) + + +class TestCpuTime: + def test_sleeping_child_burns_almost_no_cpu(self): + s = measure.measure(python_child("import time; time.sleep(0.5)")) + # Interpreter startup costs some CPU, but far less than the wall time. + assert s.cpu_s < s.wall_s + + def test_busy_child_burns_cpu_close_to_wall_time(self): + s = measure.measure( + python_child( + """ + import time + end = time.perf_counter() + 0.5 + while time.perf_counter() < end: + pass + """ + ) + ) + assert s.cpu_s > 0.3 + + def test_includes_grandchild_cpu(self): + # The SDK shims are bash wrappers around a real binary, so a + # measurement that misses grandchildren would report near-zero CPU for + # every SDK operation. + busy = ( + "import time\n" + "end = time.perf_counter() + 0.5\n" + "while time.perf_counter() < end: pass\n" + ) + s = measure.measure( + [ + "/bin/sh", + "-c", + f"{sys.executable} -c {subprocess.list2cmdline([busy])}", + ] + ) + assert s.cpu_s > 0.3, "grandchild CPU was not folded into the parent's rusage" + + +def ballast_child(mb: int) -> list[str]: + """A child that allocates ``mb`` MiB and touches every page of it.""" + return python_child( + f""" + ballast = bytearray({mb} * 1024 * 1024) + ballast[::4096] = b'x' * len(ballast[::4096]) + """ + ) + + +class TestPeakRss: + def test_reports_ballast_in_bytes(self): + # Allocate ~200 MB and confirm the figure is in bytes, not kilobytes. + # Getting the unit wrong is a 1024x error that a loose bound catches. + s = measure.measure( + python_child( + """ + ballast = bytearray(200 * 1024 * 1024) + ballast[::4096] = b'x' * len(ballast[::4096]) + """ + ) + ) + assert 150 * 2**20 < s.max_rss_bytes < 1200 * 2**20 + + def test_larger_allocation_reports_larger_peak(self): + def peak(mb: int) -> int: + return measure.measure(ballast_child(mb)).max_rss_bytes + + assert peak(200) > peak(20) + 100 * 2**20 + + def test_a_fat_measuring_process_does_not_inflate_a_small_child(self): + # The failure this guards against does not look like a failure. On + # Linux a forked child inherits the parent's resident-set accounting + # and exec does not clear it, so every command cheaper than the pytest + # process reported the pytest process's memory instead of its own -- + # a stable, plausible number that reads as "no regression" forever. + # + # Holding real ballast here is the only way to reproduce it: with a + # slim parent the bug is invisible, which is exactly why it reached CI. + lean = measure.measure(ballast_child(20)).max_rss_bytes + ballast = bytearray(400 * 2**20) + try: + ballast[::4096] = b"x" * len(ballast[::4096]) + fat = measure.measure(ballast_child(20)).max_rss_bytes + finally: + del ballast + assert fat < lean + 100 * 2**20, ( + f"measuring from a 400 MiB process reported {fat / 2**20:.0f} MiB " + f"for a child that reads {lean / 2**20:.0f} MiB from a lean one" + ) + + def test_reports_the_floor_under_the_reading(self): + # A peak RSS cannot be measured below the memory of whatever forked + # the process, so the floor travels with the sample and callers can + # tell a censored reading from a genuinely small one. + s = measure.measure(ballast_child(200)) + assert 0 < s.rss_floor_bytes < 100 * 2**20 + assert not s.rss_is_floored + + def test_a_reading_at_the_floor_is_marked_censored(self): + floored = Sample( + wall_ns=1, + cpu_s=0.0, + max_rss_bytes=12 * 2**20, + exit_code=0, + rss_floor_bytes=12 * 2**20, + ) + assert floored.rss_is_floored + assert not Sample( + wall_ns=1, + cpu_s=0.0, + max_rss_bytes=80 * 2**20, + exit_code=0, + rss_floor_bytes=12 * 2**20, + ).rss_is_floored + + def test_includes_grandchild_memory(self): + alloc = ballast_child(200)[2] + s = measure.measure( + [ + "/bin/sh", + "-c", + f"{sys.executable} -c {subprocess.list2cmdline([alloc])}", + ] + ) + assert s.max_rss_bytes > 150 * 2**20 + + +class TestFailureHandling: + def test_non_zero_exit_raises_by_default(self): + with pytest.raises(MeasurementError, match="exited 3"): + measure.measure(python_child("raise SystemExit(3)")) + + def test_error_includes_child_stderr(self): + with pytest.raises(MeasurementError, match="disaster strikes"): + measure.measure( + python_child( + "import sys; sys.stderr.write('disaster strikes'); sys.exit(1)" + ) + ) + + def test_check_false_returns_the_failing_sample(self): + s = measure.measure(python_child("raise SystemExit(7)"), check=False) + assert s.exit_code == 7 + + def test_missing_executable_raises(self): + with pytest.raises(MeasurementError, match="could not start"): + measure.measure(["/nonexistent/definitely-not-a-real-binary"]) + + def test_timeout_kills_and_raises(self): + with pytest.raises(MeasurementError, match="measurement timeout"): + measure.measure(python_child("import time; time.sleep(30)"), timeout_s=0.5) + + def test_large_output_does_not_deadlock(self): + # os.wait4 does not drain pipes. If stdout were a pipe, a child writing + # more than the pipe buffer would block forever and hang the job. + s = measure.measure( + python_child("import sys; sys.stdout.write('x' * 5_000_000)") + ) + assert s.exit_code == 0 + + +class TestMetricAccess: + def test_metric_lookup_matches_fields(self): + s = Sample(wall_ns=1_500_000, cpu_s=0.25, max_rss_bytes=2**20, exit_code=0) + assert s.metric("wall") == 1_500_000 + assert s.metric("cpu") == 0.25 + assert s.metric("rss") == 2**20 + + def test_unknown_metric_raises(self): + s = Sample(wall_ns=1, cpu_s=1.0, max_rss_bytes=1, exit_code=0) + with pytest.raises(KeyError): + s.metric("bogus") + + def test_every_declared_metric_is_retrievable(self): + s = Sample(wall_ns=1, cpu_s=1.0, max_rss_bytes=1, exit_code=0) + for name in measure.METRICS: + assert isinstance(s.metric(name), float) + assert name in measure.METRIC_LABELS + + def test_formatting(self): + cases = [ + ("wall", 1_500_000.0, "1.5 ms"), + ("cpu", 0.25, "0.250 s"), + ("rss", float(2**21), "2.0 MiB"), + ] + assert [measure.format_metric(n, v) for n, v, _ in cases] == [ + e for _, _, e in cases + ] + + def test_formatting_rejects_unknown_metric(self): + with pytest.raises(KeyError): + measure.format_metric("bogus", 1.0) diff --git a/xtest/test_bench_runner.py b/xtest/test_bench_runner.py new file mode 100644 index 000000000..3b9e9f4f1 --- /dev/null +++ b/xtest/test_bench_runner.py @@ -0,0 +1,447 @@ +"""Tests for the paired round loop and the gate it feeds. + +No subprocesses and no platform: the measurement function and the clock are +both injected, so a whole 40-round cell runs in microseconds and a planted +regression is exactly the size we planted. + +The last class here is the one that matters most. A benchmark gate that has +never been shown to catch a planted regression -- and to *ignore* a trivial +one -- is not yet known to work. +""" + +from __future__ import annotations + +import math +import random +from collections.abc import Callable +from pathlib import Path + +import pytest + +from perf import stats +from perf.measure import Sample +from perf.runner import ( + Arm, + BenchConfig, + Budget, + BudgetExhausted, + Invocation, + analyze, + run_cell, +) + +BASELINE_WALL_S = 1.0 +BASELINE_RSS = 100_000_000 +BASELINE_CPU = 0.8 + + +def arm(role: str, key: str, output: Path | None = None) -> Arm: + """An arm whose argv is a single token, so ``FakeRuns`` can recognize it. + + ``role`` is what the runner keys samples by ("baseline"/"candidate"); + ``key`` is the stand-in for the build. + """ + return Arm(role, f"sdk@{key}", Invocation([key], {}, output)) + + +def config(**overrides: object) -> BenchConfig: + """A config small enough to run fast, still valid for a verdict.""" + base: dict[str, object] = { + "min_rounds": stats.MIN_USABLE_ROUNDS, + "max_rounds": 40, + "warmup": 2, + "n_resamples": 400, + } + return BenchConfig(**(base | overrides)) # pyright: ignore[reportArgumentType] + + +class FakeRuns: + """A stand-in for ``measure`` that returns scripted, noisy samples. + + ``ratio_for`` maps the invocation's first argv element to a multiplier on + the baseline cost, so a caller plants an effect by naming the arms. + """ + + def __init__( + self, + ratio_for: dict[str, float], + *, + noise: float = 0.05, + seed: int = 7, + rss_floor: int = 0, + ) -> None: + self.ratio_for = ratio_for + self.noise = noise + #: Readings below this clip up to it, the way a real measurement floor + #: behaves: the number is the floor's, not the command's. + self.rss_floor = rss_floor + self.rng = random.Random(seed) + #: Every argv[0] seen, in call order. The interleaving is visible here. + self.calls: list[str] = [] + #: Simulated seconds consumed, for tests that drive the clock from it. + self.elapsed = 0.0 + + def __call__( + self, argv: list[str], env: dict[str, str], **kwargs: object + ) -> Sample: + del env, kwargs + key = argv[0] + self.calls.append(key) + ratio = self.ratio_for[key] + # Lognormal jitter: latency is positive and multiplicative, so noise + # on the log scale is the honest model of a noisy runner. + jitter = math.exp(self.rng.gauss(0.0, self.noise)) + wall = BASELINE_WALL_S * ratio * jitter + self.elapsed += wall + return Sample( + wall_ns=int(wall * 1e9), + cpu_s=BASELINE_CPU * ratio * jitter, + max_rss_bytes=max(int(BASELINE_RSS * ratio * jitter), self.rss_floor), + exit_code=0, + rss_floor_bytes=self.rss_floor, + ) + + +def clock_from(runs: FakeRuns) -> Callable[[], float]: + """A clock that advances only as simulated work happens.""" + return lambda: runs.elapsed + + +def run( + ratio: float, + *, + cfg: BenchConfig | None = None, + noise: float = 0.05, + seed: int = 7, + cell_id: str = "cell", + control: bool = False, + sdk: str = "", + rss_floor: int = 0, +): + """Run one cell where the candidate costs ``ratio`` times the baseline.""" + runs = FakeRuns( + {"base": 1.0, "cand": ratio}, noise=noise, seed=seed, rss_floor=rss_floor + ) + result = run_cell( + cell_id, + arm("baseline", "base"), + arm("candidate", "cand"), + cfg or config(), + control=control, + sdk=sdk, + clock=clock_from(runs), + run=runs, + ) + return result, runs + + +class TestRoundLoop: + def test_arms_are_paired_every_round(self): + _, runs = run(1.0) + assert runs.calls.count("base") == runs.calls.count("cand") + # Every consecutive pair holds one of each: that is what pairing means. + pairs = [set(runs.calls[i : i + 2]) for i in range(0, len(runs.calls), 2)] + assert all(p == {"base", "cand"} for p in pairs) + + def test_order_within_rounds_is_shuffled(self): + _, runs = run(1.0) + firsts = runs.calls[::2] + assert "base" in firsts and "cand" in firsts, ( + "a fixed within-round order lets the second arm inherit the first " + "one's cache state" + ) + + def test_warmup_rounds_are_discarded(self): + cfg = config(warmup=3, max_rounds=stats.MIN_USABLE_ROUNDS) + result, runs = run(1.0, cfg=cfg) + assert result.n_rounds == stats.MIN_USABLE_ROUNDS + assert result.n_warmup == 3 + # Warm-up rounds ran, they just are not in the samples. + assert len(runs.calls) == 2 * (3 + stats.MIN_USABLE_ROUNDS) + + def test_interleaving_is_reproducible_for_a_seed(self): + _, a = run(1.0) + _, b = run(1.0) + assert a.calls == b.calls + + def test_cells_do_not_share_an_interleaving(self): + _, a = run(1.0, cell_id="encrypt-1KiB") + _, b = run(1.0, cell_id="decrypt-1KiB") + assert a.calls != b.calls, "cells sharing one order would correlate their noise" + + def test_output_is_removed_before_each_run(self, tmp_path: Path): + out = tmp_path / "out.tdf" + out.write_bytes(b"stale") + seen: list[bool] = [] + runs = FakeRuns({"base": 1.0, "cand": 1.0}) + + def observe(argv: list[str], env: dict[str, str], **kwargs: object) -> Sample: + if argv[0] == "base": + # What the arm that owns this output sees when it starts. + seen.append(out.exists()) + out.write_bytes(b"produced") + return runs(argv, env, **kwargs) + + run_cell( + "cell", + arm("baseline", "base", out), + arm("candidate", "cand"), + config(max_rounds=stats.MIN_USABLE_ROUNDS), + clock=clock_from(runs), + run=observe, + ) + assert not any(seen), "a stale output makes round 2 measure an overwrite" + + def test_samples_are_collected_for_every_metric(self): + result, _ = run(1.0) + for name in ("baseline", "candidate"): + for metric in ("wall", "cpu", "rss"): + assert len(result.samples[name][metric]) == result.n_rounds + + +class TestStopping: + def test_stops_early_on_precision_when_quiet(self): + result, _ = run(1.0, noise=0.005) + assert result.stopped_because == "precision" + assert result.n_rounds < 40 + + def test_runs_to_max_rounds_when_noisy(self): + result, _ = run(1.0, noise=0.4) + assert result.stopped_because == "max_rounds" + assert result.n_rounds == 40 + + def test_never_stops_before_min_rounds(self): + cfg = config(min_rounds=25, max_rounds=40) + result, _ = run(1.0, cfg=cfg, noise=0.0001) + assert result.n_rounds >= 25 + + def test_deadline_stops_the_loop(self): + runs = FakeRuns({"base": 1.0, "cand": 1.0}, noise=0.3) + clock = clock_from(runs) + result = run_cell( + "cell", + arm("baseline", "base"), + arm("candidate", "cand"), + config(warmup=0, max_rounds=200), + deadline=clock() + 60.0, # each round costs ~2 simulated seconds + clock=clock, + run=runs, + ) + assert result.stopped_because == "budget" + assert result.elapsed_s <= 60.0, "a round we could not finish was started" + + def test_warmup_gives_up_when_the_budget_runs_out(self): + # The budget's end is absolute, so warm-ups that run past it are + # spending the *following* cells' time -- and then reaching the + # measured loop with nothing left, paying the whole cost of the cell + # for no data at all. Stop at the deadline and say where it went. + runs = FakeRuns({"base": 1.0, "cand": 1.0}) + clock = clock_from(runs) + with pytest.raises(BudgetExhausted, match="warm-up"): + run_cell( + "cell", + arm("baseline", "base"), + arm("candidate", "cand"), + config(warmup=10), + deadline=clock() + 4.0, # each round costs ~2 simulated seconds + clock=clock, + run=runs, + ) + assert len(runs.calls) < 2 * 10, "warm-up ran past its own deadline" + + def test_budget_below_min_usable_rounds_refuses_a_verdict(self): + runs = FakeRuns({"base": 1.0, "cand": 1.0}) + clock = clock_from(runs) + with pytest.raises(BudgetExhausted, match="below the"): + run_cell( + "cell", + arm("baseline", "base"), + arm("candidate", "cand"), + config(warmup=0), + deadline=clock() + 4.0, + clock=clock, + run=runs, + ) + + +class TestBenchConfigValidation: + def test_rejects_min_rounds_below_the_usable_floor(self): + with pytest.raises(ValueError, match="min_rounds"): + BenchConfig(min_rounds=stats.MIN_USABLE_ROUNDS - 1) + + def test_rejects_max_below_min(self): + with pytest.raises(ValueError, match="max_rounds"): + BenchConfig(min_rounds=20, max_rounds=10) + + def test_rejects_a_threshold_that_is_not_a_ratio(self): + with pytest.raises(ValueError, match="ratio above 1.0"): + BenchConfig(threshold=0.9) + + def test_rejects_unknown_gated_metrics(self): + with pytest.raises(ValueError, match="unknown gated metrics"): + BenchConfig(gated_metrics=("wall", "iops")) + + def test_target_half_width_is_a_third_of_the_log_threshold(self): + cfg = BenchConfig(threshold=1.15) + assert cfg.target_half_width_log == pytest.approx(math.log(1.15) / 3) + + +class TestBudget: + def test_divides_remaining_time_evenly(self): + now = 1000.0 + budget = Budget(300.0, 3, clock=lambda: now) + assert budget.next_deadline() == pytest.approx(now + 100.0) + + def test_unused_time_flows_to_later_cells(self): + now = [0.0] + budget = Budget(300.0, 3, clock=lambda: now[0]) + budget.next_deadline() + now[0] = 10.0 # the first cell stopped early on precision + # 290s left over two cells, not the 100s it would have got by + # dividing up front. + assert budget.next_deadline() == pytest.approx(155.0) + + def test_never_hands_out_a_deadline_in_the_past(self): + now = [0.0] + budget = Budget(10.0, 2, clock=lambda: now[0]) + now[0] = 60.0 + assert budget.next_deadline() == pytest.approx(60.0) + assert budget.remaining_s == 0.0 + + def test_rejects_a_budget_with_no_cells(self): + with pytest.raises(ValueError, match="at least one cell"): + Budget(10.0, 0) + + +class TestGateOnPlantedEffects: + """The critical check: does the gate fire when it should, and only then? + + Each case runs a real cell through the real statistics; only the + measurement is simulated. The A/A control cell is included exactly as a + live run would include it, so the noise floor is assessed the same way. + """ + + def gate(self, candidate_ratio: float, *, noise: float = 0.05, seed: int = 11): + cfg = config(max_rounds=40) + control, _ = run( + 1.0, cfg=cfg, noise=noise, seed=seed, cell_id="aa", control=True + ) + measured, _ = run( + candidate_ratio, cfg=cfg, noise=noise, seed=seed + 1, cell_id="encrypt" + ) + return analyze([control, measured], cfg) + + def test_planted_25_percent_slowdown_is_caught(self): + gate = self.gate(1.25) + assert gate.should_fail + assert "encrypt/wall" in gate.regressions + c = gate.comparisons["encrypt/wall"] + assert c.verdict is stats.Verdict.REGRESSION + assert c.ci_low > 1.15, "the interval must exclude the threshold, not just 1.0" + assert c.ratio == pytest.approx(1.25, rel=0.1) + + def test_planted_3_percent_slowdown_is_ignored(self): + gate = self.gate(1.03) + assert not gate.should_fail + assert gate.comparisons["encrypt/wall"].verdict is not stats.Verdict.REGRESSION + + def test_no_effect_does_not_fire(self): + gate = self.gate(1.0) + assert not gate.should_fail + assert not gate.regressions + + def test_planted_speedup_is_reported_not_failed(self): + gate = self.gate(0.7) + assert not gate.should_fail + assert gate.comparisons["encrypt/wall"].verdict is stats.Verdict.IMPROVED + assert "encrypt/wall" in gate.improvements + + def test_the_control_cell_never_fails_the_build(self): + # Both arms of the control are the same build, so any verdict it + # reaches is the harness's own error, not a regression in anything. + gate = self.gate(1.25) + assert not any(k.startswith("aa/") for k in gate.regressions) + + def test_a_regression_in_an_ungated_metric_does_not_fail(self): + cfg = config(max_rounds=40, gated_metrics=("wall",)) + control, _ = run(1.0, cfg=cfg, cell_id="aa", control=True) + measured, _ = run(1.4, cfg=cfg, seed=12, cell_id="encrypt") + gate = analyze([control, measured], cfg) + # CPU time moved with everything else and is reported as such; it + # simply is not allowed to turn the build red. + assert gate.comparisons["encrypt/cpu"].verdict is stats.Verdict.REGRESSION + assert "encrypt/cpu" not in gate.regressions + assert "encrypt/wall" in gate.regressions + + def test_rss_pinned_to_the_measurement_floor_cannot_report_pass(self): + # A command whose peak sits at the floor is not measured, it is + # clipped -- and both arms clip to the same number. That produces a + # ratio of exactly 1.000 with a vanishing interval, which is the most + # convincing PASS the harness can emit and carries no information. + cfg = config(max_rounds=40) + floor = 4 * BASELINE_RSS + control, _ = run(1.0, cfg=cfg, cell_id="aa", control=True, rss_floor=floor) + measured, _ = run(1.25, cfg=cfg, seed=12, cell_id="encrypt", rss_floor=floor) + gate = analyze([control, measured], cfg) + + rss = gate.comparisons["encrypt/rss"] + assert rss.ratio == pytest.approx(1.0), "the floor clipped both arms" + assert rss.verdict is stats.Verdict.INCONCLUSIVE + assert "floor" in rss.note + assert "encrypt/rss" not in gate.regressions + assert "encrypt/rss" not in gate.improvements + # Wall clock is untouched by a memory floor and still does its job. + assert "encrypt/wall" in gate.regressions + + def test_rss_above_the_floor_is_still_gated(self): + cfg = config(max_rounds=40) + control, _ = run( + 1.0, cfg=cfg, cell_id="aa", control=True, rss_floor=BASELINE_RSS // 10 + ) + measured, _ = run( + 1.25, cfg=cfg, seed=12, cell_id="encrypt", rss_floor=BASELINE_RSS // 10 + ) + gate = analyze([control, measured], cfg) + assert "encrypt/rss" in gate.regressions + + def test_each_sdk_is_judged_against_its_own_control(self): + # One control per SDK: they are different harness paths with different + # floors. Judging go's cells against java's control judges them + # against a noise floor that was never measured for them -- and with + # a single run-level control, whichever SDK happened to be last wins. + cfg = config(max_rounds=40) + go_aa, _ = run( + 1.0, cfg=cfg, noise=0.02, cell_id="go-aa", control=True, sdk="go" + ) + go_cell, _ = run( + 1.0, cfg=cfg, noise=0.02, seed=12, cell_id="go-encrypt", sdk="go" + ) + # java's runner was noisy enough that it could not resolve the + # threshold; its cells must not claim a clean bill of health. + java_aa, _ = run( + 1.0, + cfg=cfg, + noise=0.5, + seed=13, + cell_id="java-aa", + control=True, + sdk="java", + ) + java_cell, _ = run( + 1.0, cfg=cfg, noise=0.02, seed=14, cell_id="java-encrypt", sdk="java" + ) + gate = analyze([go_aa, go_cell, java_aa, java_cell], cfg) + + assert len(gate.noise_by_control) == 2, "one noise floor per SDK" + assert gate.comparisons["go-encrypt/wall"].verdict is stats.Verdict.PASS + assert ( + gate.comparisons["java-encrypt/wall"].verdict is stats.Verdict.INCONCLUSIVE + ), "java's own control had no power, whatever go's control managed" + + def test_a_run_with_no_control_cannot_report_pass(self): + cfg = config(max_rounds=40) + measured, _ = run(1.0, cfg=cfg, cell_id="encrypt") + gate = analyze([measured], cfg) + assert gate.noise is not None and gate.noise.underpowered + assert gate.comparisons["encrypt/wall"].verdict is stats.Verdict.INCONCLUSIVE + assert not gate.should_fail, "an unassessed run warns; it does not fail" diff --git a/xtest/test_bench_stats.py b/xtest/test_bench_stats.py new file mode 100644 index 000000000..20f077da0 --- /dev/null +++ b/xtest/test_bench_stats.py @@ -0,0 +1,344 @@ +"""Unit tests for the benchmark decision logic in ``perf/stats.py``. + +These are pure-function tests: no platform, no SDK, no subprocess. They run in +``check.yml`` alongside lint, because a regression gate whose statistics are +wrong is worse than no gate at all -- it either cries wolf until it is muted, +or stays quiet while performance rots. + +The two properties that matter most are covered by +``test_pure_noise_false_positive_rate_is_controlled`` (the gate does not fire +on a runner that is merely noisy) and +``test_planted_regression_is_detected`` (it does fire on a real slowdown). +""" + +import math + +import numpy as np +import pytest + +from perf import stats +from perf.stats import Verdict + +# Typical CI-runner dispersion for a CLI invocation: roughly +/-8% round to +# round. Large enough to be realistic, small enough that 30 rounds can resolve +# a 15% effect. +NOISE_SIGMA = 0.08 +ROUNDS = 30 +# Bootstrap resamples for tests. Lower than the production default to keep the +# repeated-trial tests quick; the estimates are still stable to ~1%. +RESAMPLES = 999 + + +def synth( + rng: np.random.Generator, + true_ratio: float, + *, + n: int = ROUNDS, + sigma: float = NOISE_SIGMA, +) -> tuple[np.ndarray, np.ndarray]: + """Return (baseline, candidate) samples with a known multiplicative effect. + + Both arms get independent lognormal noise around a shared base cost, which + is the structure the real harness produces: a per-round shared component + (runner speed) that cancels, plus independent per-invocation jitter. + """ + base = 1.0 * rng.lognormal(0.0, sigma, n) + baseline = base * rng.lognormal(0.0, sigma, n) + candidate = base * true_ratio * rng.lognormal(0.0, sigma, n) + return baseline, candidate + + +def gate_one( + comparison: stats.PairedComparison, + *, + control: stats.PairedComparison | None = None, + threshold: float = 1.15, +) -> stats.GateResult: + """Run a single comparison through the full run-level gate.""" + cells = {"cell": comparison} + if control is not None: + cells["control"] = control + return stats.apply_multiplicity_control( + cells, + controls=all_under_one_control(cells) if control is not None else None, + threshold=threshold, + ) + + +def all_under_one_control( + cells: dict[str, stats.PairedComparison], key: str = "control" +) -> dict[str, str]: + """Map every cell to the single control cell, as a one-SDK run does.""" + return dict.fromkeys(cells, key) + + +def quiet_control(seed: int = 7) -> stats.PairedComparison: + """An A/A control from a well-behaved runner, tight enough to have power.""" + rng = np.random.default_rng(seed) + # More rounds and lower jitter than a real cell, so the control does not + # itself become the limiting factor in tests about other things. + b, c = synth(rng, 1.0, n=80, sigma=0.03) + return stats.compare(b, c, seed=seed, n_resamples=RESAMPLES) + + +class TestLogRatios: + def test_recovers_exact_ratio(self): + d = stats.log_ratios([2.0, 4.0], [3.0, 6.0]) + assert np.allclose(np.exp(d), 1.5) + + def test_rejects_mismatched_lengths(self): + with pytest.raises(ValueError, match="same length"): + stats.log_ratios([1.0, 2.0], [1.0]) + + @pytest.mark.parametrize("bad", [0.0, -1.0]) + def test_rejects_non_positive(self, bad: float): + # A zero or negative duration is a broken measurement, not a fast one. + with pytest.raises(ValueError, match="positive"): + stats.log_ratios([1.0, bad], [1.0, 1.0]) + + def test_rejects_non_finite(self): + with pytest.raises(ValueError, match="finite"): + stats.log_ratios([1.0, math.inf], [1.0, 1.0]) + + def test_empty_is_empty(self): + assert stats.log_ratios([], []).size == 0 + + +class TestCompare: + def test_point_estimate_tracks_true_ratio(self): + rng = np.random.default_rng(0) + b, c = synth(rng, 1.25, n=200) + r = stats.compare(b, c, seed=0, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(1.25, rel=0.05) + + def test_interval_covers_truth(self): + rng = np.random.default_rng(1) + b, c = synth(rng, 1.25, n=200) + r = stats.compare(b, c, seed=1, n_resamples=RESAMPLES) + assert r.ci_low < 1.25 < r.ci_high + + def test_too_few_rounds_yields_no_interval(self): + rng = np.random.default_rng(2) + b, c = synth(rng, 2.0, n=3) + r = stats.compare(b, c, seed=2, n_resamples=RESAMPLES) + assert r.n_rounds == 3 + assert math.isnan(r.ci_low) + assert "at least" in r.note + + def test_identical_inputs_give_unit_ratio_and_no_significance(self): + v = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + r = stats.compare(v, v, seed=0, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(1.0) + assert r.p_value == 1.0 + + def test_constant_offset_has_degenerate_interval(self): + # Every round shows exactly a 2x slowdown: there is no sampling + # variability, so the interval collapses onto the point estimate + # rather than blowing up in the BCa jackknife. + b = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + c = [2 * x for x in b] + r = stats.compare(b, c, seed=0, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(2.0) + assert r.ci_low == pytest.approx(2.0) + assert r.ci_high == pytest.approx(2.0) + + def test_single_outlier_round_does_not_dominate(self): + # One stalled invocation out of 30 must not manufacture a regression; + # this is why the estimator is a median and the test is signed-rank. + rng = np.random.default_rng(3) + b, c = synth(rng, 1.0) + c = c.copy() + c[0] *= 50 + r = stats.compare(b, c, seed=3, n_resamples=RESAMPLES) + assert r.ratio == pytest.approx(1.0, abs=0.1) + assert ( + gate_one(r, control=quiet_control()).comparisons["cell"].verdict + is not Verdict.REGRESSION + ) + + +class TestDecisionRule: + def test_planted_regression_is_detected(self): + rng = np.random.default_rng(10) + b, c = synth(rng, 1.30, n=60) + g = gate_one( + stats.compare(b, c, seed=10, n_resamples=RESAMPLES), control=quiet_control() + ) + assert g.comparisons["cell"].verdict is Verdict.REGRESSION + assert g.regressions == ["cell"] + assert g.should_fail + + def test_trivial_but_real_slowdown_does_not_fire(self): + # A reproducible 3% slowdown, measured precisely enough to be + # statistically significant, is deliberately not a build failure. + rng = np.random.default_rng(11) + b, c = synth(rng, 1.03, n=400, sigma=0.02) + r = stats.compare(b, c, seed=11, n_resamples=RESAMPLES) + assert r.p_value < 0.05, "precondition: the effect is statistically real" + g = gate_one(r, control=quiet_control()) + assert g.comparisons["cell"].verdict is not Verdict.REGRESSION + assert not g.should_fail + + def test_planted_speedup_is_reported_but_never_fails(self): + rng = np.random.default_rng(12) + b, c = synth(rng, 0.70, n=60) + g = gate_one( + stats.compare(b, c, seed=12, n_resamples=RESAMPLES), control=quiet_control() + ) + assert g.comparisons["cell"].verdict is Verdict.IMPROVED + assert not g.should_fail + + def test_borderline_effect_without_power_is_inconclusive_not_pass(self): + # A 15% effect with only a handful of very noisy rounds: the honest + # answer is "cannot tell", never "no regression". + rng = np.random.default_rng(13) + b, c = synth(rng, 1.15, n=6, sigma=0.35) + g = gate_one( + stats.compare(b, c, seed=13, n_resamples=RESAMPLES), control=quiet_control() + ) + assert g.comparisons["cell"].verdict is Verdict.INCONCLUSIVE + + def test_pure_noise_false_positive_rate_is_controlled(self): + # The property the whole design exists to guarantee: on a runner with + # no real effect, the gate must almost never fire. Nominal alpha is + # 0.05, but the threshold clause should push the realized rate far + # below that. + trials, fired = 200, 0 + for seed in range(trials): + rng = np.random.default_rng(1000 + seed) + b, c = synth(rng, 1.0) + g = gate_one( + stats.compare(b, c, seed=seed, n_resamples=RESAMPLES), + control=quiet_control(), + ) + fired += g.should_fail + assert fired / trials <= 0.02, ( + f"gate fired on {fired}/{trials} pure-noise runs; " + "it will be muted in production at this rate" + ) + + def test_detects_regression_across_realistic_noise(self): + # The complement of the false-positive test: a 30% regression must be + # caught reliably, not just on a lucky seed. + trials, caught = 40, 0 + for seed in range(trials): + rng = np.random.default_rng(2000 + seed) + b, c = synth(rng, 1.30, n=40) + g = gate_one( + stats.compare(b, c, seed=seed, n_resamples=RESAMPLES), + control=quiet_control(), + ) + caught += g.should_fail + assert caught / trials >= 0.90, ( + f"only caught {caught}/{trials} real 30% regressions" + ) + + +class TestNoiseFloor: + def test_clean_control_is_trusted(self): + n = stats.assess_noise_floor(quiet_control(), threshold=1.15) + assert not n.tripped + assert not n.underpowered + + def test_missing_control_is_underpowered(self): + n = stats.assess_noise_floor(None, threshold=1.15) + assert n.underpowered + assert "no A/A control" in n.detail + + def test_wide_control_marks_run_underpowered(self): + rng = np.random.default_rng(20) + b, c = synth(rng, 1.0, n=8, sigma=0.5) + n = stats.assess_noise_floor( + stats.compare(b, c, seed=20, n_resamples=RESAMPLES), threshold=1.15 + ) + assert n.underpowered + + def test_biased_control_disables_the_gate(self): + # A control that reports a large effect against a true ratio of 1.0 + # means the harness or the runner is systematically biased. Real cells + # must still be reported, but must not turn the build red. + rng = np.random.default_rng(21) + cb, cc = synth(rng, 1.40, n=60) # A/A that "found" 40%: impossible + control = stats.compare(cb, cc, seed=21, n_resamples=RESAMPLES) + rng2 = np.random.default_rng(22) + b, c = synth(rng2, 1.40, n=60) + g = gate_one( + stats.compare(b, c, seed=22, n_resamples=RESAMPLES), control=control + ) + + assert g.noise is not None and g.noise.tripped + assert not g.trustworthy + assert g.comparisons["cell"].verdict is Verdict.REGRESSION + assert not g.should_fail, "an untrustworthy gate must not fail the build" + assert "A/A control failed" in g.summary + + def test_underpowered_run_cannot_report_pass(self): + rng = np.random.default_rng(23) + noisy_control = stats.compare( + *synth(np.random.default_rng(24), 1.0, n=8, sigma=0.4), + seed=24, + n_resamples=RESAMPLES, + ) + b, c = synth(rng, 1.0, n=40) + g = gate_one( + stats.compare(b, c, seed=23, n_resamples=RESAMPLES), control=noisy_control + ) + assert g.comparisons["cell"].verdict is Verdict.INCONCLUSIVE + assert not g.should_fail + + +class TestMultiplicityControl: + def test_bh_is_monotone_and_bounded(self): + raw = [0.001, 0.01, 0.03, 0.2, 0.7] + adj = stats.benjamini_hochberg(raw) + pairs = zip(adj, raw, strict=True) + assert all(a >= r - 1e-12 for a, r in pairs), "adjustment never shrinks p" + assert adj == sorted(adj), "monotone in the sorted input" + assert all(a <= 1.0 for a in adj) + + def test_bh_passes_nan_through(self): + adj = stats.benjamini_hochberg([0.01, float("nan"), 0.02]) + assert math.isnan(adj[1]) + assert all(math.isfinite(a) for a in (adj[0], adj[2])) + + def test_correction_suppresses_lone_lucky_cell(self): + # 20 pure-noise cells: without BH one of them firing is expected. + cells = {} + for i in range(20): + rng = np.random.default_rng(3000 + i) + b, c = synth(rng, 1.0) + cells[f"cell{i}"] = stats.compare(b, c, seed=i, n_resamples=RESAMPLES) + cells["control"] = quiet_control() + g = stats.apply_multiplicity_control( + cells, controls=all_under_one_control(cells) + ) + assert not g.should_fail + assert g.regressions == [] + + def test_control_is_excluded_from_the_gate(self): + rng = np.random.default_rng(30) + b, c = synth(rng, 1.0, n=40) + g = gate_one( + stats.compare(b, c, seed=30, n_resamples=RESAMPLES), control=quiet_control() + ) + assert "control" not in g.regressions + assert g.comparisons["control"].p_adjusted is None + + def test_ungated_metric_is_reported_but_cannot_fail(self): + rng = np.random.default_rng(31) + b, c = synth(rng, 1.5, n=60) + cells = { + "wall": stats.compare(b, c, seed=31, n_resamples=RESAMPLES), + "cpu": stats.compare(b, c, seed=32, n_resamples=RESAMPLES), + "control": quiet_control(), + } + g = stats.apply_multiplicity_control( + cells, gated={"wall"}, controls=all_under_one_control(cells) + ) + assert g.comparisons["cpu"].verdict is Verdict.REGRESSION + assert g.regressions == ["wall"], "cpu is reported but never gates" + + def test_empty_run_is_not_a_failure(self): + g = stats.apply_multiplicity_control({}) + assert not g.should_fail + assert g.regressions == [] diff --git a/xtest/test_benchmarks.py b/xtest/test_benchmarks.py new file mode 100644 index 000000000..0e4197455 --- /dev/null +++ b/xtest/test_benchmarks.py @@ -0,0 +1,91 @@ +"""SDK performance regression cells. + +One test per cell: an operation at a payload size, measuring the newest +installed release against the branch build on the same runner, in the same +round, in a randomized order. + +**These tests do not assert.** Each one records its raw samples and passes. +The verdict cannot be reached cell by cell: the multiplicity correction is +computed across every gated cell in the run, and the A/A control can +invalidate all of them at once. The gate therefore runs once in +``pytest_sessionfinish`` (see ``conftest.py``), which fails the session on a +confirmed regression. + +Nothing is collected here without ``--bench``; see ``conftest.py``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import NoReturn + +import pytest + +import abac +from fixtures import bench +from perf import report, runner +from perf.cells import BenchCell + +pytestmark = pytest.mark.benchmark + + +def test_sdk_performance( + bench_cell: BenchCell, + bench_config: runner.BenchConfig, + bench_arms: bench.ArmResolver, + bench_payloads: dict[str, Path], + bench_ciphertexts: bench.CiphertextFactory, + bench_budget: runner.Budget, + bench_recorder: report.BenchmarkRecorder, + attribute_default_rsa: abac.Attribute, + tmp_dir: Path, +) -> None: + """Measure one cell and record it; the session-wide gate decides. + + A cell that cannot be measured -- a missing build, two builds that would + not be doing the same work, a budget that ran out -- is skipped *and* + recorded as skipped, so that a quiet report is visibly quiet rather than + indistinguishable from a clean one. + """ + + def bail(reason: str) -> NoReturn: + bench_recorder.skip(bench_cell.id, reason) + pytest.skip(reason) + + try: + arms = bench_arms(bench_cell.sdk) + except bench.ArmSelectionError as e: + bail(str(e)) + + problem = bench.comparability_problem(arms) + if problem: + bail(problem) + + ct_file = ( + bench_ciphertexts(arms, bench_cell.payload.label) + if bench_cell.operation == "decrypt" + else None + ) + baseline, candidate = bench.build_arms( + bench_cell, + arms, + pt_file=bench_payloads[bench_cell.payload.label], + ct_file=ct_file, + tmp_dir=tmp_dir, + attr_values=attribute_default_rsa.value_fqns, + ) + + try: + result = runner.run_cell( + bench_cell.id, + baseline, + candidate, + bench_config, + deadline=bench_budget.next_deadline(), + control=bench_cell.control, + sdk=bench_cell.sdk, + ) + except runner.BudgetExhausted as e: + bail(str(e)) + + bench_recorder.record(result) diff --git a/xtest/test_sdk_commands.py b/xtest/test_sdk_commands.py new file mode 100644 index 000000000..9b01c0fc4 --- /dev/null +++ b/xtest/test_sdk_commands.py @@ -0,0 +1,149 @@ +"""Unit tests for the SDK CLI command builders in ``tdfs.py``. + +``SDK.encrypt_command`` / ``SDK.decrypt_command`` are the single definition of +the ``XT_WITH_*`` contract: both ``SDK.encrypt``/``SDK.decrypt`` and the +benchmark harness build their invocations through them. Pinning the argv and +env here means a change to that contract shows up as a failing assertion +rather than as a benchmark silently measuring a different operation than the +functional suite. + +No platform and no real SDK -- the builders only need ``cli.sh`` to exist, so +these run against a stub tree in ``tmp_path``. +""" + +from pathlib import Path + +import pytest + +import tdfs + + +@pytest.fixture +def sdk(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tdfs.SDK: + """An SDK pointing at a stub ``cli.sh`` that is never executed.""" + cli = tmp_path / "sdk" / "go" / "dist" / "main" / "cli.sh" + cli.parent.mkdir(parents=True) + cli.write_text("#!/bin/sh\nexit 0\n") + monkeypatch.chdir(tmp_path) + return tdfs.SDK("go", "main") + + +class TestEncryptCommand: + def test_positional_arguments(self, sdk: tdfs.SDK): + argv, _ = sdk.encrypt_command(Path("in.txt"), Path("out.tdf")) + assert argv == [sdk.path, "encrypt", "in.txt", "out.tdf", "ztdf"] + + def test_mime_type_defaults_on(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command(Path("in.txt"), Path("out.tdf")) + assert env == {"XT_WITH_MIME_TYPE": "application/octet-stream"} + + def test_empty_mime_type_omits_the_variable(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command(Path("in.txt"), Path("out.tdf"), mime_type="") + assert "XT_WITH_MIME_TYPE" not in env + + def test_attributes_are_comma_joined(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command( + Path("in.txt"), + Path("out.tdf"), + attr_values=[ + "https://e.com/attr/a/value/1", + "https://e.com/attr/b/value/2", + ], + ) + assert env["XT_WITH_ATTRIBUTES"] == ( + "https://e.com/attr/a/value/1,https://e.com/attr/b/value/2" + ) + + def test_empty_attribute_list_omits_the_variable(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command(Path("in.txt"), Path("out.tdf"), attr_values=[]) + assert "XT_WITH_ATTRIBUTES" not in env + + def test_assertions(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command( + Path("in.txt"), Path("out.tdf"), assert_value="[{}]" + ) + assert env["XT_WITH_ASSERTIONS"] == "[{}]" + + def test_target_mode(self, sdk: tdfs.SDK): + _, env = sdk.encrypt_command( + Path("in.txt"), Path("out.tdf"), target_mode="4.3.0" + ) + assert env["XT_WITH_TARGET_MODE"] == "4.3.0" + + def test_ecwrap_container_maps_to_ztdf_plus_a_flag(self, sdk: tdfs.SDK): + argv, env = sdk.encrypt_command( + Path("in.txt"), Path("out.tdf"), container="ztdf-ecwrap" + ) + assert argv[-1] == "ztdf", "the CLI format argument is the simple container" + assert env["XT_WITH_ECWRAP"] == "true" + + def test_target_mode_survives_ecwrap(self, sdk: tdfs.SDK): + # The XT_WITH_TARGET_MODE guard tests the *simplified* format, and + # ztdf-ecwrap simplifies to ztdf, so target mode applies to both. + _, env = sdk.encrypt_command( + Path("in.txt"), + Path("out.tdf"), + container="ztdf-ecwrap", + target_mode="4.3.0", + ) + assert env["XT_WITH_TARGET_MODE"] == "4.3.0" + assert env["XT_WITH_ECWRAP"] == "true" + + +class TestDecryptCommand: + def test_positional_arguments(self, sdk: tdfs.SDK): + argv, env = sdk.decrypt_command(Path("in.tdf"), Path("out.txt")) + assert argv == [sdk.path, "decrypt", "in.tdf", "out.txt", "ztdf"] + assert env == {}, "a plain decrypt sets no XT_WITH_* overrides" + + def test_assertion_verification_keys(self, sdk: tdfs.SDK): + _, env = sdk.decrypt_command( + Path("in.tdf"), Path("out.txt"), assert_keys="{keys}" + ) + assert env["XT_WITH_ASSERTION_VERIFICATION_KEYS"] == "{keys}" + + def test_verify_assertions_only_set_when_disabled(self, sdk: tdfs.SDK): + _, on = sdk.decrypt_command(Path("in.tdf"), Path("out.txt")) + _, off = sdk.decrypt_command( + Path("in.tdf"), Path("out.txt"), verify_assertions=False + ) + assert "XT_WITH_VERIFY_ASSERTIONS" not in on + assert off["XT_WITH_VERIFY_ASSERTIONS"] == "false" + + def test_ecwrap_flag(self, sdk: tdfs.SDK): + _, env = sdk.decrypt_command(Path("in.tdf"), Path("out.txt"), ecwrap=True) + assert env["XT_WITH_ECWRAP"] == "true" + + def test_kas_allowlist(self, sdk: tdfs.SDK): + _, env = sdk.decrypt_command( + Path("in.tdf"), + Path("out.txt"), + kasallowlist="http://localhost:8080", + ignore_kas_allowlist=True, + ) + assert env["XT_WITH_KAS_ALLOWLIST"] == "http://localhost:8080" + assert env["XT_WITH_IGNORE_KAS_ALLOWLIST"] == "true" + + def test_ecwrap_container_maps_to_ztdf(self, sdk: tdfs.SDK): + argv, _ = sdk.decrypt_command( + Path("in.tdf"), Path("out.txt"), container="ztdf-ecwrap" + ) + assert argv[-1] == "ztdf" + + +class TestDeterminism: + def test_builders_are_pure(self, sdk: tdfs.SDK): + # The benchmark builds a command once and runs it many times; a + # builder that mutated shared state would make round N differ from + # round 1 and show up as a phantom regression. + args = (Path("in.txt"), Path("out.tdf")) + kwargs = {"container": "ztdf-ecwrap", "attr_values": ["a"]} + first = sdk.encrypt_command(*args, **kwargs) + second = sdk.encrypt_command(*args, **kwargs) + assert first == second + + def test_no_side_effects_on_the_filesystem(self, sdk: tdfs.SDK, tmp_path: Path): + sdk.encrypt_command(Path("in.txt"), Path("out.tdf")) + sdk.decrypt_command(Path("in.tdf"), Path("out.txt")) + assert not (tmp_path / "out.tdf").exists() + assert not (tmp_path / "out.txt").exists() diff --git a/xtest/uv.lock b/xtest/uv.lock index 5ee4556c6..c2452db75 100644 --- a/xtest/uv.lock +++ b/xtest/uv.lock @@ -313,6 +313,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "numpy" +version = "2.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/80/db0b4559e57ec36362bedbb05530a87fafbcb6067708c946967a41d449e7/numpy-2.5.2.tar.gz", hash = "sha256:d482d171c406ae88c5b19cad3b6a1c4c5209f886ab74bc44c2c865c23f52d860", size = 20773161, upload-time = "2026-08-09T13:48:27.962Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/f8/c3b222bf075b50afd8e949a07a15c4b312a4a84bd8102a332bcd953cbbb4/numpy-2.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d787cf769c3baeb5f6235e778edb52c08dfa923789b5958f28e6450f96107cb1", size = 16885180, upload-time = "2026-08-09T13:46:03.939Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/2c1d4b1987795a92b5bbf7c24fe249ab96aa2573ab0d7604802c189d7b86/numpy-2.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:24b9dc2e3d84aa58523798805194e23e736f3f6ce2d1a5b92583ae734e6dbda8", size = 11907878, upload-time = "2026-08-09T13:46:07.045Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ee/d08226fc858044355983a6e5b94f08ff6f3969e0a2b160a4a89f0ddb3445/numpy-2.5.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:9e9413326d726c2545bfa65d2c0876871e8d8386e77f992c1d426e180bbd4323", size = 5354922, upload-time = "2026-08-09T13:46:10.04Z" }, + { url = "https://files.pythonhosted.org/packages/94/f0/6d3d933056440ebbc5e6bad92065fc6c26a48a84a36b1208580e94eea76c/numpy-2.5.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:60e902ac295855348a5ca2ea4c89108989a9f5fddfad3dfc0a8f36b10358567e", size = 6679168, upload-time = "2026-08-09T13:46:12.275Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3b/ecd49dd90033cceb2704d88ca905d4d7d89b0e8c739608754ffd325fa820/numpy-2.5.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e500dc868e9313530ce12ba470fe50ff3afe3d62993ed6eff652dacd555b65", size = 15624501, upload-time = "2026-08-09T13:46:15.322Z" }, + { url = "https://files.pythonhosted.org/packages/c7/99/461bd36dbdfac6c1c53efa370bd55a83227542d0d118f1677dbf1a3dacd5/numpy-2.5.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318b9a4c845dbea06708a29c84ee429cc3065048db34cdb799047643492050ee", size = 16713701, upload-time = "2026-08-09T13:46:18.949Z" }, + { url = "https://files.pythonhosted.org/packages/f9/9c/2b251df9e8a5d647b62b0cbc1b90a91850c1cf4859ecb532fd0b4eacff6c/numpy-2.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:34c319e2963be042673fb46570501b2f06c41924e17e3563d58646b4380dfb68", size = 16986065, upload-time = "2026-08-09T13:46:23.006Z" }, + { url = "https://files.pythonhosted.org/packages/8f/25/20de43f53ff1390534a124475055a19f01fe10c920a0fd11b8e18d6d6052/numpy-2.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f06571a052127dc1b4e8b83029b4d1b20daa2b64a31cdd181fc6bc774e9000eb", size = 18470031, upload-time = "2026-08-09T13:46:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/56/5e/0c577ca308d6da5eb79b546ba10bbe5b60148192194e2da060913b1de4f1/numpy-2.5.2-cp314-cp314-win32.whl", hash = "sha256:2cc779226e476d1e1f08c74068c419e60f41a9e0e069c92f6671d31d5c985e98", size = 6121028, upload-time = "2026-08-09T13:46:30.046Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/7bcbd5b11f94199073320410cddcbb80cee62415bfeb540874b265c2d922/numpy-2.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:7587f53dfbd5edc0f7b87c6217b4c6d2d1f2ef9c3da70bc1315e7db5f8d7ec9d", size = 12597627, upload-time = "2026-08-09T13:46:32.886Z" }, + { url = "https://files.pythonhosted.org/packages/87/bc/4d0b06fba0da90ccc75af62823cb9dcedb6c9ea0cffa058cb2c9ee773a77/numpy-2.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:3e4c367352d3747784248a227fbec218e193b56f7e6692e3b64fc805478ecfdf", size = 10680414, upload-time = "2026-08-09T13:46:36.036Z" }, + { url = "https://files.pythonhosted.org/packages/cd/17/f429aac9dc08833a0d0f188eba38c532a751b1a1f2ca6018a37b455cb321/numpy-2.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b879fb674276e331513fb136b78dbc6bd3c848309e0d841cfd63be3896c4cfc1", size = 12026967, upload-time = "2026-08-09T13:46:39.084Z" }, + { url = "https://files.pythonhosted.org/packages/ca/9f/d0849de96a2a4ceaa16662f18ee13eaa9c0aa418269fdc8c4857c56b11da/numpy-2.5.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:fd0d703772bba096843785bd38371e31bb4a0c1151497ad5739d182114a73f7f", size = 5473874, upload-time = "2026-08-09T13:46:42.075Z" }, + { url = "https://files.pythonhosted.org/packages/89/3c/8df216d4a4a5422a3de045301cf7df8ea47286d76f5cb7160b0128ac26b7/numpy-2.5.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:3a2f061cebd9e3d23bdcfaaded5e2293a4c6a5b60fa42df85d410a725ce621bf", size = 6789276, upload-time = "2026-08-09T13:46:44.387Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3a/20d7e9891c4ddfadd6ff8d95bf4b29f353d8e1770553de2099880551dfb9/numpy-2.5.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6df895598c0edcb41030126c89e0f353b07d93238116143b7405e937359736c4", size = 15659154, upload-time = "2026-08-09T13:46:47.538Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d6/f3aa3d2688bf501b858835c6bd087ae9b51a56ae6fca8e2b0990abd177af/numpy-2.5.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ab3d4a901f844ea836c3e80bf463c6a27d7f3c14e8e292fcf28d348b25b9bce", size = 16748909, upload-time = "2026-08-09T13:46:51.442Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8f/1c5cae8d2baf86ab802ae97a00be55bc7e21ebc11b12bbc33376c5f05342/numpy-2.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cebc2d6dbb605a7703d59751dea4bd6b0ab127a5a4338a6f432df1936fef8b26", size = 17027685, upload-time = "2026-08-09T13:46:55.095Z" }, + { url = "https://files.pythonhosted.org/packages/5c/27/71d3467404aedc1c24ce79610f91b52b0b0f466c43a701aa56fc75c145ab/numpy-2.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eaca7ff36f0f52e2111ec71f169d8fd3e889e7ddc0d2592e0d703fd8d3ce8fac", size = 18501181, upload-time = "2026-08-09T13:46:59.09Z" }, + { url = "https://files.pythonhosted.org/packages/14/2f/42921d27c40aea7e077f4a423ae509fd9220b028cd787bafefd8ab2b3a5f/numpy-2.5.2-cp314-cp314t-win32.whl", hash = "sha256:ddf47472af2e4280d79bac82304f5e80150211f1b9e614b760061d5fdfbb6eba", size = 6271085, upload-time = "2026-08-09T13:47:01.903Z" }, + { url = "https://files.pythonhosted.org/packages/75/e6/bad5f5d56de9b1971bac959963dda276d35c40f1854475005434bbe08692/numpy-2.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:44ef9675d908e65f9953063837c3277730f3f4437615a4cdab67b366cabaf884", size = 12787971, upload-time = "2026-08-09T13:47:04.963Z" }, + { url = "https://files.pythonhosted.org/packages/df/05/f608795cb34391acd67e38d94a3c36abd8d8576293a3a80727d7595c372c/numpy-2.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:eaa088384c46f519dacb93b7ec483a6d6b19a4a2085ae4f25ab9b1c43d387d1e", size = 10750306, upload-time = "2026-08-09T13:47:07.976Z" }, + { url = "https://files.pythonhosted.org/packages/33/c6/28de0191c5f82b7d42a0a51390ba98587048aa93a39fafb05bdbe6e8d00c/numpy-2.5.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:078f9b027b478c9379b9677babbf0f8b8f1ecfada27636d7b9a93990c638739f", size = 16885274, upload-time = "2026-08-09T13:47:11.439Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d1/973ca116000d244897e468ea1aff30b589e5022e3c8744b71706fe33bd57/numpy-2.5.2-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:50a68f4bacd8a2b33d8da3d2269d0d78500f86ea582e4786dc10f5ef2c2c6842", size = 11907846, upload-time = "2026-08-09T13:47:15.128Z" }, + { url = "https://files.pythonhosted.org/packages/78/d9/8c4b3937ef204cb2fd88d389ccd0f265a2ffb11f35a01d2064cf46714bd6/numpy-2.5.2-cp315-cp315-macosx_14_0_arm64.whl", hash = "sha256:e79aba74ffaf5f78a050d777c184cddf8fdffabab38acf5f3ef1fecbc17895d6", size = 5354892, upload-time = "2026-08-09T13:47:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/b6ee65ea2999fdb7023935e108e6fb776ee4082aa15f159acfa857e578c8/numpy-2.5.2-cp315-cp315-macosx_14_0_x86_64.whl", hash = "sha256:9a0731745a72a184490a582fb4af2533512bd071ace67785b5fdffc0ae58dce8", size = 6679309, upload-time = "2026-08-09T13:47:20.456Z" }, + { url = "https://files.pythonhosted.org/packages/43/f3/acb18d8b137a393c8e7803a8c994c9e64bde3930692a69d826993113a159/numpy-2.5.2-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ec954036759bcee3aa484f8603bd9c14f3e776293b85578b8734c2d72777c69", size = 15625850, upload-time = "2026-08-09T13:47:24.365Z" }, + { url = "https://files.pythonhosted.org/packages/a9/bf/a8e9bb0db815a0e265b5744ebedd3af0bd5faad8604e5b50a1cd012f3c91/numpy-2.5.2-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc649493697006bc90614a5f0bbc8cb3cb1866715c474e473694968d7e6b99ab", size = 16713664, upload-time = "2026-08-09T13:47:27.965Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c3/6e913736b3dd6582344af32418b5fb9dab34282e8a8174ae1d54ceb0fc13/numpy-2.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:cf7de32f486e4ac9e2d93b810f9e9ac72a728dd46a32a0bb403222f27f653514", size = 16986749, upload-time = "2026-08-09T13:47:31.541Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/7d3b23eff5c7428ef6c01e6f7052bb60d504c4d33e317b36b8959c24ad97/numpy-2.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2ffa7bacab3e2ee1b19ed31766bb60bb380b68c23f051e199c5cc598afd68710", size = 18470495, upload-time = "2026-08-09T13:47:35.364Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a4/68a321d825374f6eb677ffe8ef8c6b9a328304e6fd2e39d9530822776607/numpy-2.5.2-cp315-cp315-win32.whl", hash = "sha256:6b588cc8f902d6bff201c19fd00c43ab8545671e3554d014e12e14139e5e8617", size = 6120696, upload-time = "2026-08-09T13:47:38.561Z" }, + { url = "https://files.pythonhosted.org/packages/c8/23/deafbb1700f79fae9cd1e91220f133d124cc267de1b584da3fbf6db2f6cd/numpy-2.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:07d4e89f3a9ab0a9ba24264ccdb642b3dd951b2281e8883a5481a4aa79cc31a7", size = 12597324, upload-time = "2026-08-09T13:47:41.401Z" }, + { url = "https://files.pythonhosted.org/packages/33/cd/3272ba105e3bbbdaeb11357eda31e7a6825ffe159e8171665660299a948f/numpy-2.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:a610dc7e3c52edd39c2bc2375ff9c3fd59cb3ad00e4472d36f83bc1457145788", size = 10680466, upload-time = "2026-08-09T13:47:44.873Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/58370637b1bb70a5c9ce2b43f4b521ccb224e36ccb76a6596b17ae4b447c/numpy-2.5.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:40f4d451aed46a8046a1aae41c4e55fb3612273df9c502480135e1501576a34b", size = 16993947, upload-time = "2026-08-09T13:47:48.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/93/2abcb807712b289d6d60fe4cf30532f98974a8396d885650f3ba5a13026e/numpy-2.5.2-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c081cbe16ba1ab53078e5ff29013621e33c509eedab055775d956427712c236e", size = 12025331, upload-time = "2026-08-09T13:47:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3a/2898e003a5fbaf87e76c039b4ee1f5eb390471b4ffe74887c1f34c4e791e/numpy-2.5.2-cp315-cp315t-macosx_14_0_arm64.whl", hash = "sha256:0090ccdd57ec2703e9b49d0bf554767370581c1dd0a6b2bb2b2d9def317d042a", size = 5472336, upload-time = "2026-08-09T13:47:55.403Z" }, + { url = "https://files.pythonhosted.org/packages/61/a5/23f69d07c544597b29758b31b55c27dc9d541012a2c1496189fef702aec2/numpy-2.5.2-cp315-cp315t-macosx_14_0_x86_64.whl", hash = "sha256:6a9bb119fb8dd21ba30b3f0e555b7e2b081bd9883af21ec9c1c633d161cda3a8", size = 6788387, upload-time = "2026-08-09T13:47:58.192Z" }, + { url = "https://files.pythonhosted.org/packages/15/ea/c0dbdbcf22f43782510a3e492dd3da73c6112b69cac8929d16d127536fc4/numpy-2.5.2-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a839318485284a6fb31be4f8f2c91c8f2cb22f4543c4a8903f12b0671ffe07cc", size = 15667096, upload-time = "2026-08-09T13:48:01.562Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5e/29c73c31748cdb0f7566642125ba17fd5b56780cddf891b085dab27e4466/numpy-2.5.2-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba0a474801b8dc67b66bf465548abc90e82b44d2611b5770f33008dcabffe8ec", size = 16751730, upload-time = "2026-08-09T13:48:05.706Z" }, + { url = "https://files.pythonhosted.org/packages/47/95/02501e8454796bb58dadf7a99d3181e0b464bf264e1003039572f9779fac/numpy-2.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0a4035ae1129ff8777f08bfbd44f1e5d8e9c049ce0c2dd78fc0d92c13e7251c0", size = 17038686, upload-time = "2026-08-09T13:48:09.627Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b5/53a681d91b5c82687067d8ea5035e02d917b5509d6f334cb06484a954714/numpy-2.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:77843ca236b777e67f8d6b3660ea116e499612703a0ecd7093f316201eb9d8e2", size = 18507727, upload-time = "2026-08-09T13:48:13.744Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/6e11443f7b64ee376c860506091103bf68f92d2cab9e8d96d4501babf07c/numpy-2.5.2-cp315-cp315t-win32.whl", hash = "sha256:7354826bc6f8f69402e9b7fe28d15fcd34feebd74f856f111585c5b0c9fb0251", size = 6269775, upload-time = "2026-08-09T13:48:17.543Z" }, + { url = "https://files.pythonhosted.org/packages/f1/18/195d6b86cd72dbbc501edfa778005fa6b87afd34c153e46028cd3a0938f4/numpy-2.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:e5651f3f87add730ee6608d915009e19c911fba0cb000c7e3ea994b7d768eb12", size = 12782559, upload-time = "2026-08-09T13:48:21.023Z" }, + { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -592,6 +643,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/19/016553f86f207450aebebc2b2b5088d086b901cc8186c02ac4284db3bd88/ruff-0.15.16-py3-none-win_arm64.whl", hash = "sha256:8cd61783afb39638a7133ef0d2dfb1e91277593962f81b5a8423eb0b888a6121", size = 11134555, upload-time = "2026-06-04T16:33:00.136Z" }, ] +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + [[package]] name = "smmap" version = "5.0.3" @@ -652,6 +734,7 @@ dependencies = [ { name = "jsonschema" }, { name = "jsonschema-specifications" }, { name = "markupsafe" }, + { name = "numpy" }, { name = "packaging" }, { name = "pluggy" }, { name = "pycparser" }, @@ -665,6 +748,7 @@ dependencies = [ { name = "referencing" }, { name = "requests" }, { name = "rpds-py" }, + { name = "scipy" }, { name = "smmap" }, { name = "typing-extensions" }, { name = "urllib3" }, @@ -694,6 +778,7 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.25.1" }, { name = "jsonschema-specifications", specifier = ">=2025.9.1" }, { name = "markupsafe", specifier = ">=3.0.3" }, + { name = "numpy", specifier = ">=2.5.2" }, { name = "packaging", specifier = ">=26.2" }, { name = "pluggy", specifier = ">=1.6.0" }, { name = "pycparser", specifier = ">=3.0" }, @@ -709,6 +794,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.34.2" }, { name = "rpds-py", specifier = ">=2026.5.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.15.16" }, + { name = "scipy", specifier = ">=1.18.0" }, { name = "smmap", specifier = ">=5.0.3" }, { name = "typing-extensions", specifier = ">=4.15.0" }, { name = "urllib3", specifier = ">=2.7.0" }, From a54102626fbe65bd27021f52dccc092192363f83 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 14 Aug 2026 13:22:52 -0400 Subject: [PATCH 3/4] docs(xtest): explain the benchmark harness in perf/README.md --- xtest/AGENTS.md | 1 + xtest/perf/README.md | 399 +++++++++++++++++++++++++++++++++++++++++ xtest/perf/__init__.py | 4 + 3 files changed, 404 insertions(+) create mode 100644 xtest/perf/README.md diff --git a/xtest/AGENTS.md b/xtest/AGENTS.md index 7b1e7b899..04f02dc12 100644 --- a/xtest/AGENTS.md +++ b/xtest/AGENTS.md @@ -15,6 +15,7 @@ fixture system. | `conftest.py` | `pytest_addoption` + the encrypt/decrypt SDK parametrization. Defines `--sdks`, `--sdks-encrypt`, `--sdks-decrypt`, `--containers`, `--no-audit-logs`. | | `fixtures/` | Module-scoped pytest fixtures: `attributes.py`, `keys.py`, `audit.py`, `assertions.py`, `kas.py`, `encryption.py`, `obligations.py`. | | `tdfs.py` | SDK abstraction layer — wraps the `cli.sh` shims under `sdk//dist//`. | +| `perf/` | Paired A/B performance regression benchmarks (opt-in via `--bench`). **Read `perf/README.md` before changing anything in here** — the design decisions fail silently when undone. | | `sdk/{go,java,js}/dist//` | SDK CLI builds. Installed by `otdf-sdk-mgr install` (see `../otdf-sdk-mgr/AGENTS.md`). | | `test.env` | Default endpoint and client-credential env vars. Source with `set -a && source test.env && set +a`. | diff --git a/xtest/perf/README.md b/xtest/perf/README.md new file mode 100644 index 000000000..9eb0c7d4b --- /dev/null +++ b/xtest/perf/README.md @@ -0,0 +1,399 @@ +# SDK performance regression benchmarks + +A paired A/B benchmark for the OpenTDF SDK CLIs. It answers one question: +**did this change make the SDK measurably and meaningfully slower?** + +Two builds — normally the newest installed release and the branch build — +are measured on the *same* runner, interleaved round by round, and only their +*ratio* is reported. Nothing is ever compared against a stored historical +number. + +It runs nightly (one runner per SDK) and on `workflow_dispatch` with +`run-benchmarks` checked. It never runs on pull requests: 30 minutes of serial +measurement is too slow for a PR gate, and a PR runner is the noisiest place to +measure. + +> **Dispatching it by hand:** set the `*-ref` inputs to `main latest`, not the +> default `main`. The nightly cron resolves `main latest` on its own, but an +> explicit `main` installs only the branch build — no release to use as a +> baseline — and every cell skips. The run fails rather than passing empty +> (see [NOTHING MEASURED](#the-verdicts)), but it will have wasted 45 minutes +> to tell you that. + +- **Section 1 — [Reading a result](#1-reading-a-result)** is for developers on + the SDKs and the platform: your build got flagged, what does that mean. +- **Section 2 — [Maintaining the harness](#2-maintaining-the-harness)** is for + whoever changes this code: how it works and why it is shaped this way. + +--- + +## 1. Reading a result + +### Where the output is + +| Artifact | Where | Contents | +| --- | --- | --- | +| Job summary | The Actions run page | The table below, plus the verdict | +| `bench-` artifact | Run artifacts | `.json` with **every raw per-round sample**, and an HTML report | +| Terminal | Job log tail | One-line summary and the JSON path | + +The JSON is the useful one. It holds each cell's full per-round vectors for both +arms, so a surprising verdict can be re-analysed offline instead of by re-running +a 30-minute job to look at the same numbers again. + +### The table + +``` +| cell | metric | baseline | candidate | ratio (95% CI) | p (BH) | n | verdict | +| go-encrypt-1MiB | wall clock | 412.3 ms | 498.1 ms | 1.208x [1.171, 1.245] | <0.001 | 22 | **REGRESSION** | +``` + +- **cell** — `--`, plus `-control` for the A/A cell. + Payload sizes are 1 KiB, 1 MiB, and 32 MiB. +- **ratio** — candidate ÷ baseline. `1.208x` means the candidate took 20.8% + longer. Below 1.0 means faster. +- **95% CI** — the bootstrap interval on that ratio. Its *width* is how precisely + this run could measure; a wide interval means a noisy runner, not a big change. +- **p (BH)** — one-sided p-value, Benjamini–Hochberg adjusted across the run. +- **n** — paired rounds actually measured (20–60; the loop stops early once the + interval is narrow enough). + +### The verdicts + +**REGRESSION** — the CI lower bound exceeds the threshold (default **1.15x**, +i.e. 15% slower) *and* the adjusted p < 0.05. Both clauses are required, and +neither is redundant: the threshold alone would fire on a reproducible 0.5% +slowdown nobody cares about, and significance alone would fire on noise often +enough to be ignored within a week. This fails the job. + +**PASS** — not a regression, *and* the run had enough precision to have found +one. "We looked and found nothing" only counts when we could have found +something. + +**IMPROVED** — the same test in the other direction. Never fails anything. + +**inconclusive** — the run could not decide. Common reasons, all shown in the +note beside the verdict: +- the runner was too noisy for this cell's interval to be usable; +- the A/A noise floor was wider than the 15% effect being gated on, so a real + regression of that size could not have been distinguished from noise; +- peak RSS hit the measurement floor (see below); +- too few rounds completed inside the time budget. + +**Inconclusive is not a pass.** It means the question was not answered. If a +change you expect to be performance-sensitive comes back inconclusive on every +cell, the run told you nothing and re-running it is reasonable. + +**NOTHING MEASURED** — no cell produced a comparison at all, usually because +only one build was installed so there was no baseline to compare against. This +**fails the job**. An empty run and a clean run have the same empty list of +regressions, so without this a benchmark that had quietly stopped measuring +would keep reporting a green tick. The "Not measured" section of the report +lists the reason for each cell. + +### The A/A control + +Each SDK gets a control cell that compares the baseline build **against itself** +through the identical pipeline. Its true ratio is exactly 1.0 by construction, so +whatever it reports is the harness's own error on this runner. It does two jobs: + +- If the control *trips* — its own A/A comparison looks like a real effect — then + something is systematically biased and **the whole run stops being able to fail + the build**. Results are still reported, marked untrustworthy. +- Its interval width is the run's **noise floor**: the smallest effect this + runner could have resolved. If the floor is wider than the threshold, cells + report inconclusive rather than PASS. + +In a multi-SDK run each SDK is judged against *its own* control — go's harness +path says nothing about java's. `noise_floor_by_control` in the JSON has each +one; the top-level `noise_floor` is the worst of them. + +### Gated vs ungated metrics + +| Metric | Gated? | Why | +| --- | --- | --- | +| wall clock | yes | What users experience | +| peak RSS | yes | Regressions here are real and invisible in timing | +| CPU time | **no** | Noisiest of the three on a shared runner, and a real CPU regression shows up in wall clock anyway | + +Ungated rows are labelled `(ungated)` and reported for context only. They cannot +fail the build. + +Peak RSS additionally gets **censored** when a cell's readings sit at the +measurement floor (the RSS of the process that forked the command). Both arms +clip to the same value there, producing a `1.000x` ratio with a tight interval — +the most convincing-looking PASS the harness can emit, and completely meaningless. +Censored cells report inconclusive with the floor named in the note. + +### My build was flagged. Now what? + +1. **Read the CI column, not just the ratio.** A `1.20x [1.02, 1.41]` is a very + different claim from `1.20x [1.19, 1.21]`. +2. **Check the control row.** If the A/A cell for your SDK also looks strange, + suspect the runner before your code. +3. **Look at which cells fired.** Only the 1 KiB cells means startup cost — + process boot, package resolution, TLS handshake, token fetch. Only 32 MiB + means throughput — the crypto and IO path. Both means something structural. +4. **Reproduce locally.** The comparison is self-contained; it does not need CI. + +```bash +cd xtest && set -a && source test.env && set +a + +# whatever two builds you want, side by side under sdk//dist/ +uv run pytest --bench --sdks go \ + --bench-baseline go@v0.29.0 \ + --bench-candidate go@main \ + -v test_benchmarks.py +``` + +Useful knobs while investigating: + +| Option | Default | Use | +| --- | --- | --- | +| `--bench-threshold` | `1.15` | Smallest slowdown worth failing on | +| `--bench-min-rounds` / `--bench-max-rounds` | `20` / `60` | Rounds per cell | +| `--bench-warmup` | `5` | Discarded rounds paying one-time costs | +| `--bench-budget-seconds` | `1500` | Wall-clock allowance shared by all cells | +| `--bench-seed` | `0` | Payloads, round order, bootstrap. Fix it to reproduce | +| `--bench-out` | `test-results/benchmarks` | JSON destination | +| `--bench-no-gate` | off | Measure and report, never fail | + +A local run is noisier than CI unless the machine is otherwise idle. Close +things; the noise floor will tell you whether you succeeded. + +### What this benchmark cannot tell you + +- **Anything about absolute speed.** A number from a GitHub-hosted runner is not + comparable to a number from your laptop or from last week's runner. Only + within-run ratios mean anything. +- **Anything about trends.** There is no history and no stored baseline. Each run + is a self-contained experiment. +- **Anything about a slowdown under 15%** by default. That is the price of not + crying wolf on a shared runner. +- **Anything about your change specifically** if the baseline moved too — the + comparison is release-vs-`main`, so it catches whatever landed on `main`. + +--- + +## 2. Maintaining the harness + +### Module map + +| File | Responsibility | +| --- | --- | +| `cells.py` | The experiment matrix: payload sizes, `BenchCell`, `cells_for()`. No pytest, no `tdfs` | +| `measure.py` | Wall/CPU/RSS for one invocation, via `os.wait4` | +| `_launcher.py` | The separate process that actually forks the measured command | +| `runner.py` | The paired round loop, the stopping rule, the budget, `analyze()` | +| `stats.py` | Pure functions: log-ratios, bootstrap CI, Wilcoxon, BH, the decision rule | +| `report.py` | Session recorder, JSON artifact, step-summary markdown | +| `../fixtures/bench.py` | The pytest glue: arm selection, payloads, ciphertexts, budget | +| `../test_benchmarks.py` | One test per cell. **Records; never asserts** | +| `../conftest.py` | `--bench*` options, cell parametrization, the session-finish gate | + +Offline tests, no platform and no subprocesses needed: + +```bash +cd xtest +uv run pytest -q test_bench_stats.py test_bench_measure.py \ + test_bench_runner.py test_bench_arms.py +``` + +These run on every PR via `check.yml`, so the harness is exercised continuously +even though the benchmark itself runs nightly. + +### The design, and why + +#### Ratios within a run, never comparison against history + +CPU models vary, tenancy is shared, and steal time is unbounded on a hosted +runner. Storing a baseline and diffing against it produces false alarms until +people mute the job. Both builds are measured on the same runner and the +statistic is the within-round ratio, so runner speed is a shared factor that +divides out. + +#### Interleaved rounds, randomized within the round + +Running all of A then all of B lands every drift effect — a noisy neighbour +arriving, thermal throttling, the page cache warming — entirely on one arm, where +it reads as a difference between builds. Both arms run once per round instead. +The order *within* a round is shuffled because a fixed order is itself a +confounder: whichever arm goes second inherits the first one's cache state. + +The shuffle is seeded per cell (`f"{seed}:{cell_id}"`), so a rerun reproduces the +interleaving exactly while different cells do not share one order — which would +correlate their noise. + +#### Log-ratios + +`d_i = ln(candidate_i) - ln(baseline_i)`. Logs make ratios symmetric (a 2x +slowdown and a 2x speedup are equal and opposite) and additive, which is what +the median and the bootstrap want. Everything is exponentiated back for reporting. + +#### Stopping on precision, never on significance + +> This is the single easiest thing here to "optimize" into invalidity. + +The loop stops when the CI is narrow enough. It must never stop when the p-value +gets small. Peeking at p and stopping the moment it crosses alpha is optional +stopping: you get a fresh chance to cross the line every round and only ever stop +on the lucky side, which inflates the false-positive rate far past nominal. +Attained CI *width* is driven by the dispersion of the differences rather than +their location, so it is approximately ancillary to the effect being tested and +stopping on it does not bias the verdict. + +`_precise_enough()` therefore looks only at interval width, never at where the +interval sits. It also uses `not (width <= target)` rather than `width > target`, +because a NaN width must read as "keep going" and `NaN > target` is `False`. + +#### Both clauses of the decision rule + +A cell is a regression iff the CI lower bound exceeds `threshold` **and** the +BH-adjusted p is below alpha. Clause 1 alone fires on real-but-trivial effects +measured precisely; clause 2 alone fires on noise roughly alpha of the time per +cell, and a run has enough cells that "roughly alpha" becomes "most nights". + +#### Separate BH families + +Gated keys are corrected as their own family. Ungated metrics get a family of +their own so they still carry a reportable verdict. Adjusting the gated metrics +against metrics nobody gates on would only make a real regression harder to +confirm. Controls and censored keys are excluded from correction entirely — an +A/A cell is not a hypothesis about the candidate. + +#### One A/A control per SDK, running first + +A control measures a particular SDK's harness path. `cells_for()` emits each +SDK's control first, because a run that overruns its budget loses whatever is at +the end: losing one comparison leaves the rest trustworthy, losing the control +leaves nothing trustworthy, since without a noise floor no cell may report PASS. + +`GateResult.noise` is the *worst* control in the run, not the average. A single +tripped control means the harness may be biased on this runner, and averaging +that away with two quiet ones is exactly the reassurance the control exists to +withhold. + +#### Measurement isolation (`_launcher.py`) + +On Linux a forked child inherits the parent's resident-set accounting and +`execve` does not clear it, so `ru_maxrss` comes back as +`max(child's true peak, parent's RSS at fork time)`. Measured from a pytest +process holding numpy, scipy and a session of samples, every invocation would +report *pytest's* ~165 MiB instead of its own — a stable `1.000x` ratio that +reads as "no regression". + +`posix_spawn` and `sh -c 'exec ...'` do **not** help; both were measured and both +inherit the same floor, because an exec is too late. The only fix is to fork from +a process holding nothing, which is all `_launcher.py` is for. It reports its own +RSS as the floor alongside each reading, which is what powers censoring. + +Two things in that file look wrong and are not: +- `except BaseException` in the forked child — letting a `SystemExit` or + `KeyboardInterrupt` unwind past there would run the *parent's* atexit handlers + and flush its buffers a second time, from a process that exists only to exec. +- `os.killpg(..., SIGKILL)` on timeout — signalling the group is the point; + leaving a wedged JVM behind would hold the runner until the job timeout. + +`os.wait4` rather than `resource.getrusage(RUSAGE_CHILDREN)`, because the latter +is a process-lifetime high-water mark: once one big child has run, every later +delta reads zero. + +#### Everything except the build is pinned + +Both arms get the same plaintext, the same attribute (explicit RSA, so an arm +does not silently switch to EC), the same container, and the same target mode. +`comparability_problem()` refuses the comparison outright when the two builds +disagree on `hexless`, `hexaflexible`, or `autoconfigure` — a timing difference +there is a difference in *work*, not in speed. + +For decrypt, both arms read one ciphertext produced by the baseline. If each arm +decrypted its own output, a difference in how the two builds *write* a TDF would +show up as a difference in how fast they read one. + +#### Baselines must be final releases + +`SDK.is_released()` accepts `v0.29.0-rc.1`, and `semver()` parses it to the same +`(0, 29, 0)` as the final release — so ordering by semver alone leaves them tied +and the directory listing breaks the tie. That is a baseline nobody chose, and it +differs run to run. Baseline selection uses `is_final_release()`, which matches +only a plain `vX.Y.Z`. + +#### Payloads are seeded per payload, not per run + +`tmp_dir` persists between runs. With one RNG stream shared across the payloads, a +partially-cached set skips some `randbytes` calls and shifts the stream for every +payload after it — so a rerun measures different bytes than the run it claims to +be comparable with. Each payload derives from `f"{seed}:{label}"` instead. + +Content is random rather than repetitive because compressible input would let an +SDK that happens to compress look faster for reasons unrelated to crypto. + +#### Cells record; the session gates + +The verdict cannot be reached cell by cell — the multiplicity correction spans +the run and the A/A control can invalidate all of it at once. So +`test_sdk_performance` never asserts. `pytest_sessionfinish` runs `analyze()` +once, writes the artifacts **unconditionally and before gating** (a run about to +fail is exactly the one whose raw numbers someone wants), and only then sets the +exit status. + +A cell that cannot be measured is skipped *and* recorded as skipped, so a quiet +report is visibly quiet rather than indistinguishable from a clean one. If +*every* cell skips, `GateResult.nothing_measured` fails the run: `--bench` is an +explicit request for a measurement, and answering it with a green tick and an +empty table is the one outcome nobody inspects. + +The bench job installs `go` on every runner even when it is not the SDK under +measurement, because `otdfctl` provisions the attributes and KAS registry that +every cell needs and `conftest.py` loads it at import time. `OTDFCTL_HEADS` must +name *go's* head, not the matrix SDK's. + +#### Collection and isolation + +Benchmark cells are **deselected** without `--bench`, via +`pytest_collection_modifyitems` and the `benchmark` marker. They are not +parametrized over an empty list — `empty_parameter_set_mark` would turn that into +one *skipped* item per test, which reads as a benchmark nobody asked for. + +`--bench` refuses to run under `pytest-xdist`. Parallel workers contend for the +CPU under measurement. + +### Adding to it + +**A new payload size** — add a `Payload` to `PAYLOADS` in `cells.py`. Note that +`CONTROL_PAYLOAD = PAYLOADS[1]`, so inserting at the front moves the control. +Cell count per SDK is `1 + 2 × len(PAYLOADS)`; the 1500s budget is divided +across all of them, so adding sizes makes every cell poorer unless the budget +grows too. + +**A new metric** — add it to `METRICS` and `METRIC_LABELS` in `measure.py`, teach +`Sample.metric()` and `format_metric()` about it, and decide whether it belongs +in `BenchConfig.gated_metrics`. Default to ungated until it has shown a usable +noise floor over several nights. + +**A new operation** — extend `operation_type` and `cells_for()` in `cells.py`, +then handle it in `build_arms()` in `fixtures/bench.py`. If it needs an input +produced by the baseline, follow `CiphertextFactory`: build it once, from the +baseline only, and share it between the arms. + +**A new SDK** — nothing here needs to change; it comes from `--sdks` and the +matrix in `xtest.yml`. + +**A new comparability hazard** — add the feature name to +`_COMPARABILITY_FEATURES`. Cheap to add, and the failure it prevents (comparing +two builds doing different amounts of work) is invisible in the output. + +### Invariants — do not break these + +1. Never stop the round loop on a p-value. +2. Never compare against a stored historical number. +3. Never let a cell assert; the gate is run-level. +4. Never report PASS without a noise floor establishing the run had the power to + fail. +5. Never let the two arms differ in anything but the build. +6. Never run the measured command from a process holding memory. +7. Never run the benchmark in parallel with anything, including itself. +8. Never let a run that measured nothing report success. + +Every one of these fails *silently* and *plausibly* when broken: the numbers +still look like numbers. That is why they are written down. diff --git a/xtest/perf/__init__.py b/xtest/perf/__init__.py index 5ff6e1479..2def84a11 100644 --- a/xtest/perf/__init__.py +++ b/xtest/perf/__init__.py @@ -9,4 +9,8 @@ - ``stats``: the paired statistical comparison and its decision rule. - ``runner``: the round loop that produces paired samples. - ``report``: JSON artifacts and GitHub step-summary markdown. + +``README.md`` in this directory covers how to read a result and why the harness +is shaped the way it is. Read it before changing anything here: most of the +design decisions fail silently and plausibly when undone. """ From 3481193e5426fa89d220a8f904a8acd5a0b368f5 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 14 Aug 2026 13:22:52 -0400 Subject: [PATCH 4/4] fix(xtest): fail benchmark runs that measure nothing, and give every bench runner otdfctl Two bugs the first dispatched benchmark run exposed. A run where every cell skipped reported "No regressions" and exited 0: an empty run and a clean run have the same empty regression list, so a benchmark that has quietly stopped measuring can pass indefinitely. GateResult grows a nothing_measured property, the summary says NOTHING MEASURED instead of describing a noise floor it never established, and pytest_sessionfinish fails the run. The bench job only installed the SDK under measurement, but conftest.py loads otdfctl at import time to provision attributes and the KAS registry, so the java and js runners died during collection on a missing sdk/go/dist/main/ otdfctl.sh. Every bench runner now configures and builds go for otdfctl, and OTDFCTL_HEADS points at go's heads rather than the matrix SDK's. --- .github/workflows/xtest.yml | 40 +++++++++++++++++++++++++++++++++---- xtest/conftest.py | 7 ++++++- xtest/perf/README.md | 9 +++++++++ xtest/perf/stats.py | 21 +++++++++++++++++++ xtest/test_bench_stats.py | 21 +++++++++++++++++-- 5 files changed, 91 insertions(+), 7 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 740a3648d..5488e9274 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -37,7 +37,7 @@ on: required: false type: boolean default: false - description: "Run the SDK performance regression benchmarks (adds ~45m per SDK)" + description: "Run the SDK performance regression benchmarks (adds ~45m per SDK). Needs two builds per SDK: set the *-ref inputs to 'main latest', since a bare 'main' installs no release to use as a baseline and every cell will skip." workflow_call: inputs: platform-ref: @@ -838,8 +838,10 @@ jobs: with: node-version: "22.x" + # Not gated on matrix.sdk: every bench runner needs otdfctl now, and + # leaving these outputs empty on the java and js runners would send + # setup-cli-tool off to make its own platform checkout to build it from. - name: Capture platform otdfctl location - if: matrix.sdk == 'go' id: platform-otdfctl run: |- if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then @@ -870,8 +872,25 @@ jobs: platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} + # otdfctl provisions the attributes and KAS registry every cell needs, + # whichever SDK is under measurement, and conftest.py loads it at import + # time. The go runner already has it from the step above; without this + # the java and js runners fail during collection, before a single + # measurement is taken. + - name: Configure otdfctl + id: configure-otdfctl + if: matrix.sdk != 'go' + uses: ./otdftests/xtest/setup-cli-tool + with: + path: otdftests/xtest/sdk + sdk: go + version-info: "${{ needs.resolve-versions.outputs.go }}" + platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} + platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} + + # Unconditional: every bench runner builds go now, either as the SDK + # under measurement or as otdfctl. - name: Cache Go modules - if: matrix.sdk == 'go' uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 with: path: | @@ -921,6 +940,14 @@ jobs: BUF_INPUT_HTTPS_USERNAME: opentdf-bot BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + - name: Build otdfctl + if: matrix.sdk != 'go' && fromJson(steps.configure-otdfctl.outputs.heads)[0] != null + run: make + working-directory: otdftests/xtest/sdk/go + env: + BUF_INPUT_HTTPS_USERNAME: opentdf-bot + BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + ######## MEASURE ############# # --locked --no-build: install exactly what uv.lock pins, and run no # setup scripts doing it. A benchmark that measured a differently @@ -949,7 +976,12 @@ jobs: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" SCHEMA_FILE: "manifest.schema.json" PLATFORM_TAG: main - OTDFCTL_HEADS: ${{ steps.configure-sdk.outputs.heads }} + # go's heads, not the matrix SDK's: conftest reads this to locate + # otdfctl under sdk/go/dist//, so pointing it at java's or + # js's head names a directory that does not exist. + OTDFCTL_HEADS: >- + ${{ matrix.sdk == 'go' && steps.configure-sdk.outputs.heads + || steps.configure-otdfctl.outputs.heads }} # The benchmark never touches the audit-log fixture; asserting on # logs would also add file IO to the measured path. DISABLE_AUDIT_ASSERTIONS: "1" diff --git a/xtest/conftest.py b/xtest/conftest.py index 903c54724..5604bf0c1 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -425,7 +425,12 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int): if config.getoption("--bench-no-gate", default=False): return - if gate.should_fail: + # A run that measured nothing fails too, and not only one that found a + # regression. --bench is an explicit request for a measurement; answering + # it with a green tick and an empty table is the one outcome nobody + # inspects, so a benchmark that has quietly stopped measuring can survive + # indefinitely. Every reason a cell skips is already in the report. + if gate.should_fail or gate.nothing_measured: session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/xtest/perf/README.md b/xtest/perf/README.md index 9eb0c7d4b..49d933d90 100644 --- a/xtest/perf/README.md +++ b/xtest/perf/README.md @@ -91,6 +91,15 @@ regressions, so without this a benchmark that had quietly stopped measuring would keep reporting a green tick. The "Not measured" section of the report lists the reason for each cell. +> One cause looks like a bug and is not. If an SDK's newest release tags the same +> commit as `main` — java sat at `v0.18.0 == main == dev == 57d070b0` through +> August 2026 — then `main latest` resolves both arms to one SHA, `otdf-sdk-mgr` +> installs a single build, and every cell skips with *"no final release to compare +> against; installed: main"*. The message is true from where the harness stands, but +> the release it is looking for does exist; the two arms are just the same code. +> Check with `otdf-sdk-mgr versions resolve main latest` — one entry back +> instead of two means there is nothing to measure until `main` moves. + ### The A/A control Each SDK gets a control cell that compares the baseline build **against itself** diff --git a/xtest/perf/stats.py b/xtest/perf/stats.py index 2f3e49255..c7bb71576 100644 --- a/xtest/perf/stats.py +++ b/xtest/perf/stats.py @@ -368,6 +368,19 @@ class GateResult: def should_fail(self) -> bool: return self.trustworthy and bool(self.regressions) + @property + def nothing_measured(self) -> bool: + """True if the run produced no comparisons at all. + + Not the same thing as "no regressions", though the two are identical + from the outside: both have an empty ``regressions`` list. A run where + every cell was skipped -- no baseline installed, an SDK that would not + build -- reports the cheerful summary of a clean one, which is how a + benchmark that quietly stopped measuring anything survives for months. + Callers gate on this separately. + """ + return not self.comparisons + def apply_multiplicity_control( comparisons: dict[str, PairedComparison], @@ -519,6 +532,14 @@ def _verdict_for( def _summarize(result: GateResult, noise: NoiseFloor, threshold: float) -> str: + if result.nothing_measured: + # Before the noise check: with nothing measured there is no control + # either, and "the A/A control failed" would misdescribe a run that + # never got as far as running one. + return ( + "NOTHING MEASURED: no cell produced a comparison, so this run says " + "nothing about performance either way." + ) if noise.tripped: return ( f"INCONCLUSIVE: the A/A control failed its own comparison. {noise.detail}. " diff --git a/xtest/test_bench_stats.py b/xtest/test_bench_stats.py index 20f077da0..71a94d9ae 100644 --- a/xtest/test_bench_stats.py +++ b/xtest/test_bench_stats.py @@ -338,7 +338,24 @@ def test_ungated_metric_is_reported_but_cannot_fail(self): assert g.comparisons["cpu"].verdict is Verdict.REGRESSION assert g.regressions == ["wall"], "cpu is reported but never gates" - def test_empty_run_is_not_a_failure(self): + def test_empty_run_reports_no_regressions(self): g = stats.apply_multiplicity_control({}) - assert not g.should_fail + assert not g.should_fail, "nothing measured is not a regression" assert g.regressions == [] + + def test_empty_run_says_it_measured_nothing(self): + # The dangerous case: an empty run and a clean run have the same empty + # regression list, so without this the report of a benchmark that + # skipped every cell is indistinguishable from one that passed. + g = stats.apply_multiplicity_control({}) + assert g.nothing_measured + assert "NOTHING MEASURED" in g.summary + assert "no regressions" not in g.summary.lower() + + def test_a_run_with_comparisons_measured_something(self): + rng = np.random.default_rng(33) + b, c = synth(rng, 1.0, n=40) + g = gate_one( + stats.compare(b, c, seed=33, n_resamples=RESAMPLES), control=quiet_control() + ) + assert not g.nothing_measured