diff --git a/src/backend/services/analytics/Cargo.toml b/src/backend/services/analytics/Cargo.toml index 4c7d673c1..5a0f8b46b 100644 --- a/src/backend/services/analytics/Cargo.toml +++ b/src/backend/services/analytics/Cargo.toml @@ -83,3 +83,6 @@ tower = { version = "0.5", features = ["util"] } # `clickhouse::test::Mock` for exercising ClickHouse read paths (the # contract-version sweep) against an in-process server in unit tests. clickhouse = { workspace = true, features = ["test-util"] } +# Paused-time `#[tokio::test]` for the semaphore acquire-timeout refusals — +# the 2 s acquire window elapses instantly instead of being slept through. +tokio = { workspace = true, features = ["test-util"] } diff --git a/src/backend/services/analytics/src/api/metric_drilldown.rs b/src/backend/services/analytics/src/api/metric_drilldown.rs index 1372cfb7e..7000c2fbd 100644 --- a/src/backend/services/analytics/src/api/metric_drilldown.rs +++ b/src/backend/services/analytics/src/api/metric_drilldown.rs @@ -152,6 +152,20 @@ async fn acquire_export_permit() -> Result .map_err(|_| export_busy()) } +async fn acquire_query_permit() -> Result, CanonicalError> { + tokio::time::timeout(QUERY_ACQUIRE_TIMEOUT, QUERY_SEMAPHORE.acquire()) + .await + .map_err(|_| { + tracing::warn!( + capacity = MAX_CONCURRENT_QUERIES, + available = QUERY_SEMAPHORE.available_permits(), + "metric drilldown query capacity exhausted" + ); + query_busy() + })? + .map_err(|_| query_busy()) +} + async fn collect_export_rows( state: &Arc, validated: &ValidatedMetricDrilldown, @@ -166,12 +180,17 @@ async fn collect_export_rows( .map_err(|_| export_limit("Export exceeded the execution time limit."))??; verify_evidence_snapshot(&state.ch, &validated.plan.relation, &validated.snapshot_id).await?; - if rows.len() > MAX_EXPORT_ROWS { + enforce_export_row_limit(rows.len())?; + Ok(rows) +} + +fn enforce_export_row_limit(rows: usize) -> Result<(), CanonicalError> { + if rows > MAX_EXPORT_ROWS { return Err(export_limit(format!( "Export exceeds the {MAX_EXPORT_ROWS} row limit." ))); } - Ok(rows) + Ok(()) } async fn serialize_export( @@ -230,10 +249,7 @@ async fn fetch_rows( ) -> Result, CanonicalError> { // INVARIANT: the permit is held across the awaited ClickHouse execution and // byte collection below — the hold is the MAX_CONCURRENT_QUERIES cap. - let _permit = tokio::time::timeout(QUERY_ACQUIRE_TIMEOUT, QUERY_SEMAPHORE.acquire()) - .await - .map_err(|_| query_busy())? - .map_err(|_| query_busy())?; + let _permit = acquire_query_permit().await?; let (sql, params) = compile_query(req)?; let base = state .ch @@ -333,4 +349,54 @@ mod tests { axum::http::StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn the_export_row_limit_refuses_only_past_the_cap() { + assert!(enforce_export_row_limit(MAX_EXPORT_ROWS).is_ok()); + let refused = enforce_export_row_limit(MAX_EXPORT_ROWS + 1) + .err() + .map(|error| error.status_code()); + assert_eq!( + refused, + Some(axum::http::StatusCode::TOO_MANY_REQUESTS.as_u16()) + ); + } + + #[tokio::test(start_paused = true)] + async fn export_permits_refuse_past_the_concurrency_cap() { + let mut held = Vec::new(); + for _ in 0..MAX_CONCURRENT_EXPORTS { + held.extend(acquire_export_permit().await.ok()); + } + assert_eq!(held.len(), MAX_CONCURRENT_EXPORTS); + let refused = acquire_export_permit() + .await + .err() + .map(|error| error.status_code()); + assert_eq!( + refused, + Some(axum::http::StatusCode::TOO_MANY_REQUESTS.as_u16()) + ); + drop(held); + assert!(acquire_export_permit().await.is_ok()); + } + + #[tokio::test(start_paused = true)] + async fn query_permits_refuse_past_the_concurrency_cap() { + let mut held = Vec::new(); + for _ in 0..MAX_CONCURRENT_QUERIES { + held.extend(acquire_query_permit().await.ok()); + } + assert_eq!(held.len(), MAX_CONCURRENT_QUERIES); + let refused = acquire_query_permit() + .await + .err() + .map(|error| error.status_code()); + assert_eq!( + refused, + Some(axum::http::StatusCode::TOO_MANY_REQUESTS.as_u16()) + ); + drop(held); + assert!(acquire_query_permit().await.is_ok()); + } } diff --git a/src/ingestion/tools/seed/PROFILE.md b/src/ingestion/tools/seed/PROFILE.md index 53a3041fe..85a725a3a 100644 --- a/src/ingestion/tools/seed/PROFILE.md +++ b/src/ingestion/tools/seed/PROFILE.md @@ -17,7 +17,7 @@ builder that writes `manifest.json`, so the two cannot disagree. | realm | `insight` | | anchor_date | `2026-06-30` | | data_window | `2026-05-02..2026-06-30` | -| seed_revision | `07365a1509ac7db6` | +| seed_revision | `ef801f8b54393aa7` | | manifest_version | 1 | `anchor_date` is the last day carrying seeded activity. It is resolved diff --git a/src/ingestion/tools/seed/insight_seed/generators/git.py b/src/ingestion/tools/seed/insight_seed/generators/git.py index 917a42abc..5b6cf4a4e 100644 --- a/src/ingestion/tools/seed/insight_seed/generators/git.py +++ b/src/ingestion/tools/seed/insight_seed/generators/git.py @@ -12,7 +12,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING -from ..profiles import TEAM_PROFILES, Person +from ..profiles import DEV_LEAD_UUID, TEAM_PROFILES, Person from .base import ( bulk_insert, clamp, @@ -52,6 +52,24 @@ # change_type_label multiIf; anything else renders as the raw value. CHANGE_TYPES = ("added", "modified", "renamed", "deleted") +# Deliberately hostile — and clearly synthetic — commit messages for the +# drilldown-export escaping scenario (#1603 scenario 11). The gold evidence +# model surfaces a commit's message as the drilldown "Title" cell, so these +# cover every value class a spreadsheet consumer can mishandle: the four +# formula-prefix bytes ('=', '+', '-', '@') and a value with an embedded tab +# and an embedded newline (which must stay inside one CSV cell). Only the +# FIRST few commits the dev lead generates carry one; every other commit +# keeps the column default, so no row count, metric value, or other +# person's evidence changes. +HOSTILE_COMMIT_MESSAGES = ( + "=SUM(A1:A9) synthetic title", + "+A1 synthetic title", + "-2+3 synthetic title", + "@macro synthetic title", + "tab\tinside synthetic title", + "newline\ninside synthetic title", +) + def _eligible(roster: Sequence[Person]) -> list[Person]: """Persons whose team profile has any git weight.""" @@ -96,10 +114,18 @@ def seed_class_git_commits( "lines_added", "lines_removed", "data_source", + # Appended after the long-standing columns: the link-parity tests read + # this table positionally, so a mid-tuple insertion moves their fields. + "message", "_version", ] rows: list[tuple[object, ...]] = [] version = 1 + # Dealt to the dev lead's first commits in generation order (a merge + # commit is skipped — gold's evidence model filters those out, so a + # message on one would never reach the drilldown). Deterministic across + # re-seeds because the commit stream itself is. + hostile_messages = list(HOSTILE_COMMIT_MESSAGES) for p in _eligible(roster): persona = persona_multiplier(p.uuid) weight = TEAM_PROFILES[p.team or ""].weights["github"] @@ -113,6 +139,9 @@ def seed_class_git_commits( # LOC per commit capped at ≤200 by construction. added = float(rng.randint(2, 180)) removed = float(rng.randint(0, 80)) + message = "" + if p.uuid == DEV_LEAD_UUID and not is_merge and hostile_messages: + message = hostile_messages.pop(0) rows.append( ( tenant_uuid, @@ -128,6 +157,7 @@ def seed_class_git_commits( added, removed, "insight_github", + message, version, ) ) diff --git a/tests/pyproject.toml b/tests/pyproject.toml index d3aa77c34..05af5ffa1 100644 --- a/tests/pyproject.toml +++ b/tests/pyproject.toml @@ -82,6 +82,7 @@ markers = [ "requires_ingestion: needs a stand whose manifest declares the 'ingestion' capability; skipped with a reason when it does not", "requires_catalogue(*parts): needs rows the analytics seed writes ('table_columns', 'definition_override'); skipped with a reason on a stand seeded without that step", "requires_service_principal: needs a stand whose authenticator token listener this runner can reach, so a service principal can be obtained; skipped with a reason when it cannot", + "rebuild_lane: the serialized rebuild lane — the test triggers a scoped dbt rebuild of one gold evidence relation through the stand's own seed image, so it needs docker beside the local compose stand and a run nothing else shares. OPT-IN: skipped unless --rebuild-lane is passed (tests/stand/conftest.py)", # Quality vectors — every stand api/ui test carries exactly one: a module-level # pytestmark where the whole module shares a vector, per-test markers throughout # a mixed module (never both — markers are additive, so a default plus an diff --git a/tests/stand/api/analytics/test_drilldown.py b/tests/stand/api/analytics/test_drilldown.py index d710fc5bd..8d1763c55 100644 --- a/tests/stand/api/analytics/test_drilldown.py +++ b/tests/stand/api/analytics/test_drilldown.py @@ -1,7 +1,7 @@ """`/v1/metric-drilldown` and its export — the evidence behind a metric value. - POST /v1/metric-drilldown 200 · 400 empty-entity · 403 hidden person - POST /v1/metric-drilldown/export 200 CSV/XLSX · 400 empty-entity + POST /v1/metric-drilldown 200 · 400 bad selection/cursor · 403 hidden · 404 unknown + POST /v1/metric-drilldown/export 200 CSV/XLSX · 400 bad selection · 403 hidden · 404 unknown The 415 half is in `test_request_contracts.py`, swept over every body route. @@ -20,14 +20,29 @@ Exports cover one metric per evidence presentation rather than the whole catalogue: a presentation is all an export can differ by. `EXPORT_SHAPES` names -them, and the metrics not in it are deliberately not exported here. +them, and the metrics not in it are deliberately not exported here. On top of +the shape sweep, `git.commits` is exported once more cell for cell: the seed +plants hostile commit titles (formula prefixes, an embedded tab and newline), +and the parity case asserts the export's actual neutralization contract +against the page the same selection returns. + +The refusal catalogue (#1603 scenario 6) closes the file: every way a request +can be unservable — tampered pagination cursors included — is refused with a +reason a caller can tell from the others, and never with a partial page. The +export route must refuse an out-of-scope person exactly as the paged read does +(#1603 scenario 13), because a file download is the one response a caller +walks away with. """ from __future__ import annotations +import base64 import csv +import datetime as dt import io +import json import math +import uuid import warnings import zipfile from collections.abc import Mapping, Sequence @@ -40,6 +55,7 @@ from insight_stand.api import JsonValue from ..schemas import ( + PROBLEM_CONTENT_TYPE, MetricDefinitionListResponse, MetricResultsResponse, PeriodView, @@ -47,6 +63,7 @@ ) from ..schemas.analytics import ( MetricDrilldownCapability, + MetricDrilldownColumnType, MetricDrilldownResponse, ) from .drilldown_matrix import EXPORT_SHAPES, MATRIX, Expectation, Tier @@ -77,6 +94,17 @@ #: metric to reach, and no lookup can turn it into a 404 on the way. _EMPTY_ENTITY_ID = "" +#: Together these send 150 values built from 50 distinct ones — past the +#: service's declared per-filter cap (`MAX_FILTER_VALUES` in the drilldown +#: domain's `dto.rs`) as sent, yet far under it once deduplicated, so only a +#: cap checked before dedup can refuse the request. +_FILTER_DISTINCT_VALUES = 50 +_FILTER_VALUE_REPEATS = 3 + +#: A dimension no metric declares: `normalize_key` accepts the spelling, so the +#: refusal is attributable to the declaration check and nothing earlier. +_UNDECLARED_DIMENSION = "undeclared_dimension" + @dataclass(frozen=True) class _Walk: @@ -301,6 +329,137 @@ def _xlsx_rows(content: bytes) -> int: ) +_SSML = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}" + +#: The CSV neutralization contract, read from `csv_safe_cell` in +#: `src/backend/services/analytics/src/domain/metric_drilldown/export.rs`: a +#: cell whose FIRST byte could start a spreadsheet formula (`=` `+` `-` `@`) or +#: shift the cell's content on paste (tab, CR, LF, space) is prefixed with a +#: single quote. Embedded occurrences are untouched — RFC 4180 quoting keeps +#: them inside the cell instead. +_CSV_NEUTRALIZED_PREFIXES = ("=", "+", "-", "@", "\t", "\r", "\n", " ") + +#: What the seed plants on a handful of the dev lead's commit titles +#: (`HOSTILE_COMMIT_MESSAGES` in `src/ingestion/tools/seed/insight_seed/ +#: generators/git.py`), named here only to prove the selection under test +#: actually contains each hostile class — never as an expectation of content. +_HOSTILE_PREFIXES = ("=", "+", "-", "@") +_HOSTILE_EMBEDDED = ("\t", "\n") + + +def _assert_csv_cell_matches(text: str, value: object, where: str) -> None: + """One exported CSV cell against the paged value it must carry.""" + if value is None: + assert text == "", f"{where}: null must export as an empty cell, got {text!r}" + elif isinstance(value, bool): + assert text == str(value).lower(), f"{where}: {text!r} is not {value}" + elif isinstance(value, str): + expected = f"'{value}" if value.startswith(_CSV_NEUTRALIZED_PREFIXES) else value + assert text == expected, ( + f"{where}: exported {text!r}, expected {expected!r} — content must arrive intact, " + "neutralized exactly per the export's own prefix rule" + ) + else: + assert isinstance(value, int | float), f"{where}: unexpected paged value {value!r}" + # A negative number starts with `-`, so it is neutralized like text. + bare = text.removeprefix("'") + assert bare and _close(float(bare), float(value)), ( + f"{where}: exported {text!r} does not carry the paged number {value!r}" + ) + + +def _shared_strings(workbook: zipfile.ZipFile) -> list[str]: + if "xl/sharedStrings.xml" not in workbook.namelist(): + return [] + root = ElementTree.fromstring(workbook.read("xl/sharedStrings.xml")) + return [ + "".join(node.text or "" for node in item.iter(f"{_SSML}t")) + for item in root.findall(f"{_SSML}si") + ] + + +def _xlsx_column_index(reference: str) -> int: + index = 0 + for character in reference: + if character.isdigit(): + break + index = index * 26 + (ord(character) - ord("A") + 1) + return index - 1 + + +def _xlsx_cell_matrix(content: bytes, width: int) -> list[list[str | float | bool | None]]: + """Every worksheet cell decoded, with the inertness contract asserted. + + A formula cell would carry an `f` element (and a cached result typed + `str`); asserting neither exists in ANY cell is what "stored as data, not + as a formula" means at the file level. Text arrives as a shared or inline + string, blanks as valueless cells, numbers and dates as numeric `v`. + """ + with zipfile.ZipFile(io.BytesIO(content)) as workbook: + strings = _shared_strings(workbook) + sheet = ElementTree.fromstring(workbook.read("xl/worksheets/sheet1.xml")) + matrix: list[list[str | float | bool | None]] = [] + for row in sheet.iter(f"{_SSML}row"): + cells: list[str | float | bool | None] = [None] * width + for cell in row.findall(f"{_SSML}c"): + reference = cell.get("r", "") + where = f"cell {reference or ''}" + assert cell.find(f"{_SSML}f") is None, f"{where}: exported as a formula" + kind = cell.get("t", "n") + assert kind != "str", f"{where}: typed as a formula's cached string result" + index = _xlsx_column_index(reference) + assert 0 <= index < width, f"{where}: outside the {width} exported columns" + value_node = cell.find(f"{_SSML}v") + if kind == "s": + assert value_node is not None and value_node.text is not None, ( + f"{where}: shared string without an index" + ) + cells[index] = strings[int(value_node.text)] + elif kind == "inlineStr": + cells[index] = "".join(node.text or "" for node in cell.iter(f"{_SSML}t")) + elif kind == "b": + cells[index] = value_node is not None and value_node.text == "1" + elif value_node is None or value_node.text is None: + cells[index] = None + else: + cells[index] = float(value_node.text) + matrix.append(cells) + return matrix + + +#: `ExcelDateTime` serial numbers count days from this epoch (the 1900 date +#: system with its historical two-day offset), so a date cell decodes without +#: any expectation about which calendar day it holds. +_XLSX_EPOCH = dt.date(1899, 12, 30) + + +def _assert_xlsx_cell_matches( + cell: str | float | bool | None, + value: object, + column_type: MetricDrilldownColumnType, + where: str, +) -> None: + """One decoded XLSX cell against the paged value it must carry.""" + if value is None: + assert cell is None, f"{where}: null arrived as {cell!r}" + elif column_type is MetricDrilldownColumnType.date and isinstance(value, str): + assert isinstance(cell, float), f"{where}: date arrived as {cell!r}, not a serial" + day = _XLSX_EPOCH + dt.timedelta(days=cell) + assert day.isoformat() == value, f"{where}: serial {cell!r} decodes to {day}, not {value!r}" + elif isinstance(value, bool): + assert cell is value, f"{where}: boolean arrived as {cell!r}" + elif isinstance(value, str): + assert cell == value, ( + f"{where}: exported {cell!r}, expected {value!r} — XLSX applies no prefix, " + "so the string must arrive byte-identical" + ) + else: + assert isinstance(value, int | float), f"{where}: unexpected paged value {value!r}" + assert isinstance(cell, float) and _close(cell, float(value)), ( + f"{where}: exported {cell!r} does not carry the paged number {value!r}" + ) + + def _numbers(rows: Sequence[Mapping[str, object]], column: str, metric_key: str) -> list[float]: values: list[float] = [] for index, row in enumerate(rows): @@ -642,6 +801,79 @@ def test_drilldown_export_carries_every_row( assert _xlsx_rows(xlsx_response.content) == len(walk.rows) + 1 +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.versatility +def test_export_cells_match_the_page_and_hostile_values_stay_inert( + api: ApiClient, stand_manifest: Manifest +) -> None: + """#1603 scenario 11 — export cell parity and escaping. + + The count sweep above proves nothing about content, and content is where + an export can betray its reader: a commit title is attacker-shaped free + text, and a spreadsheet will happily execute one that starts with `=`. The + seed plants clearly synthetic titles covering each formula prefix plus an + embedded tab and newline, so the `git.commits` selection is the one whose + export has something to get wrong. + + Both formats are compared cell for cell against the page the same + selection returns — both sides of every comparison come from the service. + The escaping asserted is the backend's actual contract (`csv_safe_cell`): + a CSV cell's first byte in the prefix set earns a leading single quote, + everything else round-trips byte-identical, and RFC 4180 quoting keeps + embedded tabs and newlines inside their cell, so the row count is the + page's. XLSX applies no prefix at all: every value arrives byte-identical + as a shared or inline string, and no cell is a formula. + """ + walk = _walk(api, stand_manifest, GIT_COMMITS, limit=_PAGE_LIMIT, page_budget=_PAGE_BUDGET) + assert walk.complete, "the hostile selection must be small enough to walk whole" + keys = walk.column_keys + assert "title" in keys, f"no title column in {keys} — nowhere for a hostile value to surface" + + titles = [row["title"] for row in walk.rows if isinstance(row["title"], str)] + for prefix in _HOSTILE_PREFIXES: + assert any(title.startswith(prefix) for title in titles), ( + f"no evidence title begins with {prefix!r} — the stand predates the hostile-title " + "seed, so this test would prove nothing; re-seed it" + ) + for embedded in _HOSTILE_EMBEDDED: + assert any(embedded in title for title in titles), ( + f"no evidence title embeds {embedded!r} — the stand predates the hostile-title " + "seed, so this test would prove nothing; re-seed it" + ) + + request = _request_for(stand_manifest, GIT_COMMITS) + + csv_response = _export(api, request, "csv") + assert csv_response.status_code == 200, f"csv: {csv_response.text[:300]}" + csv_rows = list(csv.reader(io.StringIO(csv_response.content.decode("utf-8-sig")))) + assert csv_rows[0] == [column.label for column in walk.first.columns] + assert len(csv_rows) == len(walk.rows) + 1, ( + f"CSV parsed to {len(csv_rows) - 1} rows against {len(walk.rows)} paged rows — " + "an embedded newline escaped its cell" + ) + for row_index, (page_row, csv_row) in enumerate(zip(walk.rows, csv_rows[1:], strict=True)): + assert len(csv_row) == len(keys), ( + f"CSV row {row_index} has {len(csv_row)} cells for {len(keys)} columns — " + "an embedded delimiter escaped its cell" + ) + for key, text in zip(keys, csv_row, strict=True): + _assert_csv_cell_matches(text, page_row[key], f"CSV row {row_index} column {key!r}") + + xlsx_response = _export(api, request, "xlsx") + assert xlsx_response.status_code == 200, f"xlsx: {xlsx_response.text[:300]}" + matrix = _xlsx_cell_matrix(xlsx_response.content, len(keys)) + assert len(matrix) == len(walk.rows) + 1 + assert matrix[0] == [column.label for column in walk.first.columns] + for row_index, (page_row, sheet_row) in enumerate(zip(walk.rows, matrix[1:], strict=True)): + for column_index, column in enumerate(walk.first.columns): + _assert_xlsx_cell_matches( + sheet_row[column_index], + page_row[column.key], + column.type, + f"XLSX row {row_index} column {column.key!r}", + ) + + @pytest.mark.requires_seed("dev_lead") @pytest.mark.xfail( strict=False, @@ -817,3 +1049,306 @@ def test_drilldown_400_for_a_key_that_is_not_a_person_id( assert response.status_code == 400, ( f"{path} answered {response.status_code} to a {label}: {response.text[:300]}" ) + + +def _refusal(response: ApiResponse, status: int, *needles: str) -> ProblemDocument: + """The response as the bare problem document it must be, and nothing else. + + `ProblemDocument` forbids extra fields, so a body that carried rows or a + `next_cursor` alongside the error — a partial response — fails the parse + instead of passing as a refusal. The needles are each class's own words in + the document, which is what makes one refusal distinguishable from the + next; a needle-less call asserts only the envelope. + """ + assert response.status_code == status, ( + f"expected {status}, got {response.status_code}: {response.text[:300]}" + ) + document = response.parse(ProblemDocument) + assert document.status == status + for needle in needles: + assert needle in response.text, ( + f"refused, but not for the asserted reason {needle!r}: {response.text[:300]}" + ) + return document + + +def _send(api: ApiClient, path: str, request: dict[str, JsonValue]) -> ApiResponse: + """`request` as-is to the paged route, reshaped by `_export` for the other.""" + if path == DRILLDOWN_EXPORT: + return _export(api, request, "csv") + return api.post(path, json_body=request) + + +@pytest.fixture(scope="session") +def issued_cursor(lead_session: PersonaSession, stand_manifest: Manifest) -> str: + """A pagination cursor the service genuinely issued, for tampering with. + + Each tampering case flips exactly one envelope field of a real cursor, so + its refusal is attributable to that field rather than to a cursor the + service would never have produced. Session-scoped: one issued cursor + serves every case, and issuing one costs a full drilldown request. + """ + response = lead_session.client.post(DRILLDOWN, json_body=_seeded_request(stand_manifest)) + assert response.status_code == 200, f"status={response.status_code} {response.text[:300]}" + cursor = response.parse(MetricDrilldownResponse).next_cursor + assert cursor is not None, ( + "the seeded git.commits selection fit one page at limit=1, so no cursor was issued " + "and the tampering cases have nothing genuine to start from" + ) + return cursor + + +def _tampered(cursor: str, **overrides: JsonValue) -> str: + """A genuinely issued cursor, re-encoded with named envelope fields replaced. + + Both halves mirror the service's `cursor.rs`: a url-safe unpadded base64 + JSON envelope. Asserting the field exists before overriding keeps a + backend rename from silently turning a tamper into an ignored extra field + — the test would then refuse for the wrong reason and still pass. + """ + envelope = json.loads(base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))) + assert isinstance(envelope, dict), f"cursor envelope is not an object: {envelope!r}" + for field in overrides: + assert field in envelope, f"cursor envelope has no {field!r}: {sorted(envelope)}" + envelope.update(overrides) + return base64.urlsafe_b64encode(json.dumps(envelope).encode()).decode().rstrip("=") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.parametrize( + ("label", "cursor"), + [ + ("not base64", "@@not-a-cursor@@"), + ("base64 of non-JSON", base64.urlsafe_b64encode(b"not json").decode().rstrip("=")), + ("JSON without the envelope fields", base64.urlsafe_b64encode(b"{}").decode().rstrip("=")), + ], +) +@pytest.mark.reliability +def test_drilldown_refuses_a_malformed_cursor( + api: ApiClient, stand_manifest: Manifest, label: str, cursor: str +) -> None: + """#1603 scenario 6 — a cursor that never was one is refused as malformed. + + All three spellings — undecodable, decodable to non-JSON, JSON missing the + envelope's fields — are one class: nothing here was ever issued. The reason + is the class's own words, so a caller can tell a corrupted cursor from one + that outlived its snapshot, which asks for a restart rather than a bug + report. + """ + response = api.post(DRILLDOWN, json_body=_seeded_request(stand_manifest, cursor=cursor)) + _refusal(response, 400, "cursor is malformed") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.reliability +def test_drilldown_refuses_a_wrong_version_cursor( + api: ApiClient, stand_manifest: Manifest, issued_cursor: str +) -> None: + """#1603 scenario 6 — a genuine cursor with its version flipped is refused. + + The version field is the envelope's compatibility escape hatch. Flipping it + on an otherwise genuine cursor pins that the service reads the field rather + than accepting whatever decodes — and the reason names the version, not + malformedness, so a cursor from a different deployment generation is + distinguishable from corruption. + """ + response = api.post( + DRILLDOWN, + json_body=_seeded_request(stand_manifest, cursor=_tampered(issued_cursor, version=2)), + ) + _refusal(response, 400, "cursor version is unsupported") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.reliability +def test_drilldown_refuses_a_cursor_replayed_against_another_selection( + api: ApiClient, stand_manifest: Manifest, issued_cursor: str +) -> None: + """#1603 scenario 6 — a cursor is bound to the selection that issued it. + + The replay keeps the metric — so the evidence relation and its snapshot + stay valid — and drops the filter and display dimension the cursor was + issued under, leaving the selection fingerprint as the only check that can + fire. That is the point: accepting the cursor would resume a filtered walk + inside an unfiltered one and quietly skip every row the old page ordering + had already passed. + + A replay against a different METRIC is refused too, but lands as + `EVIDENCE_SNAPSHOT_EXPIRED` whenever the two evidence relations differ, + because the snapshot check runs before the fingerprint comparison — this + case keeps the metric so the fingerprint refusal itself is pinned. + """ + request = _request_for(stand_manifest, GIT_COMMITS, limit=1, cursor=issued_cursor) + response = api.post(DRILLDOWN, json_body=request) + _refusal(response, 400, "cursor does not match the metric selection") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.reliability +def test_drilldown_refuses_a_cursor_from_an_expired_snapshot( + api: ApiClient, stand_manifest: Manifest, issued_cursor: str +) -> None: + """#1603 scenario 6 — a snapshot no longer the table's is a failed precondition. + + `snapshot_id` is the evidence table's UUID, so a genuine cursor with a + random one swapped in is exactly what a caller holds after the evidence + was rebuilt mid-walk. The documented refusal is the precondition violation + `EVIDENCE_SNAPSHOT_EXPIRED`, not an invalid argument: the cursor was + well-formed and honestly held, the world moved on. Its reason code is what + tells a client "restart the walk" apart from "your cursor is garbage". + """ + tampered = _tampered(issued_cursor, snapshot_id=str(uuid.uuid4())) + response = api.post(DRILLDOWN, json_body=_seeded_request(stand_manifest, cursor=tampered)) + _refusal(response, 400, "EVIDENCE_SNAPSHOT_EXPIRED") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.parametrize("path", [DRILLDOWN, DRILLDOWN_EXPORT], ids=["drilldown", "export"]) +@pytest.mark.reliability +def test_drilldown_refuses_an_undeclared_filter_dimension( + api: ApiClient, stand_manifest: Manifest, path: str +) -> None: + """#1603 scenario 6 — a filter on a dimension the metric never declared. + + Passing it through instead would make the filter a probe into the evidence + relation's real columns. The refusal names the `filters.dimension` field, + which is what separates it from the same words said about a display + dimension. + """ + request = _request_for( + stand_manifest, + GIT_COMMITS, + filters=[{"dimension": _UNDECLARED_DIMENSION, "values": ["anything"]}], + ) + response = _send(api, path, request) + _refusal(response, 400, "filters.dimension", "is not declared by the metric") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.parametrize("path", [DRILLDOWN, DRILLDOWN_EXPORT], ids=["drilldown", "export"]) +@pytest.mark.reliability +def test_drilldown_refuses_a_duplicated_filter_dimension( + api: ApiClient, stand_manifest: Manifest, path: str +) -> None: + """#1603 scenario 6 — the same dimension filtered twice is refused, not merged. + + Two filters on one dimension have no single meaning — intersection empties + the page, union widens it — and either silent choice would be read as a + statement about the data. Both value lists are individually valid, so the + refusal is attributable to the duplication alone. + """ + request = _request_for( + stand_manifest, + GIT_COMMITS, + filters=[ + {"dimension": "source", "values": ["github"]}, + {"dimension": "source", "values": ["gitlab"]}, + ], + ) + response = _send(api, path, request) + _refusal(response, 400, "duplicate dimension filter") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.parametrize("path", [DRILLDOWN, DRILLDOWN_EXPORT], ids=["drilldown", "export"]) +@pytest.mark.reliability +def test_drilldown_refuses_a_filter_over_the_declared_value_cap( + api: ApiClient, stand_manifest: Manifest, path: str +) -> None: + """#1603 scenario 6 — a value list past the per-filter cap is refused up front. + + The cap is checked before values are deduplicated, so the refusal is about + the request's size as sent — a caller cannot smuggle an oversized list past + it with repeats, and the query never compiles an IN-list this long. The + list is deliberately repeats of a few distinct values: it would dedup to + well under the cap, so a 400 here pins the before-dedup ordering rather + than passing for either. + """ + values: list[JsonValue] = [ + f"value_{index:03d}" for index in range(_FILTER_DISTINCT_VALUES) + ] * _FILTER_VALUE_REPEATS + request = _request_for( + stand_manifest, + GIT_COMMITS, + filters=[{"dimension": "source", "values": values}], + ) + response = _send(api, path, request) + _refusal(response, 400, "filters.values", "between 1 and 100 values are required") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.parametrize("path", [DRILLDOWN, DRILLDOWN_EXPORT], ids=["drilldown", "export"]) +@pytest.mark.reliability +def test_drilldown_refuses_an_undeclared_display_dimension( + api: ApiClient, stand_manifest: Manifest, path: str +) -> None: + """#1603 scenario 6 — a display dimension the metric never declared. + + A projected column reaches the response (and the export file), so an + unchecked one would be a column probe with the answer written into the + page. The refusal names `display_dimensions`, distinguishing it from the + filter-side twin of the same message. + """ + request = _request_for(stand_manifest, GIT_COMMITS, display_dimensions=[_UNDECLARED_DIMENSION]) + response = _send(api, path, request) + _refusal(response, 400, "display_dimensions", "is not declared by the metric") + + +@pytest.mark.requires_seed("dev_lead") +@pytest.mark.parametrize("path", [DRILLDOWN, DRILLDOWN_EXPORT], ids=["drilldown", "export"]) +@pytest.mark.reliability +def test_drilldown_refuses_an_unknown_metric_key( + api: ApiClient, stand_manifest: Manifest, path: str +) -> None: + """#1603 scenario 6 — a well-formed key that names no metric is a 400. + + The same classification `/v1/metric-results` uses: the catalogue loader + refuses a key it does not carry as an `UNAVAILABLE` field violation before + the drilldown validation's own not-found fallback can fire, so on the wire + an unknown key is a 400 on both operations. The key is shaped to pass + `normalize_metric_key`, so the refusal is the catalogue's and not a + spelling rejection dressed as one. + """ + request = _request_for(stand_manifest, "stand.does_not_exist") + response = _send(api, path, request) + _refusal(response, 400, "unknown or unavailable metric key") + + +@pytest.mark.requires_seed("dev_lead", "sales_ic") +@pytest.mark.security +def test_drilldown_export_refuses_an_out_of_scope_person_identically( + api: ApiClient, stand_manifest: Manifest +) -> None: + """#1603 scenario 13 — both export formats refuse exactly as the paged read. + + Same outsider and selection as + `test_git_commit_drilldown_refuses_a_person_out_of_scope`. An export that + answered differently — a softer status, an empty file, or a filename via + content-disposition — would make the export path the cheap way around the + visibility gate, and a file is the one response a caller walks away with. + Identical means identical: status, problem class, reason and context are + compared field-for-field, and the body must parse as a bare problem + document — zero evidence bytes, no file offered. + """ + outsider = stand_manifest.fixture("sales_ic") + request = _seeded_request(stand_manifest, entity_id=outsider.uuid) + + paged = api.post(DRILLDOWN, json_body=request) + paged_document = _refusal(paged, 403) + + for file_format in ("csv", "xlsx"): + response = _export(api, request, file_format) + document = _refusal(response, 403) + assert response.content_type.startswith(PROBLEM_CONTENT_TYPE), ( + f"{file_format}: a refusal served as {response.content_type!r}, not a problem document" + ) + assert response.headers.get("content-disposition") is None, ( + f"{file_format}: a refusal must not offer a file, got " + f"{response.headers.get('content-disposition')!r}" + ) + assert (document.type, document.title, document.detail, document.context) == ( + paged_document.type, + paged_document.title, + paged_document.detail, + paged_document.context, + ), f"{file_format}: the export's refusal differs from the paged read's" diff --git a/tests/stand/api/analytics/test_drilldown_rebuild.py b/tests/stand/api/analytics/test_drilldown_rebuild.py new file mode 100644 index 000000000..7ec01f638 --- /dev/null +++ b/tests/stand/api/analytics/test_drilldown_rebuild.py @@ -0,0 +1,318 @@ +"""`POST /v1/metric-drilldown` while its evidence table is rebuilt — the live half +of the snapshot guard. + + POST /v1/metric-drilldown 200 · 400 EVIDENCE_SNAPSHOT_EXPIRED once the build rotates + +`test_drilldown.py` covers the routes' contract against a stand at rest. This +module is the one case that cannot live there: it CHANGES the stand mid-test, +by triggering the same rebuild a deployment runs — a scoped +`dbt run --select git_metric_evidence` through the stand's own seed image, the +exact mechanism `test-stand seed` uses. dbt's table materialization swaps the +relation atomically (a fresh table exchanged over the old name), so the +rebuild rotates the ClickHouse table UUID that every continuation token pins +as its `snapshot_id`, and a cursor issued before the swap must be refused +afterwards. That refusal-under-a-real-rebuild is what scenario 7 claims; +refusing a hand-tampered token is scenario 6's, in `test_drilldown.py`. + +Why the paged read and not the export: both operations re-verify the snapshot +through the same `verify_evidence_snapshot` call, but an export does it inside +one request, and nothing outside the process can schedule a rebuild into that +window deterministically. A cursor is the same pinned snapshot made to SPAN +requests, so "rebuild, then present the pre-rebuild snapshot" becomes an exact +sequence rather than a race — and a refusal there is the refusal the export +path shares. + +The lane is serialized by construction and opt-in by policy. The suite runs in +a single pytest process (no xdist in tests/uv.lock), so nothing in this run +holds a cursor while the rebuild happens; the `rebuild_lane` marker keeps the +test out of every run that did not pass `--rebuild-lane`, because a second run +sharing the stand could be mid-walk when the UUID rotates — and because the +trigger shells out to docker, which only exists beside the local compose +stand. The stand is left equivalent: the rebuild re-materializes the same +deterministic SQL over unchanged silver and identity data, and the final +reconciliation in the test is the proof. +""" + +from __future__ import annotations + +import os +import subprocess +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import pytest +from insight_stand import ApiClient, Manifest, analytics_path +from insight_stand.api import JsonValue +from insight_stand.stand import CANDIDATE_ENV_FILES, ENV_FILE_ENV + +from ..schemas import MetricResultsResponse, PeriodView, ProblemDocument +from ..schemas.analytics import MetricDrilldownResponse + +pytestmark = pytest.mark.reliability + +DRILLDOWN = analytics_path("/v1/metric-drilldown") +METRIC_RESULTS = analytics_path("/v1/metric-results") + +#: The metric under rebuild and the dbt model serving its evidence +#: (`registry.yaml`: source `git` → `evidence_ref: git_metric_evidence`). +#: `git.commits` because the seed guarantees it rows for `dev_lead`, and its +#: reconciliation rule is the simplest one there is: one evidence row per +#: counted commit. +GIT_COMMITS = "git.commits" +EVIDENCE_MODEL = "git_metric_evidence" + +_REPO_ROOT = Path(__file__).resolve().parents[4] + +#: The compose project `dev-compose.sh` runs every stand under unless an +#: `--instance` renamed it; an instance-named stand overrides this to +#: `insight-` so the rebuild lands on the stand under test. +_COMPOSE_PROJECT_ENV = "INSIGHT_STAND_COMPOSE_PROJECT" +_DEFAULT_COMPOSE_PROJECT = "insight" + +#: Wall-clock ceiling for the rebuild container. Dominated by the one dbt +#: model build; generous because a cold run may build the seed image first. +_REBUILD_TIMEOUT_SECONDS = 900 + +_PAGE_LIMIT = 250 +_PAGE_BUDGET = 40 + + +def _stand_env_file() -> Path: + """The env file of the stand this run is aimed at — same resolution as + `insight_stand.stand`, so the rebuild targets the stand the requests hit.""" + override = (os.environ.get(ENV_FILE_ENV) or "").strip() + candidates = ( + [Path(override)] if override else [_REPO_ROOT / name for name in CANDIDATE_ENV_FILES] + ) + for candidate in candidates: + if candidate.is_file(): + return candidate + pytest.fail( + "rebuild_lane: no compose env file found " + f"(tried {', '.join(str(c) for c in candidates)}) — the rebuild can only target the " + "local compose stand, brought up by ./dev-compose.sh test-stand up" + ) + + +def _rebuild_evidence_relation() -> None: + """Rebuild ONLY the walked evidence relation, through the deployed path. + + `insight_seed.silver.apply_ch_migrations` is what `insight-seed gold` (and + the k8s clickhouse-migrate Hook Job) runs; narrowing its dbt selection to + the one model is the only difference from `test-stand seed gold`. Never + the silver step — that regenerates rows — and never direct DDL, which + would prove a hand-rolled swap rather than the deployment's. + + Mirrors `cmd_seed` in dev-compose.sh: same compose file, seed profile and + uid mapping, minus `--build` (the stand's own seed built the image, and + the ingestion tree is bind-mounted so the dbt project is current anyway). + """ + code = ( + "from insight_seed.silver import apply_ch_migrations; " + f"apply_ch_migrations(dbt_select={EVIDENCE_MODEL!r})" + ) + project = os.environ.get(_COMPOSE_PROJECT_ENV, "").strip() or _DEFAULT_COMPOSE_PROJECT + command = [ + "docker", + "compose", + "--project-name", + project, + "--env-file", + str(_stand_env_file()), + "-f", + "docker-compose.yml", + "--profile", + "seed", + "run", + "--rm", + "--no-deps", + "--entrypoint", + "python", + "seed-sample", + "-c", + code, + ] + env = {**os.environ, "SEED_UID": str(os.getuid()), "SEED_GID": str(os.getgid())} + try: + completed = subprocess.run( + command, + cwd=_REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=_REBUILD_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + # The timeout killed the docker CLI, not the container dockerd runs + # for it — without this the rebuild keeps mutating the stand past the + # ceiling the failure message claims to enforce. + _remove_seed_containers(project) + pytest.fail( + f"rebuild of {EVIDENCE_MODEL} exceeded {_REBUILD_TIMEOUT_SECONDS}s and its " + "container was force-removed — re-seed before trusting other results" + ) + if completed.returncode != 0: + pytest.fail( + f"rebuild of {EVIDENCE_MODEL} failed (exit {completed.returncode}); the model " + "either rebuilt or kept its previous build, never half of each — dbt swaps " + f"atomically.\nstderr tail: {completed.stderr[-2000:]}" + ) + + +def _remove_seed_containers(project: str) -> None: + """Force-remove any seed-sample container of the stand's compose project. + + Best effort: failing to remove must not mask the timeout failure that + called this, so errors are reported by the caller's message alone. + """ + listed = subprocess.run( + [ + "docker", + "ps", + "--quiet", + "--filter", + f"label=com.docker.compose.project={project}", + "--filter", + "label=com.docker.compose.service=seed-sample", + ], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + container_ids = listed.stdout.split() + if container_ids: + subprocess.run( + ["docker", "rm", "--force", *container_ids], + capture_output=True, + timeout=30, + check=False, + ) + + +def _request( + manifest: Manifest, *, limit: int | None = None, cursor: str | None = None +) -> dict[str, JsonValue]: + start, _, end = manifest.data_window.partition("..") + request: dict[str, JsonValue] = { + "metric_key": GIT_COMMITS, + "entity": {"type": "person", "id": manifest.fixture("dev_lead").uuid}, + "period": {"from": start, "to": end}, + "filters": [], + "display_dimensions": [], + } + if limit is not None: + request["limit"] = limit + if cursor is not None: + request["cursor"] = cursor + return request + + +def _walk(api: ApiClient, manifest: Manifest) -> list[Mapping[str, Any]]: + """Every evidence row of the selection, first page to last.""" + rows: list[Mapping[str, Any]] = [] + cursor: str | None = None + for _ in range(_PAGE_BUDGET): + response = api.post( + DRILLDOWN, json_body=_request(manifest, limit=_PAGE_LIMIT, cursor=cursor) + ) + assert response.status_code == 200, f"status={response.status_code} {response.text[:300]}" + page = response.parse(MetricDrilldownResponse) + rows.extend(row.values for row in page.rows) + cursor = page.next_cursor + if cursor is None: + return rows + pytest.fail(f"{GIT_COMMITS}: still paging after {_PAGE_BUDGET} pages of {_PAGE_LIMIT}") + + +def _period_value(api: ApiClient, manifest: Manifest) -> float | None: + """The scalar the dashboard shows for the same selection — the walk's oracle.""" + start, _, end = manifest.data_window.partition("..") + person_id = manifest.fixture("dev_lead").uuid + response = api.post( + METRIC_RESULTS, + json_body={ + "entity": {"type": "person", "ids": [person_id]}, + "period": {"from": start, "to": end}, + "metrics": [{"metric_key": GIT_COMMITS, "filters": [], "views": [{"view": "period"}]}], + }, + ) + assert response.status_code == 200, f"status={response.status_code} {response.text[:300]}" + result = response.parse(MetricResultsResponse) + assert len(result.metrics) == 1 + views = result.metrics[0].root.views + assert len(views) == 1 + assert isinstance(views[0].root, PeriodView) + values = views[0].root.values + assert len(values) == 1 + assert values[0].entity_id == person_id + return values[0].value + + +@pytest.mark.rebuild_lane +@pytest.mark.requires_seed("dev_lead") +def test_a_rebuild_between_pages_expires_the_cursor_and_never_mixes_builds( + api: ApiClient, stand_manifest: Manifest +) -> None: + """#1603 scenario 7 — a walk interrupted by a real evidence rebuild. + + Sequence: take page one of `git.commits` at limit 1 (so a continuation + token must exist), rebuild `git_metric_evidence` through the seed image, + then present the pre-rebuild token. The service must refuse with the + documented failed-precondition — 400 carrying `EVIDENCE_SNAPSHOT_EXPIRED` + in a problem document — because honouring it would resume a row order the + new build no longer defines: the mixed-builds page scenario 7 forbids. + + The refusal is also the proof the rebuild ROTATED the snapshot: had the + swap kept the table UUID, the stale token would answer 200 and fail here. + + Then the recovery a caller is entitled to: a fresh walk (new first page, + new tokens) completes and still reconciles row-for-row with the metric + value, which pins two things at once — the rebuilt table serves one + consistent build, and the stand is content-identical for every test that + follows this one. + """ + first = api.post(DRILLDOWN, json_body=_request(stand_manifest, limit=1)) + assert first.status_code == 200, f"status={first.status_code} {first.text[:300]}" + page = first.parse(MetricDrilldownResponse) + assert page.rows, f"{GIT_COMMITS}: the seed guarantees dev_lead commit evidence" + assert page.next_cursor is not None, ( + f"{GIT_COMMITS}: a one-row page of a multi-row selection must continue — " + "without a token there is no in-flight walk to interrupt" + ) + pre_rebuild_row = page.rows[0].values + + _rebuild_evidence_relation() + + # The finally holds the equivalence proof: the rebuild has already mutated + # the stand, so whether or not the refusal fires as documented, the run + # must still establish that other tests face the same content — a refusal + # failure alone would otherwise leave that unverified exactly when it + # matters most. A failure inside finally supersedes the refusal failure, + # which is the right precedence: changed content invalidates more. + try: + stale = api.post( + DRILLDOWN, json_body=_request(stand_manifest, limit=1, cursor=page.next_cursor) + ) + assert stale.status_code == 400, ( + f"a pre-rebuild cursor answered {stale.status_code}: {stale.text[:300]} — a 200 " + "here is a page resumed across two builds" + ) + assert stale.parse(ProblemDocument).status == 400 + assert "EVIDENCE_SNAPSHOT_EXPIRED" in stale.text, ( + f"refused, but not with the documented precondition: {stale.text[:300]}" + ) + finally: + rows = _walk(api, stand_manifest) + assert pre_rebuild_row in rows, ( + "the pre-rebuild first row is gone from the rebuilt evidence — the rebuild " + "changed content, so the stand is no longer the one other tests were written " + "against" + ) + period = _period_value(api, stand_manifest) + assert period == len(rows), ( + f"{GIT_COMMITS}: {len(rows)} evidence rows against a metric value of {period} " + "after the rebuild" + ) diff --git a/tests/stand/conftest.py b/tests/stand/conftest.py index a91776f10..6f7117fd1 100644 --- a/tests/stand/conftest.py +++ b/tests/stand/conftest.py @@ -141,6 +141,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: f"(default: ${MANIFEST_PATH_ENV}, else {MANIFEST_PATH})" ), ) + parser.addoption( + "--rebuild-lane", + action="store_true", + default=False, + help=( + "run the tests marked rebuild_lane, which trigger a scoped dbt rebuild of a gold " + "evidence relation through the stand's own seed image. Off by default: they need " + "docker beside the local compose stand, and they mutate shared stand state mid-run, " + "so the run they join must not share the stand with anything else" + ), + ) def pytest_configure(config: pytest.Config) -> None: @@ -233,6 +244,11 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item * `requires_seed` — a missing fixture means the stand was seeded wrong and the session aborts. That check lives in `pytest_collection_finish`, AFTER `-m` deselection, so it judges only the tests that will run. + * `rebuild_lane` — OPT-IN, skipped unless `--rebuild-lane` was passed. + Neither a capability nor a seeding fact: these tests mutate shared stand + state (a scoped dbt rebuild of a gold evidence relation) and shell out to + docker, which only works against the local compose stand. A skip rather + than a deselect, so `-ra` keeps the lane visible with its reason. """ vectors = quality_vectors(config.getini("markers")) misvectored = { @@ -250,6 +266,20 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item "leaves both on the item and breaks -m selection).\n" + "\n".join(lines) ) + if not config.getoption("--rebuild-lane"): + for item in items: + if item.get_closest_marker("rebuild_lane") is not None: + item.add_marker( + pytest.mark.skip( + reason=( + "rebuild_lane: opt-in only — pass --rebuild-lane. The test " + "rebuilds a gold evidence relation through the stand's own seed " + "image, so it needs docker beside the local compose stand and a " + "run nothing else shares." + ) + ) + ) + try: manifest = _manifest() except ManifestError as exc: