diff --git a/demo/datasets.md b/demo/datasets.md index b43e9fba..ae6bcbf3 100644 --- a/demo/datasets.md +++ b/demo/datasets.md @@ -97,19 +97,19 @@ df['tip_pct'] = (df['tip_amount'] / df['fare_amount'].replace(0, float('nan'))) Type these into Claude Code with the tallyman MCP server running: ``` -Use catalog_load_parquet to load /tmp/nyc_taxi/yellow_tripdata_2024-01.parquet, -name it yellow_jan_2024. Prompt: "NYC yellow cab trips January 2024". +Use catalog_import_source to import /tmp/nyc_taxi/yellow_tripdata_2024-01.parquet +under the alias yellow_jan_2024. Prompt: "NYC yellow cab trips January 2024". ``` ``` -Create a named entry trips_by_zone that joins yellow_jan_2024 with the taxi zone -lookup at /tmp/nyc_taxi/taxi_zone_lookup.csv on PULocationID, groups by Zone, -and counts trips with mean fare. Name it trips_by_zone. +Import /tmp/nyc_taxi/taxi_zone_lookup.csv under the alias taxi_zones, then create +a named entry trips_by_zone that joins yellow_jan_2024 with taxi_zones on +PULocationID, groups by Zone, and counts trips with mean fare. ``` ``` -Use catalog_load_parquet to load /tmp/citibike/citibike_h1_2024.parquet, -name it citibike_h1. Prompt: "Citibike trips January–June 2024". +Use catalog_import_source to import /tmp/citibike/citibike_h1_2024.parquet +under the alias citibike_h1. Prompt: "Citibike trips January–June 2024". ``` ``` diff --git a/demo/storyboard.json b/demo/storyboard.json index 256712e0..a8083638 100644 --- a/demo/storyboard.json +++ b/demo/storyboard.json @@ -2,9 +2,13 @@ "project": "spike", "steps": [ { - "tool": "catalog_load_parquet", - "args": {"rel_path": "orders.parquet", "name": "orders", "prompt": "raw orders dataset"}, - "narration": "beat 1: load the orders parquet and name it" + "tool": "catalog_import_source", + "args": { + "outside_path": "${TALLYMAN_PROJECT_ROOT}/data/orders.parquet", + "alias": "orders", + "prompt": "raw orders dataset" + }, + "narration": "beat 1: import the orders parquet under a source alias" }, { "tool": "catalog_create", @@ -21,7 +25,7 @@ "code": "from tallyman_xorq.io import tracked_expr_from_alias\nt = tracked_expr_from_alias('shoe_sales')\nexpr = t.order_by('total')\n", "prompt": "verify shoe_sales totals are reasonable" }, - "narration": "beat 4: scratch verification — sort the result" + "narration": "beat 4: scratch verification \u2014 sort the result" }, { "tool": "catalog_revise", @@ -39,8 +43,14 @@ "vega_spec": { "mark": "bar", "encoding": { - "x": {"field": "region", "type": "nominal"}, - "y": {"field": "total", "type": "quantitative"} + "x": { + "field": "region", + "type": "nominal" + }, + "y": { + "field": "total", + "type": "quantitative" + } } } }, @@ -52,7 +62,7 @@ "cell_id": "PLACEHOLDER", "markdown": "## Shoe sales by region\n\nBroader bucket: boots, sneakers, sandals." }, - "narration": "beat 10: rewrite the markdown for the audience (skipped — needs cell_id resolution)", + "narration": "beat 10: rewrite the markdown for the audience (skipped \u2014 needs cell_id resolution)", "skip": true } ] diff --git a/docs/installing.md b/docs/installing.md index 95254aa0..82e23b27 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -128,7 +128,8 @@ it. To remove it later: `claude mcp remove tallyman -s project`. With the companion running (step 5) and the MCP server approved (step 6), the `mcp__tallyman__*` tools are available in chat. Try: -> Use `catalog_load_parquet` to load `orders.parquet`. +> Use `catalog_import_source` to import `orders.parquet` from the project's +> `data/` directory under the alias `orders`. > > Now use `catalog_create` to make a named entry `shoe_sales` that groups orders > by region and totals the price. diff --git a/docs/mcp-server.md b/docs/mcp-server.md index 32b0b1f2..781371f0 100644 --- a/docs/mcp-server.md +++ b/docs/mcp-server.md @@ -94,7 +94,7 @@ git revision. A non-empty cascade emits a `recalc` SSE event. | Tool | Checkpoints | SSE notify | Auto-recalc | |---|---|---|---| | `catalog_run` | yes | `new_entry`, `build_failed` | no | -| `catalog_load_parquet` | yes | `new_entry`, `build_failed`, `notebook_changed`¹ | no | +| `catalog_import_source` | yes | `new_entry`, `build_failed`, `notebook_changed`¹, `recalc` | yes | | `catalog_create` | yes | `new_entry`, `build_failed`, `notebook_changed` | no | | `catalog_revise` | yes | `new_entry`, `build_failed`, `recalc` | yes | | `catalog_alias` | yes | `alias_changed`, `notebook_changed` | no | @@ -181,16 +181,34 @@ default authoring tool for a one-off run. `events.jsonl`. - **Promote** a scratch entry to a name afterward with `catalog_alias`. -### `catalog_load_parquet(rel_path, prompt="", name="") -> dict` -Register a parquet under `/data/` as an entry by synthesizing a -`read_project_file(rel_path)` recipe — the no-code "load this file" path. With -`name`, behaves like `catalog_create` (names the entry, appends a notebook cell). -- **Params:** `rel_path` (required) — path under `data/`; `prompt` — optional - intent / cell markdown; `name` — optional alias. -- **Returns:** same as `catalog_run`, plus `alias` and `version` when named. - `{error}` if the alias already exists; `{error, error_id}` on a missing file. -- Unlike `catalog_run`, it does **not** emit a `build_ok` event; success shows - only via the notifies. +### `catalog_import_source(outside_path, alias, pinned_version=None, prompt="", schema=None, reader_options=None) -> dict` +Import a parquet or CSV into the catalog and point the source alias `alias` at +it (ADR-011). The only way a file enters: the bytes are copied into the arena, +one snapshot of them is written in file order with a `__row_order` column, and +a version of `alias` is minted. Recipes then read +`tracked_expr_from_alias(alias)`; the original path is provenance and is never +read again. +- **Params:** `outside_path` (required) — any path, `data/` is not special; + `alias` (required) — the source alias, which may not name a catalog alias; + `pinned_version` — the version you claim the file is, checked rather than + assumed; `prompt` — optional intent / cell markdown; `schema` and + `reader_options` — CSV column types and `polars.scan_csv` options, recorded on + the entry and never re-derived (ADR-011 D12). +- **Returns:** `hash`, `alias`, `version`, `created`, `row_count`, `schema`, + `path`, `digest`, `url`, plus `recalc` when advancing the alias cascaded. + `{error, error_id}` for a missing path, an alias collision, bytes another + alias already holds (the error names that alias and version), or any row of + the ADR-011 D3 table that is an error. +- Re-running it on **unchanged** bytes is a no-op that returns the current + version, so it is safe in a script. If that version's snapshot is gone, the + no-op heals it from the clone and checks it against the recorded digest; it + never rewrites the version. Re-running it on **changed** bytes mints the next + version and cascades like a revise. +- **One set of bytes is one version under one alias** (ADR-011 D1). To give a + source a second name, `catalog_create` an entry whose recipe is + `tracked_expr_from_alias("")`; it follows the source when it is + re-imported. A CSV read with other reader options is a different entry and + may be imported under its own alias (ADR-011 D12). ### `catalog_create(name, code, prompt="") -> dict` Execute and persist as a **named** entry (alias) and append a notebook cell. @@ -229,7 +247,9 @@ cell. Used after a `catalog_run` when a scratch entry earns a permanent name. - **Params:** `hash` (required) — must name an on-disk entry; `name` (required) — must be free. - **Returns:** `{hash, alias, version, url}`. `{error}` if the hash has no entry - or the name is taken. + or the name is taken, and if the entry is a source version: a source entry + belongs to its source alias, and the error names the `catalog_create` over + `tracked_expr_from_alias` that gives the source a second name (ADR-011 D1). ### `catalog_rename(old_name, new_name) -> dict` Rename an alias, preserving its full version history and notebook position. diff --git a/packages/app/src/pages/CatalogPage.tsx b/packages/app/src/pages/CatalogPage.tsx index 9c8af1d8..8da0ba97 100644 --- a/packages/app/src/pages/CatalogPage.tsx +++ b/packages/app/src/pages/CatalogPage.tsx @@ -506,8 +506,8 @@ function EntryCacheView({ project, hash }: { project: string; hash: string }) { {data.source_tracked - ? `${data.source_formatted} raw source${data.sources.length > 1 ? ` (${data.sources.length} files)` : ""}` - : "raw source size not tracked (source-identity off)"} + ? `${data.source_formatted} imported bytes, kept so the snapshot can be written again` + : "no imported bytes — this expression is a computation over other aliases"} {data.modified_at && <> · last modified {data.modified_at.slice(0, 19).replace("T", " ")}} diff --git a/plans/ADR-002-source-identity-content-hash.md b/plans/ADR-002-source-identity-content-hash.md index 7812bfad..7ff95be2 100644 --- a/plans/ADR-002-source-identity-content-hash.md +++ b/plans/ADR-002-source-identity-content-hash.md @@ -3,7 +3,30 @@ - **Status:** Accepted (2026-06-18 — default flipped `off` → `cas` in #86, with `cp --reflink=auto` on Linux and `.cas` GC wired into `reset_to`; supersedes the 2026-06-10 Proposed draft below. See the reconstruction caveat under - Consequences.) + Consequences.) **Narrowed by `plans/ADR-011-sources-are-aliases.md` + (2026-09-22, PR #218), which keeps the clone store and deletes the rest of + the machinery around it:** + - The mode switch is gone. `TALLYMAN_SOURCE_IDENTITY`, its `off` and `salt` + modes and `salted_hash` are deleted (ADR-011 D8). There is one behaviour: + an import digests the bytes and clones them. "Identity off" and "every + input is versioned" cannot both be true. + - `manifest.sources` is gone (ADR-011 D6). It was written as the retention + closure `gc_cas` walks and read by the staleness scan as a freshness + record, and that second reading is the defect ADR-011 exists to remove. + Retention is now the DAG: a source version is an entry, so its clone is + alive exactly while that entry is, and `catalog_state._live_source_digests` + collects `manifest.provenance.digest` instead. + - `digest_for` and its `source_digests.json` stat memo are gone. Nothing + digests a file outside an import any more, so the memo has no callers and + the scan touches no file at all. + - The reconstruction caveat under Consequences is resolved rather than + inherited: `recon_cas_path` no longer falls back to serving live bytes when + the clone is missing and the file has drifted. That branch is an error + (ADR-011 D9), because the live file is no longer a build input in the first + place. + What survives unchanged is the substance of this ADR: content-addressed + clones under `data/.cas/`, written copy-on-write where the + filesystem offers it, parked in the bullpen by a reset rather than unlinked. - **Context ticket:** buckaroo-data/tallyman#30 (precondition for the result-cache rubric's content-stable `content_hash` key) - **Affected code:** `src/tallyman_xorq/source_identity.py` (new), diff --git a/plans/ADR-005-intelligent-csv-import.md b/plans/ADR-005-intelligent-csv-import.md index 3bfc50f9..0be3f5f8 100644 --- a/plans/ADR-005-intelligent-csv-import.md +++ b/plans/ADR-005-intelligent-csv-import.md @@ -7,6 +7,19 @@ Amended by `plans/ADR-008-row-order-of-reads.md` (implemented in #189): INV-1's row-index column is named `__row_order` (not `original_row_order`) and INV-2's trailing `order_by` is gone, so `tallyman_read_csv` returns a plain read of an ordered copy that is keyed by the CSV's content. + Amended again by `plans/ADR-011-sources-are-aliases.md` (2026-09-22, PR #218): **this reader now runs at + import time, not in a recipe.** `tallyman_read_csv` is refused in an authored recipe alongside + `read_project_file` (ADR-011 D2) — both open a file the catalog does not own, and leaving the CSV reader + open would be a hole in the rule. A CSV enters through + `catalog_import_source(path, alias, separator=..., schema=...)`, and everything this ADR designed goes + with it: the schema DSL and its normaliser, the inference ladder (100 → 10k → whole file), the + suggestion engine and the error contract all run inside the import, over + `source_import._write_csv_snapshot`. The recipe then reads the resulting source alias. + Two consequences for the text below. The reader options are **fixed at import** and recorded on the + source entry (ADR-011 D12), so one file is read one way forever; to read it a second way, import it again + under a second alias. And the retry loop this ADR is built around — read, see the suggestion, adjust the + schema, read again — is a loop over imports, which is why ADR-011 keeps the raw bytes in the clone store + rather than only the parsed snapshot (its Open question 1). - **Affected code:** `src/tallyman_xorq/io.py` (`tallyman_read_csv`, `_polars_overrides`, `_IBIS_TO_POLARS`, a new schema-spec normaliser and a suggestion engine), `src/tallyman_mcp/server.py` (the `catalog_run` CSV diff --git a/plans/ADR-008-row-order-of-reads.md b/plans/ADR-008-row-order-of-reads.md index 6a03bbc5..3b2e7efa 100644 --- a/plans/ADR-008-row-order-of-reads.md +++ b/plans/ADR-008-row-order-of-reads.md @@ -4,6 +4,21 @@ set. Where this text says a decision is "not yet confirmed" or "Proposed", it was implemented as written; the differences between the text and the code are under "Implementation notes" below. Original status: Proposed (2026-09-18, revised 2026-09-20 in the grilling session, and again the same day after a review of PR #184, which made the fix for #168 a precondition of D2 and D7, and a third time that day after a second review: D4 and D6 were tightened, and D10 to D12 are new). Awaiting Paddy's review; nothing here is implemented. The first draft pinned row order with an engine setting. Paddy proposed baking a row-order column into every file tallyman writes and sorting every page by it. The measurements below favour that, so it is now the decision and the engine setting is the rejected alternative under D5. Amends `plans/ADR-005-intelligent-csv-import.md` INV-1 (the name and position of the row-order column) and INV-2 (the trailing `order_by`). Corrects a threshold quoted in `plans/ADR-006-read-path-loads-builds.md` decision D5 (the canonical sort) and three other places. + **Extended by `plans/ADR-011-sources-are-aliases.md` (2026-09-22, PR #218).** D2 and D7 here make a raw + `xo.deferred_read_parquet` a build error, so that every source enters through an ordered copy. ADR-011 D2 + applies the same rule to `read_project_file` and `tallyman_read_csv` themselves: there is now no way to + author a read of a file the catalog does not own, and the sanctioned route in is an explicit import. + The ordered copy is then not a separate kind of file. An imported source version is an ordinary entry + (ADR-011 D1) and the copy **is** that entry's snapshot: one parquet in + `compute_cache/result_cache/`, named by the entry's content hash, written by pyarrow in the pinned + layout of ADR-009. `compute_cache/ordered_sources/`, the `copy_key` + (`md5(digest ǀ reader signature)`) that named files in it, `manifest.ordered_copies` and the + `ensure_ordered_copy` / `existing_ordered_copy` / `recreate_ordered_copy` trio are all deleted. + What survives from `ordered_copy.py` is what describes a read rather than performs one: + `ORDERED_COPY_ROW_GROUP_ROWS`, the reader descriptors (`parquet_reader`, `csv_reader`) and the reader + signature that the source entry's hash is built from. + Everything this ADR decided about row order is unchanged: every file tallyman writes still carries + `__row_order` as a last `int64` column, `0..N-1`, and every page still sorts by it. - **Context:** the 2026-09-18 cache audit (tallyman @ `a748ea6`, buckaroo 0.15.4, xorq 0.3.26, xorq-datafusion 0.2.7). - **Tickets:** #168 (CSV sources bypass source identity; D2 and D7 depend on diff --git a/plans/ADR-011-sources-are-aliases.md b/plans/ADR-011-sources-are-aliases.md new file mode 100644 index 00000000..c83c5add --- /dev/null +++ b/plans/ADR-011-sources-are-aliases.md @@ -0,0 +1,611 @@ +# ADR: A raw input is an alias, and files enter only by an explicit import + +- **Status:** Accepted (2026-09-22), implemented. Stage 1 — the import path and + the refusals (D1, D2, D3, D5, D9, D10, D12) — is PR #217, with two later + decisions of the same day folded into D1: the snapshot is written by pyarrow + in the pinned layout, and it is cache that `ensure_materialized` re-creates + from the clone. Stage 2 — D6, D8 and the rewrite of every call site that + authored a raw file read — is PR #218, stacked on it. The fixes from reviewing + both are PR #219, stacked on #218: one set of bytes is one version under one + alias, an alias's kind matches its entries' kind, a version is named by the + alias that holds it now, and a re-import that finds a version already there + rewrites nothing but a missing snapshot, verified like any heal (D1, D3, D11, + D12 and the PR #219 implementation notes). Amends + `plans/ADR-002-source-identity-content-hash.md` (its modes, its `sources` map + and its reconstruction caveat go; the clone store stays), + `plans/ADR-005-intelligent-csv-import.md` (its reader runs at import, not in + a recipe) and `plans/ADR-008-row-order-of-reads.md` (its refusal extends to + `read_project_file`, and the ordered copy becomes the source entry's + snapshot). Still outstanding: `docs/architecture.md`, `docs/caching.md`, + `docs/expression-lifecycle.md` and `docs/system-contract.md` all describe the + deleted source axis, and are left until PR #216 lands to avoid rewriting the + same four files twice. + Written from Paddy's design session the same day, after a review of PR #189 + found that a child pinned to its parent by content hash is permanently stale + and reports itself as an UNEXPLAINED orphan. The direction is his: "treat the + orders.parquet like an alias that got updated externally", and "we want + people pointing at aliases not hashes. aliases are what gives us the dag, + hashes are brittle." +- **Reading decision labels:** a bare label such as "D5" in this document + always means this ADR's own decision. Another ADR's decision is always + written with its ADR number and a few words saying what it decides. +- **Context:** PR #189 implements ADR-007, ADR-008 and ADR-009. Reviewing it + turned up a class of defect that no local fix addresses: staleness is + computed on two axes, and the second one (a recorded source digest no longer + matches the file on disk) asks a question the system cannot always answer. + This ADR removes the axis by making raw inputs first-class catalog objects. +- **Tickets:** none filed yet. Supersedes the open design question in the #189 + review notes ("a hash-pinned child stays stale on the source axis"), and + removes the cause of the staleness scan's digest-memo wipe + (`source_identity.py:87-101`). +- **Affected code:** `src/tallyman_xorq/io.py` (`read_project_file`, + `tracked_expr_from_alias`, `pinned_expr_from_alias`, + `_note_parent_records`), `src/tallyman_xorq/source_identity.py` (`mode`, + `digest_for`, `ensure_cas_path`, `recon_cas_path`, `gc_cas`), + `src/tallyman_xorq/staleness.py` (axis 2, `_force_source_rehash`), + `src/tallyman_xorq/recalc.py` (`_classify_orphan`), + `src/tallyman_xorq/ordered_copy.py` (the copy key and its reader options), + `src/tallyman_core/aliases.py` (a second kind of alias), + `src/tallyman_core/manifest.py` (`sources`), + `src/tallyman_mcp/server.py` (`catalog_load_parquet`), + `docs/system-contract.md`. +- **Related ADRs:** `plans/ADR-002-source-identity-content-hash.md` (the `.cas` + clone store this builds on; its `off` and `salt` modes go, and so does its + `sources` map, D6), + `plans/ADR-005-intelligent-csv-import.md` (reader options and the suggestion + contract, which move to import time), + `plans/ADR-008-row-order-of-reads.md` (its D2 and D7 — every source enters + through an ordered copy, a raw read is a build error — are the precedent this + extends to `read_project_file` itself), + `plans/ADR-007-tallyman-owned-materialization.md` (D13, a file is cache only + if it can be re-created; a source version's snapshot passes that test, because + the clone of the imported bytes re-creates it). +- **Evidence:** the probes in the 2026-09-22 session, reproduced under Problem. + +## Terms + +- **Source alias:** an alias whose versions are raw input datasets rather than + computations. `orders` v1, v2, v3. +- **Source entry:** one version of a source alias, stored as an ordinary entry + with a content hash, a recipe and a manifest. +- **The arena:** the files tallyman owns — the clone store under `data/.cas/` + and the snapshots under `compute_cache/result_cache/`. A file outside the arena + is never read by a build. +- **Import:** the explicit act of copying a file into the arena and pointing a + source alias at the resulting entry. +- **Provenance path:** the outside path a version was imported from, recorded + on the source entry and never read again. + +## Problem + +Staleness has two axes (`staleness.py`). Axis 1 asks whether a followed alias +has moved; a `follow=False` parent is never stale on it, because the recipe +asked for that exact revision. Axis 2 asks whether a recorded source digest +still matches the file on disk, and it makes no such distinction — it cannot, +because `manifest.sources` does not record how the entry came to depend on the +file. + +That map is written as a retention record. `io._note_parent_records` +(`io.py:537-559`) folds a parent's source digests into its child so that +`gc_cas` keeps the clones alive for as long as any live entry needs them. The +scan then reads the same map as a freshness record. One field, two meanings. + +Probed on this branch, with a child pinned to its parent by hash: + +``` +parent 8bb2a4661c1a worthy=True sources={orders.parquet: 2d20a090} +child 1fba1bf46640 worthy=False parents=[{hash: 8bb2a466, ref: 8bb2a466, follow: False}] + sources={orders.parquet: 2d20a090} + +# orders.parquet is edited +scan: 1fba1bf46640 stale=True (source, orders.parquet, 2d20a090 → aa0263ca) +recalc: 8bb2a4661c1a rebuilt → 667e15b5ee07 + 1fba1bf46640 noop +scan: 1fba1bf46640 stale=True ... unchanged, permanently +orphan: "UNEXPLAINED stale entry — staleness with no recorded recalc error; file a bug." +``` + +The child's data is correct: it pinned that revision and still serves it. But +the recipe replays to the same content hash, `build_and_persist` early-returns +on the existing entry dir (`build.py:514`), the manifest keeps the old digest, +and no action a user can take clears the flag. `_is_self_ref` (`recalc.py:286`) +only excuses an alias-axis reason, so every project load prints "file a bug" — +which is the signal `tests/test_auto_recalc.py:352` exists to keep trustworthy. +`catalog_recalc()` with no roots defaults to every directly-stale entry +(`server.py:1176`), so the child becomes a permanent recalc root. + +Three further defects share the cause: + +- The scan forces a rehash of every source (`_force_source_rehash`), and the + forced path skips loading the memo and then overwrites `source_digests.json` + with a single entry (`source_identity.py:87-101`), so the next build re-hashes + every source in the project. +- `ensure_cas_path` (`source_identity.py:133-145`) copies the live file to a + clone named by a digest computed before the copy and never re-digests what it + wrote. A file edited mid-copy produces a clone whose name lies about its + content. +- `recon_cas_path`'s last branch (`source_identity.py:173-180`) serves the live + file when the clone is gone and the bytes have drifted, logging a warning. + An entry then returns rows that its recorded digest does not describe. + +Underneath all four is one assumption: that a file the user owns, at a path +tallyman does not control, is a legitimate build input. + +## Governing rule + +**A recipe names aliases. A build reads only files tallyman owns.** + +Everything below follows from those two sentences. + +## Decisions + +### D1. A raw input is an alias whose versions are entries + +Importing a file mints an entry and points a source alias at it. The entry is +ordinary: a content hash, a recipe, a manifest, a snapshot carrying +`__row_order`. The alias lives in the same `aliases.jsonl` as catalog aliases, +with the same head-plus-history shape, so `orders-v2` resolves through the +existing `VERSION_REF_RE` and `resolve_version_ref` with no new syntax. + +A source alias is a distinct kind. `catalog_revise` is refused on one (there is +no recipe to revise), as is promoting a diff onto it, on every surface that +offers those (the MCP tools and the companion's code-edit and promote-diff +routes, which refuse before building anything). Rename and unalias behave as +they do for catalog aliases. The kind is recorded in the alias store so +`catalog_list` can separate inputs from computations. + +**An alias's kind matches the kind of the entries it points at.** A catalog +alias never points at a source entry (one whose manifest records `provenance`), +and a source alias never points at a computed one. `set_alias` enforces it, so +it holds on every route that names an entry: `catalog_alias`, a revise, a +promoted diff, a recalc, an import. + +A source version's `provenance` keeps the name it was **imported as**. A rename +moves the alias's history to a new name and an unalias drops it, and neither +rewrites that record, so anything that names the version as it is now (the pin +reason, the heal error and the import it advises, the rebuild's replay) asks +the alias store which source alias holds it. + +Consequence: a child of a source records `{hash: , ref: +"orders", follow: True}` — the same edge shape as any catalog parent. The DAG +gains its roots. Source versions become visible in the UI, diffable against +each other, and subject to the same reset semantics as everything else. + +**A source entry is worthy, and its snapshot is the copy.** The ordered copy +stops being a thing of its own: `compute_cache/ordered_sources/` merges into +`compute_cache/result_cache/`, and the import writes one parquet file, named by +the source entry's content hash, carrying `__row_order` like every other +snapshot. `manifest.ordered_copies` and the copy key +(`md5(digest ǀ reader signature)`) both go, because the entry hash now names +the file and the reader options are recorded on the entry that used them (D12). + +That snapshot is **cache**, in the sense of ADR-007 D13 — a file is cache if +`ensure_materialized` can re-create it. The import keeps the bytes it was given +in the clone store (`data/.cas/`, open question 1) and records +the reader options on the entry (D12), which is everything needed to write the +file again, so a deleted source snapshot is re-created from the clone and +checked against the recorded `result_digest` exactly as any other re-created +snapshot is. The Cache page lists it unpinned and its delete takes it. The +outside file has nothing to do with this: it may be gone, and the re-creation +never looks at it. + +Only when the clone is gone as well are the rows unrecoverable. Then the +snapshot is the last copy, so it is pinned, the delete answers 409, and a read +of an entry whose snapshot is already gone fails with an error naming the +missing clone and the re-import that repairs it. A source version is therefore +the one kind of entry whose "cache or data" answer depends on a second file +being present, which is the price of not storing the bytes three times. + +**One set of bytes, read one way, is one source version under one alias.** +Importing bytes that another alias of the project already holds, at any version, +is an error naming that alias and version. The likely way to get there is not +knowing the bytes are already in the project, and letting two aliases share the +entry meant one entry directory carrying one alias's name in its provenance and +recipe while another alias pointed at it. (This document first said two such +imports share one entry. PR #219 reversed that after the #217/#218 review found +the second import rewrote the first alias's manifest, and a failure part-way +deleted its entry.) + +A second name for a source is a catalog entry whose recipe reads it: +`catalog_create("orders_eu", "... expr = tracked_expr_from_alias('orders')")`. +Nothing has to be added to force the hashes apart, because a source entry's hash +is an md5 of its bytes and a recipe's is xorq's hash of the expression, and the +new entry follows `orders`, so a re-import advances it. The rule keys on the entry +hash, so a CSV read two ways is still two imports under two aliases (D12), and +it is per project: two projects importing one file each hold their own entry, +snapshot and clone under the same hash. + +### D2. Files enter only by an explicit import + +`read_project_file` is refused in an authored recipe, with the same treatment +ADR-008 D7 gives a raw `xo.deferred_read_parquet`: a build error naming the +import call to use instead. It survives only inside the recipe the importer +generates for a source entry. + +After the import, the outside path is provenance. It is recorded on the source +entry and never read again. Deleting, moving or editing the original file has +no effect on any build. + +### D3. One function, and its full case table + +```python +update_and_depend(outside_path, alias, pinned_version=None, **reader_options) +``` + +The official, and only, way to advance a source alias: + +| state | behaviour | +|---|---| +| alias absent | import, mint v1, return v1 | +| `pinned_version=None`, bytes differ from head | mint the next version, return it | +| `pinned_version=None`, bytes equal head | no-op, return head | +| `pinned_version=N` exists, digest matches | no-op, return vN | +| `pinned_version=N` exists, digest differs | error: the file is not the version you claimed | +| `pinned_version` = head+1, bytes differ | mint it, return it | +| `pinned_version` beyond head+1 | error: versions cannot be skipped | +| `pinned_version=N` < head, digest matches | return vN; the head does not move | +| bytes match a version older than the head | error: see D11 | +| bytes are a version of another alias | error: one set of bytes, one alias (D1) | + +"Bytes" in this table means the bytes read with the given reader options, which +is what the entry hash covers (D12). A row that ends on a version that already +exists (the no-ops, and a new alias over an entry no alias holds any more) never +rewrites that entry: its recipe, build and manifest are the record of the import +that minted it. If its snapshot is gone, the import heals it the way +`ensure_materialized` does, from the clone (restored from the caller's bytes +first when it is gone too) and checked against the recorded `result_digest`, so +a reader that now parses the bytes differently is recorded as an unfaithful heal +rather than becoming the version's rows. + +**Considered and not shipped: `import_once_and_depend(outside_path, alias)`.** +An idempotent form for a standalone script that wants the same data wherever it +runs: import and mint v1 if the alias is absent, and otherwise return v1 +without reading `outside_path` at all. It is recorded here because the design +session settled its semantics and it may come back when scripts outside the +catalog need to carry their own data. It is not in the shipped API, because D4 +keeps imports out of recipes and nothing else has asked for it. If it returns, +note that it pins v1 forever by design, which is what makes it reproducible and +also what keeps it out of recipes. + +### D4. A recipe never imports + +Import is a step of its own: the MCP tool of D10, or a script the user runs. A +recipe reads `tracked_expr_from_alias("orders")`. + +The reason is the shape of the edge. An import call inside a recipe would have +to resolve to a fixed version to stay reproducible, so every recipe carrying its +own import would record `follow=False` and nothing in the catalog would ever +pick up new data. Keeping imports out of recipes makes following the default and +pinning the exception. + +### D5. A bare content hash is refused in a recipe + +`pinned_expr_from_alias` accepts `"-v"` only. A bare content hash gets +the error #166 already gives for a bare alias, naming the version reference to +use instead. + +Every parent edge then names an alias, followed or pinned at a version, and no +opaque hash appears in a recipe or in `manifest.parents`. An entry with no alias +— one built by `catalog_run` — must be named before anything can build on it. + +This alone removes the defect under Problem: the child that pinned +`8bb2a4661c1a` could not have been written. + +### D6. Staleness has one axis + +Delete axis 2. An entry is stale when a followed alias has moved, and that is +the only reason. + +`manifest.sources` goes with it, rather than merely narrowing. It exists as the +retention closure `gc_cas` walks, and once every input is an entry the closure +is the DAG: a source version's file is alive exactly while its entry is alive, +which the parent edges already record. `manifest.ordered_copies` goes for the +same reason (D1 makes the copy a snapshot), and `_note_parent_records` +(`io.py:537`) — the function that folds a parent's sources and copies into its +child, and so the immediate cause of the defect under Problem — is deleted +entirely. + +Deleted with them: `_force_source_rehash`, `source_digests.json`, `digest_for`'s +stat memo, the `unknown` verdict for a removed source file, and the source-axis +branch of `_classify_orphan`. + +### D7. An import is a catalog operation + +`update_and_depend` minting a version takes a checkpoint, emits the same events +a revise does, and triggers auto-recalc on the same per-project switch. A source +advancing is indistinguishable downstream from an alias being revised, so +`followers_of`, `descendant_cone` and the recalc walk work unchanged. + +A `reset_to` rewinds source aliases along with everything else, because +`aliases.jsonl` is tracked in the catalog repo. The clones of rewound versions +stay in `.cas` or the bullpen, so a reset forward restores them. + +### D8. Source identity has one mode + +`TALLYMAN_SOURCE_IDENTITY` goes. Import always digests and always clones; there +is no configuration under which a raw input is unversioned. ADR-002's `off` and +`salt` modes are deleted along with `salted_hash`. + +### D9. Ingest verifies what it wrote + +`ensure_cas_path` digests the clone after writing and fails if it does not match +the name. `recon_cas_path`'s live-bytes fallback becomes an error: a version +tallyman promised and then lost is a failure, not a downgrade to whatever is on +disk now. + +### D10. `catalog_load_parquet` is replaced + +The new MCP tool is `catalog_import_source(outside_path, alias, +pinned_version=None, **reader_options)`, mapping to `update_and_depend`. The old +tool's semantics change too much to keep the name: today re-running it with an +existing alias is an error, and under this ADR it mints the next version. + +### D11. Version history is append-only and monotonic + +An import whose bytes match a version older than the head is an error, naming +`reset_to` as the way back, and `pinned_expr_from_alias("-v")` as the +way to read the old version without moving the head. Importing the bytes under +a different alias is not offered, because D1 refuses it. Two version numbers +never denote the same bytes read the same way, and a version number never moves +backwards while history is intact. + +The case is unlikely — it takes a source file that was edited and then restored +exactly — and the error keeps the alternative (a v3 whose content equals v1's, +or a head that jumps backwards) out of the model. + +### D12. Reader options are fixed at import + +A CSV's delimiter, schema overrides and inference settings are named once, in +the import call, and recorded on the source entry. Two recipes cannot read one +file two ways; import it twice under two aliases. That is the one case in which +the same bytes are under two aliases, and it is two entries, because the entry +hash covers the reader options. + +Because the hash covers them, an import that leaves them out names another +entry. Every message that advises an import therefore prints the recorded +options with it (`source_import.import_call`, in the MCP tool's `schema=` and +`reader_options=` shapes), and a pinned import refused for naming another entry +says the reader options may be what differs. + +This removes the hash fork found in the #215 review, where a function-valued +reader option's `repr` carries a memory address and re-derives a new copy key on +every build. Options are evaluated once, at import, and stored. + +## Testing + +- A child of a source alias is stale after an import advances it, and clear + after recalc. The permanently-stale case of Problem cannot be constructed, + because D5 refuses the recipe. +- Every row of D3's `update_and_depend` table, including the four errors. (The + `import_once_and_depend` test this list first named went with the function, + which D3 records as considered and not shipped.) +- The same bytes under a second alias are refused, naming the alias and version + that hold them, and the first alias's entry is left byte for byte as it was. +- Two projects importing one file each hold their own entry, snapshot and + clone; a heal, a clone sweep or a re-import in one never reaches the other. +- `catalog_alias` of a source entry is refused, as is a source alias pointed at + a computed entry. +- After a rename, the pin reason and the heal error name the alias the version + is under now, and running the advised import repairs that version. +- A re-import of a version whose snapshot is gone rebuilds nothing, rewrites + nothing of the entry, and records an unfaithful heal when the rows differ. +- The advised import for a CSV carries its schema and reader options and, run + as written, repairs the version. +- An authored recipe calling `read_project_file` fails to build, and the error + names the import call. +- An authored recipe passing a bare content hash to `pinned_expr_from_alias` + fails, and the error names `-v`. +- A source alias refuses `catalog_revise` and refuses a promoted diff, through + the MCP tools and through the companion's routes alike. +- `reset_to` rewinds a source alias to v1 and a reset forward restores v2 from + the bullpen. +- Editing the outside file after import changes nothing: the same build, the + same hash, the same rows. +- Deleting the outside file after import changes nothing. +- A clone whose bytes do not match its name fails the import (D9), and a + missing clone with a drifted source raises rather than serving live bytes. +- A deleted source snapshot is re-created from the clone, byte for byte, with + the outside file gone; a child reads through it; the Cache page lists the row + unpinned and its delete is taken. With the clone gone too the row is pinned, + the delete answers 409, and a read raises an error naming the clone and the + re-import. +- A source snapshot is written by pyarrow: format version 2.6, a page index, + row groups of `ORDERED_COPY_ROW_GROUP_ROWS`, the same as a computed snapshot's + in a project holding both. A parquet import touches no polars and keeps the + file's types; a CSV import never collects the frame. +- One scan of a project with N sources performs zero digests. + +## Consequences + +- Every entry's hash changes, because a source entry's hash now stands between + a recipe and its data. The corpus is rebuilt once, as it is for ADR-007 D9. + Nothing is migrated: per the project rule, existing catalogs are rebuilt and + existing recipes are rewritten. In this repo that is 309 `read_project_file` + references across 55 test files, plus nine `src/` modules — the largest + single piece of work in the ADR, and mechanical. +- The staleness scan stops touching the filesystem outside the arena, so its + cost becomes a read of `aliases.jsonl` and the manifests. The digest-memo + defect disappears rather than being fixed. +- A source's history becomes inspectable: which versions existed, when each was + imported, what each was imported from, and which entries were built on each. + Diffing v1 against v2 uses the existing diff, since both are entries. +- Data no longer arrives by being dropped in `data/`. That is a real loss of + convenience, and the import step is the price of the model. +- `docs/architecture.md`, `docs/caching.md`, `docs/expression-lifecycle.md` and + `docs/system-contract.md` all describe the source axis and need rewriting. +- ADR-002 is narrowed rather than superseded: its clone store survives, its + modes and its `sources`-as-freshness reading do not. + +## Implementation notes + +**Stage 1 (PR #217, 2026-09-22): the import path and the refusals — D1, D2, D3, +D5, D9, D10, D12.** What the code does that this document did not say: + +- **A source entry's content hash is `md5("source||")`, + truncated to xorq's 12 hex.** It cannot come from `build_expr`, because the + generated recipe reads the snapshot and the snapshot is named by the hash. The + bytes and the reader options are the whole identity (`source_import.py`). +- **The generated recipe still calls `read_project_file`**, and a contextvar + (`_SOURCE_ENTRY`) is what makes that call legal and resolves it to the entry's + own snapshot. `result_cache._recipe_expr` sets the same contextvar when it + reconstructs a source entry. The path in the recipe is provenance; nothing + opens it. +- **pyarrow writes the snapshot; polars only parses a CSV.** `result_cache/` + holds one shape of parquet file, so the import writes through the settings + every computed snapshot uses (`materialize._PARQUET_OPTIONS`, via + `materialize.write_pinned_parquet`), in row groups of + `ORDERED_COPY_ROW_GROUP_ROWS`. A parquet source needs no parser: pyarrow reads + it with `ParquetFile.iter_batches`, which also means it keeps the types the + file has (polars rewrote a `date32` as a timestamp, a `time32[ms]` as a + `time64[ns]` and a map as a list of structs, #197). A CSV is parsed by polars, + which is the only reader that holds the file's row order and applies the schema + DSL and inference ladder of ADR-005, and its batches go to the same writer + through `collect_batches`, so nothing collects the whole frame. +- **A newer polars would not have helped.** Checked 2026-09-22: 1.40.1 is + installed, 1.44.2 is the latest on PyPI, and neither exposes the parquet + format version or a page-index option — `sink_parquet` writes format 1.0 and + the request is pola-rs/polars#12752, still open. `use_pyarrow=True` is just + pyarrow doing the write. So there is no polars upgrade that reaches the pinned + layout, and writing the file twice to re-encode it was rejected. +- **The pinned layout costs size on high-cardinality numeric columns.** On the + 1.5M-row `tests/big_parquet.py` fixture (`id` sequential int64, `g` 200 + values, `v` random float) the snapshot goes from 16,188,593 bytes (polars) to + 25,495,419 (pyarrow), +57%, with the same 13 row groups of 122,880 rows. + Measured cause: pyarrow dictionary-encodes by default, one 122,880-row chunk + of int64 is ~983 KB, just under its 1 MB `dictionary_pagesize_limit`, so the + dictionary never overflows to PLAIN and the file carries both. The same data + is 16,205,749 bytes with `use_dictionary=False`, and 17,827,724 with + 1,048,576-row groups (where the limit does kick in). That is a question about + `_PARQUET_OPTIONS` and about whether the two row-group sizes should be one, + which is ADR-009's to answer for every snapshot at once; nothing here writes a + source snapshot differently from a computed one to avoid it. Also measured: + the page index is present either way — polars writes one without being asked — + so the format version and the encoder are what actually differed. +- **`SNAPSHOT_FORMAT_VERSION` stays 1**, as it did for the same change to the + ordered copies in #215, even though the bytes of a source snapshot do change. + A source entry imported before this change records the digest of the + polars-written file, so re-creating its snapshot now writes different bytes + and the heal is recorded as unfaithful, attributed to the recipe rather than to + the format. Bumping the version would attribute that one case correctly and + would misattribute every computed entry, whose layout did not change; per the + project rule the corpus is rebuilt instead, so there is no such entry. +- **Provenance lives in one manifest field**, `manifest.provenance` + (`{alias, version, path, digest, suffix, reader, imported_at}`), and its + presence is what makes an entry a source entry. `manifest.sources` stays empty + for one, so `catalog_state._live_source_digests` had to learn to keep a clone + alive from `provenance.digest` as well — otherwise a reset retires the only + copy of the imported bytes. + Its `alias` and `version` are the name the version was imported as (D1): a + rename or unalias does not rewrite them. +- **`entry_staleness` skips axis 2 for a source entry** rather than reporting it + unknown. A small piece of D6, forced: every source entry would otherwise carry + a permanent "source axis unknown". +- **`data/` stops being special.** An import takes any path. A relative path + resolves against the working directory, and the replay CLI expands + `${TALLYMAN_PROJECT_ROOT}` in storyboard arguments so `demo/storyboard.json` + can name a file shipped with the project. +- **Open question 1 is answered "keep both".** The imported bytes stay in + `data/.cas/` beside the ordered snapshot, because ADR-005's + suggestion-and-retry contract has to re-read the file as imported and the + outside path may be gone. Every import therefore stores the data twice, and + the second copy now earns its keep twice over: it is also what makes the + snapshot cache rather than data (D1). Dropping the clone would not halve the + storage, it would move the same bytes into the pinned column. +- **`tallyman_read_csv` is refused alongside `read_project_file`.** D2 names only + the latter, but both open a file the catalog does not own, and leaving the CSV + reader open would be a hole in the rule. +- A source entry keeps a real `xorq_build/`, so `load_entry_expr`, the diff and + the viewer treat it as an ordinary entry. Nothing reads it on the normal path: + a worthy entry whose snapshot exists is served by a bare read of that file. + +**Deliberate debt, now paid.** `TALLYMAN_LEGACY_FILE_READS=1` disabled D2's +refusal for the whole suite so stage 1 could land before the call-site rewrite. +Stage 2 deletes the variable, the `tests/conftest.py` line that set it and the +branch in `io.py` that read it. There is no way to author a raw file read. + +**Stage 2 (PR #218, 2026-09-23): D6, D8 and the call-site rewrite.** What the +code does that this document did not say: + +- **`manifest.sources` is deleted, not narrowed.** D6 says the field reverts to + being the retention closure; in the event it has no readers left at all, since + a source version is an entry and the closure is the DAG. `dependents.sources_of` + goes with it, and `io._note_parent_records` — the function whose folding of a + parent's digests into its child is the direct cause of the defect under + Problem — is deleted outright. +- **`StaleReason.axis` survives as a field that is always `"alias"`.** Deleting + it would change the shape of every staleness reason in the API and the UI for + no gain; a one-value field reads the same and says the axis is no longer a + choice. +- **The fixture split is `orders_parquet` and `orders_src`.** The first is the + file, for a test about the bytes; the second imports it and returns the alias, + for a test that needs a recipe. The alias is `orders_src` rather than `orders` + because several tests already create a catalog alias called `orders`, and a + name is one kind or the other and never both. +- **An import must not depend on which project is active.** `update_and_depend(path, + alias, project=X)` failed whenever `X` was not the active project: the generated + recipe's `read_project_file` resolved the ambient project while `_SOURCE_ENTRY` + named `X`, so the importer's own recipe hit the refusal written for authored + recipes. The read follows the entry being minted. Every call site passed before + the rewrite because the `project` fixture also activates its project; the cache + lab, which warms xorq in a project of its own without activating it, is what + exposed it. +- **Two test expectations changed because the behaviour is now right**, not to + keep them green. The page-load profiler measures three entries where it + measured two, because an imported source version is an entry. And the reset + round-trip has to import its second file *after* the first checkpoint: import + it before, and its source entry survives the reset back, so no clone is left + referenced only by a retired entry and the case ADR-007 D13 is about stops + being exercised at all. Arranged correctly, that test is the proof that a + backward reset parks a clone in the bullpen rather than unlinking it — the + failure mode that deleting `manifest.sources` could otherwise have caused + silently. + +**Review fixes (PR #219, 2026-09-24).** Found reviewing #217 and #218; each +landed test first. What the code does that the decisions above did not say: + +- **Duplicate bytes were a shared entry.** Stage 1 let a second alias import + bytes another alias held, on the grounds that the hash made it one entry. The + second import re-ran `_mint` over that entry, rewriting the first alias's + provenance and recipe, and a failure part-way deleted the entry directory the + first alias pointed at. D1 now refuses the import (`_refuse_bytes_held_elsewhere`). +- **The kind rule lives in `set_alias`** (D1), not in each tool, after + `catalog_alias` put a catalog name on a source entry and opened it to + `catalog_revise`. The companion's `PUT /api/code` and promote-diff routes had + no refusal at all: they built the entry, failed to point the source alias at + it and answered 500 with the entry left under no alias. They now refuse with + the tools' text (`aliases.source_alias_refusal`) before building, answering 409. +- **A version is named by the alias that holds it now** + (`source_import.current_source_version`). Naming it by `provenance.alias` + made the heal advice mint a second alias under the old name after a rename, + and made `scripts/rebuild_native_catalog.py` replay a renamed source under the + old name: children reading the new name failed to rebuild, and the order of + the source's versions was lost. +- **A repair import heals** (D3). It used to go through `_mint`, which rebuilt + the entry, rewrote the manifest and recorded whatever digest it wrote: a + probe with a reader that drops a row went from 10 rows to 9 under the same + hash with nothing recorded. An entry directory without a manifest, which a + crash part-way through an import leaves, is not an entry, and a re-import + writes it again. +- **Advised imports carry the reader options** (D12). Without them the heal + error's advice for a CSV named another entry and was refused. + +## Open questions + +1. ~~**Does a source version keep its raw bytes as well as its snapshot?**~~ + **Answered 2026-09-22: keep both.** A CSV imported with the wrong schema + would otherwise be fixable only from the outside file, which may be gone, + and ADR-005's suggestion-and-retry contract assumes the bytes can be + re-read. Keeping the clone also decides D1's other half: the snapshot is + re-creatable from it, so it is cache in the sense of ADR-007 D13 rather than + irreplaceable data. Every import stores the data twice; the clone is + copy-on-write where the filesystem offers it, so the initial cost is near + zero, but the snapshot is a real second materialization. See the + implementation notes for the measured sizes. +2. **Directories and multi-file datasets.** A dataset that arrives as + `orders/part-0000.parquet`, `part-0001.parquet`, … rather than one file: + does `update_and_depend("~/exports/orders/", "orders")` import the directory + as one version, concatenating the parts in a defined order, or is import + strictly one file to one alias? The snapshot is one parquet either way, so + this is a question about the reader and about how part order is fixed, not + about the store. Defer until a real dataset needs it; refuse a directory + with a clear error until then. diff --git a/scripts/perf_report.py b/scripts/perf_report.py index 3f931fbc..ef38deb0 100644 --- a/scripts/perf_report.py +++ b/scripts/perf_report.py @@ -346,6 +346,18 @@ def generate_datasets(data_dir: Path, scale: float) -> list[dict]: return out +def import_datasets(data_dir: Path) -> None: + """Point a source alias at each generated dataset (ADR-011 D2). + + A recipe never opens a file, so the corpus enters the catalog by import. Re-running with the same + bytes is a no-op, which is what makes the whole script re-runnable against a warm workdir. + """ + from tallyman_xorq.source_import import update_and_depend + + for name in ("events", "events_lowcard", "wide_a", "wide_b", "agg_small"): + update_and_depend(data_dir / f"{name}.parquet", name, project=PROJECT) + + # --------------------------------------------------------------------------- # catalog entries # --------------------------------------------------------------------------- @@ -354,49 +366,52 @@ def generate_datasets(data_dir: Path, scale: float) -> list[dict]: ENTRY_ORDER = ["events_scan", "lowcard_scan", "union_wide", "small_limit", "agg_by_plate", "agg_nunique"] -def entry_code(key: str, hashes: dict[str, str], scale: float) -> str: +def entry_code(key: str, scale: float) -> str: limit = max(int(SMALL_LIMIT * scale), 1_000) codes = { - "events_scan": "from tallyman_xorq.io import read_project_file\nexpr = read_project_file('events.parquet')\n", + "events_scan": ( + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('events')\n" + ), "lowcard_scan": ( - "from tallyman_xorq.io import read_project_file\n" - "expr = read_project_file('events_lowcard.parquet')\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('events_lowcard')\n" ), "union_wide": ( - "from tallyman_xorq.io import read_project_file\n" - "a = read_project_file('wide_a.parquet')\n" - "b = read_project_file('wide_b.parquet')\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "a = tracked_expr_from_alias('wide_a')\n" + "b = tracked_expr_from_alias('wide_b')\n" "expr = a.union(b)\n" ), "small_limit": ( - f"from tallyman_xorq.io import read_project_file\n" - f"expr = read_project_file('events.parquet').limit({limit})\n" + f"from tallyman_xorq.io import tracked_expr_from_alias\n" + f"expr = tracked_expr_from_alias('events').limit({limit})\n" ), # Grouped approximate distinct count. The exact `nunique()` here is the # datafusion memory bomb (#46: ~37KB transient/group); `approx_nunique` # is the bounded-state alternative an entry should use. Kept on its own # small dataset regardless; keep it OFF the 4M-row events table. "agg_nunique": ( - "from tallyman_xorq.io import read_project_file\n" - "t = read_project_file('agg_small.parquet')\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "t = tracked_expr_from_alias('agg_small')\n" "expr = t.group_by('plate').aggregate(\n" " n_tickets=t.count(),\n" " n_days=t.issue_ts.approx_nunique(),\n" " n_states=t.state.approx_nunique(),\n" ")\n" ), - } - if key == "agg_by_plate": - events_hash = hashes["events_scan"] - return ( + # Chains off events_scan by ALIAS. A recipe never names a hash (ADR-011 D5), so child_build + # points an alias at each entry as it lands and the next one reads it. + "agg_by_plate": ( "from tallyman_xorq.io import tracked_expr_from_alias\n" - f"t = tracked_expr_from_alias('{events_hash}')\n" + "t = tracked_expr_from_alias('events_scan')\n" "expr = t.group_by('plate').aggregate(\n" " n_tickets=t.count(),\n" " first_seen=t.issue_ts.min(),\n" " last_seen=t.issue_ts.max(),\n" ")\n" - ) + ), + } return codes[key] @@ -409,15 +424,16 @@ def child_build(key: str, workdir: Path, scale: float) -> None: import psutil sampler = RssSampler(psutil.Process(), interval=0.025) - hashes = _load_hashes(workdir) - code = entry_code(key, hashes, scale) + code = entry_code(key, scale) + from tallyman_core.aliases import set_alias from tallyman_xorq.build import build_and_persist t0 = time.monotonic() res = build_and_persist(PROJECT, code) wall = time.monotonic() - t0 sampler.stop() + set_alias(PROJECT, key, res.content_hash) # the next entry in ENTRY_ORDER chains off this alias print( json.dumps( { @@ -757,6 +773,7 @@ def main() -> int: # -- datasets ------------------------------------------------------------ log.info("phase: datasets") results["datasets"] = generate_datasets(data_dir(PROJECT), args.scale) + import_datasets(data_dir(PROJECT)) # -- entry builds (fresh process each) ----------------------------------- log.info("phase: entry builds") diff --git a/scripts/rebuild_native_catalog.py b/scripts/rebuild_native_catalog.py index a328174b..055417e1 100644 --- a/scripts/rebuild_native_catalog.py +++ b/scripts/rebuild_native_catalog.py @@ -19,6 +19,14 @@ remaps every hash-keyed artifact through an ``old_hash -> new_hash`` table built as it goes: +* a SOURCE entry (ADR-011: a version of a source alias, whose rows are an imported + file) is not re-exec'd at all — its generated recipe reads the very snapshot the + rebuild just wiped, and a raw file read is a build error anywhere else. It is + replayed through ``update_and_depend``, from the provenance path when that still + holds the imported bytes and from the clone under ``data/.cas`` when it does not, + under the source alias that holds it now (a rename leaves ``provenance.alias`` + naming the alias it was imported as) and pinned to its version there. + Its hash is a function of the bytes and the reader options, so it is preserved; * recipes that chain by ALIAS (``tracked_expr_from_alias("citibike")``) survive untouched — the alias is re-pointed at the rebuilt parent before the child is built; * recipes that chain by literal HASH (``tracked_expr_from_alias("763193211746")``) are @@ -64,6 +72,7 @@ import shutil import sys from dataclasses import dataclass, field +from pathlib import Path # 12-char lowercase-hex content hash (the entry-dir / pointer naming). _HASH_RE = re.compile(r"\b[0-9a-f]{12}\b") @@ -78,6 +87,8 @@ class OldCatalog: recipes: dict[str, str] = field(default_factory=dict) # old_hash -> expr.py text aliases: dict[str, str] = field(default_factory=dict) # alias -> current old_hash history: dict[str, list[str]] = field(default_factory=dict) # alias -> [old_hash, ...] + kinds: dict[str, str] = field(default_factory=dict) # alias -> "catalog" | "source" (ADR-011 D1) + provenance: dict[str, dict] = field(default_factory=dict) # old_hash -> manifest.provenance (source entries) charts: dict[str, str] = field(default_factory=dict) # old_hash -> vega-lite spec text prompts: dict[str, list[dict]] = field(default_factory=dict) # old_hash -> [{prompt, at}, ...] post_processing: dict[str, str] = field(default_factory=dict) # name -> source @@ -99,6 +110,7 @@ def read_old_catalog(project: str) -> OldCatalog: Each section reads from its decomposed file if present, else falls back to ``catalog.yaml``. """ + from tallyman_core.aliases import CATALOG_KIND # noqa: PLC0415 from tallyman_core.paths import catalog_dir # noqa: PLC0415 cat = catalog_dir(project) @@ -116,6 +128,12 @@ def read_old_catalog(project: str) -> OldCatalog: for d in sorted(entries.iterdir()): if d.is_dir() and (d / "expr.py").is_file() and (d / "manifest.json").is_file(): oc.recipes[d.name] = (d / "expr.py").read_text() + try: + provenance = (json.loads((d / "manifest.json").read_text()) or {}).get("provenance") + except (OSError, ValueError): + provenance = None + if provenance: + oc.provenance[d.name] = provenance pj = d / "prompts.jsonl" if pj.is_file(): oc.prompts[d.name] = [json.loads(x) for x in pj.read_text().splitlines() if x.strip()] @@ -133,6 +151,7 @@ def read_old_catalog(project: str) -> OldCatalog: rec = json.loads(line) oc.aliases[rec["alias"]] = rec["latest"] oc.history[rec["alias"]] = rec.get("history", []) + oc.kinds[rec["alias"]] = rec.get("kind", CATALOG_KIND) elif (cat / "aliases.json").is_file(): oc.aliases = json.loads((cat / "aliases.json").read_text()) ah = cat / "alias_history.json" @@ -180,6 +199,8 @@ def parse_deps( aliases: dict[str, str], history: dict[str, list[str]], known: set[str], + provenance: dict | None = None, + kinds: dict[str, str] | None = None, ) -> set[str]: """The old hashes this recipe depends on, as the build resolved them. @@ -188,8 +209,19 @@ def parse_deps( revise — ``tracked_expr_from_alias`` of one's own alias, #74), the dependency is the PREVIOUS revision in the alias history, not the current latest (which would be this very entry, a false self-cycle); otherwise it is the alias's current hash. + + A SOURCE entry reads nothing, but its versions are minted in order — ``update_and_depend`` + refuses to skip one — so it depends on the previous version of the source alias that holds it + (``source_version_in``). That is not always ``provenance["alias"]``, the name it was imported as: + a rename moves the history to a new name and leaves the provenance behind. """ + from tallyman_xorq.source_import import source_version_in # noqa: PLC0415 + out: set[str] = set() + if provenance: + held = source_version_in(history, kinds, self_hash, provenance["alias"]) + if held is not None and held[1] > 1: + out.add(history[held[0]][held[1] - 2]) for ref in _FROM_CAT_RE.findall(expr_text): if ref in aliases: hist = history.get(ref, []) @@ -206,12 +238,17 @@ def parse_deps( def toposort( - recipes: dict[str, str], aliases: dict[str, str], history: dict[str, list[str]] | None = None + recipes: dict[str, str], + aliases: dict[str, str], + history: dict[str, list[str]] | None = None, + provenance: dict[str, dict] | None = None, + kinds: dict[str, str] | None = None, ) -> list[str]: """Dependency order (parents before children) over the tracked_expr_from_alias graph.""" history = history or {} + provenance = provenance or {} known = set(recipes) - dmap = {h: parse_deps(t, h, aliases, history, known) for h, t in recipes.items()} + dmap = {h: parse_deps(t, h, aliases, history, known, provenance.get(h), kinds) for h, t in recipes.items()} order: list[str] = [] placed: set[str] = set() while len(placed) < len(recipes): @@ -237,6 +274,75 @@ def sub(m: re.Match) -> str: return _HASH_RE.sub(sub, expr_text) +def replay_import( + project: str, + provenance: dict, + *, + into: tuple[str, int] | None = None, + prompt: str | None = None, + log=print, +) -> str: + """Re-import a source version and return its content hash (ADR-011). + + A source entry cannot be re-exec'd: its generated recipe reads its own snapshot, which the rebuild + has just wiped, and ``read_project_file`` is a build error outside that recipe. What it CAN do is + the import again, which is deterministic — the entry hash is ``md5("source||")``, + so the same bytes under the same reader options mint the same entry, whichever file they are read + from. + + Reads the provenance path when it still holds the imported bytes, and otherwise the clone under + ``data/.cas``, which survives the rebuild because it lives under ``data/`` and only the catalog + bookkeeping is wiped. Taking the clone changes the provenance path the rebuilt entry records (it + then names the clone), which is logged; it changes nothing else. + + *into* is ``(alias, version)``: the source alias that holds the version in the old catalog and + its place in that alias's history (``source_version_in``). The import goes under that alias, + which after a rename is not ``provenance["alias"]`` (the name it was imported as), and is pinned + to that version, so a replay out of order is an error instead of one version's bytes minted as + another's number. None when no source alias holds it: it goes under the name it was imported + as, unpinned, and the final alias write keeps that name only if the old catalog has it. + """ + from tallyman_core.paths import data_dir # noqa: PLC0415 + from tallyman_xorq import source_identity as si # noqa: PLC0415 + from tallyman_xorq.ordered_copy import _spec_from_json # noqa: PLC0415 + from tallyman_xorq.source_import import update_and_depend # noqa: PLC0415 + + imported = f"{provenance['alias']}-v{provenance['version']}" + alias, version = into if into is not None else (provenance["alias"], None) + if into is None: + name = f"the version imported as {imported}, which no source alias holds" + else: + name = f"{alias}-v{version}" + ("" if f"{alias}-v{version}" == imported else f" (imported as {imported})") + + digest, suffix = provenance["digest"], provenance.get("suffix", "") + original = Path(provenance["path"]) + clone = data_dir(project) / ".cas" / f"{digest}{suffix}" + if original.is_file() and si._digest_file(original) == digest: + src = original + elif clone.is_file(): + src = clone + log(f" {name}: {original} no longer holds the imported bytes; re-importing from {clone.name}") + else: + raise RuntimeError( + f"cannot rebuild {name}: {original} no longer has the imported bytes and their clone {clone} is " + "missing, so there is nothing to import" + ) + + reader = provenance["reader"] + schema = _spec_from_json(reader["schema"]) if reader["kind"] == "csv" else None + options = dict(reader.get("scan_kwargs") or {}) if reader["kind"] == "csv" else {} + out = update_and_depend( + src, + alias, + version, + project=project, + prompt=prompt, + schema=schema, + **options, + ) + return out["hash"] + + def rebuild_project(project: str, *, dry_run: bool = False, log=print) -> dict[str, str]: """Re-exec a project's recipes into the native store. Returns old->new hash map. @@ -254,14 +360,16 @@ def rebuild_project(project: str, *, dry_run: bool = False, log=print) -> dict[s from tallyman_core.post_processing import write_post_processing # noqa: PLC0415 from tallyman_core.summary_stats import write_stat # noqa: PLC0415 from tallyman_xorq import build_and_persist # noqa: PLC0415 + from tallyman_xorq.source_import import source_version_in # noqa: PLC0415 oc = read_old_catalog(project) if not oc.recipes: raise RuntimeError(f"no rebuildable entries (expr.py + manifest.json) found in project {project!r}") - order = toposort(oc.recipes, oc.aliases, oc.history) + order = toposort(oc.recipes, oc.aliases, oc.history, oc.provenance, oc.kinds) log( - f"project {project!r}: {len(oc.recipes)} entries, {len(oc.aliases)} aliases, " - f"{len(oc.charts)} charts, {len(oc.post_processing)} post-processing, {len(oc.stats)} stats" + f"project {project!r}: {len(oc.recipes)} entries ({len(oc.provenance)} imported sources), " + f"{len(oc.aliases)} aliases, {len(oc.charts)} charts, " + f"{len(oc.post_processing)} post-processing, {len(oc.stats)} stats" ) log(f"build order (topological): {order}") @@ -301,18 +409,29 @@ def rebuild_project(project: str, *, dry_run: bool = False, log=print) -> dict[s remap: dict[str, str] = {} for old_hash in order: - recipe = oc.recipes[old_hash].replace("${TALLYMAN_PROJECT_ROOT}", root) - recipe = rewrite_hash_refs(recipe, remap) # fix literal parent-hash refs prompts = oc.prompts.get(old_hash, []) first = prompts[0].get("prompt") if prompts else None - res = build_and_persist(project, recipe, prompt=first) - remap[old_hash] = res.content_hash - tag = "" if res.content_hash == old_hash else f" (rehashed -> {res.content_hash})" + provenance = oc.provenance.get(old_hash) + imported_into = None # the alias an import pointed at the entry itself + if provenance is not None: + held = source_version_in(oc.history, oc.kinds, old_hash, provenance["alias"]) + new_hash = replay_import(project, provenance, into=held, prompt=first, log=log) + imported_into = held[0] if held is not None else provenance["alias"] + else: + recipe = oc.recipes[old_hash].replace("${TALLYMAN_PROJECT_ROOT}", root) + recipe = rewrite_hash_refs(recipe, remap) # fix literal parent-hash refs + new_hash = build_and_persist(project, recipe, prompt=first).content_hash + remap[old_hash] = new_hash + tag = "" if new_hash == old_hash else f" (rehashed -> {new_hash})" log(f" built {old_hash}{tag}") for name in revises_at.get(old_hash, []): - al.set_alias(project, name, res.content_hash) + # The import already pointed the alias it went under. Any other alias that holds the entry + # is pointed here, with the kind it had: a second source alias over the same bytes has to + # be on this version before its own next version replays, pinned to follow it. + if name != imported_into: + al.set_alias(project, name, new_hash, kind=oc.kinds.get(name, al.CATALOG_KIND)) if len(prompts) > 1: # build wrote only the first; carry the rest - p = prompts_path(project, res.content_hash) + p = prompts_path(project, new_hash) p.parent.mkdir(parents=True, exist_ok=True) p.write_text("".join(json.dumps(x) + "\n" for x in prompts)) @@ -321,6 +440,7 @@ def rebuild_project(project: str, *, dry_run: bool = False, log=print) -> dict[s project, {name: remap.get(h, h) for name, h in oc.aliases.items()}, {name: [remap.get(x, x) for x in hs] for name, hs in oc.history.items()}, + {name: oc.kinds.get(name, al.CATALOG_KIND) for name in oc.aliases}, ) for old_h, spec in oc.charts.items(): diff --git a/scripts/rebuild_parking_catalog.py b/scripts/rebuild_parking_catalog.py index a9e3038f..822336b3 100644 --- a/scripts/rebuild_parking_catalog.py +++ b/scripts/rebuild_parking_catalog.py @@ -42,7 +42,8 @@ from dataclasses import dataclass from pathlib import Path -# The three raw sources the recipes read via `read_project_file(...)`. +# The three raw files. Each is imported once (ADR-011 D2) under a source alias named after it with a +# "_src" suffix, and the tier-0 recipes read that alias; a recipe never opens a file. SOURCES = ( "camera_select_columns.parquet", "camera_violations.parquet", @@ -61,16 +62,17 @@ class Step: # is built before any dependent reads it, so `tracked_expr_from_alias(alias)` resolves to the # intended head. Codes are the recipes recovered from the original entry dirs. STEPS: tuple[Step, ...] = ( - # --- Tier 0: raw sources (read_project_file) --- + # --- Tier 0: one catalog alias per imported source --- Step( "camera_select_columns", "create", - "from tallyman_xorq.io import read_project_file\nexpr = read_project_file('camera_select_columns.parquet')\n", + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('camera_select_columns_src')\n", ), Step( "camera_select_columns", "revise", - "from tallyman_xorq.io import read_project_file\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" "import xorq.vendor.ibis as ibis\n" "def parse_issue_date(t):\n" " return (\n" @@ -80,29 +82,31 @@ class Step: " .else_(t.issue_date.to_date('%m/%d/%Y'))\n" " .end()\n" " )\n" - "expr = read_project_file('camera_select_columns.parquet').mutate(issue_date=parse_issue_date)\n", + "expr = tracked_expr_from_alias('camera_select_columns_src').mutate(issue_date=parse_issue_date)\n", ), Step( "camera_violations", "create", - "from tallyman_xorq.io import read_project_file\nexpr = read_project_file('camera_violations.parquet')\n", + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('camera_violations_src')\n", ), Step( "camera_violations", "revise", - "from tallyman_xorq.io import read_project_file\n" - "expr = read_project_file('camera_violations.parquet').limit(10_000_000)\n", + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('camera_violations_src').limit(10_000_000)\n", ), Step( "camera_violations", "revise", - "from tallyman_xorq.io import read_project_file\n" - "expr = read_project_file('camera_violations.parquet').limit(5_000_000)\n", + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('camera_violations_src').limit(5_000_000)\n", ), Step( "parking_plt_only4", "create", - "from tallyman_xorq.io import read_project_file\nexpr = read_project_file('parking_plt_only4.parquet')\n", + "from tallyman_xorq.io import tracked_expr_from_alias\n" + "expr = tracked_expr_from_alias('parking_plt_only4_src')\n", ), # --- Tier 1: depend on tier-0 aliases --- Step( @@ -312,12 +316,17 @@ class Step: ) -def _stage_sources(src_dir: Path, project: str, rows: int) -> None: - """Place the raw sources into /data/, truncated to `rows` if > 0.""" +def _stage_sources(src_dir: Path, project: str, rows: int) -> list[Path]: + """Place the raw sources into /data/, truncated to `rows` if > 0; return where they landed. + + ``data/`` is not special any more (ADR-011): the files are staged here only so the rebuild is + self-contained and re-runnable, and they enter the catalog by the import that follows. + """ from tallyman_core.paths import data_dir dst = data_dir(project) dst.mkdir(parents=True, exist_ok=True) + staged: list[Path] = [] for name in SOURCES: src = src_dir / name if not src.is_file(): @@ -345,6 +354,8 @@ def _stage_sources(src_dir: Path, project: str, rows: int) -> None: else: shutil.copy2(src, target) print(f" staged {name}: full copy") + staged.append(target) + return staged def _verify(project: str) -> None: @@ -387,7 +398,7 @@ def rebuild(src_dir: Path, project: str, rows: int) -> None: ensure_project(project) ensure_catalog_repo(project) genesis(project) - # read_project_file/tracked_expr_from_alias inside recipe code resolve the active project with + # tracked_expr_from_alias inside recipe code resolves the active project with # no explicit arg, and resolve_project() reads the active_project file *before* # TALLYMAN_PROJECT (env is only a one-shot seed when the file is absent). Write # the file so resolution is deterministic even when a different project was @@ -395,7 +406,15 @@ def rebuild(src_dir: Path, project: str, rows: int) -> None: set_active_project(project) print(f"staging sources (rows={'full' if not rows else rows}):") - _stage_sources(src_dir, project, rows) + staged = _stage_sources(src_dir, project, rows) + + print(f"importing {len(staged)} sources:") + from tallyman_xorq.source_import import update_and_depend + + for path in staged: + out = update_and_depend(path, f"{path.stem}_src", project=project, prompt="rebuild: import") + print(f" {out['alias']}-v{out['version']} -> {out['hash']} ({out['row_count']} rows)") + checkpoint_catalog(project, "rebuild: import sources") print(f"building {len(STEPS)} entries:") for i, step in enumerate(STEPS, 1): diff --git a/scripts/spike_csv_source_identity.py b/scripts/spike_csv_source_identity.py deleted file mode 100644 index 423b65fd..00000000 --- a/scripts/spike_csv_source_identity.py +++ /dev/null @@ -1,77 +0,0 @@ -"""ADR-008 evidence (D2, D7) and #168: is a CSV root's ordered intermediate fixed under a content hash? - -``tallyman_read_csv`` reads a CSV through an ordered parquet intermediate. That file's key is -``md5(absolute path | schema | reader options)``, not content, and it is overwritten in place when the CSV's mtime -changes. The script edits a CSV, re-runs the identical recipe, and reports the build hash before and after, and what -the first version's frozen build returns afterwards, for three shapes of the root expression: - -1. today: a read of the intermediate with a trailing ``order_by("original_row_order")``; -2. ADR-008 D7 without the fix for #168: a plain read of the same intermediate (no sort, so no snapshot, no digest); -3. the fix for #168: digest the CSV, clone it to a content-named path, and key the intermediate on the clone. - -Raw xorq builds, no tallyman build machinery, so case 1 reports the hash only: in a real build the root is worthy -because of its Sort, and its baked snapshot goes on serving the old rows. An unchanged hash also means -``build_and_persist`` returns the existing entry, so no new version is created at all. - - uv run python scripts/spike_csv_source_identity.py -""" - -from __future__ import annotations - -import os -import tempfile -import time -from pathlib import Path - -HOME = Path(tempfile.mkdtemp(prefix="spike_csv_identity_")) -os.environ["TALLYMAN_HOME"] = str(HOME) # the ordered intermediates live under it; set before tallyman is imported - -import xorq.api as xo # noqa: E402 -from xorq.ibis_yaml.compiler import build_expr, load_expr # noqa: E402 - -from tallyman_xorq import source_identity as si # noqa: E402 -from tallyman_xorq.io import _ordered_csv_parquet, tallyman_read_csv # noqa: E402 - -BUILDS = HOME / "builds" -CAS = HOME / "cas" -ORIGINAL = "id,amount\n1,10\n2,20\n3,30\n" -EDITED = "id,amount\n1,10\n2,999\n3,30\n4,40\n" - - -def today(csv: Path): - return tallyman_read_csv(str(csv)) - - -def plain_read(csv: Path): - return xo.deferred_read_parquet(str(_ordered_csv_parquet(str(csv), None, {}))) - - -def keyed_on_clone(csv: Path): - clone = CAS / f"{si._digest_file(csv)}{csv.suffix}" - if not clone.exists(): - clone.write_bytes(csv.read_bytes()) # tallyman's ensure_cas_path makes a copy-on-write clone here - return xo.deferred_read_parquet(str(_ordered_csv_parquet(str(clone), None, {}))) - - -def main() -> None: - CAS.mkdir() - csv = HOME / "sales.csv" - cases = ( - ("1. today (trailing order_by kept)", today, False), - ("2. ADR-008 D7 without the #168 fix", plain_read, True), - ("3. keyed on a content-named clone", keyed_on_clone, True), - ) - for label, root, reread in cases: - csv.write_text(ORIGINAL) - v1 = Path(build_expr(root(csv), builds_dir=BUILDS)) - time.sleep(0.05) # so the edit changes the CSV's mtime - csv.write_text(EDITED) - v2 = Path(build_expr(root(csv), builds_dir=BUILDS)) - print(f"{label}: hash before the edit {v1.name}, after {v2.name}, same hash: {v1.name == v2.name}") - if reread: - rows = load_expr(v1).execute().amount.tolist() - print(f" V1's frozen build, re-read after the edit: {rows} (built from {[10, 20, 30]})") - - -if __name__ == "__main__": - main() diff --git a/scripts/spike_reset_roundtrip.py b/scripts/spike_reset_roundtrip.py index e9c1c4ec..72a8f97b 100644 --- a/scripts/spike_reset_roundtrip.py +++ b/scripts/spike_reset_roundtrip.py @@ -4,9 +4,11 @@ - the entry directory: moved to the bullpen, copied back by a forward reset; - the snapshots under ``compute_cache/``: the same, driven by a git-tracked list of the files that existed; -- the content-addressed clone of the source under ``data/.cas/``: DELETED by ``gc_cas``, not moved. +- the content-addressed clone of an imported source under ``data/.cas/``: MOVED to the bullpen by ``gc_cas`` when + no surviving entry needs it, and copied back by a forward reset. -Step s1 has an entry over ``orders.parquet``. Step s2 adds a cheap entry and a worthy entry over ``extra.parquet``. +Step s1 has an entry over an imported ``orders.parquet``. Step s2 imports ``extra.parquet`` and adds a cheap entry +and a worthy entry over it. The script resets to s1, resets forward to s2, and reads both s2 entries. ``extra.parquet`` is never touched. It then empties the cache, which is the cold state of ADR-007 D7, and reads the worthy entry again so that it has to heal. @@ -32,14 +34,15 @@ from tallyman_core.paths import compute_cache_dir # noqa: E402 from tallyman_xorq import build_and_persist # noqa: E402 from tallyman_xorq.result_cache import cached_result_expr # noqa: E402 +from tallyman_xorq.source_import import update_and_depend # noqa: E402 PROJECT = "spike" -def recipe(source: str, tail: str = "") -> str: +def recipe(alias: str, tail: str = "") -> str: return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file({source!r}, project={PROJECT!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({alias!r}, project={PROJECT!r})\n" f"expr = t{tail}\n" ) @@ -66,10 +69,12 @@ def main() -> None: pd.DataFrame({"region": ["a", "b", "a"], "price": [1.0, 2.0, 3.0]}).to_parquet(data / "orders.parquet") pd.DataFrame({"k": ["x", "y", "x", "z"], "v": [1.0, 2.0, 3.0, 4.0]}).to_parquet(data / "extra.parquet") - build_and_persist(PROJECT, recipe("orders.parquet")) + update_and_depend(data / "orders.parquet", "orders_src", project=PROJECT) + build_and_persist(PROJECT, recipe("orders_src")) s1 = cs.checkpoint_catalog(PROJECT, "s1") - cheap = build_and_persist(PROJECT, recipe("extra.parquet", ".filter(t.v > 1)")) - worthy = build_and_persist(PROJECT, recipe("extra.parquet", ".group_by('k').aggregate(s=t.v.sum())")) + update_and_depend(data / "extra.parquet", "extra_src", project=PROJECT) + cheap = build_and_persist(PROJECT, recipe("extra_src", ".filter(t.v > 1)")) + worthy = build_and_persist(PROJECT, recipe("extra_src", ".group_by('k').aggregate(s=t.v.sum())")) s2 = cs.checkpoint_catalog(PROJECT, "s2") print(f"at s2, clones: {clones()}") @@ -80,7 +85,7 @@ def main() -> None: print(f"after the reset to s1, clones: {clones()}") cs.reset_to(PROJECT, s2) print(f"after the reset forward to s2, clones: {clones()}; extra.parquet is unchanged on disk") - read("cheap entry, which reads the clone on every read", cheap.content_hash) + read("cheap entry, which reads its source's snapshot on every read", cheap.content_hash) read("worthy entry, whose snapshot came back from the bullpen", worthy.content_hash) for snapshot in compute_cache_dir(PROJECT).rglob("*.parquet"): diff --git a/src/tallyman_cli/main.py b/src/tallyman_cli/main.py index cf27b747..6b036f73 100644 --- a/src/tallyman_cli/main.py +++ b/src/tallyman_cli/main.py @@ -198,6 +198,20 @@ def run_mcp(project: str | None) -> None: mcp_main() +def _expand_project_root(value, project: str | None): + """Expand ``${TALLYMAN_PROJECT_ROOT}`` in a storyboard string argument. + + A storyboard that imports a file shipped under the project (``catalog_import_source``) has to name an absolute + path, since ``data/`` is not a special location any more; the placeholder is how it stays portable. + """ + if not isinstance(value, str) or not project: + return value + from tallyman_core.paths import project_dir + from tallyman_xorq.portable import PLACEHOLDER + + return value.replace(PLACEHOLDER, str(project_dir(project))) + + @cli.command("replay") @click.argument("storyboard", type=click.Path(exists=True, dir_okay=False)) @click.option("--project", default=None, help="Project name override.") @@ -216,14 +230,18 @@ def replay_storyboard(storyboard: str, project: str | None, delay: float, stop_o { "project": "spike", "steps": [ - {"tool": "catalog_load_parquet", "args": {...}, "narration": "..."}, + {"tool": "catalog_import_source", "args": {...}, "narration": "..."}, ... ] } Each step calls the named tool from tallyman_mcp.server with `args` as - kwargs. Use this for stage rehearsal, fallback recordings, or as a - deterministic regression test of the full storyboard. + kwargs. A string argument may contain ``${TALLYMAN_PROJECT_ROOT}``, expanded + to the project's directory, so a storyboard that imports a file shipped with + the project stays portable across machines and projects. + + Use this for stage rehearsal, fallback recordings, or as a deterministic + regression test of the full storyboard. """ import importlib import json @@ -247,7 +265,7 @@ def replay_storyboard(storyboard: str, project: str | None, delay: float, stop_o click.echo(f"[{i}/{len(steps)}] (skipped: {step.get('tool')})") continue tool_name = step.get("tool") - args = step.get("args", {}) + args = {k: _expand_project_root(v, sb_project) for k, v in step.get("args", {}).items()} narration = step.get("narration", "") tool_fn = getattr(mcp_module, tool_name, None) if not callable(tool_fn): diff --git a/src/tallyman_companion/app.py b/src/tallyman_companion/app.py index 2656236f..54530e79 100644 --- a/src/tallyman_companion/app.py +++ b/src/tallyman_companion/app.py @@ -347,8 +347,7 @@ def add(key: str, label: str, path: Path, kind: str, reclaimable: bool, detail: # single-user scale; revisit if it ever paginates. from datetime import datetime, timezone # noqa: PLC0415 - from tallyman_core.paths import data_dir as _data_dir # noqa: PLC0415 - from tallyman_xorq.dependents import dependents_index, parents_of, sources_of # noqa: PLC0415 + from tallyman_xorq.dependents import dependents_index, parents_of # noqa: PLC0415 manifest: dict = {} try: @@ -356,18 +355,27 @@ def add(key: str, label: str, path: Path, kind: str, reclaimable: bool, detail: except (OSError, ValueError): pass - # Raw source files the recipe reads. None under source-identity mode=off — - # kept distinct from "reads no files" so the UI can say "not tracked". - src_digests = sources_of(project, content_hash) + # The raw bytes this entry holds, which is a question only a source version has an answer to + # (ADR-011 D6: a computed entry reads aliases, never a file). For one it is the clone of the + # imported file under data/.cas — the copy that makes the snapshot re-creatable — listed under the + # provenance path it came from, which may be long gone. sources: list[dict] = [] source_bytes = 0 - if src_digests is not None: - dd = _data_dir(project) - for rel in sorted(src_digests): - sp = dd / rel - sz = _path_size(sp) - source_bytes += sz - sources.append({"path": rel, "bytes": sz, "formatted": _fmt_bytes(sz), "exists": sp.exists()}) + provenance = manifest.get("provenance") + if provenance: + from tallyman_core.manifest import SourceProvenance # noqa: PLC0415 + from tallyman_xorq.source_import import source_clone_path # noqa: PLC0415 + + clone = source_clone_path(project, SourceProvenance.model_validate(provenance)) + source_bytes = _path_size(clone) + sources.append( + { + "path": provenance["path"], + "bytes": source_bytes, + "formatted": _fmt_bytes(source_bytes), + "exists": clone.exists(), + } + ) # Last-modified across the entry's on-disk artifacts (a snapshot self-heal # makes this later than created_at). @@ -406,7 +414,7 @@ def add(key: str, label: str, path: Path, kind: str, reclaimable: bool, detail: "diff_cache_formatted": _fmt_bytes(diff_cache_bytes), "source_bytes": source_bytes, "source_formatted": _fmt_bytes(source_bytes), - "source_tracked": src_digests is not None, + "source_tracked": bool(sources), "sources": sources, "parents": parents, "children": children, @@ -1315,6 +1323,12 @@ def _resolve(v: int): column_config_overrides = compute_column_config_overrides(a_expr.schema(), b_expr.schema(), keys) target_alias = f"diff_{alias}_v{a_idx}_v{b_idx}" + # The generated name can already be a source alias: refuse before building the diff, as the MCP tool does. + from tallyman_core.aliases import source_alias_refusal # noqa: PLC0415 + + refusal = source_alias_refusal(project, target_alias, "the target of a promoted diff") + if refusal: + raise HTTPException(409, refusal) keys_repr = repr(keys) code = textwrap.dedent(f"""\ # auto-generated — diff of {alias} V{a_idx} → V{b_idx} @@ -1649,6 +1663,12 @@ async def put_code(project: str, alias: str, payload: dict): prev_hash = _get_alias(project, alias) if prev_hash is None: raise HTTPException(404, f"alias {alias!r} not found") + # A source alias's versions are imported files: refuse before building, as catalog_revise does (ADR-011 D1). + from tallyman_core.aliases import source_alias_refusal as _source_alias_refusal # noqa: PLC0415 + + refusal = _source_alias_refusal(project, alias, "revised") + if refusal: + raise HTTPException(409, refusal) code = payload.get("code", "") if not code.strip(): raise HTTPException(400, "code is empty") @@ -1668,8 +1688,8 @@ async def put_code(project: str, alias: str, payload: dict): msg = ( f"this revision references its own alias {alias!r}, which makes an opaque, " f"permanently-stale (follows-its-own-head) entry. Write a self-contained recipe: " - f"inline the source (read_project_file / read_csv …) for a source-shaped entry, or " - f"reference the previous version by hash with pinned_expr_from_alias({prev_hash!r}) " + f"start from the source alias (tracked_expr_from_alias) for a source-shaped entry, or " + f"reference the previous version with pinned_expr_from_alias('{alias}-v{len(history_for(project, alias))}') " # noqa: E501 f"for an expensive one." ) rec = _record_error(project, code=code, message=msg, tool="api_code") diff --git a/src/tallyman_core/__init__.py b/src/tallyman_core/__init__.py index ce22a053..06142164 100644 --- a/src/tallyman_core/__init__.py +++ b/src/tallyman_core/__init__.py @@ -1,12 +1,17 @@ from tallyman_core import notebook from tallyman_core.aliases import ( # noqa: I001 (keep notebook import first) + CATALOG_KIND, + SOURCE_KIND, AliasExists, + AliasKindMismatch, AliasNotFound, alias_for_hash, + alias_kind, get_alias, history_for, load_aliases, load_history, + load_kinds, remove_alias, rename_alias, set_alias, @@ -78,7 +83,12 @@ __all__ = [ "AliasExists", + "AliasKindMismatch", "AliasNotFound", + "CATALOG_KIND", + "SOURCE_KIND", + "alias_kind", + "load_kinds", "CellNotFound", "ChartSpecError", "Manifest", diff --git a/src/tallyman_core/aliases.py b/src/tallyman_core/aliases.py index b17b143a..4cab14bc 100644 --- a/src/tallyman_core/aliases.py +++ b/src/tallyman_core/aliases.py @@ -4,9 +4,23 @@ alias_map: {alias: latest_hash} alias_history: {alias: [hash_v1, hash_v2, ...]} # oldest first - -Both live in a tracked ``aliases.jsonl`` in the catalog repo — one line per -alias, ``{"alias", "latest", "history": [...]}``. The native store tracks the + alias_kinds: {alias: "catalog" | "source"} + +An alias has a **kind** (ADR-011 D1). A *catalog* alias names a computation, +revised by ``catalog_revise``. A *source* alias names a raw input dataset whose +versions are imported files, advanced only by ``catalog_import_source``; there is +no recipe to revise and no diff to promote onto it. Both live in the same store +with the same head-plus-history shape, so ``orders-v2`` resolves through the one +``VERSION_REF_RE``, and a name is one kind or the other but never both. + +The kind is recorded here, on the alias; whether an entry is a *source entry* (a +version of an imported file) is recorded on the entry, by its manifest carrying +``provenance``. ``set_alias`` ties the two together: **an alias's kind matches the +kind of the entries it points at**, so a catalog name never heads an import and a +source name never heads a recipe's result, whichever route set it. + +All three live in a tracked ``aliases.jsonl`` in the catalog repo — one line per +alias, ``{"alias", "latest", "history": [...], "kind"}``. The native store tracks the file directly, so alias state clones and versions with the catalog and ``reset_to``'s ``git reset`` rolls it back with the rest of the tree (no separate reconcile). Before the native cut this had to be smuggled into @@ -24,7 +38,8 @@ import re from tallyman_core.fsutil import atomic_write_text -from tallyman_core.paths import catalog_dir, ensure_project +from tallyman_core.manifest import read_manifest +from tallyman_core.paths import catalog_dir, ensure_project, entry_dir class AliasExists(ValueError): @@ -35,6 +50,17 @@ class AliasNotFound(KeyError): pass +class AliasKindMismatch(ValueError): + """An alias of one kind aimed at the other kind: a name that already belongs to the other kind of alias, or an + entry whose kind is not the alias's (ADR-011 D1).""" + + +# A catalog alias names a computation; a source alias names an imported dataset. +CATALOG_KIND = "catalog" +SOURCE_KIND = "source" +_KINDS = (CATALOG_KIND, SOURCE_KIND) + + # "-v" — the version-reference syntax pinned_expr_from_alias accepts # (#166), matching the V1…Vn vocabulary the UI and catalog_diff use. Alias # names matching it are rejected at creation/rename so a name can never @@ -80,16 +106,17 @@ def _aliases_file(project: str): return catalog_dir(project) / "aliases.jsonl" -def _read(project: str) -> tuple[dict[str, str], dict[str, list[str]]]: - """Read the alias map + history out of the tracked ``aliases.jsonl``. +def _read(project: str) -> tuple[dict[str, str], dict[str, list[str]], dict[str, str]]: + """Read the alias map, history and kinds out of the tracked ``aliases.jsonl``. - One line per alias: ``{"alias", "latest", "history": [...]}``. The native - store tracks this file directly, so ``reset_to``'s ``git reset`` rolls alias - state back with the rest of the tree (no separate reconcile). + One line per alias: ``{"alias", "latest", "history": [...], "kind"}``. The + native store tracks this file directly, so ``reset_to``'s ``git reset`` rolls + alias state back with the rest of the tree (no separate reconcile). """ p = _aliases_file(project) alias_map: dict[str, str] = {} history: dict[str, list[str]] = {} + kinds: dict[str, str] = {} if p.exists(): for line in p.read_text().splitlines(): if not line.strip(): @@ -97,12 +124,15 @@ def _read(project: str) -> tuple[dict[str, str], dict[str, list[str]]]: rec = json.loads(line) alias_map[rec["alias"]] = rec["latest"] history[rec["alias"]] = rec.get("history", []) - return alias_map, history + kinds[rec["alias"]] = rec.get("kind", CATALOG_KIND) + return alias_map, history, kinds -def _write(project: str, aliases: dict[str, str], history: dict[str, list[str]]) -> None: - """Persist the alias map + history to ``aliases.jsonl`` (sorted for a stable - diff; an empty map removes the file).""" +def _write( + project: str, aliases: dict[str, str], history: dict[str, list[str]], kinds: dict[str, str] +) -> None: + """Persist the alias map, history and kinds to ``aliases.jsonl`` (sorted for a + stable diff; an empty map removes the file).""" ensure_project(project) p = _aliases_file(project) p.parent.mkdir(parents=True, exist_ok=True) @@ -110,7 +140,15 @@ def _write(project: str, aliases: dict[str, str], history: dict[str, list[str]]) p.unlink(missing_ok=True) return body = "".join( - json.dumps({"alias": name, "latest": aliases[name], "history": history.get(name, [])}) + "\n" + json.dumps( + { + "alias": name, + "latest": aliases[name], + "history": history.get(name, []), + "kind": kinds.get(name, CATALOG_KIND), + } + ) + + "\n" for name in sorted(aliases) ) atomic_write_text(p, body) @@ -124,6 +162,16 @@ def load_history(project: str) -> dict[str, list[str]]: return _read(project)[1] +def load_kinds(project: str) -> dict[str, str]: + """``{alias: "catalog" | "source"}`` for every alias in the project (ADR-011 D1).""" + return _read(project)[2] + + +def alias_kind(project: str, name: str) -> str | None: + """The kind of *name*, or None when no such alias exists.""" + return _read(project)[2].get(name) + + def get_alias(project: str, name: str) -> str | None: return load_aliases(project).get(name) @@ -180,53 +228,145 @@ def previous_version(project: str, content_hash: str, alias: str | None = None) return hist[idx] if 0 <= idx < len(hist) else None -def set_alias(project: str, name: str, content_hash: str, *, expect_exists: bool | None = None) -> dict: +def source_alias_refusal(project: str, name: str, doing: str) -> str | None: + """The error for an operation that would give *name* a computed version, or None when *name* is not a source alias. + + A source alias names an imported dataset (ADR-011 D1). Its versions are files, not recipes, so there is nothing + to revise and nothing to promote a diff onto, and a catalog alias may not take its name either. Every surface + that can aim such an operation at a name (the MCP tools, the companion's routes) refuses with this text before it + builds anything, so no entry is left behind under no alias. *doing* completes "it cannot be …". + """ + if alias_kind(project, name) != SOURCE_KIND: + return None + return ( + f"{name!r} is a source alias — its versions are imported files, not recipes — so it cannot be {doing}. " + f"To give it new data, import the file again with catalog_import_source(, {name!r}). To build on it, " + f"write a recipe over tracked_expr_from_alias({name!r}) under a different name." + ) + + +def _entry_kind(project: str, content_hash: str) -> str | None: + """The kind of alias the entry *content_hash* may take, or None when there is no readable manifest to tell. + + ``SOURCE_KIND`` for a source entry (its manifest records ``provenance``: it is a version of an imported file), + ``CATALOG_KIND`` for any other entry, which a recipe computed. None when the entry directory or its manifest is + missing or unreadable, which ``set_alias`` treats as "do not check": the alias bookkeeping is used, and tested, + with hashes that name no entry. + """ + try: + manifest = read_manifest(entry_dir(project, content_hash)) + except (OSError, ValueError): # no entry, no manifest yet, or one that does not parse + return None + return SOURCE_KIND if manifest.provenance is not None else CATALOG_KIND + + +def _entry_kind_refusal(project: str, name: str, content_hash: str, entry_kind: str) -> str: + """The ``AliasKindMismatch`` message for pointing *name* at an entry of the other kind, *entry_kind*.""" + if entry_kind == SOURCE_KIND: + held = version_of_hash(project, content_hash) + if held is None: # imported, but its source alias has since been removed + return ( + f"{content_hash!r} is a source entry (a version of an imported file) that no source alias holds, so " + f"the catalog alias {name!r} cannot point at it: a source version has no recipe to revise. To name " + f"it, import its file again with catalog_import_source(, {name!r})." + ) + src, version = held + steer = f"from tallyman_xorq.io import tracked_expr_from_alias\nexpr = tracked_expr_from_alias({src!r})" + return ( + f"{content_hash!r} is {src}-v{version}, a version of the source alias {src!r} (an imported file, not a " + f"computation), so the catalog alias {name!r} cannot point at it: a source version has no recipe to " + f"revise. To give the source a second name, create a catalog entry that reads it: " + f"catalog_create({name!r}, {steer!r}). That entry follows {src!r} when it is imported again." + ) + return ( + f"{content_hash!r} is a computed entry (a recipe's result), so the source alias {name!r} cannot point at it: " + f"a source alias's versions are imported files, and it advances only by catalog_import_source(, " + f"{name!r}). To name this entry, give it a catalog alias under another name." + ) + + +def set_alias( + project: str, + name: str, + content_hash: str, + *, + expect_exists: bool | None = None, + kind: str = CATALOG_KIND, +) -> dict: """Point `name` at `content_hash` and append to history. expect_exists: None — accept either fresh or existing alias. True — error if alias does NOT exist (`catalog_revise` semantics). False — error if alias DOES exist (`catalog_create`/`catalog_alias` semantics). + + `kind` is what the caller is setting: ``CATALOG_KIND`` for a computation (the + default, so every existing caller keeps its meaning) or ``SOURCE_KIND`` for a + version of an imported dataset. A name that already belongs to the other kind + raises ``AliasKindMismatch`` — the collision is refused both ways (ADR-011 D1). + + So does an entry of the other kind: a catalog alias may not point at a source + entry (one whose manifest records ``provenance``), since it would then offer + ``catalog_revise`` on a version with no recipe, and a source alias may not + point at a computed entry, since its versions are imported files. The check + is here rather than in each tool so that every route that names an entry + (``catalog_alias``, a revise, a promoted diff, a recalc, an import) + keeps the alias's kind and its entries' kind the same. A hash whose manifest + cannot be read is not checked. """ + if kind not in _KINDS: + raise ValueError(f"alias kind {kind!r}; expected one of {_KINDS}") ensure_project(project) validate_alias_name(name) - aliases, history = _read(project) + aliases, history, kinds = _read(project) exists = name in aliases if expect_exists is True and not exists: raise AliasNotFound(name) if expect_exists is False and exists: raise AliasExists(name) + if exists and kinds.get(name, CATALOG_KIND) != kind: + raise AliasKindMismatch( + f"{name!r} is a {kinds.get(name, CATALOG_KIND)} alias and cannot be set as a {kind} alias; " + "a name is one kind or the other" + ) + entry_kind = _entry_kind(project, content_hash) + if entry_kind is not None and entry_kind != kind: + raise AliasKindMismatch(_entry_kind_refusal(project, name, content_hash, entry_kind)) aliases[name] = content_hash + kinds[name] = kind history.setdefault(name, []) # Avoid duplicate consecutive entries. if not history[name] or history[name][-1] != content_hash: history[name].append(content_hash) - _write(project, aliases, history) + _write(project, aliases, history, kinds) return { "name": name, "hash": content_hash, "version": len(history[name]), + "kind": kind, } def rename_alias(project: str, old_name: str, new_name: str) -> dict: validate_alias_name(new_name) - aliases, history = _read(project) + aliases, history, kinds = _read(project) if old_name not in aliases: raise AliasNotFound(old_name) if new_name in aliases: raise AliasExists(new_name) aliases[new_name] = aliases.pop(old_name) history[new_name] = history.pop(old_name, []) - _write(project, aliases, history) + kinds[new_name] = kinds.pop(old_name, CATALOG_KIND) # a rename carries the kind over + _write(project, aliases, history, kinds) return {"old": old_name, "new": new_name, "hash": aliases[new_name]} def remove_alias(project: str, name: str) -> None: - aliases, history = _read(project) + aliases, history, kinds = _read(project) aliases.pop(name, None) history.pop(name, None) - _write(project, aliases, history) + kinds.pop(name, None) + _write(project, aliases, history, kinds) diff --git a/src/tallyman_core/catalog_state.py b/src/tallyman_core/catalog_state.py index 475d7ac4..33e8913d 100644 --- a/src/tallyman_core/catalog_state.py +++ b/src/tallyman_core/catalog_state.py @@ -173,7 +173,12 @@ def _cas_bullpen(project: str) -> Path: def _live_source_digests(project: str) -> set[str] | None: - """The union of every surviving entry's ``manifest.sources`` digests, or None when a manifest can't be read. + """Every clone digest a surviving entry needs, or None when a manifest can't be read. + + One way an entry refers to a clone: ``manifest.provenance.digest``, the bytes an import copied in (ADR-011 D1). + The clone is the only record of the file as imported and a re-import is the only way back, so the retention + closure is simply which source versions survive — which is the DAG, and the reason ADR-011 D6 could delete + ``manifest.sources``, the map a build used to fold up from everything it read. An unreadable manifest means the sweep would run on partial information, and a clone wrongly retired is the only frozen copy of somebody's bytes, so callers skip the sweep. @@ -184,11 +189,11 @@ def _live_source_digests(project: str) -> set[str] | None: live: set[str] = set() for h in read_tallyman_state(project)["entry_hashes"]: try: - sources = read_manifest(entry_dir(project, h)).sources + manifest = read_manifest(entry_dir(project, h)) except Exception: return None - if sources: - live.update(sources.values()) + if manifest.provenance is not None: + live.add(manifest.provenance.digest) return live @@ -384,8 +389,8 @@ def _retire_cas_clones(project: str) -> int: ``.cas`` lives under ``data/`` — outside the catalog git repo — so the ``git reset`` above cannot roll it back. A clone is data, the only frozen copy of the bytes an entry was built from once the live file is edited, so - nothing deletes one: a reset forward brings it back (``restore_from_bullpen``). Liveness is the union of every - surviving entry's ``manifest.sources`` digests. Conservative and best-effort: if any entry's manifest can't be + nothing deletes one: a reset forward brings it back (``restore_from_bullpen``). Liveness is every surviving + source version's ``manifest.provenance.digest``. Conservative and best-effort: if any entry's manifest can't be read we skip the sweep, and a failure here never aborts the reset. """ from tallyman_xorq import source_identity # lazy: avoid a core->xorq import cycle diff --git a/src/tallyman_core/manifest.py b/src/tallyman_core/manifest.py index ffc5cbea..77f21aa5 100644 --- a/src/tallyman_core/manifest.py +++ b/src/tallyman_core/manifest.py @@ -27,6 +27,30 @@ class ParentRef(BaseModel): follow: bool +class SourceProvenance(BaseModel): + """Where a source version's bytes came from, recorded once at import (ADR-011 D1, D12). + + A manifest that carries one IS a source entry: a version of a source alias, whose rows are an imported file + rather than a computation. ``path`` is the **provenance path** — the outside path the bytes were imported from, + recorded here and never read again, so deleting or editing that file changes nothing. ``digest`` is the md5 of + the imported bytes, which names their clone in ``data/.cas/`` and, with ``reader``, determines + the entry's content hash. ``reader`` is the reader options the import used (``{"kind": "parquet"}``, or a CSV's + schema spec and ``scan_csv`` options): fixed at import and never re-derived at build time. + + ``alias`` and ``version`` are the name the version was **imported as**, which is history too: a rename moves the + alias's history to a new name and an unalias drops it, and neither rewrites this record. Anything that names the + version as it is now asks the alias store (``source_import.current_source_version``). + """ + + alias: str + version: int + path: str + digest: str + suffix: str + reader: dict + imported_at: str + + class Manifest(BaseModel): content_hash: str project: str @@ -60,17 +84,12 @@ class Manifest(BaseModel): # heal can say the engine changed instead of blaming the recipe. snapshot_format: int | None = None engine_versions: dict[str, str] | None = None - # Ordered copies of sources this entry's plan reads (ADR-007 D13, ADR-008 D2), keyed by the copy's file stem: - # ``{"source": rel_or_abs_path, "digest": md5 of the source, "suffix": ".csv", "reader": {"kind": "parquet"|"csv", - # ...}, "content_digest": "arrow-sha256:..."}``. What ``ensure_materialized`` needs to make a deleted copy again - # from its clone, and the digest the new copy is checked against. - ordered_copies: dict[str, dict] | None = None - # rel data path -> content md5, recorded when a source-identity mode is - # active (tallyman_xorq.source_identity); absent under mode=off. - sources: dict[str, str] | None = None # Resolved tracked_expr_from_alias parent edges ({hash, ref, follow}), recorded at build # time so the inter-entry DAG survives #73/#74; absent for root entries (#84). parents: list[ParentRef] | None = None + # Set only on a source entry: the import that minted it (ADR-011 D1). Its presence is what makes the entry a + # source version — worthy, with its imported bytes as its snapshot, no parents and no recipe to revise. + provenance: SourceProvenance | None = None def write_manifest(entry_path: Path, manifest: Manifest) -> Path: diff --git a/src/tallyman_core/marimo_export.py b/src/tallyman_core/marimo_export.py index ef11c8b3..e8b3d827 100644 --- a/src/tallyman_core/marimo_export.py +++ b/src/tallyman_core/marimo_export.py @@ -6,7 +6,7 @@ as the final expression so Marimo displays the DataFrame inline. The generated file is self-contained: it sets TALLYMAN_PROJECT via -``os.environ.setdefault`` so ``read_project_file`` / ``tracked_expr_from_alias`` resolve +``os.environ.setdefault`` so ``tracked_expr_from_alias`` resolves correctly when the user runs it outside the tallyman-notebooks environment. """ diff --git a/src/tallyman_core/paths.py b/src/tallyman_core/paths.py index 0add6e0c..f7e919cf 100644 --- a/src/tallyman_core/paths.py +++ b/src/tallyman_core/paths.py @@ -205,10 +205,11 @@ def entry_view_build_dir(project: str, content_hash: str) -> Path: def compute_cache_dir(project: str) -> Path: - """Per-project directory of files tallyman can make again: snapshots and ordered copies of sources. + """Per-project directory of files tallyman can make again: one snapshot per worthy entry. - ``result_cache/.parquet`` holds each worthy entry's snapshot and - ``ordered_sources/.parquet`` each source's ordered copy (ADR-007 D2, D13). + ``result_cache/.parquet`` holds each worthy entry's snapshot (ADR-007 D2, D13), + including a source version's, whose rows are an imported file rather than a computation and whose + snapshot is re-created from the clone of those bytes (ADR-011 D1). Lives inside the catalog dir so it travels with the project, and is untracked by git (content-addressed parquet). Everything in it is cache, so anything may delete it: ``ensure_materialized`` makes a missing file again and checks it. A reset leaves it diff --git a/src/tallyman_mcp/server.py b/src/tallyman_mcp/server.py index b6605b32..c78803ff 100644 --- a/src/tallyman_mcp/server.py +++ b/src/tallyman_mcp/server.py @@ -14,6 +14,7 @@ from tallyman_core import ( AliasExists, + AliasKindMismatch, AliasNotFound, CellNotFound, ChartSpecError, @@ -253,6 +254,16 @@ def _auto_recalc_after_head_advance(project: str, name: str, *, tool: str) -> di return report +def _source_alias_refusal(project: str, name: str, doing: str) -> str | None: + """The error for an operation aimed at a source alias, or None when *name* is not one (ADR-011 D1). + + The text lives in ``tallyman_core.aliases.source_alias_refusal`` so the companion's routes refuse with it too. + """ + from tallyman_core.aliases import source_alias_refusal # noqa: PLC0415 + + return source_alias_refusal(project, name, doing) + + @mcp.tool() @_tag_project def catalog_run(code: str, prompt: str = "") -> dict: @@ -263,10 +274,9 @@ def catalog_run(code: str, prompt: str = "") -> dict: PREFERRED pattern: - from tallyman_xorq.io import read_project_file, tracked_expr_from_alias + from tallyman_xorq.io import tracked_expr_from_alias import xorq.vendor.ibis as ibis - t = tracked_expr_from_alias("alias_name") # named catalog entry by alias (records lineage) - t = read_project_file("file.parquet") # raw file under /data/ + t = tracked_expr_from_alias("alias_name") # a catalog entry OR an imported source, by alias expr = t.filter(t.col > 0).group_by("region").aggregate(n=t.count()) THREE NAMESPACES — mixing these up is the #1 build failure: @@ -274,7 +284,7 @@ def catalog_run(code: str, prompt: str = "") -> dict: import xorq.api as xo # backends: xo.memtable, xo.connect import xorq.vendor.ibis as ibis # the expression API: ibis._, ibis.cases, # ibis.window, ibis.literal, ibis.coalesce, ibis.desc - from tallyman_xorq.io import read_project_file, tracked_expr_from_alias, tallyman_read_csv # reading data + from tallyman_xorq.io import tracked_expr_from_alias, pinned_expr_from_alias # reading data t.mutate(pk=ibis._.a + "_" + ibis._.b) # deferred string concat via ibis._ # NEVER `import xorq` then `xorq.`: bare xorq.read_parquet / xorq.memtable / @@ -282,34 +292,29 @@ def catalog_run(code: str, prompt: str = "") -> dict: # NEVER bare `import ibis` / `from ibis ...`: it builds but fails at save time # with a vendored-Expr class error. Always `import xorq.vendor.ibis as ibis`. # Math is a COLUMN METHOD: col.sin(), col.log(), col.sqrt() — not ibis.sin(col). - # Read data only via read_project_file / tracked_expr_from_alias / tallyman_read_csv (no - # xo.read_parquet / xo.deferred_read_parquet / ibis.read_parquet: a raw read is a build error). + # Read data ONLY via tracked_expr_from_alias / pinned_expr_from_alias. A recipe never opens + # a file: read_project_file, tallyman_read_csv, xo.read_parquet, xo.deferred_read_parquet and + # ibis.read_parquet are all build errors. Import the file first (catalog_import_source). ONLY BACKEND — xorq's built-in datafusion; there is NO duckdb. Do not use `ibis.duckdb`, a duckdb connection, `.sql()`, or `con.register()`. Build - everything with the ibis expression API over read_project_file/tracked_expr_from_alias. - - DATA SOURCING — four functions, each with a distinct role: - - `tracked_expr_from_alias("name")` reads a catalog entry by alias and records it - as a parent in the lineage DAG. Use this for normal recipe chaining — the - standard way to build on top of another catalog entry. - - `read_project_file("file.parquet")` reads raw files under `/data/`. - Use this only for raw files visible on disk (not catalog aliases). The file - is ingested once into a copy in file order with a last column, `__row_order` - (see ROW ORDER below). - - `tallyman_read_csv("/abs/path/to/file.csv", schema=...)` reads a CSV, in file - order, with a last column ``__row_order`` (see ROW ORDER below). Editing the CSV - and re-running the recipe creates a NEW entry; the old one keeps its rows. - Use this for ALL CSV ingests instead of ``xo.deferred_read_csv``. - - `pinned_expr_from_alias()` reads a catalog entry by - content hash or explicit version reference (e.g. "shoe_sales-v2") and - records a pinned (follow=False) parent edge. Use when you want to stay on - a specific version — recalc will find the entry but won't advance it when - the parent alias moves. A bare alias is rejected (#166): a pin must name - the same entry forever, so say which version you mean. - - When in doubt, call `catalog_list` first. A name that looks like a file - is often actually an alias — `read_project_file` on an alias raises - `ProjectDataNotFound`. + everything with the ibis expression API over tracked_expr_from_alias. + + DATA SOURCING — a recipe names aliases and never a file or a hash (ADR-011): + - `tracked_expr_from_alias("name")` reads an alias — a catalog entry or an + imported source — and records it as a parent in the lineage DAG. It follows + the alias: when the parent is revised or the source re-imported, recalc + advances this entry. The standard way to build on anything. + - `pinned_expr_from_alias("name-vN")` reads one version and records a pinned + (follow=False) parent edge, so recalc finds the entry but never advances it. + A bare alias is rejected (#166) and so is a bare content hash: a pin must + name the same entry forever, and it must say which alias and which version. + - A RAW FILE IS NOT A RECIPE INPUT. Import it once, with + `catalog_import_source("/abs/path/orders.parquet", "orders")`, and read the + source alias. The import copies the bytes into the catalog, so editing or + deleting the original afterwards changes nothing; re-importing an edited + file mints `orders` v2 and everything downstream goes stale. + - When in doubt, call `catalog_list` first. ROW ORDER — every entry carries a last column, `__row_order`: @@ -357,19 +362,21 @@ def catalog_run(code: str, prompt: str = "") -> dict: LOADING RAW CSV FILES — always inspect first: - Use ``tallyman_read_csv`` (not ``xo.deferred_read_csv``) for all CSV - ingests. It adds a last ``__row_order`` column holding each row's - position in the file (0..N-1): + Import CSVs with ``catalog_import_source``, which reads the file once, in + file order, with a last ``__row_order`` column holding each row's position + (0..N-1), and mints a source alias the recipe then reads: - from tallyman_xorq.io import tallyman_read_csv - import xorq.vendor.ibis as ibis - schema = ibis.schema({"Date": "date", "Close": "float64"}) - t = tallyman_read_csv("/abs/path/to/file.csv", schema=schema) + catalog_import_source("/abs/path/to/file.csv", "prices", + schema={"Date": "date", "Close": "float64"}) + # then, in the recipe: + from tallyman_xorq.io import tracked_expr_from_alias + t = tracked_expr_from_alias("prices") - Extra keyword arguments are forwarded to ``polars.scan_csv`` as reader - options (``separator``, ``skip_rows``, ``null_values``, ``quote_char``, - ``has_header``, ``encoding``, ...), e.g. - ``tallyman_read_csv(path, schema=schema, separator=";", skip_rows=2)``. + Extra keyword arguments to the import are forwarded to ``polars.scan_csv`` + as reader options (``separator``, ``skip_rows``, ``null_values``, + ``quote_char``, ``has_header``, ``encoding``, ...). They are recorded on the + source entry and never re-derived, so one file is read exactly one way; to + read it a second way, import it again under a second alias. Before writing a schema, run `head -5 ` to verify the actual column names and order. Do NOT guess — yfinance and similar sources @@ -383,7 +390,7 @@ def catalog_run(code: str, prompt: str = "") -> dict: - The first column is often literally named "Price" in raw yfinance output, not "Date". - ALWAYS use an explicit schema with tallyman_read_csv — without one, type + ALWAYS pass an explicit schema when importing a CSV — without one, type inference silently misassigns types (e.g. date columns become timestamps). DATE-ONLY COLUMNS — use 'date', never 'timestamp': @@ -522,8 +529,8 @@ def catalog_run(code: str, prompt: str = "") -> dict: - fabricate placeholder values that look like real output (e.g. a memtable of zeros) when you cannot compute the requested result — return an error or a clearly-labelled column instead. - - pass a project= argument to read_project_file/tracked_expr_from_alias; the active - project is implicit. + - pass a project= argument to tracked_expr_from_alias; the active project + is implicit. Args: code: A self-contained Python script that binds `expr`. @@ -544,39 +551,105 @@ def catalog_run(code: str, prompt: str = "") -> dict: @mcp.tool() @_tag_project -def catalog_load_parquet(rel_path: str, prompt: str = "", name: str = "") -> dict: - """Register a parquet file from the project's data/ directory as a catalog entry. - - Use this for simple "load this file" steps. The agent does not need to write - any xorq code — pass a path relative to the project's `data/` directory. +def catalog_import_source( + outside_path: str, + alias: str, + pinned_version: int | None = None, + prompt: str = "", + schema: dict | list | None = None, + reader_options: dict | None = None, +) -> dict: + """Import a data file into the catalog and point a source alias at it (ADR-011). + + THE ONLY WAY A FILE ENTERS THE CATALOG. A recipe never opens a file: it reads + an alias. This copies the bytes in, writes one snapshot of them in file order + with a `__row_order` column, and mints a version of the source alias `alias`, + which recipes then read with `tracked_expr_from_alias(alias)`. + + After the import the original path is provenance only. Editing, moving or + deleting it changes nothing. To pick up new data, run this again on the same + alias: the bytes differ, so it mints the next version, and every entry + downstream goes stale and is recalculated (the same cascade a revise triggers). + + Re-running it with UNCHANGED bytes is a no-op that returns the current version, + so it is safe in a script that runs repeatedly. Args: - rel_path: Path relative to `/data/`. Example: "orders.parquet". + outside_path: Any path to a parquet or CSV file. It does not have to live + under the project's `data/` directory. + alias: The source alias. Must not already name a catalog alias, and the + bytes must not already be a version of another alias — the error + names that alias. For a second name, catalog_create an entry whose + recipe is `tracked_expr_from_alias()`. + pinned_version: The version you claim this file is. Given, it is checked: + if the file is that version the call is a no-op, and if it is not you + get an error instead of an accidental new version. Use it in a script + that must reproduce a known state. prompt: Optional human-readable description (the user's intent). - name: Optional alias to register the entry under. If provided, the - entry behaves like one created by `catalog_create` (named, appended - to the notebook). Errors if the alias already exists. + schema: For a CSV, the column types — a dict `{"Date": "date", ...}`, or + the positional DSL `[["Date", "date"], ["&rest", "infer"]]`. ALWAYS + pass one for a CSV; see the LOADING RAW CSV FILES section of + `catalog_run` for how to inspect the file first. + reader_options: For a CSV, `polars.scan_csv` options — `separator`, + `skip_rows`, `null_values`, `quote_char`, `has_header`, `encoding`. + Recorded on the entry and never re-derived, so the file is read one + way forever; to read it a second way, import it again under a second + alias. Returns: - Same shape as catalog_run, plus `alias`/`version` when `name` is set. + dict with keys: hash, alias, version, created, row_count, schema, path, + digest, url — plus `recalc` when advancing the alias cascaded. """ + from tallyman_xorq.source_import import SourceImportError, update_and_depend # noqa: PLC0415 + project = _resolve_active_project() - if name and get_alias(project, name) is not None: - return {"error": f"alias {name!r} already exists. Use catalog_revise to update it."} - code = f"from tallyman_xorq.io import read_project_file\nexpr = read_project_file({rel_path!r})\n" - out = _run_and_record(project, code, prompt, tool="catalog_load_parquet") - if "error" in out: - return out - out.pop("_build", None) - if name: - info = set_alias(project, name, out["hash"], expect_exists=False) - notebook.append(project, name, markdown=prompt or "") - out["alias"] = info["name"] - out["version"] = info["version"] - _notify("new_entry", content_hash=out["hash"], alias=name, version=info["version"]) + if schema is not None and isinstance(schema, list): + schema = tuple(tuple(cell) for cell in schema) + try: + out = update_and_depend( + outside_path, + alias, + pinned_version, + project=project, + prompt=prompt or None, + schema=schema, + **(reader_options or {}), + ) + except (SourceImportError, BuildError, OSError) as exc: + rec = record_error(project, code="", message=str(exc), prompt=prompt or None, tool="catalog_import_source") + record_event( + project, + "build_error", + session=SESSION_ID, + tool="catalog_import_source", + prompt=prompt or None, + message=str(exc), + error_id=rec["id"], + ) + _notify("build_failed", error_id=rec["id"], tool="catalog_import_source") + return {"error": str(exc), "error_id": rec["id"]} + + out["url"] = _entry_url(project, out["hash"]) + if not out["created"]: + return out # nothing moved, so nothing to record, notify or cascade + + # An import is a catalog operation (ADR-011 D7): it emits the events a revise emits, and the dispatch-boundary + # decorator commits it — the head advance and any cascade — as ONE revision. + record_event( + project, "alias_set", session=SESSION_ID, tool="catalog_import_source", + alias=alias, version=out["version"], hash=out["hash"], prompt=prompt or None, + ) + if out["version"] == 1: + notebook.append(project, alias, markdown=prompt or "") _notify("notebook_changed") else: - _notify("new_entry", content_hash=out["hash"]) + carried = carry_forward_entry_config(project, history_for(project, alias)[-2], out["hash"]) + if carried: + out["carried_over"] = carried + _notify("new_entry", content_hash=out["hash"], alias=alias, version=out["version"]) + recalc_report = _auto_recalc_after_head_advance(project, alias, tool="catalog_import_source") + if recalc_report is not None: + out["recalc"] = recalc_report return out @@ -642,6 +715,9 @@ def catalog_create(name: str, code: str, prompt: str = "") -> dict: Same shape as catalog_run, plus `alias` and `version`. """ project = _resolve_active_project() + refusal = _source_alias_refusal(project, name, "created as a catalog entry") + if refusal: + return {"error": refusal} if get_alias(project, name) is not None: return {"error": f"alias {name!r} already exists. Use catalog_revise to update it."} try: @@ -679,11 +755,11 @@ def catalog_revise(name: str, code: str, prompt: str = "") -> dict: (`tracked_expr_from_alias("")` / `pinned_expr_from_alias("")`) is rejected (#135) — it makes an opaque, follows-its-own-head entry. Two good shapes instead: - - source-shaped entry -> inline the source: `read_project_file(...)` / - `read_csv` plus the transforms, then your change. + - source-shaped entry -> start from the source alias: + `tracked_expr_from_alias("")` plus the transforms, then your change. - expensive parent (Aggregate/Join/Sort/window/UDF) -> reference the - previous version by its content HASH: `pinned_expr_from_alias("")`, - which reads its baked snapshot instead of re-running the computation. + previous version: `pinned_expr_from_alias("-v")`, which reads its + baked snapshot instead of re-running the computation. Code conventions and gotchas are documented in `catalog_run`. @@ -693,6 +769,9 @@ def catalog_revise(name: str, code: str, prompt: str = "") -> dict: prompt: Optional human-readable description of what changed. """ project = _resolve_active_project() + refusal = _source_alias_refusal(project, name, "revised") + if refusal: + return {"error": refusal} prev_hash = get_alias(project, name) if prev_hash is None: return {"error": f"alias {name!r} does not exist. Use catalog_create to create it."} @@ -703,8 +782,8 @@ def catalog_revise(name: str, code: str, prompt: str = "") -> dict: msg = ( f"this revision references its own alias {name!r}, which makes an opaque, " f"permanently-stale (follows-its-own-head) entry. Write a self-contained recipe: " - f"inline the source (read_project_file / read_csv …) for a source-shaped entry, or " - f"reference the previous version by hash with pinned_expr_from_alias({prev_hash!r}) " + f"start from the source alias (tracked_expr_from_alias) for a source-shaped entry, or " + f"reference the previous version with pinned_expr_from_alias('{name}-v{len(history_for(project, name))}') " f"for an expensive one." ) rec = record_error(project, code=code, message=msg, prompt=prompt or None, tool="catalog_revise") @@ -749,13 +828,25 @@ def catalog_alias(hash: str, name: str) -> dict: Use this when an entry was created via `catalog_run` and you decide post-hoc that it deserves a name. Errors if the alias already exists. + + The entry must be a computed one. A source entry (a version of an imported + file) belongs to its source alias and cannot take a catalog name (ADR-011 + D1); to give a source a second name, `catalog_create` an entry whose recipe + is `tracked_expr_from_alias("")`, which follows the source + when it is imported again. """ project = _resolve_active_project() + refusal = _source_alias_refusal(project, name, "given to another entry") + if refusal: + return {"error": refusal} if not entry_dir(project, hash).exists(): return {"error": f"no catalog entry for hash {hash!r}"} if get_alias(project, name) is not None: return {"error": f"alias {name!r} already exists"} - info = set_alias(project, name, hash, expect_exists=False) + try: + info = set_alias(project, name, hash, expect_exists=False) + except AliasKindMismatch as exc: # a source entry: the message names its source alias-version and the steer + return {"error": str(exc)} notebook.append(project, name) _notify("alias_changed", content_hash=hash, alias=name, version=info["version"]) _notify("notebook_changed") @@ -994,6 +1085,9 @@ def catalog_promote_diff(name: str, va: int = -2, vb: int = -1, alias: str | Non from tallyman_xorq.result_cache import cached_result_expr project = _resolve_active_project() + refusal = _source_alias_refusal(project, alias or "", "the target of a promoted diff") if alias else None + if refusal: + return {"error": refusal} hashes = history_for(project, name) if not hashes: return {"error": f"alias {name!r} has no history"} @@ -1024,6 +1118,9 @@ def _resolve(v): column_config_overrides = compute_column_config_overrides(a_expr.schema(), b_expr.schema(), keys) target_alias = alias or f"diff_{name}_v{a_idx}_v{b_idx}" + refusal = _source_alias_refusal(project, target_alias, "the target of a promoted diff") + if refusal: + return {"error": refusal} keys_repr = repr(keys) code = textwrap.dedent(f"""\ # auto-generated — diff of {name} V{a_idx} → V{b_idx} @@ -1696,7 +1793,7 @@ def ds_modeling_workflow(dataset: str, target: str = "") -> str: tgt = target or "" return ( f"Build a modeling workflow on {dataset!r} predicting {tgt!r} using the tallyman catalog tools.\n" - "1. catalog_load_parquet the dataset under a name.\n" + "1. catalog_import_source the dataset file under a source alias.\n" "2. catalog_create an engineered-feature entry — new computed columns first, " "ibis.cases() for encodings, and add a stable row id for splitting/diffing.\n" "3. Fit the model AS A CATALOG ENTRY with xorq.ml (deferred_fit_predict_sklearn + " diff --git a/src/tallyman_xorq/build.py b/src/tallyman_xorq/build.py index 4638d664..72fffed7 100644 --- a/src/tallyman_xorq/build.py +++ b/src/tallyman_xorq/build.py @@ -118,8 +118,8 @@ def _user_imports_bare_ibis(code: str) -> bool: return False -# Read helpers the model reaches for on the wrong namespace. The project-aware -# way is read_project_file/tracked_expr_from_alias/tallyman_read_csv. +# Read helpers the model reaches for on the wrong namespace. A recipe reads an alias +# (tracked_expr_from_alias / pinned_expr_from_alias), never a file. _READ_FNS = frozenset( {"read_parquet", "read_csv", "read_in_memory", "read_delta", "deferred_read_parquet", "deferred_read_csv"} ) @@ -168,7 +168,7 @@ def _ibis_import_hint(exc_msg: str, code: str = "") -> str: - `module 'xorq' has no attribute 'X'` — xorq is not the API entrypoint (`import xorq.api as xo`); `_` lives on `xorq.vendor.ibis`; - `module 'ibis' has no attribute 'X'` — math is a column method, reads - go through read_project_file/tracked_expr_from_alias, `ibis.case` is now `ibis.cases`; + go through tracked_expr_from_alias, `ibis.case` is now `ibis.cases`; - `cannot import name 'X' from 'tallyman_xorq.io'` — invented loader; - any duckdb reach — there is no duckdb backend, only datafusion; - `'Table' object has no attribute 'X'` — guessed a column name. @@ -197,11 +197,9 @@ def _ibis_import_hint(exc_msg: str, code: str = "") -> str: elif name in _READ_FNS: hints.append( f"`xorq.{name}` does not exist — `xorq` is not the API entrypoint " - "(`import xorq.api as xo`). Read data with `from tallyman_xorq.io " - "import read_project_file, tracked_expr_from_alias, tallyman_read_csv`; " - "use `tallyman_read_csv(abs_path, schema=...)` for CSVs (the only " - "supported CSV ingest) and `read_project_file(rel_path)` for a " - "parquet file under /data/." + "(`import xorq.api as xo`), and a recipe does not open a file at all. Import the " + "file once with `catalog_import_source('', '')` and read it with " + "`from tallyman_xorq.io import tracked_expr_from_alias`." ) else: hints.append( @@ -217,31 +215,31 @@ def _ibis_import_hint(exc_msg: str, code: str = "") -> str: hints.append(f"math/elementwise functions are column methods: `col.{name}()`, not `ibis.{name}(col)`.") elif name in {"read_parquet", "read_csv", "read_table"}: hints.append( - f"`ibis.{name}` does not exist — read data with " - "`from tallyman_xorq.io import read_project_file, tracked_expr_from_alias, tallyman_read_csv`; " - "use `tallyman_read_csv(abs_path, schema=...)` for CSVs." + f"`ibis.{name}` does not exist — a recipe reads an alias, not a file. Import the file " + "once with `catalog_import_source('', '')`, then " + "`from tallyman_xorq.io import tracked_expr_from_alias`." ) elif name == "case": hints.append("`ibis.case` is gone — use `ibis.cases((cond, val), ..., else_=default)`.") else: hints.append( f"`ibis.{name}` does not exist on `xorq.vendor.ibis`. Many operations are column methods " - "(`col.method()`), and data is read via read_project_file/tracked_expr_from_alias." + "(`col.method()`), and data is read via tracked_expr_from_alias." ) m = re.search(r"cannot import name '(\w+)' from 'tallyman_xorq\.io'", exc_msg) if m: hints.append( - f"`tallyman_xorq.io` has no `{m.group(1)}` — it exports `read_project_file` " - "(raw files under /data/), `tracked_expr_from_alias` (alias → records lineage), " - "`pinned_expr_from_alias` (hash or 'name-vN' version ref, pinned), and `tallyman_read_csv` " - "(CSV ingest with a stable __row_order)." + f"`tallyman_xorq.io` has no `{m.group(1)}` — it exports `tracked_expr_from_alias` " + "(an alias, followed, recording the parent edge) and `pinned_expr_from_alias` " + "('-v', pinned). A file is not a recipe input: import it with " + "`catalog_import_source('', '')` first." ) if "duckdb" in exc_msg.lower(): hints.append( "there is no duckdb backend here — the only backend is xorq's built-in datafusion. " - "Build with the ibis expression API over read_project_file/tracked_expr_from_alias sources; " + "Build with the ibis expression API over tracked_expr_from_alias sources; " "do not use duckdb, `.sql()`, or `con.register()`." ) @@ -279,9 +277,10 @@ def _ibis_import_hint(exc_msg: str, code: str = "") -> str: def _csv_direct_read_check(expr) -> None: """Raise BuildError if the recipe calls xo.deferred_read_csv directly. - tallyman_read_csv is the only supported CSV ingest path — it goes through source identity and writes an ordered - copy with a stable ``__row_order``, so the entry's rows are fixed under its hash and paging is repeatable. A raw - deferred_read_csv bypasses that and gives a nondeterministic row order above datafusion's repartition threshold. + A CSV enters the catalog by an import, which digests it, clones the bytes and writes one ordered snapshot of + them (ADR-011 D1), so the entry's rows are fixed under its hash and paging is repeatable. A raw + deferred_read_csv bypasses all of it and gives a nondeterministic row order above datafusion's repartition + threshold. """ try: from xorq.expr.relations import Read @@ -295,10 +294,10 @@ def _csv_direct_read_check(expr) -> None: return # if the walk fails, don't block the build if csv_nodes: raise BuildError( - "xo.deferred_read_csv is not allowed in tallyman recipes — " - "use tallyman_read_csv(abs_path, schema=...) instead. " - "tallyman_read_csv ingests the CSV once into an ordered copy with a stable __row_order, so the " - "entry's rows are fixed under its hash; deferred_read_csv gives " + "xo.deferred_read_csv is not allowed in tallyman recipes — a recipe never opens a file. Import the " + "CSV once, catalog_import_source('', '', schema=...), and read it with " + "tracked_expr_from_alias(''). The import writes one ordered snapshot with a stable " + "__row_order, so the entry's rows are fixed under its hash; deferred_read_csv gives a " "nondeterministic row order above datafusion's repartition threshold." ) @@ -306,9 +305,9 @@ def _csv_direct_read_check(expr) -> None: def _raw_parquet_read_check(expr, project: str) -> None: """Raise BuildError if the recipe reads a parquet file that tallyman did not write (ADR-008 D12). - Such a read has no digest, no clone and no ordered copy, so an entry built on it has no ``__row_order`` to page by. - Tallyman's own files are the snapshots and ordered copies under the project's ``compute_cache/``; everything else - goes through ``read_project_file``. + Such a read has no digest and no clone, so an entry built on it has no ``__row_order`` to page by and no record + of which bytes it was built from. Tallyman's own files are the snapshots under the project's ``compute_cache/``; + everything else enters by an import (ADR-011 D2). """ from xorq.common.utils.graph_utils import walk_nodes from xorq.expr.relations import Read @@ -324,9 +323,9 @@ def _raw_parquet_read_check(expr, project: str) -> None: continue raise BuildError( f"the recipe reads {path} with xo.deferred_read_parquet, which is not allowed: the file gets no " - "content digest, no clone and no ordered copy, so the entry would have no __row_order to page by. " - "Read a parquet file under /data/ with read_project_file('.parquet'), and a catalog " - "entry with tracked_expr_from_alias('')." + "content digest and no clone, so the entry would have no __row_order to page by and no record of " + f"which bytes it was built from. Import the file first — catalog_import_source({str(path)!r}, " + "'') — and read it, like any other parent, with tracked_expr_from_alias('')." ) @@ -410,7 +409,7 @@ def build_and_persist( return _build_and_persist(project, code, expr_name, prompt) -def _reading(expr, project: str, ordered: dict[str, dict]) -> str: +def _reading(expr, project: str) -> str: """What a cheap entry reads, in words, for the message that says it must keep ``__row_order`` (ADR-008 D3).""" from xorq.common.utils.graph_utils import walk_nodes from xorq.expr.relations import Read @@ -419,13 +418,12 @@ def _reading(expr, project: str, ordered: dict[str, dict]) -> str: paths = [dict(r.read_kwargs).get("hash_path") for r in walk_nodes(Read, expr)] paths = [Path(str(p)) for p in paths if p] - return describe_read(project, paths[0], ordered) if len(paths) == 1 else "a file" + return describe_read(project, paths[0]) if len(paths) == 1 else "a file" def _build_and_persist(project: str, code: str, expr_name: str, prompt: str | None) -> BuildResult: from xorq.ibis_yaml.compiler import build_expr, load_expr - from tallyman_xorq import ordered_copy as oc from tallyman_xorq._git_state_guard import install_git_state_guard from tallyman_xorq.materialize import SNAPSHOT_FORMAT_VERSION, engine_versions, materialize, snapshot_path from tallyman_xorq.result_cache import stream_row_count @@ -436,22 +434,16 @@ def _build_and_persist(project: str, code: str, expr_name: str, prompt: str | No # from the long-lived server) and abort the whole build. Make it best-effort. install_git_state_guard() - # Collect source digests while user code imports (read_project_file notes each - # file it reads), and the ordered copies each source was ingested into (what a - # manifest needs so ensure_materialized can make one again). + # Collect the resolved tracked_expr_from_alias parent edges while user code imports (#84). They are the whole + # record of what the entry reads: a recipe names aliases and never a file (ADR-011 D2), so there are no source + # digests to fold up and no ordered copies to record. from tallyman_xorq import parent_capture as pc - from tallyman_xorq import source_identity as si - collect_token = si.begin_collect() parent_token = pc.begin_collect() - copies_token = oc.begin_collect() try: module, tmp_script = _import_script(code) finally: - sources = si.end_collect(collect_token) - # Resolved tracked_expr_from_alias parent edges captured during import (#84). parents = pc.end_collect(parent_token) - ordered = oc.end_collect(copies_token) expr_obj = getattr(module, expr_name, None) if expr_obj is None: names = ", ".join(n for n in dir(module) if not n.startswith("_")) @@ -460,8 +452,9 @@ def _build_and_persist(project: str, code: str, expr_name: str, prompt: str | No # Advisory nondeterminism lint (#88) on the author's expression, surfaced on the result, never fatal. lint_warnings = _nondeterminism_warnings(expr_obj) - # Fatal: raw reads are banned. tallyman_read_csv and read_project_file are the only ingest paths (each writes an - # ordered copy with a stable __row_order); a raw deferred_read_csv or deferred_read_parquet bypasses them. + # Fatal: raw reads are banned. A file enters the catalog by an import and a recipe reads the source alias + # (ADR-011 D2); read_project_file and tallyman_read_csv refuse in io.py, and these two catch the xorq readers + # that would otherwise go straight to a file with no digest, no clone and no __row_order. _csv_direct_read_check(expr_obj) _raw_parquet_read_check(expr_obj, project) @@ -479,7 +472,7 @@ def _build_and_persist(project: str, code: str, expr_name: str, prompt: str | No expr_obj, project, verdict=verdict, - reading=_reading(expr_obj, project, ordered) if not verdict.worthy else None, + reading=_reading(expr_obj, project) if not verdict.worthy else None, ) except (InMemoryReadError, CacheNodeError, RowOrderError) as exc: raise BuildError(str(exc)) from exc @@ -505,10 +498,6 @@ def _build_and_persist(project: str, code: str, expr_name: str, prompt: str | No raise BuildError(f"build_expr failed: {exc}{hint}\n{traceback.format_exc()}") from exc content_hash = build_path.name - if si.mode() == "salt" and sources: - # xorq's hash is path-identity only; mixing the source digests in - # makes the entry hash content-sensitive (same shape, 12 hex). - content_hash = si.salted_hash(content_hash, sources) target = entry_dir(project, content_hash) if target.exists(): @@ -641,8 +630,6 @@ def _build_and_persist(project: str, code: str, expr_name: str, prompt: str | No nonreproducible_columns=differing or None, snapshot_format=SNAPSHOT_FORMAT_VERSION, engine_versions=engine_versions(), - ordered_copies=ordered or None, - sources=sources or None, parents=parents or None, ) write_manifest(target, manifest) diff --git a/src/tallyman_xorq/dependents.py b/src/tallyman_xorq/dependents.py index cf1433c9..18ce5c83 100644 --- a/src/tallyman_xorq/dependents.py +++ b/src/tallyman_xorq/dependents.py @@ -7,9 +7,8 @@ walk back — not the column lineage — to find the dependents of a changed entry. This is that thin reader over the recorded manifests — the single seam every consumer (recalc, staleness) goes through instead of touching manifest fields -directly: a child's parents, its source-leaf digests, the reverse index (a -parent's children), and the topologically-ordered descendant cone a recalc -rebuilds in dependency order. +directly: a child's parents, the reverse index (a parent's children), and the +topologically-ordered descendant cone a recalc rebuilds in dependency order. """ from __future__ import annotations @@ -44,17 +43,6 @@ def references_own_alias(project: str, content_hash: str, alias: str) -> bool: return any(p.ref == alias for p in parents_of(project, content_hash)) -def sources_of(project: str, content_hash: str) -> dict[str, str] | None: - """The recorded source-leaf digests of one entry (``{rel_path: digest}``). - - ``None`` when the entry was built under source-identity ``off`` (no digests - recorded) — kept distinct from ``{}`` (identity on, but the recipe read no - raw files), because staleness treats ``None`` as an unevaluable axis rather - than silently "fresh". - """ - return read_manifest(entry_dir(project, content_hash)).sources - - def build_dag(project: str) -> dict[str, list[ParentRef]]: """Forward edges for every live entry: ``{child_hash: [ParentRef, ...]}``. diff --git a/src/tallyman_xorq/io.py b/src/tallyman_xorq/io.py index 9263aff5..4f17b31d 100644 --- a/src/tallyman_xorq/io.py +++ b/src/tallyman_xorq/io.py @@ -19,12 +19,44 @@ class ProjectDataNotFound(FileNotFoundError): pass +def _source_entry_read(fn: str, path: str): + """The expression a raw read resolves to (ADR-011 D2). + + A file enters the catalog by an explicit import and by nothing else, so a raw read is a build error in an + authored recipe. It survives in exactly one place: the recipe the importer generates for a source entry, where + it resolves to that entry's own snapshot. + + The entry being minted names its own project, and the read follows it rather than the ambient active project: + ``update_and_depend(path, alias, project=X)`` is an override, so an import must work whichever project happens + to be active. Resolving the ambient one instead made the importer's own recipe hit the refusal below. + """ + from tallyman_xorq.source_import import source_entry_context + + ctx = source_entry_context() + if ctx is not None: + from xorq.expr.api import deferred_read_parquet + + from tallyman_xorq.materialize import snapshot_path + + owner, content_hash = ctx + return deferred_read_parquet(str(snapshot_path(owner, content_hash))) + from tallyman_xorq.build import BuildError + + raise BuildError( + f"{fn}({path!r}) reads a file tallyman does not own, which is not allowed in a recipe: the entry's rows " + "would depend on a file anyone can edit or delete, and nothing would record which bytes it was built from. " + f"Import the file first — catalog_import_source({path!r}, '') — and then read it in the recipe with " + "tracked_expr_from_alias(''), which follows the alias, or pinned_expr_from_alias('-v') to " + "pin one version of it." + ) + + def project_path(rel_path: str, project: str | None = None, must_exist: bool = True) -> Path: """Resolve `rel_path` against the project's data dir. Raises if absent. - Pass ``must_exist=False`` for digest-pinned reconstruction: it serves the frozen - ``.cas`` clone (``recon_cas_path``) and must not require the live source to still - be present. The traversal guard still applies regardless. + Pass ``must_exist=False`` to name a file that may not be there. The traversal + guard still applies regardless. Nothing in a build calls this: a recipe reads + aliases, and ``data/`` stopped being special with ADR-011 D2. """ proj = resolve_project(project) base = data_dir(proj) @@ -40,45 +72,17 @@ def project_path(rel_path: str, project: str | None = None, must_exist: bool = T def read_project_file(rel_path: str, project: str | None = None): - """Load a raw data file from `/data/` as a xorq expression. - - Use this to bring raw input files (parquet, csv, etc.) into the catalog's - expression layer. This is the root of the dependency DAG — the file has no - catalog hash, no recipe, no lineage entry. Everything built on top of it - flows through tracked_expr_from_alias. - - The file is never read directly (ADR-008 D2). It goes through source - identity (tallyman_xorq.source_identity): its content digest is taken, in - `cas` mode it is cloned to `data/.cas/`, and in `salt` mode - the digest is recorded for build_and_persist to mix into the entry hash. - Then an *ordered copy* of it is written under the project's `compute_cache/` - (tallyman_xorq.ordered_copy): the same rows in file order plus a last column, - `__row_order`, `0..N-1`, which pages sort by. The returned expression is a - plain read of that copy, so the entry's hash covers the source's content - (the copy's name is a function of the digest) and editing the file forks it. - """ - from xorq.expr.api import deferred_read_parquet + """The raw read of a parquet file, which only a source entry's own recipe may do (ADR-011 D2). - from tallyman_xorq import ordered_copy as oc - from tallyman_xorq import source_identity as si + In an authored recipe this raises a ``BuildError`` naming the import to use instead. A file enters the + catalog through ``catalog_import_source`` / ``update_and_depend`` and by no other route, and a recipe + reads the resulting source alias with ``tracked_expr_from_alias``. - proj = resolve_project(project) - reader = oc.parquet_reader() - recorded = _reconstructing_source_digest(proj, rel_path) - if recorded is not None: - # Reconstruction (#115): this read_project_file is re-running an already-built entry's recipe (expr.py), not - # authoring a new entry. Resolve to the copy of the bytes the entry was BUILT from, named by the digest in its - # manifest.sources, instead of re-digesting the live file, which may have been edited in place since. Note - # that same frozen digest so a child build reconstructing this entry records it. - si.note_source(rel_path, recorded) - return deferred_read_parquet(str(oc.existing_ordered_copy(proj, digest=recorded, reader=reader))) - path = project_path(rel_path, proj) - digest = si.digest_for(proj, path) - if si.mode() != "off": - si.note_source(rel_path, digest) - source = si.ensure_cas_path(proj, path, digest) if si.mode() == "cas" else path - copy = oc.ensure_ordered_copy(proj, source, digest=digest, rel=rel_path, reader=reader) - return deferred_read_parquet(str(copy)) + It survives as a callable because the recipe the importer generates for a source entry calls it, and + a contextvar (``source_import._SOURCE_ENTRY``) resolves that call to the entry's own snapshot. The + path in the generated recipe is provenance; nothing opens it. + """ + return _source_entry_read("read_project_file", rel_path) # ibis primitive -> polars dtype, for reading a CSV with an explicit schema. @@ -372,7 +376,7 @@ def _suggest_schema_dsl(src: Path, scan_kwargs: dict, reserved: tuple[str, ...] return repr(pairs) -def _materialize_ordered(src: Path, schema, scan_kwargs: dict, tmp_path: Path) -> None: +def _materialize_ordered(src: Path, schema, scan_kwargs: dict, tmp_path: Path, *, write) -> None: """Read *src* (a CSV) into a row-order-stable parquet at *tmp_path* (#143). Inference mode (no schema, or a spec with inferred columns) escalates the @@ -385,11 +389,14 @@ def _materialize_ordered(src: Path, schema, scan_kwargs: dict, tmp_path: Path) - The last column is ``__row_order`` (ADR-008 D2). A CSV that already has a column of that name has it overwritten, which is the right outcome for a file tallyman exported. + + *write* takes the parsed LazyFrame of one rung of the ladder and puts it on + disk. There is one caller, the import, and it passes a writer that streams + the rows into pyarrow so a source snapshot has the layout every other + snapshot has (ADR-011 D1). """ import polars as pl - from tallyman_xorq.ordered_copy import _WRITE - # Header (names only) drives the schema plan; infer_schema_length=0 reads # just the header line, no type sampling. header = list(pl.scan_csv(str(src), infer_schema_length=0, **scan_kwargs).collect_schema().names()) @@ -434,7 +441,7 @@ def _ordered(infer_len): last_exc = None for infer_len in ladder: try: - _ordered(infer_len).sink_parquet(str(tmp_path), **_WRITE) + write(_ordered(infer_len), tmp_path) return except pl.exceptions.ComputeError as exc: last_exc = exc @@ -445,42 +452,24 @@ def _ordered(infer_len): def tallyman_read_csv(path: str, schema=None, project: str | None = None, **kwargs): - """Read a CSV into a xorq expression with a stable ``__row_order``. - - Use this instead of ``xo.deferred_read_csv`` for all CSV ingests. The CSV - goes through source identity like a parquet source (``read_project_file``): - its content digest is taken, it is cloned to ``data/.cas/``, and an ordered - copy of the clone is written under the project's ``compute_cache/``. polars - reads the clone (``scan_csv -> with_row_index -> sink_parquet``), which - preserves the file's row order (unlike datafusion's parallel scan, whose row - order is nondeterministic above the repartition threshold, about 10 MB) and - numbers the rows in a last column, ``__row_order``, ``0..N-1``. See - ``plans/ADR-008-row-order-of-reads.md``. - - The CSV is read exactly once, at ingest, and the returned expression is a - plain ``deferred_read_parquet`` of the copy: no sort, one row-order column. - Because the copy is named by the CSV's content and the reader options, - editing the CSV and running the same recipe gives a new content hash, and the - earlier entry keeps the rows it was built from (#168). + """The raw read of a CSV, refused for the same reason ``read_project_file`` is (ADR-011 D2). + + D2 names ``read_project_file``; this one goes with it, because both open a file the catalog does not + own and leaving the CSV reader open would be a hole in the rule. Import the CSV instead — + ``catalog_import_source(path, alias, separator=..., schema=...)`` — and read the source alias. The + reader options move to the import call and are recorded on the entry (ADR-011 D12), so the file is + read one way forever; to read it a second way, import it again under a second alias. + + The CSV reader itself did not go anywhere: the schema DSL, the inference ladder and the row-order + guarantee of ADR-005 and ADR-008 D2 all run at import time now (``source_import._write_csv_snapshot`` + over ``_materialize_ordered``). Args: path: Absolute path to the CSV file. - schema: Optional ibis schema for the columns (same as deferred_read_csv). - When omitted, polars infers types. + schema: Accepted and unused; the import takes it. project: Project name override (defaults to the active project). - **kwargs: Forwarded to ``polars.scan_csv`` — reader options such as - ``separator``, ``skip_rows``, ``null_values``, ``quote_char``, - ``has_header``, ``encoding``. They participate in the copy's key, - so changing one re-ingests. ``infer_schema_length`` and - ``schema_overrides`` are managed internally (see ``_RESERVED_SCAN_KWARGS``) - and rejected — they would collide with the values every internal - ``scan_csv`` call already sets. + **kwargs: Accepted and unused; the import takes them. """ - from xorq.expr.api import deferred_read_parquet - - from tallyman_xorq import ordered_copy as oc - from tallyman_xorq import source_identity as si - reserved = [k for k in _RESERVED_SCAN_KWARGS if k in kwargs] if reserved: raise ValueError( @@ -489,74 +478,7 @@ def tallyman_read_csv(path: str, schema=None, project: str | None = None, **kwar "(100 -> 10k -> whole-file); to pin column types pass schema= (an ibis schema, " "plain dict, or tuple-of-tuples), never schema_overrides." ) - proj = resolve_project(project) - reader = oc.csv_reader(schema, kwargs) - src = Path(path) - rel = _relative_to_data(proj, src) - recorded = _reconstructing_source_digest(proj, rel) - if recorded is not None: - si.note_source(rel, recorded) - return deferred_read_parquet(str(oc.existing_ordered_copy(proj, digest=recorded, reader=reader))) - digest = si.digest_for(proj, src) - if si.mode() != "off": - si.note_source(rel, digest) - source = si.ensure_cas_path(proj, src, digest) if si.mode() == "cas" else src - copy = oc.ensure_ordered_copy(proj, source, digest=digest, rel=rel, reader=reader) - return deferred_read_parquet(str(copy)) - - -def _relative_to_data(proj: str, path: Path) -> str: - """The name a source is recorded under in ``manifest.sources``: relative to the project's data dir when it sits - there, the absolute path otherwise (a CSV can live anywhere).""" - try: - return str(path.resolve().relative_to(data_dir(proj).resolve())) - except ValueError: - return str(path) - - -def _reconstructing_source_digest(proj: str, rel_path: str) -> str | None: - """The digest *rel_path* was built with, if a recipe for *proj* is being - reconstructed on this call stack (#115); otherwise None. - - Reads the per-entry ``{rel_path: digest}`` map ``result_cache._recipe_expr`` - threads through ``_RECON_SOURCES``. Returns None outside reconstruction (the build - path), when the reconstructing entry belongs to a different project, or when this - source was not recorded — read_project_file then falls through to its live-file path. - """ - from tallyman_xorq.result_cache import _RECON_SOURCES - - ctx = _RECON_SOURCES.get() - if ctx is None: - return None - recon_proj, sources = ctx - if recon_proj != proj: - return None - return sources.get(rel_path) - - -def _note_parent_records(proj: str, content_hash: str) -> None: - """Fold the parent's recorded source digests, and its ordered copies when its graph is inlined, into the build. - - The child's ``manifest.sources`` is the closure record ``gc_cas`` walks to keep clones alive, so it carries every - digest its parent recorded, and the child keeps its own leaves alive even if the parent entry is later evicted. - A CHEAP parent's graph is inlined into the child's build (a worthy parent is a bare read of its snapshot), so the - child reads that parent's ordered copies directly and needs the records that let ``ensure_materialized`` make a - deleted one again (ADR-007 D13). ``note_source`` and ``note_ordered_copy`` are no-ops outside a build's collect - window, so this costs nothing on a plain read. - """ - from tallyman_core import read_manifest - from tallyman_xorq import ordered_copy as oc - from tallyman_xorq import source_identity as si - - try: - manifest = read_manifest(entry_dir(proj, content_hash)) - except (OSError, ValueError): - return - for rel_path, digest in (manifest.sources or {}).items(): - si.note_source(rel_path, digest) - if manifest.cache_worthy is False: - for key, record in (manifest.ordered_copies or {}).items(): - oc.note_ordered_copy(key, record) + return _source_entry_read("tallyman_read_csv", path) def tracked_expr_from_alias(alias: str, project: str | None = None): @@ -606,7 +528,6 @@ def tracked_expr_from_alias(alias: str, project: str | None = None): from tallyman_xorq import parent_capture as pc pc.note_parent(content_hash, ref=alias, follow=True) - _note_parent_records(proj, content_hash) return cached_result_expr(proj, content_hash) @@ -618,31 +539,47 @@ def pinned_expr_from_alias(ref: str, project: str | None = None): not advance it when the parent alias moves. Use this when you want to stay on a specific version of a parent rather than following alias changes. - Accepts a content hash or an explicit version reference ``"-v"`` - (1-based into the alias history — the V1…Vn the UI shows). A bare alias is - rejected (#166): it reads like a pin but resolves to whatever the head - happens to be when the recipe is built, so the recipe text under-determines - the entry. A pinned reference must denote the same entry forever. + Accepts an explicit version reference ``"-v"`` only (1-based into the + alias history — the V1…Vn the UI shows). Two other forms are rejected: + + - a bare alias (#166), which reads like a pin but resolves to whatever the head + happens to be when the recipe is built, so the recipe text under-determines + the entry. A pinned reference must denote the same entry forever; + - a bare content hash (ADR-011 D5). Every parent edge names an alias, followed + or pinned at a version, so no opaque hash appears in a recipe or in + ``manifest.parents``. An entry with no alias — one built by ``catalog_run`` — + must be named (``catalog_alias``) before anything can build on it. The parent edge is suppressed during reconstruction (same as tracked_expr_from_alias) so re-running a child's recipe doesn't accidentally write to the manifest. Args: - ref: A content hash, or a version reference like "shoe_sales-v2". + ref: A version reference like "shoe_sales-v2". project: Project name override (defaults to active TALLYMAN_PROJECT). """ - from tallyman_core.aliases import VERSION_REF_RE, history_for, resolve_version_ref + from tallyman_core.aliases import VERSION_REF_RE, history_for, resolve_version_ref, version_of_hash from tallyman_xorq.result_cache import _RECONSTRUCTING, _resolve_noncyclic_hash, cached_result_expr proj = resolve_project(project) if entry_dir(proj, ref).exists(): - content_hash = ref + named = version_of_hash(proj, ref) + steer = ( + f"Pin it as {named[0] + '-v' + str(named[1])!r} instead." + if named + else f"Entry {ref} heads no alias history, so there is no version to name: give it a name with " + "catalog_alias first, then pin it as '-v'." + ) + raise ProjectDataNotFound( + f"pinned_expr_from_alias({ref!r}) is a bare content hash, which a recipe may not name (ADR-011 D5): " + f"a parent edge names an alias and a version of it, so the DAG is readable and a hash never leaks into " + f"a recipe. {steer}" + ) elif get_alias(proj, ref) is not None: raise ProjectDataNotFound( f"pinned_expr_from_alias({ref!r}) is a bare alias, which would silently pin " f"whatever the head happens to be right now (#166). Pin an exact version — " - f"a content hash, or {ref + '-v'!r} — or use tracked_expr_from_alias({ref!r}) " - "to follow the alias." + f"{ref + '-v'!r} — or use tracked_expr_from_alias({ref!r}) to follow the " + "alias. A bare content hash is not the way either (ADR-011 D5)." ) else: content_hash = resolve_version_ref(proj, ref) @@ -662,5 +599,4 @@ def pinned_expr_from_alias(ref: str, project: str | None = None): from tallyman_xorq import parent_capture as pc pc.note_parent(content_hash, ref=ref, follow=False) - _note_parent_records(proj, content_hash) return cached_result_expr(proj, content_hash) diff --git a/src/tallyman_xorq/materialize.py b/src/tallyman_xorq/materialize.py index ec47542f..7ae3d8f1 100644 --- a/src/tallyman_xorq/materialize.py +++ b/src/tallyman_xorq/materialize.py @@ -11,8 +11,8 @@ that fixes the layout of the file (ADR-009 D3) and numbers them in a last column, ``__row_order`` (ADR-008 D2), and returns the content digest of the file it wrote, read back (ADR-009 D2). -``ensure_materialized`` is the one entry point that makes files exist: every snapshot, ordered copy and clone an -entry's plan reads is on disk before anything executes (ADR-007 D5). +``ensure_materialized`` is the one entry point that makes files exist: every snapshot an entry's plan reads is on +disk before anything executes (ADR-007 D5). """ from __future__ import annotations @@ -34,7 +34,7 @@ # The snapshot format (ADR-009 D3). The row-group size decides the batch boundaries an entry built on this file sees, # and an ungrouped float total depends on them (#187), so the row-group size and the materialization connection's batch # size are part of the reproducibility contract: changing either is a corpus rebuild, and the version below stands for -# both (and for the layout of the ordered copies of sources, ``ordered_copy.ORDERED_COPY_ROW_GROUP_ROWS``). +# both (and for the row groups a source snapshot is written in, ``ordered_copy.ORDERED_COPY_ROW_GROUP_ROWS``). SNAPSHOT_ROW_GROUP_ROWS = 1_048_576 SNAPSHOT_BATCH_SIZE = 8192 SNAPSHOT_FORMAT_VERSION = 1 @@ -104,6 +104,43 @@ class Materialized: differing_columns: list[str] = field(default_factory=list) +def row_groups(batches, rows: int): + """The rows of *batches*, in order, as tables of *rows* rows each; the last one may be shorter. + + The one place a stream is cut into row groups, so a file's layout is a function of its rows and not of how the + producer batched them (ADR-009 D3). Memory is bounded by one row group. + """ + pending: list[pa.RecordBatch] = [] + pending_rows = 0 + for batch in batches: + if not batch.num_rows: + continue + pending.append(batch) + pending_rows += batch.num_rows + while pending_rows >= rows: + table = pa.Table.from_batches(pending) + yield table.slice(0, rows) + tail = table.slice(rows) + pending, pending_rows = tail.to_batches(), tail.num_rows + if pending_rows: + yield pa.Table.from_batches(pending) + + +def write_pinned_parquet(batches, schema: pa.Schema, dest: Path, *, row_group_rows: int) -> None: + """Write *batches*, in order, to *dest* in the pinned parquet settings of ``_PARQUET_OPTIONS`` (ADR-009 D3). + + Every file under ``compute_cache/`` goes through here or through ``_stream_to_parquet``, so ``result_cache/`` + holds one shape of parquet: the same format version, page index and compression, whether the rows were computed + by an entry's build or parsed out of an imported file (ADR-011 D1). Each row group is combined into contiguous + arrays first, so the bytes do not depend on the producer's chunking. + """ + with pq.ParquetWriter(dest, schema, **_PARQUET_OPTIONS) as writer: + for table in row_groups(batches, row_group_rows): + if not table.schema.equals(schema, check_metadata=False): + table = table.cast(schema) + writer.write_table(table.combine_chunks(), row_group_size=row_group_rows) + + def _stream_to_parquet(expr, dest: Path) -> tuple[int, pa.Schema]: """Write the rows of *expr* to *dest* in the snapshot format, numbering them in a last ``__row_order`` column. @@ -120,9 +157,7 @@ def _stream_to_parquet(expr, dest: Path) -> tuple[int, pa.Schema]: written = 0 with pq.ParquetWriter(dest, out_schema, **_PARQUET_OPTIONS) as writer: - - def flush(table: pa.Table) -> None: - nonlocal written + for table in row_groups(reader, SNAPSHOT_ROW_GROUP_ROWS): n = table.num_rows numbered = table.select(names).append_column( pa.field(ROW_ORDER, pa.int64()), pa.array(np.arange(written, written + n, dtype=np.int64)) @@ -131,21 +166,6 @@ def flush(table: pa.Table) -> None: numbered = numbered.cast(out_schema) writer.write_table(numbered.combine_chunks(), row_group_size=SNAPSHOT_ROW_GROUP_ROWS) written += n - - pending: list[pa.RecordBatch] = [] - pending_rows = 0 - for batch in reader: - if not batch.num_rows: - continue - pending.append(batch) - pending_rows += batch.num_rows - while pending_rows >= SNAPSHOT_ROW_GROUP_ROWS: - table = pa.Table.from_batches(pending) - flush(table.slice(0, SNAPSHOT_ROW_GROUP_ROWS)) - tail = table.slice(SNAPSHOT_ROW_GROUP_ROWS) - pending, pending_rows = tail.to_batches(), tail.num_rows - if pending_rows: - flush(pa.Table.from_batches(pending)) return written, out_schema @@ -222,8 +242,12 @@ def _heal(project: str, content_hash: str) -> None: def _recreate(project: str, owner_hash: str, path: Path) -> None: - """Make a missing file the owner's plan reads again, by the rule for its class (ADR-007 D13).""" - from tallyman_xorq import ordered_copy as oc + """Make a missing file the owner's plan reads again (ADR-007 D13). + + There is one class of file to make again: another entry's snapshot, named by its content hash, made + by recursing on it. A source's ordered copy was the second class and is gone — a source is an entry, + so its rows come back through this same path (``_heal_a_source`` under ``_ensure``), ADR-011 D1. + """ from tallyman_xorq.build import BuildError if path.parent == snapshots_dir(project): @@ -236,8 +260,6 @@ def _recreate(project: str, owner_hash: str, path: Path) -> None: "catalog, so it cannot be made again" ) ensure_materialized(project, parent) - elif oc.is_ordered_copy_path(project, path): - oc.recreate_ordered_copy(project, owner_hash, path) else: raise BuildError( f"entry {owner_hash} in {project!r} reads {path}, which tallyman did not write and cannot make again" @@ -246,6 +268,84 @@ def _recreate(project: str, owner_hash: str, path: Path) -> None: raise BuildError(f"entry {owner_hash} in {project!r}: {path} is still missing after it was made again") +def _source_provenance(project: str, content_hash: str): + """The import recorded on the entry, or None when it is not a source version.""" + from tallyman_core import read_manifest + from tallyman_core.paths import entry_dir + + try: + return read_manifest(entry_dir(project, content_hash)).provenance + except (OSError, ValueError): + return None + + +def _source_names(project: str, content_hash: str, provenance) -> tuple[tuple[str, int] | None, str]: + """``(held, imported)``: how a message names a source version (ADR-011 D1). + + ``held`` is ``(alias, version)`` as the alias store has it now, or None when no source alias holds the entry + (``source_import.current_source_version``). ``imported`` is the ``-v`` the version was imported as, + from ``provenance``: history, which a rename or an unalias leaves behind. A message names the version by + ``held``, and mentions ``imported`` only where the two differ. + """ + from tallyman_xorq.source_import import current_source_version + + return current_source_version(project, content_hash, provenance), f"{provenance.alias}-v{provenance.version}" + + +def _heal_a_source(project: str, content_hash: str) -> bool: + """Re-create a source version's snapshot from the clone of the bytes it was imported from (ADR-011 D1). + + A source snapshot is cache in the sense of ADR-007 D13 — ``ensure_materialized`` can re-create it — because the + clone under ``data/.cas`` holds the imported bytes and the entry records the reader options that read them. It + cannot go through ``materialize``: the entry's build reads the very snapshot that is missing, so the rows come + from the clone instead. What is written is then verified against the recorded ``result_digest`` like any other + re-created snapshot. + + Only a clone that is gone as well makes the version unrecoverable, and then the error names the file that is + missing and the re-import that repairs it: the alias and version that hold the entry now (``_source_names``), so + the advised call re-imports into the version it names rather than minting an alias under the name the version + was imported as, and the reader options the entry recorded (``source_import.import_call``), without which a CSV + names another entry. When no source alias holds it there is no version to re-import into, and the error says what + an import of the same bytes would do instead. Returns whether this entry is a source version, so the general + path can stop. + """ + from tallyman_core.catalog_state import project_lock + from tallyman_xorq.build import BuildError + from tallyman_xorq.result_cache import _verify_self_heal + from tallyman_xorq.source_import import import_call, rewrite_source_snapshot, source_clone_path + + provenance = _source_provenance(project, content_hash) + if provenance is None: + return False + with project_lock(project): + if snapshot_path(project, content_hash).exists(): # a peer healed it while we waited + return True + clone = source_clone_path(project, provenance) + if not clone.is_file(): + held, imported = _source_names(project, content_hash, provenance) + lost = ( + f"cannot be made again: {snapshot_path(project, content_hash)} is not on disk and neither is the " + f"clone of the imported bytes, {clone}." + ) + if held is None: + raise BuildError( + f"the data imported as {imported} (entry {content_hash}), which no source alias holds now, " + f"{lost} Importing the same bytes again, " + f"{import_call(provenance.path, None, provenance.reader)}, writes these rows again as a new " + "version of ." + ) + alias, version = held + also = "" if f"{alias}-v{version}" == imported else f", imported as {imported}" + raise BuildError( + f"the data of {alias}-v{version} (entry {content_hash}{also}) {lost} Import them again with " + f"{import_call(provenance.path, alias, provenance.reader, pinned_version=version)}." + ) + digest = rewrite_source_snapshot(project, content_hash, provenance) + perf_log.debug("ensure_materialized re-imported %s from %s", content_hash, clone.name) + _verify_self_heal(project, content_hash, digest) + return True + + def _ensure(project: str, content_hash: str) -> bool: """``ensure_materialized``, returning whether the entry is worthy (the caller usually needs to know).""" from tallyman_xorq.result_cache import _resolve_result_plan, cache_worthy @@ -253,6 +353,8 @@ def _ensure(project: str, content_hash: str) -> bool: worthy = cache_worthy(project, content_hash) if worthy and snapshot_path(project, content_hash).exists(): return True + if _heal_a_source(project, content_hash): + return worthy plan = _resolve_result_plan(project, content_hash) for path in plan.reads: if not path.exists(): @@ -268,8 +370,8 @@ def ensure_materialized(project: str, content_hash: str) -> None: 1. A worthy entry whose snapshot exists is done, and no build is loaded. 2. Otherwise load the entry's build (the plan is kept in the existing LRU) and collect every file its ``Read`` nodes point at. - 3. Re-create each that is missing by the rule for its class: a snapshot by recursing on the hash in its file name, - an ordered copy of a source from its clone, a clone from the live source while the bytes still match. + 3. Re-create each that is missing: every one is another entry's snapshot, made by recursing on the hash in its + file name. A source version is re-created from the clone of its imported bytes (``_heal_a_source``). 4. If the entry is worthy, heal its own snapshot and verify it. Every caller that composes or executes an entry goes through here, so nothing ever runs over a file that is @@ -282,17 +384,33 @@ def pinned_reason(project: str, content_hash: str) -> str | None: """Why the entry's snapshot must not be deleted, or None when it may be (ADR-009 D6, ADR-007 D12). A snapshot is pinned when it cannot be made again faithfully: the recipe is not reproducible (two runs at create - time gave different digests), or a heal already produced different rows than were built. The Cache page's delete - leaves such a file alone and says why. ``compute_cache/`` as a whole is still deletable by definition. + time gave different digests), a heal already produced different rows than were built, or it is a source + version whose clone of the imported bytes is gone (ADR-011 D1 — with the clone it is ordinary cache, made again + from those bytes). The Cache page's delete leaves such a file alone and says why, naming a source version by the + alias that holds it now (``_source_names``). ``compute_cache/`` as a whole is still deletable by definition. """ from tallyman_core import read_manifest from tallyman_core.errors import list_errors from tallyman_core.paths import entry_dir + from tallyman_xorq.source_import import source_clone_path try: manifest = read_manifest(entry_dir(project, content_hash)) except (OSError, ValueError): manifest = None + if manifest is not None and manifest.provenance is not None: + clone = source_clone_path(project, manifest.provenance) + if not clone.is_file(): + held, imported = _source_names(project, content_hash, manifest.provenance) + if held is None: + name = f"the source version imported as {imported}, which no source alias holds now" + else: + current = f"{held[0]}-v{held[1]}" + name = f"the source version {current}" + ("" if current == imported else f" (imported as {imported})") + return ( + f"this file is the last copy of {name}: the clone of the bytes imported from " + f"{manifest.provenance.path} is gone from {clone}, so nothing can make it again and it is kept" + ) if manifest is not None and manifest.reproducible is False: columns = ", ".join(manifest.nonreproducible_columns or []) return ( diff --git a/src/tallyman_xorq/ordered_copy.py b/src/tallyman_xorq/ordered_copy.py index e1edcd7e..fb52c081 100644 --- a/src/tallyman_xorq/ordered_copy.py +++ b/src/tallyman_xorq/ordered_copy.py @@ -1,63 +1,38 @@ -"""Ordered copies of sources (ADR-008 D2, ADR-007 D13): the one way a file enters a recipe. - -A source (a parquet file or a CSV under the project) is read through its content-addressed clone (``data/.cas``), and -polars writes a parquet copy of it, in file order, with one more column at the end: ``__row_order``, ``0..N-1``. That -copy is what a recipe reads. Nothing reads the source or the clone directly, so: - -- every file tallyman reads carries the column that pages sort by; -- the copy is keyed by the source's content digest and the reader options, so editing a source and running the same - recipe forks the entry's hash (#168) and the old entry keeps the rows it was built from; -- the copy is cache (ADR-007 D13). It lives under ``compute_cache/``, so a project that is packed, cloned or emptied - loses it, and ``ensure_materialized`` makes it again from the clone with the reader options the manifest records, - then checks it against the content digest recorded when it was first written. - -The layout of the copy is part of the reproducibility contract (ADR-009 D3, #187): an ungrouped float total depends on -the row-group boundaries of the file it reads, so the row-group size below is frozen with the snapshot format version. +"""Reader options, and the row-group size a source snapshot is written in. + +An import names how a file is read — once, in the import call — and records it on the source entry +(ADR-011 D12). This module holds that record's shape: ``parquet_reader`` / ``csv_reader`` build it, +``_spec_to_json`` / ``_spec_from_json`` round-trip a CSV schema spec through JSON, and +``_reader_signature`` is the part of it the source entry's content hash covers, so two recipes cannot +read one file two ways and a function-valued option cannot fork the hash on every build (#215). + +What used to be here was the **ordered copy** of ADR-008 D2: every source read through its clone into a +second parquet store, ``compute_cache/ordered_sources/.parquet``, keyed by +``md5(digest ǀ reader signature)``. ADR-011 D1 subsumes it. A raw input is an entry now, the entry hash +names its one snapshot under ``compute_cache/result_cache/``, and the reader options live on the entry +that used them — so the second store, the copy key, and the machinery that made a deleted copy again +(``ensure_ordered_copy``, ``recreate_ordered_copy``, the per-build ``ordered_copies`` manifest record) +all have nothing left to address. ``ensure_materialized`` re-creates a source snapshot from the clone +instead (``materialize._heal_a_source``). + +The layout of that snapshot is still part of the reproducibility contract (ADR-009 D3, #187): an +ungrouped float total depends on the row-group boundaries of the file it reads, so the row-group size +below is frozen with the snapshot format version. """ from __future__ import annotations -import contextvars -import hashlib import json -import logging -import os -import uuid from pathlib import Path -from tallyman_xorq.row_order import ROW_ORDER - -perf_log = logging.getLogger("tallyman.perf") - -# Pinned polars parquet-write settings for an ordered copy. Held constant so the layout, and therefore any float total -# computed straight from a source, is reproducible. Changing one is a corpus rebuild (SNAPSHOT_FORMAT_VERSION). +# Row-group size of a source snapshot. Held constant so the layout, and therefore any float total +# computed straight from a source, is reproducible. Changing it is a corpus rebuild +# (``materialize.SNAPSHOT_FORMAT_VERSION``). ORDERED_COPY_ROW_GROUP_ROWS = 122_880 -_WRITE = { - "compression": "zstd", - "compression_level": 3, - "row_group_size": ORDERED_COPY_ROW_GROUP_ROWS, - "statistics": True, -} - -ORDERED_COPIES_DIRNAME = "ordered_sources" - - -class SourceUnavailable(FileNotFoundError): - """A file the entry needs cannot be made again: its clone is gone and the live source is not the bytes it read.""" - - -def ordered_copies_dir(project: str) -> Path: - from tallyman_core.paths import compute_cache_dir - - return compute_cache_dir(project) / ORDERED_COPIES_DIRNAME - - -def is_ordered_copy_path(project: str, path: Path) -> bool: - return Path(path).parent == ordered_copies_dir(project) # --------------------------------------------------------------------------- -# readers: what the manifest records so a copy can be made again +# readers: what the entry records so its snapshot can be written again # --------------------------------------------------------------------------- @@ -68,8 +43,9 @@ def parquet_reader() -> dict: def csv_reader(schema, scan_kwargs: dict) -> dict: """The reader options of a CSV source, in a form that goes into JSON and can be replayed. - ``lossless`` is False when a ``scan_csv`` option does not survive JSON (the copy is then keyed correctly but cannot - be made again from the manifest alone, and the error says so). + ``lossless`` is False when a ``scan_csv`` option does not survive JSON; the import refuses such a + call outright (``source_import._reader_for``), because an entry that cannot say how its file was + read cannot write its snapshot again. """ try: json.dumps(scan_kwargs, sort_keys=True) @@ -104,237 +80,20 @@ def _spec_from_json(doc): def _reader_signature(reader: dict) -> str: + """The part of a reader a source entry's content hash covers (``source_import.source_entry_hash``).""" return json.dumps({k: v for k, v in reader.items() if k != "lossless"}, sort_keys=True) -def copy_key(digest: str, reader: dict) -> str: - """The copy's file stem: a function of the source's content and the reader options, nothing else.""" - return hashlib.md5(f"{digest}|{_reader_signature(reader)}".encode()).hexdigest() # noqa: S324 — a cache key - - -# --------------------------------------------------------------------------- -# collecting the records a build needs for its manifest -# --------------------------------------------------------------------------- - -_collector: contextvars.ContextVar[dict[str, dict] | None] = contextvars.ContextVar( - "tallyman_ordered_copy_collector", default=None -) - - -def begin_collect() -> contextvars.Token: - return _collector.set({}) - - -def note_ordered_copy(key: str, record: dict) -> None: - bag = _collector.get() - if bag is not None: - bag[key] = record - - -def end_collect(token: contextvars.Token) -> dict[str, dict]: - bag = _collector.get() or {} - _collector.reset(token) - return dict(bag) - - # --------------------------------------------------------------------------- -# writing +# naming a Read in an error # --------------------------------------------------------------------------- -def _write_parquet_copy(src: Path, dest: Path) -> None: - import polars as pl - - lf = pl.scan_parquet(str(src)) - columns = [c for c in lf.collect_schema().names() if c != ROW_ORDER] # an existing __row_order is overwritten - lf.select(columns).with_row_index(ROW_ORDER).select([*columns, pl.col(ROW_ORDER).cast(pl.Int64)]).sink_parquet( - str(dest), **_WRITE - ) - - -def _write_copy(src: Path, reader: dict, dest: Path) -> None: - if reader["kind"] == "parquet": - _write_parquet_copy(src, dest) - return - from tallyman_xorq.io import _materialize_ordered - - _materialize_ordered(src, _spec_from_json(reader["schema"]), dict(reader["scan_kwargs"]), dest) - - -def _digest_sidecar(path: Path) -> Path: - return path.with_suffix(".digest") - - -def _content_digest_of(path: Path) -> str: - """The copy's content digest: the sidecar written with it, or a read-back when the sidecar is gone.""" - from tallyman_xorq.digest import content_digest - - sidecar = _digest_sidecar(path) - try: - return sidecar.read_text().strip() - except OSError: - digest = content_digest(path) - try: - sidecar.write_text(digest) - except OSError: - pass - return digest - - -def _write_atomically(src: Path, reader: dict, target: Path) -> str: - """Write the copy to a unique temp name beside its destination, replace, and return its content digest.""" - from tallyman_xorq.digest import content_digest - - target.parent.mkdir(parents=True, exist_ok=True) - tmp = target.with_name(f"{target.stem}.{uuid.uuid4().hex}.tmp") - try: - _write_copy(src, reader, tmp) - os.replace(tmp, target) - finally: - tmp.unlink(missing_ok=True) - digest = content_digest(target) - _digest_sidecar(target).write_text(digest) - return digest - - -def ensure_ordered_copy(project: str, source: Path, *, digest: str, rel: str, reader: dict) -> Path: - """The ordered copy of *source* (whose content digest is *digest*), written if it is not on disk yet. - - *source* is the file polars reads: the clone in cas mode, the live file otherwise. The copy is recorded for the - build in progress, so its manifest can make it again. - """ - from tallyman_core.catalog_state import project_lock - - key = copy_key(digest, reader) - target = ordered_copies_dir(project) / f"{key}.parquet" - if not target.exists(): - with project_lock(project): - if not target.exists(): # a peer may have written it while we waited - _write_atomically(source, reader, target) - note_ordered_copy( - key, - { - "source": rel, - "digest": digest, - "suffix": Path(source).suffix, - "reader": reader, - "content_digest": _content_digest_of(target), - }, - ) - return target - - -def existing_ordered_copy(project: str, *, digest: str, reader: dict) -> Path: - """The path a copy of a source with this digest and reader has, without touching the source (reconstruction).""" - return ordered_copies_dir(project) / f"{copy_key(digest, reader)}.parquet" - - -# --------------------------------------------------------------------------- -# making one again -# --------------------------------------------------------------------------- - - -def _live_source(project: str, source: str) -> Path: - from tallyman_core.paths import data_dir - - p = Path(source) - return p if p.is_absolute() else data_dir(project) / source - - -def _clone_or_live(project: str, record: dict) -> Path: - """The file to re-read a source from: its clone, made again from the live source if that still has the bytes.""" - from tallyman_core.paths import data_dir - from tallyman_xorq import source_identity as si - - digest, suffix = record["digest"], record.get("suffix", "") - clone = data_dir(project) / ".cas" / f"{digest}{suffix}" - if clone.exists(): - return clone - live = _live_source(project, record["source"]) - if live.is_file() and si._digest_file(live) == digest: - return si.ensure_cas_path(project, live, digest) if si.mode() == "cas" else live - raise SourceUnavailable( - f"the source file {record['source']!r} cannot be read the way it was when the entry was built: its " - f"content-addressed clone ({digest}{suffix}) is gone and the file on disk no longer has those bytes. The " - "rows the entry was built from are unrecoverable; rebuild the entry from the current file." - ) - - -def _find_record(project: str, owner_hash: str, key: str) -> dict | None: - """The record of the copy *key*: from the owner's manifest, else from any entry in the catalog that recorded it. - - The build that first reads a source records its copy, and entries are never deleted, so the record is always in - some manifest, whichever path (``tracked_expr_from_alias``, a diff, ``cached_result_expr`` in a recipe) carried the - copy into the owner's plan. A copy's key is a function of the source digest and the reader options, so any - manifest's record of it makes the same file. - """ - from tallyman_core.manifest import read_manifest - from tallyman_core.paths import entries_dir, entry_dir - - def records_of(entry: Path) -> dict: - try: - return read_manifest(entry).ordered_copies or {} - except (OSError, ValueError): - return {} - - record = records_of(entry_dir(project, owner_hash)).get(key) - if record is not None: - return record - base = entries_dir(project) - for entry in sorted(base.iterdir()) if base.is_dir() else []: - record = records_of(entry).get(key) - if record is not None: - return record - return None - - -def recreate_ordered_copy(project: str, owner_hash: str, path: Path) -> None: - """Make a deleted ordered copy again for the entry *owner_hash*, and check it against its recorded digest. - - Uses the reader options recorded by the entry that wrote the copy (``_find_record``). A copy whose digest differs - from the one recorded when it was first written is served all the same, but never silently (ADR-007 D5): a durable - error record and a warning. - """ - from tallyman_core.catalog_state import project_lock - - key = Path(path).stem - record = _find_record(project, owner_hash, key) - if record is None: - raise SourceUnavailable( - f"entry {owner_hash} reads {path.name} but no manifest in the catalog records the source it was made " - "from, so it cannot be made again; rebuild the entry" - ) - reader = record["reader"] - if reader.get("lossless") is False: - raise SourceUnavailable( - f"the reader options of {record['source']!r} were not JSON-serializable, so {path.name} cannot be made " - "again from the manifest; rebuild the entry" - ) - with project_lock(project): - if path.exists(): - return - src = _clone_or_live(project, record) - digest = _write_atomically(src, reader, path) - if digest != record["content_digest"]: - message = ( - f"the ordered copy of {record['source']!r} was made again with content digest {digest}, not the " - f"{record['content_digest']} recorded when it was first written (a change in polars, or a source that is " - "not what it was)" - ) - perf_log.warning("recreate_ordered_copy %s: %s", key, message) - try: - from tallyman_core.errors import record_error - - record_error(project, code="unfaithful_ordered_copy", message=message, hash=owner_hash) - except Exception: - perf_log.debug("unfaithful ordered copy record failed for %s", key, exc_info=True) - - -def describe_read(project: str, path: Path, records: dict[str, dict] | None = None) -> str: +def describe_read(project: str, path: Path) -> str: """A phrase naming what a Read of *path* is, for an error that has to say what an entry reads. - ``records`` are the ordered copies the build in progress collected, so a copy is named by the source it was made - from and not by its key. + Every file a build reads is a snapshot now, so the phrase names the entry that owns it and its + alias; anything else is named by its file name and is, by construction, a bug. """ from tallyman_core.aliases import alias_for_hash from tallyman_xorq.materialize import snapshots_dir @@ -343,7 +102,4 @@ def describe_read(project: str, path: Path, records: dict[str, dict] | None = No if path.parent == snapshots_dir(project): alias = alias_for_hash(project, path.stem) return f"entry {path.stem}" + (f" ({alias})" if alias else "") - record = (records or {}).get(path.stem) - if record is not None: - return f"the source {record['source']}" return f"the file {path.name}" diff --git a/src/tallyman_xorq/result_cache.py b/src/tallyman_xorq/result_cache.py index 032c677d..0b23eb48 100644 --- a/src/tallyman_xorq/result_cache.py +++ b/src/tallyman_xorq/result_cache.py @@ -67,36 +67,6 @@ def cache_worthy(project: str, content_hash: str) -> bool: # machine-locking memory blowup into a fast, clear error instead. _MAX_RECON_DEPTH = 64 -# The source digests ({rel_path: digest} from manifest.sources) the entry being -# reconstructed on this stack was BUILT from, threaded into read_project_file so a cold -# read resolves each source to the frozen data/.cas/ clone it was built from — -# not a re-digest of the (possibly edited-in-place) live file (#115). Set per-entry in -# _recipe_expr alongside _RECONSTRUCTING and OVERWRITTEN (then restored) by each nested -# reconstruction, so a grandparent's read_project_file sees the grandparent's recorded -# sources. Carries the project so a read_project_file resolving a *different* project falls -# through to the live path. None outside reconstruction (the build path); an entry that -# recorded no sources (mode=off / pre-#86) yields an empty map, so read_project_file also -# falls through. -_RECON_SOURCES: contextvars.ContextVar[tuple[str, dict] | None] = contextvars.ContextVar( - "_recon_sources", default=None -) - - -def _recorded_sources(project: str, content_hash: str) -> dict: - """The entry's recorded ``manifest.sources`` ({rel_path: digest}), or ``{}``. - - Empty when the manifest is unreadable or the entry recorded no sources (mode=off, - or a build predating #86); read_project_file then falls through to its live-file path. - """ - from tallyman_core import read_manifest - from tallyman_core.paths import entry_dir - - try: - return read_manifest(entry_dir(project, content_hash)).sources or {} - except (OSError, ValueError): - return {} - - def _resolve_noncyclic_hash(project: str, requested: str, content_hash: str) -> str: """The hash ``tracked_expr_from_alias`` should load, stepped out of any reconstruction cycle. @@ -137,16 +107,16 @@ def _resolve_noncyclic_hash(project: str, requested: str, content_hash: str) -> def _recipe_expr(project: str, content_hash: str): """Re-import the entry's persisted recipe (``expr.py``) as a live expression. - Symmetric with the original build's ``_import_script`` — same source code, - same source-identity handling — so the reconstructed expression is - structurally identical to what was built. The recipe's ``read_project_file`` / + Symmetric with the original build's ``_import_script`` — same source code — so the + reconstructed expression is structurally identical to what was built. The recipe's deferred readers bind to the in-process *default* backend, so the returned - expression roots there and composes with ``read_project_file`` and other - ``tracked_expr_from_alias`` results as a single backend (#75). + expression roots there and composes with other ``tracked_expr_from_alias`` results + as a single backend (#75). """ from tallyman_core.paths import entry_dir, project_dir from tallyman_xorq.build import BuildError, _import_script from tallyman_xorq.portable import PLACEHOLDER + from tallyman_xorq.source_import import in_source_recipe, is_source_entry, release_source_recipe active = _RECONSTRUCTING.get() if len(active) >= _MAX_RECON_DEPTH: @@ -158,17 +128,18 @@ def _recipe_expr(project: str, content_hash: str): code = (entry_dir(project, content_hash) / "expr.py").read_text().replace(PLACEHOLDER, str(project_dir(project))) # Mark this entry in-flight for the duration of the recipe exec, so any # tracked_expr_from_alias the recipe issues can step back out of a self-reference (#74). - # In the SAME window, pin this entry's recorded sources so any read_project_file the - # recipe issues resolves to the frozen .cas clone it was built from, not a - # re-digest of the edited-in-place live file (#115). Both contextvars must wrap - # _import_script — that is where the recipe's tracked_expr_from_alias / read_project_file run — - # and reset in this finally, NOT the sys.modules-cleanup finally below. + # The contextvar must wrap _import_script — that is where the recipe's + # tracked_expr_from_alias calls run — and reset in this finally, NOT the + # sys.modules-cleanup finally below. token = _RECONSTRUCTING.set(active | {(project, content_hash)}) - recon_token = _RECON_SOURCES.set((project, _recorded_sources(project, content_hash))) + # A source entry's recipe is the one the importer generated, and its read_project_file is the one raw read + # tallyman allows (ADR-011 D2): mark it so the read resolves to this entry's own snapshot. + source_token = in_source_recipe(project, content_hash) if is_source_entry(project, content_hash) else None try: module, tmp = _import_script(code) finally: - _RECON_SOURCES.reset(recon_token) + if source_token is not None: + release_source_recipe(source_token) _RECONSTRUCTING.reset(token) try: expr = getattr(module, _RECIPE_VAR, None) @@ -242,8 +213,8 @@ def _profile_content_token(backend) -> str: def rebind_onto(expr, target): """Collapse every backend in a loaded build onto *target*. - ``load_expr`` mints fresh backend objects per profile, so two loaded builds — or a loaded build and a recipe's - ``read_project_file`` — span distinct backend objects and composition raises "Multiple backends found". The + ``load_expr`` mints fresh backend objects per profile, so two loaded builds — or a loaded build and a source + entry's snapshot read — span distinct backend objects and composition raises "Multiple backends found". The contract allows the collapse because every profile in a tallyman build is content-identical no-arg ``xorq_datafusion``; that assumption is enforced here (ADR-006 D3): more than one distinct content profile fails loudly rather than misbinding a node onto the wrong kind of connection. A raw ``DatabaseTable`` (bundled data that diff --git a/src/tallyman_xorq/source_cache.py b/src/tallyman_xorq/source_cache.py index aa493e51..e914282c 100644 --- a/src/tallyman_xorq/source_cache.py +++ b/src/tallyman_xorq/source_cache.py @@ -8,8 +8,8 @@ The rewrite rejects: - **In-memory reads** (``read_in_memory`` or an ``ibis.memtable``). They signal that the author dropped to pandas - instead of reading a file through ``read_project_file`` or ``tallyman_read_csv``; the bytes would not round-trip - through the portable build. + instead of importing the file and reading the source alias; the bytes would not round-trip through the portable + build. - **A recipe that already contains a cache node.** A node with default storage would write under ``~/.cache/xorq``. - **A recipe that assigns to ``__row_order``**, and **a cheap entry that drops it** (ADR-008 D3, D6). @@ -41,10 +41,10 @@ class CacheNodeError(RuntimeError): _IN_MEMORY_MSG = ( "expression reads in-memory data (read_in_memory / ibis.memtable). This " "usually means the source was loaded into pandas (e.g. pd.read_csv) and " - "handed to xorq in memory, instead of a native reader. Read the " - "file with tallyman_read_csv(abs_path, schema=...) for CSVs or " - "read_project_file(rel_path) for parquet, so the source round-trips " - "through the portable build and is ingested with a stable __row_order." + "handed to xorq in memory, instead of entering the catalog as data. Import " + "the file once, catalog_import_source('', '', schema=...), " + "and read it with tracked_expr_from_alias(''), so the rows are held by " + "tallyman, round-trip through the portable build, and carry a stable __row_order." ) _CACHE_NODE_MSG = ( diff --git a/src/tallyman_xorq/source_identity.py b/src/tallyman_xorq/source_identity.py index b2d4f6e4..fc8dd4a1 100644 --- a/src/tallyman_xorq/source_identity.py +++ b/src/tallyman_xorq/source_identity.py @@ -1,42 +1,29 @@ -"""Tallyman-side source-content identity for catalog entries. - -xorq's build hash is tokenized under SnapshotStrategy, whose -``snapshot_normalize_read`` keys a Read on its *path string only* — neither -file stat nor file content reaches ``content_hash`` (verified empirically by -``tests/test_cache_lab.py::test_append_invalidates``: changing a source file -in place keeps the hash and dedups to the stale entry). xorq is a frozen -upstream for us, so content sensitivity has to be restored on our side. Two -candidate strategies, switchable at runtime so the cache lab can benchmark -them head to head: - -``TALLYMAN_SOURCE_IDENTITY`` env: - -- ``off`` (default): current behavior, path-identity only. -- ``cas``: ``read_project_file`` reads through a content-addressed clone at - ``data/.cas/``. The path xorq hashes then *is* the content - identity, every xorq-level key (build hash, snapshot cache keys) becomes - content-honest for free, and because the clone is a copy-on-write snapshot - (APFS clonefile), an old entry's recompute still reads the bytes it was - built from even after the user edits the source in place. -- ``salt``: ``read_project_file`` records each source's digest during user-code - import, and ``build_and_persist`` mixes the digests into the entry's - ``content_hash``. Builds keep their human-readable ``data/`` paths, but - xorq-level keys stay path-only — see ``result_cache`` for the snapshot-key - collision this forces us to route around. - -Digests are md5 over file content, memoized per ``(mtime_ns, size, inode)`` -in ``artifacts/source_digests.json`` so an unchanged file is hashed once, -not once per build. The memo reintroduces a stat-keyed shortcut: a swap -that preserves all three stat fields serves the stale digest (the same hole -git's index accepts). ``TALLYMAN_SOURCE_REHASH=1`` bypasses the memo. +"""The clone store: tallyman's frozen copy of every byte that was ever imported. + +An import copies the file it was given to ``data/.cas/``, named by the md5 of its +content, and verifies the copy against that name before publishing it (ADR-011 D9). The clone is what +makes a source version durable: the outside path is provenance and may be gone, the snapshot under +``compute_cache/`` is cache anyone may delete, and the clone is what re-creates the snapshot and what +ADR-005's suggestion-and-retry contract re-reads when a CSV was imported under the wrong schema. + +There is one mode (ADR-011 D8). ADR-002 made this switchable — ``off`` (path identity only), ``cas`` +(read through the clone) and ``salt`` (mix the digest into the entry hash) — so the cache lab could +benchmark them against each other. Import always digests and always clones now, because a raw input is +an entry and an entry's identity is its bytes; there is no configuration under which a raw input is +unversioned, so the env switch, the two other modes and ``salted_hash`` are gone. + +Nothing memoizes a digest either. The memo existed so a build did not re-hash an unchanged source on +every run, and it keyed on ``(mtime_ns, size, inode)``, which a same-stat in-place swap defeats. A file +is digested once now, at the moment it is imported, so the memo has nothing to save and its hole is not +worth keeping. + +``gc_cas`` retires a clone no live entry needs. Liveness is the DAG: a source version's clone is alive +exactly while its entry is (``catalog_state._live_source_digests`` over ``manifest.provenance``). """ from __future__ import annotations -import contextvars import hashlib -import json -import logging import os import shutil import subprocess @@ -44,27 +31,19 @@ import uuid from pathlib import Path -from tallyman_core import artifacts_dir, data_dir +from tallyman_core import data_dir -# Shares tallyman.perf with result_cache so the one branch where reconstruction -# can't be content-faithful surfaces on the same namespace as the #83 self-heal -# faithfulness warning. -_log = logging.getLogger("tallyman.perf") -_MODE_ENV = "TALLYMAN_SOURCE_IDENTITY" -_REHASH_ENV = "TALLYMAN_SOURCE_REHASH" -_MODES = ("off", "cas", "salt") +class CloneDigestMismatch(ValueError): + """The clone tallyman just wrote does not hash to the name it was given (ADR-011 D9).""" -def mode() -> str: - m = os.environ.get(_MODE_ENV, "cas") - if m not in _MODES: - raise ValueError(f"{_MODE_ENV}={m!r}; expected one of {_MODES}") - return m +class LostSourceVersion(FileNotFoundError): + """A version tallyman promised is gone: no clone, and the live file no longer has those bytes (ADR-011 D9).""" # --------------------------------------------------------------------------- -# Digests (memoized on stat) +# Digests # --------------------------------------------------------------------------- @@ -73,37 +52,8 @@ def _digest_file(path: Path) -> str: return hashlib.file_digest(fh, hashlib.md5).hexdigest() -def _index_path(project: str) -> Path: - return artifacts_dir(project) / "source_digests.json" - - -def digest_for(project: str, path: Path) -> str: - """Content digest of *path*, memoized per (mtime_ns, size, inode).""" - st = path.stat() - key = str(path) - stat_sig = [st.st_mtime_ns, st.st_size, st.st_ino] - idx_path = _index_path(project) - index: dict = {} - if not os.environ.get(_REHASH_ENV) and idx_path.exists(): - try: - index = json.loads(idx_path.read_text()) - except (OSError, json.JSONDecodeError): - index = {} - hit = index.get(key) - if hit and hit.get("stat") == stat_sig: - return hit["digest"] - digest = _digest_file(path) - index[key] = {"stat": stat_sig, "digest": digest} - try: - idx_path.parent.mkdir(parents=True, exist_ok=True) - idx_path.write_text(json.dumps(index, indent=2)) - except OSError: - pass # the memo is an optimization; never fail a build over it - return digest - - # --------------------------------------------------------------------------- -# cas: content-addressed source clones +# the clone store # --------------------------------------------------------------------------- @@ -131,6 +81,12 @@ def _clone(src: Path, dst: Path) -> None: def ensure_cas_path(project: str, src: Path, digest: str) -> Path: + """The clone of *src* named by *digest*, written if it is not there yet, and verified before it is published. + + The digest is computed before the copy, so a file edited mid-copy would otherwise leave a clone whose name lies + about its content — and that clone is the only frozen record of what an entry was built from. The written bytes + are re-digested and a mismatch raises ``CloneDigestMismatch`` rather than publishing the file (ADR-011 D9). + """ cas_dir = data_dir(project) / ".cas" cas_dir.mkdir(parents=True, exist_ok=True) dst = cas_dir / f"{digest}{src.suffix}" @@ -139,6 +95,12 @@ def ensure_cas_path(project: str, src: Path, digest: str) -> Path: tmp = dst.with_name(f"{dst.name}.{uuid.uuid4().hex}.tmp") try: _clone(src, tmp) + written = _digest_file(tmp) + if written != digest: + raise CloneDigestMismatch( + f"the copy of {src} hashes to {written}, not the {digest} it was named by — the file changed " + "while it was being copied. Nothing was written; try again once the file is settled." + ) os.replace(tmp, dst) finally: tmp.unlink(missing_ok=True) @@ -146,22 +108,16 @@ def ensure_cas_path(project: str, src: Path, digest: str) -> Path: def recon_cas_path(project: str, live_src: Path, digest: str) -> Path: - """The frozen ``.cas`` clone named by *digest*, for digest-pinned reconstruction (#115). - - Returns ``data/.cas/`` — the bytes the entry was built from — - WITHOUT re-digesting the live source, so a cold read after an in-place edit serves - what was built, not the edited file. The clone is normally already on disk: the - ``.cas`` GC (``gc_cas``) preserves any clone a live entry's ``manifest.sources`` - references. - - If the clone is absent — manually deleted, or a fresh cross-machine catalog clone, - since ``.cas`` lives under ``data/`` outside the catalog git repo — re-materialise - it from the live source IFF the live bytes still hash to *digest*. The check hashes - file *content* (``_digest_file``), never ``digest_for``, so the stat-keyed memo - can't certify an in-place edit that preserved ``(mtime_ns, size, inode)`` as the - original (the documented memo hole). When the clone is gone AND the live source has - drifted, the original bytes are unrecoverable: serve the live source best-effort - (never crash a read) but warn — this is the one branch that is NOT content-faithful. + """The frozen ``.cas`` clone named by *digest*, without re-digesting *live_src*. + + Returns ``data/.cas/`` — the bytes as they were imported. The clone is normally + already on disk: ``gc_cas`` keeps any clone a live entry's ``manifest.provenance`` names. + + If the clone is absent — manually deleted, or a fresh cross-machine catalog clone, since ``.cas`` + lives under ``data/`` outside the catalog git repo — re-materialise it from the live file IFF the + live bytes still hash to *digest*. When the clone is gone AND the live file has drifted, the + original bytes are unrecoverable and this raises ``LostSourceVersion`` (ADR-011 D9): a version + tallyman promised and then lost is a failure, not a downgrade to whatever is on disk now. """ cas_dir = data_dir(project) / ".cas" dst = cas_dir / f"{digest}{live_src.suffix}" @@ -169,21 +125,20 @@ def recon_cas_path(project: str, live_src: Path, digest: str) -> Path: return dst if live_src.exists() and _digest_file(live_src) == digest: return ensure_cas_path(project, live_src, digest) - _log.warning( - "recon_cas_path: frozen .cas clone %s%s missing and live source %s drifted — " - "serving live bytes; this cold read is NOT faithful to the build (#115)", - digest, - live_src.suffix, - live_src, + raise LostSourceVersion( + f"the bytes {digest}{live_src.suffix} are gone: their frozen clone under data/.cas is missing and " + f"{live_src} no longer has that content. Whatever was built from them cannot be read faithfully; " + "import the data again, or rebuild the entry from the current file." ) - return live_src def gc_cas(project: str, live_digests: set[str], *, bullpen: Path | None = None) -> int: """Retire ``.cas`` clones whose digest no live entry references. - ``live_digests`` is the union of every live entry's ``manifest.sources`` - values — the md5 the clone is named by (````). Returns the + ``live_digests`` is every surviving source version's ``manifest.provenance.digest`` — the md5 the + clone is named by (````). That set IS the retention closure now (ADR-011 D6): a + clone is alive exactly while the entry whose bytes it holds is, which the DAG already records, so + there is no separate ``manifest.sources`` map to walk. Returns the number of files retired; a no-op when the ``.cas`` dir is absent. ``.cas`` lives under ``data/``, outside the catalog git repo, so a reset's ``git reset`` never reclaims it — this is the explicit sweep, called from @@ -213,36 +168,3 @@ def gc_cas(project: str, live_digests: set[str], *, bullpen: Path | None = None) except OSError: pass # best-effort sweep; never fail a reset over it return retired - - -# --------------------------------------------------------------------------- -# salt: collect source digests during user-code import -# --------------------------------------------------------------------------- - -_collector: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar( - "tallyman_source_collector", default=None -) - - -def begin_collect() -> contextvars.Token: - return _collector.set({}) - - -def note_source(rel_path: str, digest: str) -> None: - bag = _collector.get() - if bag is not None: - bag[rel_path] = digest - - -def end_collect(token: contextvars.Token) -> dict[str, str]: - bag = _collector.get() or {} - _collector.reset(token) - return dict(bag) - - -def salted_hash(xorq_hash: str, sources: dict[str, str]) -> str: - """Mix source digests into the entry identity; same 12-hex shape as - xorq's build hash so nothing downstream (dir names, URLs, aliases) - notices the difference.""" - parts = "|".join(f"{rel}={digest}" for rel, digest in sorted(sources.items())) - return hashlib.md5(f"{xorq_hash}|{parts}".encode()).hexdigest()[: len(xorq_hash)] diff --git a/src/tallyman_xorq/source_import.py b/src/tallyman_xorq/source_import.py new file mode 100644 index 00000000..d85232fc --- /dev/null +++ b/src/tallyman_xorq/source_import.py @@ -0,0 +1,715 @@ +"""Importing a file: the one way data enters the catalog (ADR-011). + +A raw input is not something a build reads. It is an **alias** whose versions are ordinary entries, and the only way +to advance one is ``update_and_depend`` — the explicit act of copying a file into the arena (the files tallyman owns) +and pointing a source alias at the resulting entry. + +One import does four things: + +1. digests the outside file and clones its bytes to ``data/.cas/``, verified after the write + (ADR-011 D9). The clone is the imported file as it was, kept so a CSV read with the wrong schema can be imported + again without the outside file (ADR-005's suggestion-and-retry contract), and so the snapshot below can be + written again from it; +2. writes ONE parquet snapshot of it, in file order plus a last ``__row_order`` column, at + ``compute_cache/result_cache/.parquet``. A source entry is worthy and **its snapshot is the ordered + copy** — ``compute_cache/ordered_sources/`` and the copy key of ADR-008 D2 do not exist for an import, because the + entry hash names the file and the reader options are recorded on the entry that used them (ADR-011 D12); +3. writes the entry: a generated recipe, a frozen ``xorq_build/``, a schema and a manifest whose ``provenance`` + records the outside path, the digest and the reader options; +4. points the source alias at it, appending a version. + +The **content hash of a source entry is a function of its bytes and its reader options**, and of nothing else. One +such entry belongs to one alias: importing bytes another alias already holds is an error that names that alias, and a +second name for a source is a catalog entry whose recipe reads it (``_refuse_bytes_held_elsewhere``). + +That snapshot is **cache** in the sense of ADR-007 D13 — a file is cache if ``ensure_materialized`` can re-create it +— because the clone holds the bytes and the entry holds the reader options. A deleted one is written again from the +clone and verified against the recorded ``result_digest``, like any other snapshot; the Cache page offers to delete +it like any other. Only when the clone is gone as well are the rows unrecoverable, and then the snapshot is pinned +and the error names the re-import (``materialize._heal_a_source``). + +After the import the outside path is **provenance**: recorded on the entry and never read again. Deleting, moving or +editing the original file has no effect on any build. +""" + +from __future__ import annotations + +import contextvars +import hashlib +import json +import logging +import os +import shutil +import tempfile +import uuid +from datetime import datetime, timezone +from pathlib import Path + +perf_log = logging.getLogger("tallyman.perf") + +# What ``manifest.cache_worthy_why`` records for a source entry: worthy, but not because a computation earned it. +_WORTHY_WHY = "source version (imported data, not a computation)" + +# The shape of xorq's build hash, which an entry's directory name, URLs and aliases all assume. +HASH_LEN = 12 + +_CSV_SUFFIXES = frozenset({".csv", ".tsv", ".txt"}) +_PARQUET_SUFFIXES = frozenset({".parquet", ".pq"}) + +# The generated recipe of a source entry, executing on this call stack, as ``(project, content_hash)``. +# ``read_project_file`` and ``tallyman_read_csv`` are build errors in an authored recipe (ADR-011 D2) and resolve to +# this entry's snapshot inside it — the one place a raw read survives. +_SOURCE_ENTRY: contextvars.ContextVar[tuple[str, str] | None] = contextvars.ContextVar("_source_entry", default=None) + + +class SourceImportError(ValueError): + """An import that cannot be performed as asked: a bad path, an alias collision, or a version claim that is wrong.""" + + +# --------------------------------------------------------------------------- +# identity +# --------------------------------------------------------------------------- + + +def source_entry_hash(digest: str, reader: dict) -> str: + """The content hash of the entry an import mints: its bytes and its reader options, and nothing else. + + Same 12-hex shape as xorq's build hash, so nothing downstream (entry directories, URLs, aliases) notices the + difference. The same bytes under the same reader are the same entry, which is why only one alias may hold them. + """ + from tallyman_xorq.ordered_copy import _reader_signature + + payload = f"source|{digest}|{_reader_signature(reader)}" + return hashlib.md5(payload.encode()).hexdigest()[:HASH_LEN] # noqa: S324 — an identity, not a credential + + +def _readable_manifest(project: str, content_hash: str): + """The entry's manifest, or None when there is no entry or its manifest cannot be read.""" + from tallyman_core import read_manifest + from tallyman_core.paths import entry_dir + + try: + return read_manifest(entry_dir(project, content_hash)) + except (OSError, ValueError): + return None + + +def is_source_entry(project: str, content_hash: str) -> bool: + """Whether the entry is a version of a source alias — i.e. whether its manifest records an import.""" + from tallyman_core import read_manifest + from tallyman_core.paths import entry_dir + + try: + return read_manifest(entry_dir(project, content_hash)).provenance is not None + except (OSError, ValueError): + return False + + +def source_version_in( + history: dict[str, list[str]], kinds: dict[str, str] | None, content_hash: str, imported_as: str +) -> tuple[str, int] | None: + """The source alias whose history holds *content_hash*, and its 1-based version there, or None when none does. + + ``provenance.alias`` is the name a version was **imported as**: recorded once, at import, and never updated. A + rename carries the history and the kind to a new name (``aliases.rename_alias``) and an unalias drops them, so + that name is only a hint, tried first because it is usually still right. This is ``aliases.version_of_hash`` + restricted to source aliases — an import advances nothing else, so a version held only by a catalog alias has no + alias an import could repair it under. *kinds* None means the kinds are not known, and every alias counts. + """ + from tallyman_core.aliases import SOURCE_KIND + + for name in (imported_as, *history): + hashes = history.get(name, []) + if content_hash in hashes and (kinds is None or kinds.get(name) == SOURCE_KIND): + return name, hashes.index(content_hash) + 1 + return None + + +def current_source_version(project: str, content_hash: str, provenance) -> tuple[str, int] | None: + """``(alias, version)`` of the source entry *content_hash* as the project's alias store has it now, or None. + + What a message that names a source version, or advises the import that repairs one, should say: the name in + *provenance* is where it came from, and this is where it is (``source_version_in``). + """ + from tallyman_core.aliases import load_history, load_kinds + + return source_version_in(load_history(project), load_kinds(project), content_hash, provenance.alias) + + +def source_entry_context() -> tuple[str, str] | None: + """The ``(project, content_hash)`` whose generated recipe is running on this stack, or ``None``. + + The project comes from the entry being minted, not from whichever project is active: ``project=`` on an + import is an override, and an import must work while another project is active. + """ + return _SOURCE_ENTRY.get() + + +def in_source_recipe(project: str, content_hash: str): + """Run the generated recipe of *content_hash* with its raw reads resolved to that entry's snapshot (D2).""" + return _SOURCE_ENTRY.set((project, content_hash)) + + +def release_source_recipe(token) -> None: + _SOURCE_ENTRY.reset(token) + + +# --------------------------------------------------------------------------- +# reader options, fixed at import (D12) +# --------------------------------------------------------------------------- + + +def _reader_for(path: Path, schema, reader_options: dict) -> dict: + """The reader a source is read with, decided once, here, from the file's suffix and the caller's options. + + A CSV's delimiter, schema overrides and inference settings are named in the import call and recorded on the entry. + Two recipes cannot read one file two ways; import it twice under two aliases. + """ + from tallyman_xorq import ordered_copy as oc + from tallyman_xorq.io import _RESERVED_SCAN_KWARGS + + suffix = path.suffix.lower() + if suffix in _PARQUET_SUFFIXES: + if schema is not None or reader_options: + named = sorted([*reader_options, *(["schema"] if schema is not None else [])]) + raise SourceImportError( + f"{path.name} is a parquet file, whose rows and types are in the file — the reader option(s) " + f"{named} apply to a CSV only. Import it without them." + ) + return oc.parquet_reader() + if suffix in _CSV_SUFFIXES: + reserved = [k for k in _RESERVED_SCAN_KWARGS if k in reader_options] + if reserved: + raise SourceImportError( + f"{reserved} is managed internally, not a pass-through polars.scan_csv reader option. Type " + "inference escalates automatically (100 -> 10k -> whole-file); to pin column types pass schema= " + "(an ibis schema, plain dict, or tuple-of-tuples), never schema_overrides." + ) + reader = oc.csv_reader(schema, reader_options) + if reader.get("lossless") is False: + raise SourceImportError( + "a reader option of this import does not survive JSON, so the entry could not record how the file " + "was read. Pass reader options as plain values (strings, numbers, lists), not callables." + ) + return reader + raise SourceImportError( + f"tallyman imports parquet ({', '.join(sorted(_PARQUET_SUFFIXES))}) and CSV " + f"({', '.join(sorted(_CSV_SUFFIXES))}) files; {path.name} has suffix {path.suffix!r}. Convert it first." + ) + + +# --------------------------------------------------------------------------- +# the version table (D3, D11) +# --------------------------------------------------------------------------- + + +def _plan_version(alias: str, history: list[str], content_hash: str, pinned_version: int | None) -> tuple[bool, int]: + """Decide what this import does: ``(mint, version)``, or raise. + + The full case table of ADR-011 D3, plus D11's rule that history is append-only and monotonic — two version + numbers never denote the same bytes read the same way, and a version number never moves backwards while history + is intact. + """ + head = len(history) # the head's 1-based version; 0 when the alias is absent + already = history.index(content_hash) + 1 if content_hash in history else None + + if pinned_version is None: + if already == head and head: # the bytes are the head's: a no-op + return False, head + if already is not None: # the bytes are an older version's (D11) + raise SourceImportError( + f"these bytes are already {alias}-v{already}, and {alias} is at v{head}. Version history is " + f"append-only: a v{head + 1} whose rows equal v{already}'s would make the version number meaningless. " + f"To put {alias} back on v{already}, reset the catalog to the revision before v{already + 1} " + f"(tallyman reset / catalog_reset_to). To read v{already} in a recipe without moving the head, use " + f"pinned_expr_from_alias('{alias}-v{already}')." + ) + return True, head + 1 + + if pinned_version < 1: + raise SourceImportError(f"pinned_version must be 1 or greater; got {pinned_version}.") + if pinned_version > head + 1: + raise SourceImportError( + f"pinned_version={pinned_version} would skip version(s): {alias} has {head} version(s) and versions " + f"cannot be skipped, so the next one is v{head + 1}." + ) + if pinned_version <= head: + claimed = history[pinned_version - 1] + if claimed == content_hash: + return False, pinned_version # the file IS that version; the head does not move + raise SourceImportError( + f"this import is not {alias}-v{pinned_version}: that version is entry {claimed}, and this import would " + f"be entry {content_hash}. An entry is named by the bytes and the reader options together (ADR-011 D12), " + f"so either this is not the file {alias}-v{pinned_version} was imported from, or it is read with other " + f"reader options than that import used; the header of {alias}-v{pinned_version}'s recipe records them. " + f"Import without pinned_version to mint the next version." + ) + if already is not None: # pinned at head+1, but these bytes are already a version (D11) + raise SourceImportError( + f"these bytes are already {alias}-v{already}, so they cannot also be v{pinned_version}. " + f"Two version numbers never denote the same bytes read the same way." + ) + return True, pinned_version + + +def _refuse_bytes_held_elsewhere(project: str, alias: str, content_hash: str) -> None: + """Raise when another alias of *project* already has *content_hash* as one of its versions. + + One set of bytes, read one way, is one source version under one alias. The likely way to get here is not knowing + the bytes are already in the project, so the error says where they are and how to give them a second name. The + check keys on the entry hash, so a CSV read two ways is still two entries (D12), and it is per project. + """ + from tallyman_core.aliases import load_history + + for other, hashes in sorted(load_history(project).items()): + if other == alias or content_hash not in hashes: + continue + held = f"{other}-v{hashes.index(content_hash) + 1}" + raise SourceImportError( + f"these bytes are already {held} in project {project!r}, so they cannot also be imported as {alias!r}: " + f"one set of bytes is one source version under one alias. Read them with " + f"tracked_expr_from_alias({other!r}). To give them a second name, create a catalog entry over the " + f"source — catalog_create({alias!r}, \"from tallyman_xorq.io import tracked_expr_from_alias\\n" + f"expr = tracked_expr_from_alias({other!r})\") — which follows {other!r} when it is re-imported." + ) + + +# --------------------------------------------------------------------------- +# writing a source entry +# --------------------------------------------------------------------------- + + +def _numbered(batches): + """*batches*, each with a last ``__row_order`` column continuing the count over the whole stream (ADR-008 D2).""" + import numpy as np + import pyarrow as pa + + from tallyman_xorq.row_order import ROW_ORDER + + written = 0 + for batch in batches: + if not batch.num_rows: + continue + numbers = pa.array(np.arange(written, written + batch.num_rows, dtype=np.int64)) + yield batch.append_column(pa.field(ROW_ORDER, pa.int64()), numbers) + written += batch.num_rows + + +def _write_parquet_snapshot(clone: Path, dest: Path) -> None: + """Copy the parquet *clone* to *dest*, in file order, numbering the rows in a last ``__row_order``. + + pyarrow reads and writes every type a parquet file can hold, so the snapshot's schema is the imported file's + plus ``__row_order``, and a recipe over the source alias sees the types the file has. polars, which wrote this + before, turned a ``date32`` into a timestamp, a ``time32`` into a ``time64`` and a map into a list of structs + (#197). An existing ``__row_order`` is dropped and written again, last. + """ + import pyarrow as pa + import pyarrow.parquet as pq + + from tallyman_xorq.materialize import write_pinned_parquet + from tallyman_xorq.ordered_copy import ORDERED_COPY_ROW_GROUP_ROWS + from tallyman_xorq.row_order import ROW_ORDER + + with pq.ParquetFile(clone) as source: + kept = [f for f in source.schema_arrow if f.name != ROW_ORDER] + schema = pa.schema([*kept, pa.field(ROW_ORDER, pa.int64())]) + batches = source.iter_batches(columns=[f.name for f in kept]) + write_pinned_parquet(_numbered(batches), schema, dest, row_group_rows=ORDERED_COPY_ROW_GROUP_ROWS) + + +def _write_csv_snapshot(clone: Path, reader: dict, dest: Path) -> None: + """Parse the CSV *clone* with polars under the recorded reader options, and write it with pyarrow. + + polars is here because it is the only reader that holds the file's row order (datafusion's parallel scan does + not, above its repartition threshold) and the only one that applies the schema DSL and the inference ladder of + ADR-005. It does not write the file: its batches go to the same writer every other snapshot uses. + """ + from tallyman_xorq.io import _materialize_ordered + from tallyman_xorq.ordered_copy import _spec_from_json + + _materialize_ordered( + clone, _spec_from_json(reader["schema"]), dict(reader["scan_kwargs"]), dest, write=_write_frames + ) + + +def _write_frames(frame, dest: Path) -> None: + """Write the rows of the polars LazyFrame *frame* to *dest*, in order, without ever collecting the whole thing. + + ``collect_batches`` pulls the streaming engine one chunk at a time, so memory is bounded by a row group and a + source larger than RAM imports the way a big one is supposed to. The frame already carries ``__row_order`` last + (``io._materialize_ordered``), and its schema is taken from the query rather than from the first batch, so a + CSV with a header and no rows still writes a file with the right columns. + """ + import polars as pl + + from tallyman_xorq.materialize import write_pinned_parquet + from tallyman_xorq.ordered_copy import ORDERED_COPY_ROW_GROUP_ROWS + + schema = pl.DataFrame(schema=frame.collect_schema()).to_arrow().schema + batches = ( + batch + for chunk in frame.collect_batches(chunk_size=ORDERED_COPY_ROW_GROUP_ROWS) + for batch in chunk.to_arrow().to_batches() + ) + write_pinned_parquet(batches, schema, dest, row_group_rows=ORDERED_COPY_ROW_GROUP_ROWS) + + +def _write_snapshot(clone: Path, reader: dict, dest: Path) -> str: + """Write the ordered parquet of *clone* to *dest*, and return the content digest of what was written. + + One writer for both readers: pyarrow, in the pinned layout of ``materialize._PARQUET_OPTIONS`` with row groups of + ``ORDERED_COPY_ROW_GROUP_ROWS``, which ``SNAPSHOT_FORMAT_VERSION`` covers (ADR-009 D3). A parquet source needs + no parser at all; a CSV is parsed by polars and written here. polars cannot write the layout itself — 1.40.1 + (installed) and 1.44.2 (latest) expose neither the parquet format version nor the page index + (pola-rs/polars#12752) — and a file written twice to be re-encoded is worse than a file written once. + """ + from tallyman_xorq.digest import content_digest + + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(f".{dest.stem}.{uuid.uuid4().hex}.tmp") + try: + if reader["kind"] == "parquet": + _write_parquet_snapshot(clone, tmp) + else: + _write_csv_snapshot(clone, reader, tmp) + os.replace(tmp, dest) + finally: + tmp.unlink(missing_ok=True) + return content_digest(dest) + + +def source_clone_path(project: str, provenance) -> Path: + """The clone of a source version's imported bytes: ``data/.cas/``. + + The bytes as they arrived, kept beside the snapshot (ADR-011, open question 1). They are what re-creates the + snapshot when it is deleted, and what ADR-005's suggestion-and-retry contract re-reads when a CSV was imported + under the wrong schema, so the entry survives the outside file going away. + """ + from tallyman_core.paths import data_dir + + return data_dir(project) / ".cas" / f"{provenance.digest}{provenance.suffix}" + + +def import_call(path: str, alias: str | None, reader: dict, pinned_version: int | None = None) -> str: + """The ``catalog_import_source(...)`` call that imports *path* the way *reader* records it was read. + + What an error prints when the way out is an import. A CSV's entry hash covers its reader options (D12), so a call + without them names another entry and a pinned one is refused; the call therefore carries the schema and the + ``scan_csv`` options the entry recorded, in the MCP tool's own argument shapes. *alias* None prints ````, + for a version no alias holds, where the caller has to choose the name. + """ + args = [repr(path), "" if alias is None else repr(alias)] + if pinned_version is not None: + args.append(f"pinned_version={pinned_version}") + if reader.get("kind") == "csv": + spec = reader.get("schema") + if spec is not None: + cells = [[name, dtype] for name, dtype in spec["cells"]] + schema = dict(cells) if spec["form"] == "named" else cells + args.append(f"schema={schema!r}") + if reader.get("scan_kwargs"): + args.append(f"reader_options={reader['scan_kwargs']!r}") + return f"catalog_import_source({', '.join(args)})" + + +def rewrite_source_snapshot(project: str, content_hash: str, provenance) -> str: + """Write the snapshot of the source entry *content_hash* again, from its clone; return the digest written. + + ``ensure_materialized`` calls this for a source version whose file was deleted (ADR-011 D1). The reader options + are the ones recorded at import (D12), so the rows are parsed exactly as they were the first time. + """ + from tallyman_xorq.materialize import snapshot_path + + clone = source_clone_path(project, provenance) + return _write_snapshot(clone, provenance.reader, snapshot_path(project, content_hash)) + + +def _recipe(outside_path: Path, alias: str, version: int, digest: str, reader: dict) -> str: + """The generated recipe of a source entry: what the Code tab shows, and what its frozen build is made from. + + The read resolves to this entry's own snapshot (``_SOURCE_ENTRY``), not to the path in the call — the path is + provenance and is never read again. This is the one recipe in which a raw read is allowed (ADR-011 D2). + + The recipe is written once, so the alias in its header is the one it was imported as, and says so: after a + rename the entry is a version of another name (``current_source_version``). + """ + return ( + f"# Generated by catalog_import_source when the file was imported as {alias}-v{version}. Do not edit.\n" + "# A source version is data: its rows are the bytes imported from the path below, ordered and\n" + "# numbered in __row_order. The path is provenance — the build reads tallyman's own copy of it.\n" + f"# imported from: {outside_path}\n" + f"# content: md5:{digest}\n" + f"# reader: {json.dumps(reader, sort_keys=True)}\n" + "from tallyman_xorq.io import read_project_file\n" + "\n" + f"expr = read_project_file({str(outside_path)!r})\n" + ) + + +def _mint( + project: str, + outside_path: Path, + *, + digest: str, + reader: dict, + content_hash: str, + alias: str, + version: int, + prompt: str | None, +) -> dict: + """Write the entry, its snapshot and its clone. Returns ``{"row_count", "schema"}``.""" + import pyarrow.parquet as pq + + from tallyman_core import ( + Manifest, + atomic_write_text, + entry_build_dir, + entry_dir, + entry_schema_path, + project_dir, + write_manifest, + ) + from tallyman_core.manifest import SourceProvenance + from tallyman_xorq import source_identity as si + from tallyman_xorq._git_state_guard import install_git_state_guard + from tallyman_xorq.build import BuildError, _import_script + from tallyman_xorq.materialize import SNAPSHOT_FORMAT_VERSION, engine_versions, snapshot_path + from tallyman_xorq.portable import PLACEHOLDER, make_portable_inplace + from tallyman_xorq.source_cache import rewrite_for_build + from tallyman_xorq.worthiness import Verdict + + # 1. The bytes, as imported, into the arena. ensure_cas_path digests what it wrote (ADR-011 D9). + clone = si.ensure_cas_path(project, outside_path, digest) + + # 2. The one snapshot, named by the entry hash. An existing one is kept: the hash is a function of the bytes and + # the reader, so it already holds exactly these rows. + snapshot = snapshot_path(project, content_hash) + if snapshot.exists(): + from tallyman_xorq.digest import content_digest + + result_digest = content_digest(snapshot) + else: + result_digest = _write_snapshot(clone, reader, snapshot) + arrow_schema = pq.read_schema(snapshot) + row_count = pq.ParquetFile(snapshot).metadata.num_rows + + # 3. The entry: a generated recipe, its frozen build, a schema and a manifest. + install_git_state_guard() + code = _recipe(outside_path, alias, version, digest, reader) + token = in_source_recipe(project, content_hash) + try: + module, tmp_script = _import_script(code) + finally: + release_source_recipe(token) + try: + expr = getattr(module, "expr", None) + if expr is None: + raise BuildError(f"the generated recipe of {alias}-v{version} bound no 'expr'") + target = entry_dir(project, content_hash) + # A re-import that repairs a missing snapshot writes into an entry that already exists; a failure must not + # take that entry with it. + created = not target.exists() + target.mkdir(parents=True, exist_ok=True) + try: + from xorq.ibis_yaml.compiler import build_expr + + ordered = rewrite_for_build(expr, project, verdict=Verdict(True, _WORTHY_WHY)) + with tempfile.TemporaryDirectory(prefix="tallyman_source_") as builds_str: + build_path = Path(build_expr(ordered, builds_dir=Path(builds_str))) + xorq_build = entry_build_dir(project, content_hash) + xorq_build.mkdir(parents=True, exist_ok=True) + for item in build_path.rglob("*"): + dest = xorq_build / item.relative_to(build_path) + if item.is_dir(): + dest.mkdir(parents=True, exist_ok=True) + else: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(item.read_bytes()) + make_portable_inplace(xorq_build, project_dir(project)) + + (target / "expr.py").write_text(code.replace(str(project_dir(project)), PLACEHOLDER)) + schema_doc = { + "fields": [{"name": f.name, "type": str(f.type)} for f in arrow_schema], + "row_count": row_count, + } + atomic_write_text(entry_schema_path(project, content_hash), json.dumps(schema_doc, indent=2)) + write_manifest( + target, + Manifest( + content_hash=content_hash, + project=project, + prompt=prompt, + row_count=row_count, + execute_seconds=0.0, + cache_worthy=True, + cache_worthy_why=_WORTHY_WHY, + cache_bytes=snapshot.stat().st_size, + result_digest=result_digest, + snapshot_format=SNAPSHOT_FORMAT_VERSION, + engine_versions=engine_versions(), + provenance=SourceProvenance( + alias=alias, + version=version, + path=str(outside_path), + digest=digest, + suffix=outside_path.suffix, + reader=reader, + imported_at=datetime.now(timezone.utc).isoformat(), + ), + ), + ) + except Exception: + if created: + shutil.rmtree(target, ignore_errors=True) + raise + finally: + import sys + + sys.modules.pop(getattr(module, "__name__", "") or "", None) + tmp_script.unlink(missing_ok=True) + perf_log.info("import %s-v%s -> %s (%s rows, %s)", alias, version, content_hash, row_count, outside_path) + return {"row_count": row_count, "schema": schema_doc} + + + +# --------------------------------------------------------------------------- +# the one way to advance a source alias (D3) +# --------------------------------------------------------------------------- + + +def update_and_depend( + outside_path: str | Path, + alias: str, + pinned_version: int | None = None, + *, + project: str | None = None, + prompt: str | None = None, + schema=None, + **reader_options, +) -> dict: + """Import *outside_path* and point the source alias *alias* at the resulting entry (ADR-011 D3). + + The official, and only, way to advance a source alias. What happens depends on the alias's history and on whether + the caller claims a version: + + ================================================== ========================================= + state behaviour + ================================================== ========================================= + alias absent import, mint v1, return v1 + ``pinned_version=None``, bytes differ from head mint the next version, return it + ``pinned_version=None``, bytes equal head no-op, return head + ``pinned_version=N`` exists, digest matches no-op, return vN + ``pinned_version=N`` exists, digest differs error: the file is not the version claimed + ``pinned_version`` = head+1, bytes differ mint it, return it + ``pinned_version`` beyond head+1 error: versions cannot be skipped + ``pinned_version=N`` < head, digest matches return vN; the head does not move + bytes match a version older than the head error: history is append-only (D11) + bytes are a version of another alias error: one set of bytes, one alias + ================================================== ========================================= + + Args: + outside_path: Any path to a parquet or CSV file. ``data/`` is not special — a file is imported from wherever + it is, and after the import the path is provenance and is never read again. + alias: The source alias. It may not already name a catalog alias, and a catalog alias may not later take it. + Bytes another alias of the project already holds are refused; the error names that alias. + pinned_version: The version the caller claims this file is, when they want the claim checked. + project: Project name override (defaults to the active project). + prompt: Optional human-readable description, recorded on the entry. + schema: For a CSV, the column types (an ibis schema, a dict, or the tuple-of-tuples DSL). + **reader_options: For a CSV, ``polars.scan_csv`` options (``separator``, ``skip_rows``, ``null_values`` …). + They are recorded on the entry and never re-derived at build time (ADR-011 D12). + + Returns: + ``{"alias", "version", "hash", "created", "path", "digest", "row_count", "schema"}``. ``created`` is False + when the import was a no-op. An import of bytes that are already an entry rewrites nothing of that entry; if + its snapshot is gone it is healed from the clone and checked against the recorded digest, exactly as + ``ensure_materialized`` heals one, so a repair is never a way to change a version's rows. + + Raises: + SourceImportError: for every row of the table above that is an error, and for a path that is not an importable + file. + """ + from tallyman_core import ensure_project, resolve_project + from tallyman_core.aliases import SOURCE_KIND, alias_kind, history_for, set_alias, validate_alias_name + from tallyman_core.catalog_state import project_lock + from tallyman_xorq import source_identity as si + + proj = resolve_project(project) + ensure_project(proj) + + src = Path(outside_path).expanduser() + if src.is_dir(): + raise SourceImportError( + f"{src} is a directory. An import is one file to one alias; a dataset that arrives as several parts is " + "open question 2 of ADR-011 and has no defined part order yet. Concatenate the parts into one file " + "first, or import each part under its own alias." + ) + if not src.is_file(): + raise SourceImportError(f"{src} is not a file, so there is nothing to import.") + + try: + validate_alias_name(alias) + except ValueError as exc: # a name matching the '-v' version syntax (#166) + raise SourceImportError(str(exc)) from exc + kind = alias_kind(proj, alias) + if kind is not None and kind != SOURCE_KIND: + raise SourceImportError( + f"{alias!r} is a catalog alias (a computation), so it cannot also name an imported file. " + "Import under a different name, or rename the catalog alias first." + ) + + reader = _reader_for(src, schema, reader_options) + digest = si._digest_file(src) + content_hash = source_entry_hash(digest, reader) + + with project_lock(proj): + history = history_for(proj, alias) + mint, version = _plan_version(alias, history, content_hash, pinned_version) + if mint: + _refuse_bytes_held_elsewhere(proj, alias, content_hash) + from tallyman_xorq.materialize import snapshot_path + + existing = _readable_manifest(proj, content_hash) + if existing is None: + # No entry, or a directory a crash left without a manifest, which is not an entry either: write it. + written = _mint( + proj, + src, + digest=digest, + reader=reader, + content_hash=content_hash, + alias=alias, + version=version, + prompt=prompt, + ) + else: + # The entry exists, and its recipe, build and manifest are the record of the import that minted it: an + # import of the same bytes, as a repair or under a new alias, rewrites none of them. A missing snapshot + # is healed the way ensure_materialized heals any source snapshot, from the clone and checked against + # the recorded result_digest, so a reader that now parses the bytes differently is recorded as an + # unfaithful heal instead of becoming the version's rows. The caller's bytes restore the clone first + # when it is gone too; ensure_cas_path verifies them against the digest (D9). + if not snapshot_path(proj, content_hash).exists(): + from tallyman_xorq.materialize import ensure_materialized + + si.ensure_cas_path(proj, src, digest) + ensure_materialized(proj, content_hash) + from tallyman_core import entry_schema_path + + written = { + "row_count": existing.row_count, + "schema": json.loads(entry_schema_path(proj, content_hash).read_text()), + } + if mint: + set_alias(proj, alias, content_hash, kind=SOURCE_KIND) + + return { + "alias": alias, + "version": version, + "hash": content_hash, + "created": mint, + "path": str(src), + "digest": digest, + "row_count": written["row_count"], + "schema": written["schema"], + } diff --git a/src/tallyman_xorq/staleness.py b/src/tallyman_xorq/staleness.py index b9812ff6..4d8c6e94 100644 --- a/src/tallyman_xorq/staleness.py +++ b/src/tallyman_xorq/staleness.py @@ -1,52 +1,40 @@ """Decide whether a catalog entry is stale relative to its recorded inputs (#89). -Everything the consumer needs is already captured at build: ``manifest.parents`` -(the resolved ``tracked_expr_from_alias`` edges and their read-intent) and -``manifest.sources`` (each source's content digest). Nothing reads it. This is -the read-only half — no recompute — of the reactive consumer: compare the -recorded inputs against the current world and report what moved. - -Two independent axes, both computable from already-recorded data: - -1. **Upstream advanced.** A ``follow=True`` parent (an alias argument) is stale - when its alias head now resolves to a different hash than the one recorded at - build. A ``follow=False`` parent (a literal-hash pin) is never stale here — it - asked for that exact revision. -2. **Source drifted.** A recorded ``(rel_path, digest)`` is stale when the file - on disk no longer digests to the recorded value. Under ``off`` identity mode - the manifest records no digests, so this axis is reported ``unknown`` — never - silently ``fresh``. - -``result_digest`` is deliberately *not* a staleness input: a recompute-differs -entry is *nondeterministic*, not stale, and recompute cannot make it fresh — that -is the #83/#121 path, surfaced by the recompute action, not this scan. +Everything the consumer needs is already captured at build: ``manifest.parents``, the resolved +``tracked_expr_from_alias`` edges and their read-intent. This is the read-only half — no recompute — of +the reactive consumer: compare the recorded inputs against the current world and report what moved. + +**One axis** (ADR-011 D6). An entry is stale when a ``follow=True`` parent (an alias argument) resolves +to a different hash than the one recorded at build, and for no other reason. A ``follow=False`` parent +(a version pin) is never stale — it asked for that exact revision. + +There used to be a second axis: a recorded ``(rel_path, digest)`` no longer matching the file on disk. +It asked a question the system could not answer, because ``manifest.sources`` did not record *how* the +entry came to depend on the file, so a child pinned to its parent by hash read as stale forever. With +every raw input an entry of its own (ADR-011 D1), the question is the first axis: a re-import advances +the source alias, and everything following it goes stale the way it does for any other parent. The scan +therefore touches no file at all — it reads ``aliases.jsonl`` and the manifests. + +``result_digest`` is deliberately *not* a staleness input: a recompute-differs entry is +*nondeterministic*, not stale, and recompute cannot make it fresh — that is the #83/#121 path, surfaced +by the recompute action, not this scan. """ from __future__ import annotations -import os -from contextlib import contextmanager from dataclasses import dataclass, field from tallyman_core import aliases -from tallyman_xorq import source_identity from tallyman_xorq.build import list_entries -from tallyman_xorq.dependents import descendant_cone, parents_of, sources_of -from tallyman_xorq.io import ProjectDataNotFound, project_path - -# digest_for memoizes on (mtime_ns, size, inode); a same-stat in-place content -# swap would otherwise serve the cached digest and the source axis would -# false-negative. Force a faithful read for the staleness check (the env var is -# the documented bypass, source_identity.py). -_REHASH_ENV = "TALLYMAN_SOURCE_REHASH" +from tallyman_xorq.dependents import descendant_cone, parents_of @dataclass(frozen=True) class StaleReason: - axis: str # "alias" | "source" - ref: str # the alias name or the source's rel_path - was: str # the value recorded at build (parent hash / source digest) - now: str | None # the current value (alias head / current digest) + axis: str # always "alias"; the field is kept so a reason reads the same in the API and the UI + ref: str # the alias name + was: str # the parent hash recorded at build + now: str | None # the alias head now @dataclass @@ -59,54 +47,24 @@ class StaleVerdict: live: bool = True # #154: content_hash is a current alias head (set by scan) -@contextmanager -def _force_source_rehash(): - prev = os.environ.get(_REHASH_ENV) - os.environ[_REHASH_ENV] = "1" - try: - yield - finally: - if prev is None: - os.environ.pop(_REHASH_ENV, None) - else: - os.environ[_REHASH_ENV] = prev - - def entry_staleness(project: str, content_hash: str) -> StaleVerdict: """Whether *content_hash*'s own recorded inputs have moved (read-only). - Both axes read through the ``dependents`` manifest reader (``parents_of`` / - ``sources_of``) rather than the raw manifest, so that module stays the single - seam over the recorded DAG. + Reads through the ``dependents`` manifest reader (``parents_of``) rather than the raw manifest, so + that module stays the single seam over the recorded DAG. Nothing here opens a data file. """ reasons: list[StaleReason] = [] unknown: list[str] = [] - # Axis 1: a followed alias advanced past the recorded head. for parent in parents_of(project, content_hash): if not parent.follow: - continue # a hash pin is never stale on this axis + continue # a version pin is never stale head = aliases.get_alias(project, parent.ref) if head is None: unknown.append(f"alias:{parent.ref}") # alias gone — deletion is out of scope here elif head != parent.hash: reasons.append(StaleReason(axis="alias", ref=parent.ref, was=parent.hash, now=head)) - # Axis 2: a recorded source drifted on disk. - sources = sources_of(project, content_hash) - if sources is None: - unknown.append("source") # built under mode=off; the axis is unavailable - else: - with _force_source_rehash(): - for rel_path, recorded in sources.items(): - try: - current = source_identity.digest_for(project, project_path(rel_path, project)) - except ProjectDataNotFound: - unknown.append(f"source:{rel_path}") # source file removed - continue - if current != recorded: - reasons.append(StaleReason(axis="source", ref=rel_path, was=recorded, now=current)) - return StaleVerdict( content_hash=content_hash, stale=bool(reasons), diff --git a/tests/conftest.py b/tests/conftest.py index 882d6dee..f826bea4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,10 +46,38 @@ def project(isolated_home: Path) -> str: @pytest.fixture def orders_parquet(project: str) -> Path: - """Generate a deterministic shoe-orders fixture under project/data/.""" + """Generate a deterministic shoe-orders fixture under project/data/. + + The file only. A recipe cannot read it (ADR-011 D2) — depend on ``orders_src`` for that, and use + this where the test is about the file itself: editing it, importing it under an alias of the test's own (not + alongside ``orders_src``, since one set of bytes goes under one alias), or + asserting on the bytes. + """ return write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=200, seed=0) +ORDERS_SRC = "orders_src" + + +@pytest.fixture +def orders_src(orders_parquet: Path, project: str) -> str: + """The shoe-orders fixture imported as a source alias, and the alias name (ADR-011 D1). + + A recipe never opens a file: it reads an alias. Depend on this wherever a test needs a recipe over + the orders data, and read it in the recipe with ``tracked_expr_from_alias("orders_src")``. The + import mints one ordinary entry — a content hash, a manifest, a snapshot carrying ``__row_order`` + — so a project that uses this fixture has one more entry and one more alias than it did before the + rewrite, which is what the DAG gaining its roots looks like. + + The name is ``orders_src`` rather than ``orders`` because several tests already create a catalog + alias called ``orders``, and one name is one kind or the other but never both. + """ + from tallyman_xorq.source_import import update_and_depend + + update_and_depend(orders_parquet, ORDERS_SRC, project=project) + return ORDERS_SRC + + @pytest.fixture def fresh_companion_app(project: str): """Create a companion app bound to the current isolated project.""" diff --git a/tests/test_activity_log.py b/tests/test_activity_log.py index cb2e81ec..f7315938 100644 --- a/tests/test_activity_log.py +++ b/tests/test_activity_log.py @@ -16,15 +16,15 @@ _BAD = "x = 5 # never binds `expr` → BuildError" -def _ok_code(project: str, cols: str = '"region", "price", "__row_order"') -> str: +def _ok_code(project: str, src: str, cols: str = '"region", "price", "__row_order"') -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.select({cols}) """ -def test_build_error_records_event_with_traceback(project, orders_parquet, monkeypatch): +def test_build_error_records_event_with_traceback(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) out = catalog_run(_BAD) assert "error" in out @@ -40,10 +40,10 @@ def test_build_error_records_event_with_traceback(project, orders_parquet, monke assert e["code"] == _BAD -def test_alias_create_and_revise_record_alias_events(project, orders_parquet, monkeypatch): +def test_alias_create_and_revise_record_alias_events(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("shoes", _ok_code(project)) - catalog_revise("shoes", _ok_code(project, cols='"region", "__row_order"')) # different projection → V2 + catalog_create("shoes", _ok_code(project, orders_src)) + catalog_revise("shoes", _ok_code(project, orders_src, cols='"region", "__row_order"')) # different projection → V2 aliases = [e for e in read_events(project, categories=["alias"]) if e["alias"] == "shoes"] assert len(aliases) >= 2 @@ -52,10 +52,10 @@ def test_alias_create_and_revise_record_alias_events(project, orders_parquet, mo assert min(a["version"] for a in aliases) == 1 -def test_api_log_filters_by_category_and_lists_sessions(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_api_log_filters_by_category_and_lists_sessions(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_run(_BAD) # mcp build_error - catalog_create("good", _ok_code(project)) # alias_set + catalog_create("good", _ok_code(project, orders_src)) # alias_set record_event(project, "buckaroo", origin="companion", status="ok", hash="deadbeef", load_ms=12.3) c = TestClient(fresh_companion_app) @@ -72,7 +72,7 @@ def test_api_log_filters_by_category_and_lists_sessions(fresh_companion_app, pro assert ts == sorted(ts, reverse=True) -def test_events_are_session_filterable(project, orders_parquet, monkeypatch): +def test_events_are_session_filterable(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) record_event(project, "build_ok", session="aaa", tool="catalog_run", hash="h1") record_event(project, "build_ok", session="bbb", tool="catalog_run", hash="h2") @@ -80,7 +80,7 @@ def test_events_are_session_filterable(project, orders_parquet, monkeypatch): assert got and all(e["session"] == "aaa" for e in got) -def test_api_log_backfills_errors_from_errors_jsonl(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_api_log_backfills_errors_from_errors_jsonl(fresh_companion_app, project, orders_src, monkeypatch): # The Log tab must show build failures even when no event was recorded for # them — e.g. an older MCP that wrote errors.jsonl but not events.jsonl. # /api/log backfills build_error from errors.jsonl (#log), marked history. @@ -99,7 +99,7 @@ def test_api_log_backfills_errors_from_errors_jsonl(fresh_companion_app, project assert any(e.get("error_id") == rec["id"] for e in only_mcp) -def test_api_log_dedups_error_present_in_both_stores(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_api_log_dedups_error_present_in_both_stores(fresh_companion_app, project, orders_src, monkeypatch): # When the new MCP recorded a build_error event AND errors.jsonl has the # same id, it must appear once — the rich event wins over the backfill. monkeypatch.setenv("TALLYMAN_PROJECT", project) diff --git a/tests/test_aliases.py b/tests/test_aliases.py index 91d50351..a569be23 100644 --- a/tests/test_aliases.py +++ b/tests/test_aliases.py @@ -30,16 +30,16 @@ def _agg_code(project_name: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project_name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project_name!r}) expr = t.group_by("region").aggregate(n=t.count()) """ def _filter_code(project_name: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project_name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project_name!r}) filtered = t.filter(t.category == "boots") expr = filtered.group_by("region").aggregate(n=filtered.count()) """ @@ -52,7 +52,7 @@ def _filter_code(project_name: str) -> str: def test_set_alias_first_time(project: str): info = set_alias(project, "x", "h1") - assert info == {"name": "x", "hash": "h1", "version": 1} + assert info == {"name": "x", "hash": "h1", "version": 1, "kind": "catalog"} assert load_aliases(project) == {"x": "h1"} assert load_history(project) == {"x": ["h1"]} @@ -160,7 +160,7 @@ def test_rename_alias_collision(project: str): # --------------------------------------------------------------------------- -def test_catalog_create_assigns_alias(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_create_assigns_alias(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) out = catalog_create("shoe_sales", _agg_code(project), prompt="sales by region") assert "error" not in out @@ -169,7 +169,7 @@ def test_catalog_create_assigns_alias(project: str, orders_parquet: Path, monkey assert get_alias(project, "shoe_sales") == out["hash"] -def test_catalog_create_rejects_duplicate(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_create_rejects_duplicate(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) out = catalog_create("shoe_sales", _agg_code(project)) @@ -177,7 +177,7 @@ def test_catalog_create_rejects_duplicate(project: str, orders_parquet: Path, mo assert "already exists" in out["error"] -def test_catalog_revise_bumps_version(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_revise_bumps_version(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) v1 = catalog_create("shoe_sales", _agg_code(project)) v2 = catalog_revise("shoe_sales", _filter_code(project)) @@ -187,14 +187,14 @@ def test_catalog_revise_bumps_version(project: str, orders_parquet: Path, monkey assert get_alias(project, "shoe_sales") == v2["hash"] -def test_catalog_revise_rejects_missing_alias(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_revise_rejects_missing_alias(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) out = catalog_revise("nonexistent", _agg_code(project)) assert "error" in out assert "does not exist" in out["error"] -def test_catalog_alias_promotes_scratch(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_alias_promotes_scratch(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) scratch = catalog_run(_agg_code(project), prompt="exploratory") out = catalog_alias(scratch["hash"], "shoe_sales") @@ -209,7 +209,7 @@ def test_catalog_alias_rejects_missing_hash(project: str, monkeypatch): assert "error" in out -def test_catalog_alias_rejects_taken_name(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_alias_rejects_taken_name(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) scratch = catalog_run(_agg_code(project)) catalog_alias(scratch["hash"], "shoe_sales") @@ -218,7 +218,7 @@ def test_catalog_alias_rejects_taken_name(project: str, orders_parquet: Path, mo assert "error" in out -def test_catalog_rename(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_rename(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("foo", _agg_code(project)) out = catalog_rename("foo", "bar") @@ -233,7 +233,7 @@ def test_catalog_rename(project: str, orders_parquet: Path, monkeypatch): # --------------------------------------------------------------------------- -def test_catalog_run_returns_url(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_run_returns_url(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) out = catalog_run(_agg_code(project), prompt="url check") assert "url" in out @@ -241,7 +241,7 @@ def test_catalog_run_returns_url(project: str, orders_parquet: Path, monkeypatch assert project in out["url"] -def test_catalog_create_returns_url(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_create_returns_url(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) out = catalog_create("shoe_sales", _agg_code(project), prompt="url check") assert "url" in out @@ -249,7 +249,7 @@ def test_catalog_create_returns_url(project: str, orders_parquet: Path, monkeypa assert project in out["url"] -def test_catalog_revise_returns_url(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_revise_returns_url(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) out = catalog_revise("shoe_sales", _filter_code(project), prompt="url check") @@ -258,7 +258,7 @@ def test_catalog_revise_returns_url(project: str, orders_parquet: Path, monkeypa assert project in out["url"] -def test_catalog_alias_returns_url(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_alias_returns_url(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) scratch = catalog_run(_agg_code(project)) out = catalog_alias(scratch["hash"], "shoe_sales") @@ -272,7 +272,7 @@ def test_catalog_alias_returns_url(project: str, orders_parquet: Path, monkeypat # --------------------------------------------------------------------------- -def test_catalog_renders_named_with_vchip(fresh_companion_app, project: str, orders_parquet: Path): +def test_catalog_renders_named_with_vchip(fresh_companion_app, project: str, orders_src: str): build_and_persist(project, _agg_code(project)) set_alias(project, "shoe_sales", build_and_persist(project, _filter_code(project)).content_hash) c = TestClient(fresh_companion_app) @@ -283,7 +283,7 @@ def test_catalog_renders_named_with_vchip(fresh_companion_app, project: str, ord assert entries["shoe_sales"]["version"] == 1 -def test_catalog_includes_forensic_versions(fresh_companion_app, project: str, orders_parquet: Path): +def test_catalog_includes_forensic_versions(fresh_companion_app, project: str, orders_src: str): v1 = build_and_persist(project, _agg_code(project)) set_alias(project, "shoe_sales", v1.content_hash) v2 = build_and_persist(project, _filter_code(project)) @@ -300,7 +300,7 @@ def test_catalog_includes_forensic_versions(fresh_companion_app, project: str, o assert current[0]["version"] == 2 -def test_entry_detail_resolves_alias_lookup(fresh_companion_app, project: str, orders_parquet: Path): +def test_entry_detail_resolves_alias_lookup(fresh_companion_app, project: str, orders_src: str): res = build_and_persist(project, _agg_code(project)) set_alias(project, "shoe_sales", res.content_hash) c = TestClient(fresh_companion_app) @@ -311,7 +311,7 @@ def test_entry_detail_resolves_alias_lookup(fresh_companion_app, project: str, o assert body["alias"] == "shoe_sales" -def test_entry_detail_shows_forensic_history(fresh_companion_app, project: str, orders_parquet: Path): +def test_entry_detail_shows_forensic_history(fresh_companion_app, project: str, orders_src: str): v1 = build_and_persist(project, _agg_code(project)) set_alias(project, "shoe_sales", v1.content_hash) v2 = build_and_persist(project, _filter_code(project)) @@ -325,7 +325,7 @@ def test_entry_detail_shows_forensic_history(fresh_companion_app, project: str, assert len(body["forensic_history"]) == 2 -def test_catalog_renders_section_headers(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_catalog_renders_section_headers(fresh_companion_app, project: str, orders_src: str, monkeypatch): """T-13: entries API groups named/forensic/scratch with correct flags.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create, catalog_revise, catalog_run @@ -355,3 +355,133 @@ def test_api_aliases_returns_history(fresh_companion_app, project: str): assert names["a"]["hash"] == "h2" assert names["a"]["history"] == ["h1", "h2"] assert names["b"]["hash"] == "x1" + + +# --------------------------------------------------------------------------- +# ADR-011 D1 — a source alias is a distinct kind. Its versions are raw input +# datasets, so there is no recipe to revise and no diff to promote onto it, +# and a name can be one kind or the other but never both. +# --------------------------------------------------------------------------- + + +def _import(project: str, tmp_path: Path, alias: str, n_rows: int = 5) -> dict: + import pandas as pd + + from tallyman_xorq import source_import + + src = tmp_path / "outside" / f"{alias}.parquet" + src.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame({"region": [f"r{i}" for i in range(n_rows)]}).to_parquet(src) + return source_import.update_and_depend(str(src), alias, project=project) + + +def _over_source(alias: str, project: str) -> str: + return ( + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({alias!r}, project={project!r})\n" + "expr = t.group_by('region').aggregate(n=t.count())\n" + ) + + +def test_source_alias_kind_is_recorded(project: str, tmp_path: Path, monkeypatch): + """catalog_list has to be able to separate inputs from computations.""" + from tallyman_core.aliases import CATALOG_KIND, SOURCE_KIND, alias_kind + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + _import(project, tmp_path, "orders") + assert "error" not in catalog_create("by_region", _over_source("orders", project)) + + assert alias_kind(project, "orders") == SOURCE_KIND + assert alias_kind(project, "by_region") == CATALOG_KIND + assert alias_kind(project, "nope") is None + + +def test_a_catalog_alias_cannot_take_a_source_alias_name(project: str, tmp_path: Path, monkeypatch): + monkeypatch.setenv("TALLYMAN_PROJECT", project) + _import(project, tmp_path, "orders") + + out = catalog_create("orders", _over_source("orders", project)) + + assert "error" in out + assert "source alias" in out["error"] + + +def test_an_import_cannot_take_a_catalog_alias_name(project: str, tmp_path: Path, monkeypatch): + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + _import(project, tmp_path, "orders") + assert "error" not in catalog_create("by_region", _over_source("orders", project)) + other = tmp_path / "outside" / "orders.parquet" + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(other), "by_region", project=project) + + assert "catalog alias" in str(exc.value) + + +def test_set_alias_refuses_to_change_an_alias_kind(project: str): + from tallyman_core.aliases import SOURCE_KIND, AliasKindMismatch + + set_alias(project, "orders", "aaaaaaaaaaaa", kind=SOURCE_KIND) + + with pytest.raises(AliasKindMismatch): + set_alias(project, "orders", "bbbbbbbbbbbb") + + +# An alias's kind matches the kind of the entries it points at. The kind lives on the alias (aliases.jsonl); whether an +# entry is a *source entry* — a version of an imported file, recognised by its manifest carrying ``provenance`` — lives +# on the entry. set_alias is the one writer every route goes through, so it is where the two are tied together. + + +def test_set_alias_refuses_a_catalog_alias_onto_a_source_entry(project: str, tmp_path: Path, monkeypatch): + """A catalog name over an import would let catalog_revise "revise" a version that has no recipe (D1).""" + from tallyman_core.aliases import AliasKindMismatch, alias_kind + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + imported = _import(project, tmp_path, "orders") + + with pytest.raises(AliasKindMismatch) as exc: + set_alias(project, "looks_like_catalog", imported["hash"]) + + assert "orders" in str(exc.value) + assert get_alias(project, "looks_like_catalog") is None + assert alias_kind(project, "looks_like_catalog") is None + + +def test_set_alias_refuses_a_source_alias_onto_a_computed_entry(project: str, tmp_path: Path, monkeypatch): + """A source alias advances only by an import, so its head is always an imported file, never a recipe's result.""" + from tallyman_core.aliases import SOURCE_KIND, AliasKindMismatch + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + imported = _import(project, tmp_path, "orders") + computed = build_and_persist(project, _over_source("orders", project)) + + with pytest.raises(AliasKindMismatch): + set_alias(project, "orders", computed.content_hash, kind=SOURCE_KIND) + + assert history_for(project, "orders") == [imported["hash"]] + + +def test_set_alias_does_not_check_a_hash_with_no_entry(project: str): + """With no manifest there is nothing to match the kind against, so either kind is accepted.""" + from tallyman_core.aliases import SOURCE_KIND, alias_kind + + set_alias(project, "a_catalog", "aaaaaaaaaaaa") + set_alias(project, "a_source", "bbbbbbbbbbbb", kind=SOURCE_KIND) + + assert alias_kind(project, "a_catalog") == "catalog" + assert alias_kind(project, "a_source") == SOURCE_KIND + + +def test_renaming_a_source_alias_keeps_its_kind(project: str, tmp_path: Path, monkeypatch): + """Rename and unalias behave as they do for catalog aliases (D1).""" + from tallyman_core.aliases import SOURCE_KIND, alias_kind + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + _import(project, tmp_path, "orders") + + catalog_rename("orders", "raw_orders") + + assert alias_kind(project, "raw_orders") == SOURCE_KIND + assert alias_kind(project, "orders") is None diff --git a/tests/test_auto_recalc.py b/tests/test_auto_recalc.py index 33726541..73415d78 100644 --- a/tests/test_auto_recalc.py +++ b/tests/test_auto_recalc.py @@ -25,6 +25,7 @@ from tallyman_xorq.build import list_entries from tallyman_xorq.dependents import descendant_cone from tallyman_xorq.recalc import followers_of +from tallyman_xorq.source_import import update_and_depend from tallyman_xorq.staleness import scan # --------------------------------------------------------------------------- @@ -32,18 +33,18 @@ # --------------------------------------------------------------------------- -def _src_code(project: str, src: str = "orders.parquet") -> str: # a source projection +def _src_code(project: str, src: str = "orders_src") -> str: # a projection of a source alias return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({src!r}, project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.select("region", "price", "__row_order") """ -def _src_code_v2(project: str, src: str = "orders.parquet") -> str: # different graph, same source +def _src_code_v2(project: str, src: str = "orders_src") -> str: # different graph, same source return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({src!r}, project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.select("region", "price", "__row_order").mutate(extra=1) """ @@ -69,14 +70,24 @@ def _hash(result: dict) -> str: return result["hash"] -def _second_source(project: str, name: str = "widgets.parquet", *, seed: int = 1, n: int = 120): - return write_shoe_orders(data_dir(project) / name, n_rows=n, seed=seed) +WIDGETS_SRC = "widgets_src" + + +def _second_source(project: str, *, seed: int = 1, n: int = 120) -> str: + """Import a second, unrelated source alias and return its name. + + Called again with a different seed it imports different bytes under the same alias, which + advances it (ADR-011 D3) and makes whatever reads it stale. That is the replacement for + editing a file on disk, which a build never sees. + """ + path = write_shoe_orders(data_dir(project) / "widgets.parquet", n_rows=n, seed=seed) + update_and_depend(path, WIDGETS_SRC, project=project) + return WIDGETS_SRC def _clean_env(monkeypatch) -> None: - """Default-ON path: no env override, content-addressed sources.""" + """Default-ON path: no env override.""" monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") # --------------------------------------------------------------------------- @@ -84,7 +95,7 @@ def _clean_env(monkeypatch) -> None: # --------------------------------------------------------------------------- -def test_revise_cascades_to_follower_without_explicit_recalc(project, orders_parquet, monkeypatch): +def test_revise_cascades_to_follower_without_explicit_recalc(project, orders_src, monkeypatch): _clean_env(monkeypatch) a1 = _hash(catalog_create("a", _src_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) @@ -110,7 +121,7 @@ def test_revise_cascades_to_follower_without_explicit_recalc(project, orders_par # --------------------------------------------------------------------------- -def test_revise_and_cascade_is_one_undoable_revision(project, orders_parquet, monkeypatch): +def test_revise_and_cascade_is_one_undoable_revision(project, orders_src, monkeypatch): _clean_env(monkeypatch) a1 = _hash(catalog_create("a", _src_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) @@ -132,7 +143,7 @@ def test_revise_and_cascade_is_one_undoable_revision(project, orders_parquet, mo # --------------------------------------------------------------------------- -def test_flag_off_leaves_follower_stale(project, orders_parquet, monkeypatch): +def test_flag_off_leaves_follower_stale(project, orders_src, monkeypatch): _clean_env(monkeypatch) set_auto_recalc(project, False) assert auto_recalc_enabled(project) is False @@ -151,7 +162,7 @@ def test_flag_off_leaves_follower_stale(project, orders_parquet, monkeypatch): # --------------------------------------------------------------------------- -def test_revise_with_no_followers_is_one_checkpoint_empty_report(project, orders_parquet, monkeypatch): +def test_revise_with_no_followers_is_one_checkpoint_empty_report(project, orders_src, monkeypatch): _clean_env(monkeypatch) a1 = _hash(catalog_create("a", _src_code(project))) # nothing follows a step_before = current_step(project) @@ -169,7 +180,7 @@ def test_revise_with_no_followers_is_one_checkpoint_empty_report(project, orders # --------------------------------------------------------------------------- -def test_broken_downstream_stops_and_reports_revise_still_lands(project, orders_parquet, monkeypatch): +def test_broken_downstream_stops_and_reports_revise_still_lands(project, orders_src, monkeypatch): _clean_env(monkeypatch) a1 = _hash(catalog_create("a", _src_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) @@ -191,7 +202,7 @@ def test_broken_downstream_stops_and_reports_revise_still_lands(project, orders_ # --------------------------------------------------------------------------- -def test_recalc_sub_report_shape(project, orders_parquet, monkeypatch): +def test_recalc_sub_report_shape(project, orders_src, monkeypatch): _clean_env(monkeypatch) _hash(catalog_create("a", _src_code(project))) _hash(catalog_create("b", _child_code("a"))) @@ -203,7 +214,7 @@ def test_recalc_sub_report_shape(project, orders_parquet, monkeypatch): assert r["status"] == "ok" -def test_auto_recalc_report_carries_real_checkpoint_step(project, orders_parquet, monkeypatch): +def test_auto_recalc_report_carries_real_checkpoint_step(project, orders_src, monkeypatch): # The auto-path sub-report's `checkpoint_step` must be the step the cascade # actually committed in. The walk takes no checkpoint of its own (the per-op # dispatch decorator owns it, after the handler returns), so the report is @@ -226,16 +237,17 @@ def test_auto_recalc_report_carries_real_checkpoint_step(project, orders_parquet # --------------------------------------------------------------------------- -def test_orphan_stale_is_left_untouched_logged_and_classified(project, orders_parquet, monkeypatch, caplog): +def test_orphan_stale_is_left_untouched_logged_and_classified(project, orders_src, monkeypatch, caplog): _clean_env(monkeypatch) - _second_source(project, "widgets.parquet", seed=1) - a1 = _hash(catalog_create("a", _src_code(project))) # over orders.parquet + widgets = _second_source(project, seed=1) + a1 = _hash(catalog_create("a", _src_code(project))) # over orders_src b1 = _hash(catalog_create("b", _child_code("a"))) - c1 = _hash(catalog_create("c", _src_code(project, "widgets.parquet"))) + c1 = _hash(catalog_create("c", _src_code(project, widgets))) - # Make c directly source-stale for a reason unrelated to the revise of a. - _second_source(project, "widgets.parquet", seed=7, n=250) - assert scan(project)[c1].stale is True # c is directly source-stale + # Make c directly stale for a reason unrelated to the revise of a: re-import + # widgets, which advances the alias c reads. + _second_source(project, seed=7, n=250) + assert scan(project)[c1].stale is True # c is directly stale on the widgets_src edge with caplog.at_level(logging.WARNING): out = catalog_revise("a", _src_code_v2(project)) @@ -258,10 +270,10 @@ def test_orphan_stale_is_left_untouched_logged_and_classified(project, orders_pa # --------------------------------------------------------------------------- -def test_cascade_failure_persists_and_ties_back(project, orders_parquet, monkeypatch): +def test_cascade_failure_persists_and_ties_back(project, orders_src, monkeypatch): _clean_env(monkeypatch) - _second_source(project, "widgets.parquet", seed=1) - _hash(catalog_create("z", _src_code(project, "widgets.parquet"))) # unrelated alias + widgets = _second_source(project, seed=1) + _hash(catalog_create("z", _src_code(project, widgets))) # unrelated alias _hash(catalog_create("a", _src_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) (entry_dir(project, b1) / "expr.py").write_text("broken (((\n") @@ -275,7 +287,7 @@ def test_cascade_failure_persists_and_ties_back(project, orders_parquet, monkeyp # A later auto-recalc driven by an UNRELATED alias still sees b1 stale; b1 is # not a follower of z, so it lands in orphan_stale, classified explained-by-error. - out2 = catalog_revise("z", _src_code_v2(project, "widgets.parquet")) + out2 = catalog_revise("z", _src_code_v2(project, widgets)) orphans = {o["hash"]: o for o in out2["recalc"]["orphan_stale"]} assert b1 in orphans assert err["id"] in orphans[b1]["explanation"] @@ -310,7 +322,7 @@ def test_env_overrides_config_file(project, monkeypatch): assert auto_recalc_enabled(project) is False -def test_config_json_is_tracked_and_reset_restores_it(project, orders_parquet, monkeypatch): +def test_config_json_is_tracked_and_reset_restores_it(project, orders_src, monkeypatch): # config.json must be on the catalog's tracked-surface allowlist: it is swept # into checkpoints and reset_to's assert_catalog_consistent would otherwise # reject it as an unlisted tracked path. Pin both that reset doesn't raise and @@ -349,7 +361,7 @@ def test_read_config_tolerates_unreadable_file(project, monkeypatch): # --------------------------------------------------------------------------- -def test_self_referential_revise_is_not_unexplained_orphan(project, orders_parquet, monkeypatch): +def test_self_referential_revise_is_not_unexplained_orphan(project, orders_src, monkeypatch): # A self-referential head (a recipe that reads tracked_expr_from_alias of its OWN # alias) is permanently alias-stale by design (#74/#85: the self-pin can never # refresh). catalog_revise now rejects creating one (#135), but historical heads @@ -357,9 +369,9 @@ def test_self_referential_revise_is_not_unexplained_orphan(project, orders_parqu # a head as "self-referential", NOT UNEXPLAINED ("file a bug") — otherwise the # UNEXPLAINED signal becomes untrustworthy. _clean_env(monkeypatch) - _second_source(project, "widgets.parquet", seed=1) + widgets = _second_source(project, seed=1) _hash(catalog_create("tpd", _src_code(project))) # has a price column - _hash(catalog_create("z", _src_code(project, "widgets.parquet"))) # unrelated alias + _hash(catalog_create("z", _src_code(project, widgets))) # unrelated alias # catalog_revise rejects a self-referential recipe (#135), so build the self-ref # head directly to exercise the staleness classification for an existing one. @@ -372,14 +384,14 @@ def test_self_referential_revise_is_not_unexplained_orphan(project, orders_parqu # A later, unrelated revise must classify the self-ref head as explained # ("self-referential"), not UNEXPLAINED — keeping the UNEXPLAINED signal trustworthy. - out = catalog_revise("z", _src_code_v2(project, "widgets.parquet")) + out = catalog_revise("z", _src_code_v2(project, widgets)) orphans = {o["hash"]: o for o in out["recalc"]["orphan_stale"]} assert tpd_head in orphans # it IS directly (self-)stale, so it is accounted for assert "UNEXPLAINED" not in orphans[tpd_head]["explanation"] assert "self-referential" in orphans[tpd_head]["explanation"].lower() -def test_skipped_but_directly_stale_entry_ties_back_to_the_failure(project, orders_parquet, monkeypatch): +def test_skipped_but_directly_stale_entry_ties_back_to_the_failure(project, orders_src, monkeypatch): # When a cascade halts, an entry it SKIPS that is itself directly stale must # still be accounted for: recorded against the durable error store so a later # unrelated revise classifies it "explained", not a false UNEXPLAINED. @@ -389,8 +401,8 @@ def test_skipped_but_directly_stale_entry_ties_back_to_the_failure(project, orde # walk reaches first fails and halts, leaving the other SKIPPED while still # directly stale — exactly the case the skipped-record covers, order-agnostic. _clean_env(monkeypatch) - _second_source(project, "widgets.parquet", seed=1) - _hash(catalog_create("z", _src_code(project, "widgets.parquet"))) # unrelated alias + widgets = _second_source(project, seed=1) + _hash(catalog_create("z", _src_code(project, widgets))) # unrelated alias _hash(catalog_create("a", _src_code(project))) # Distinct recipes so b and c are different entries (same recipe → same hash → # one entry with two aliases), both following a. @@ -419,7 +431,7 @@ def test_skipped_but_directly_stale_entry_ties_back_to_the_failure(project, orde # the skipped-but-directly-stale entry is recorded, so error_for_hash resolves it assert error_for_hash(project, skipped) is not None - out2 = catalog_revise("z", _src_code_v2(project, "widgets.parquet")) # unrelated revise + out2 = catalog_revise("z", _src_code_v2(project, widgets)) # unrelated revise orphans = {o["hash"]: o for o in out2["recalc"]["orphan_stale"]} assert skipped in orphans assert "UNEXPLAINED" not in orphans[skipped]["explanation"] @@ -432,7 +444,7 @@ def test_skipped_but_directly_stale_entry_ties_back_to_the_failure(project, orde # --------------------------------------------------------------------------- -def test_cascade_failure_records_triggering_tool(project, orders_parquet, monkeypatch): +def test_cascade_failure_records_triggering_tool(project, orders_src, monkeypatch): # The persisted recalc failure must name the tool that actually ran. On the # auto-recalc path that is catalog_revise; "catalog_recalc" (which never ran # here) would misattribute the failure in the error banner / forensics. @@ -455,7 +467,7 @@ def test_cascade_failure_records_triggering_tool(project, orders_parquet, monkey # --------------------------------------------------------------------------- -def test_auto_recalc_scans_staleness_once(project, orders_parquet, monkeypatch): +def test_auto_recalc_scans_staleness_once(project, orders_src, monkeypatch): # auto_recalc computed verdicts via scan(), then the cone walk scanned again — # two full staleness scans (each re-digests every source) per revise. The walk # must reuse the verdicts auto_recalc already holds. @@ -508,7 +520,7 @@ def _make_divergent_husk(project) -> tuple[str, str, str]: return leaf1, husk, live -def test_revising_leaf_does_not_replay_a_superseded_husk(project, orders_parquet, monkeypatch): +def test_revising_leaf_does_not_replay_a_superseded_husk(project, orders_src, monkeypatch): # THE f2dd/cb24 REGRESSION. Revising a leaf the husk pins must recompute only # u's live head, not replay the divergent husk into a brand-new junk entry. _clean_env(monkeypatch) @@ -527,7 +539,7 @@ def test_revising_leaf_does_not_replay_a_superseded_husk(project, orders_parquet assert husk not in out["recalc"]["remap"] -def test_followers_of_excludes_superseded_husk(project, orders_parquet, monkeypatch): +def test_followers_of_excludes_superseded_husk(project, orders_src, monkeypatch): _clean_env(monkeypatch) set_auto_recalc(project, False) # advance leaf WITHOUT the cascade, to inspect the raw roots _leaf1, husk, live = _make_divergent_husk(project) @@ -538,7 +550,7 @@ def test_followers_of_excludes_superseded_husk(project, orders_parquet, monkeypa assert husk not in follows # the dead husk is not -def test_scan_excludes_husk_from_stale_but_keeps_it_in_the_dict(project, orders_parquet, monkeypatch): +def test_scan_excludes_husk_from_stale_but_keeps_it_in_the_dict(project, orders_src, monkeypatch): _clean_env(monkeypatch) set_auto_recalc(project, False) _leaf1, husk, live = _make_divergent_husk(project) @@ -550,24 +562,24 @@ def test_scan_excludes_husk_from_stale_but_keeps_it_in_the_dict(project, orders_ assert husk in v # ...but still present in the dict (the replay loop indexes every cone member) -def test_superseded_husk_is_not_an_unexplained_orphan(project, orders_parquet, monkeypatch): +def test_superseded_husk_is_not_an_unexplained_orphan(project, orders_src, monkeypatch): # The perpetual-false-alarm half: a husk must not fill orphan_stale with # "file a bug", while a genuinely-stale live head with no error still does. _clean_env(monkeypatch) set_auto_recalc(project, False) - _second_source(project, "widgets.parquet", seed=1) + widgets = _second_source(project, seed=1) _leaf1, husk, live = _make_divergent_husk(project) catalog_revise("leaf", _src_code_v2(project)) # leaf advances → husk + live both stale, none recomputed - _hash(catalog_create("z", _src_code(project, "widgets.parquet"))) # unrelated alias + _hash(catalog_create("z", _src_code(project, widgets))) # unrelated alias set_auto_recalc(project, True) - out = catalog_revise("z", _src_code_v2(project, "widgets.parquet")) + out = catalog_revise("z", _src_code_v2(project, widgets)) orphans = {o["hash"]: o for o in out["recalc"]["orphan_stale"]} assert live in orphans and "UNEXPLAINED" in orphans[live]["explanation"] # real stale head still flagged assert husk not in orphans # the husk is no longer a false UNEXPLAINED -def test_scan_driven_recalc_does_not_replay_husk(project, orders_parquet, monkeypatch): +def test_scan_driven_recalc_does_not_replay_husk(project, orders_src, monkeypatch): # The SECOND junk-site: a no-arg catalog_recalc defaults its roots to every # directly-stale entry (server.py), bypassing followers_of. The head-gate must # cover it too, or the husk churns junk here instead of on the revise path. @@ -582,26 +594,30 @@ def test_scan_driven_recalc_does_not_replay_husk(project, orders_parquet, monkey assert new == {get_alias(project, "u")} # only u's live-head recompute; no husk junk -def test_scan_driven_recalc_does_not_replay_husk_under_source_drift(project, orders_parquet, monkeypatch): - # The SOURCE-DRIFT sibling of the test above. Instead of REVISING the leaf - # (which advances its head, so the leaf drops out of the root set), drift the - # leaf's SOURCE FILE on disk. The leaf HEAD stays a directly-stale root, so +def test_scan_driven_recalc_does_not_replay_husk_under_source_reimport( + project, orders_parquet, orders_src, monkeypatch +): + # The RE-IMPORT sibling of the test above. Instead of REVISING the leaf + # (which advances its head, so the leaf drops out of the root set), advance the + # source alias the leaf reads. The leaf HEAD stays a directly-stale root, so # descendant_cone([leaf]) re-expands through the husk's follow=True parent edge # and drags the husk back into the cone. The head-gate lives on the ROOT SET, # not on cone membership, so _replay_cone rebuilds the husk anyway — the - # f2dd/cb24 junk-replay regression, on the source-drift path. + # f2dd/cb24 junk-replay regression, on the advancing-input path. _clean_env(monkeypatch) set_auto_recalc(project, False) leaf1, husk, _live = _make_divergent_husk(project) - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=200, seed=1) # drift leaf's source in place + write_shoe_orders(orders_parquet, n_rows=200, seed=1) + update_and_depend(orders_parquet, "orders_src", project=project) # orders_src v2: the leaf's input moved v = scan(project) - assert v[leaf1].stale is True # leaf head directly stale on the source axis → a recalc root + assert v[leaf1].stale is True # leaf head directly stale on the orders_src edge → a recalc root assert v[husk].stale is False and v[husk].live is False # husk gated out of the root set... assert husk in descendant_cone(project, [leaf1]) # ...but still dragged into the leaf's cone before = {e["content_hash"] for e in list_entries(project)} catalog_recalc(dry_run=False) # roots default to the scan-driven directly-stale set = [leaf] new = {e["content_hash"] for e in list_entries(project)} - before - # exactly two new entries — the leaf source-recompute and u's live-head recompute — no junk husk replay + # exactly two new entries — the leaf's rebuild against orders_src v2 and u's live-head + # recompute — no junk husk replay assert new == {get_alias(project, "leaf"), get_alias(project, "u")} diff --git a/tests/test_buckaroo.py b/tests/test_buckaroo.py index d1f0652b..8d53f952 100644 --- a/tests/test_buckaroo.py +++ b/tests/test_buckaroo.py @@ -31,24 +31,24 @@ def _code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ -def _cheap_code(project: str) -> str: # a filter over one file: a cheap entry, re-run on every read +def _cheap_code(project: str) -> str: # a filter over one source alias: a cheap entry, re-run on every read return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.filter(t.qty > 1) """ def _chain_parent_code(project: str) -> str: # Aggregate → expensive → snapshot return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ @@ -72,7 +72,7 @@ def test_unit_ensure_session_short_circuits_when_not_running(project: str): assert mgr.ensure_session("anyhash", project) is None -def test_unit_ensure_session_restarts_dead_subprocess(project: str, orders_parquet: Path, monkeypatch): +def test_unit_ensure_session_restarts_dead_subprocess(project: str, orders_src: str, monkeypatch): """If buckaroo's subprocess died since the last call (mid-session crash, OOM, signal), ensure_session attempts one restart instead of silently falling back forever. Without this, a one-time death turns the rest of @@ -124,7 +124,7 @@ def json(self): assert restart_calls["n"] == 1 -def test_unit_ensure_session_restart_throttled(project: str, orders_parquet: Path, monkeypatch): +def test_unit_ensure_session_restart_throttled(project: str, orders_src: str, monkeypatch): """If buckaroo restart fails, don't retry on every page hit — that would hammer the system with subprocess spawns. Throttle to one attempt per cooldown.""" @@ -168,18 +168,25 @@ def _running_manager() -> BuckarooManager: def test_unit_reload_project_sessions_posts_one_reload_per_entry_of_the_project( - project: str, orders_parquet: Path, monkeypatch + project: str, orders_src: str, monkeypatch ): """Klass hot-reload posts ``/reload_expr/`` for every entry of THE project (ADR-007 D6): tallyman keeps no record of open sessions, so it asks Buckaroo about each entry. Another project's entries are never touched.""" from tallyman_cli.fixtures import write_shoe_orders - from tallyman_core import data_dir, ensure_project + from tallyman_core import data_dir, ensure_project, get_alias, set_active_project + from tallyman_xorq.source_import import update_and_depend + src = get_alias(project, orders_src) # the import is an entry of this project too, so it gets a post as well a = build_and_persist(project, _code(project)).content_hash b = build_and_persist(project, _cheap_code(project)).content_hash ensure_project("other") - write_shoe_orders(data_dir("other") / "orders.parquet", n_rows=50, seed=1) + # The import's generated recipe resolves the active project, so "other" has to be active while it runs. + set_active_project("other") + update_and_depend( + write_shoe_orders(data_dir("other") / "orders.parquet", n_rows=50, seed=1), orders_src, project="other" + ) build_and_persist("other", _code("other")) + set_active_project(project) mgr = _running_manager() posted_urls: list[str] = [] @@ -193,11 +200,13 @@ def raise_for_status(self): monkeypatch.setattr(mgr._client, "post", lambda url, **kw: (posted_urls.append(url), FakeResponse())[1]) reloaded = mgr.reload_project_sessions(project) - assert reloaded == 2 - assert sorted(posted_urls) == sorted(f"{mgr.base_url}/reload_expr/{mgr.session_id_for(project, h)}" for h in (a, b)) + assert reloaded == 3 + assert sorted(posted_urls) == sorted( + f"{mgr.base_url}/reload_expr/{mgr.session_id_for(project, h)}" for h in (src, a, b) + ) -def test_unit_reload_project_sessions_returns_zero_when_not_running(project: str, orders_parquet: Path): +def test_unit_reload_project_sessions_returns_zero_when_not_running(project: str, orders_src: str): build_and_persist(project, _code(project)) mgr = BuckarooManager() # is_running is False because we never called start(): nothing is posted, nothing is reloaded. @@ -205,11 +214,13 @@ def test_unit_reload_project_sessions_returns_zero_when_not_running(project: str def test_unit_reload_project_sessions_skips_sessions_buckaroo_does_not_hold( - project: str, orders_parquet: Path, monkeypatch + project: str, orders_src: str, monkeypatch ): """A 404 (never opened, or idle-evicted) or a 400 (no longer an xorq session) from /reload_expr means "not open" (ADR-007 D6): skipped and not counted, and the entry's stat cache is left alone. Only a grid that reloaded has its stat cache cleared, so its next request recomputes the stats with the new klass.""" + from tallyman_core import get_alias + live = build_and_persist(project, _code(project)).content_hash unopened = build_and_persist(project, _cheap_code(project)).content_hash not_xorq = build_and_persist(project, _code(project) + "\nexpr = expr.mutate(_v=1)").content_hash @@ -223,6 +234,8 @@ def test_unit_reload_project_sessions_skips_sessions_buckaroo_does_not_hold( mgr.session_id_for(project, live): 200, mgr.session_id_for(project, unopened): 404, mgr.session_id_for(project, not_xorq): 400, + # The import's own entry is in the project and is posted too; nobody has it open. + mgr.session_id_for(project, get_alias(project, orders_src)): 404, } class _Response: @@ -248,7 +261,7 @@ def test_unit_status_shape(project: str): assert "session_count" not in s # there is no session record to count (ADR-007 D6) -def test_unit_ensure_session_uses_load_expr_with_xorq_build_dir(project: str, orders_parquet: Path, monkeypatch): +def test_unit_ensure_session_uses_load_expr_with_xorq_build_dir(project: str, orders_src: str, monkeypatch): """When an entry has an xorq_build/ dir (every catalog entry does), ensure_session posts to /load_expr so Buckaroo serves it via the xorq backend with push-down sort/search rather than paging over a @@ -294,7 +307,7 @@ def fake_post(url, json=None, timeout=None): def test_unit_ensure_session_worthy_entry_is_served_from_its_snapshot_no_result_parquet( - project: str, orders_parquet: Path, monkeypatch + project: str, orders_src: str, monkeypatch ): """A worthy entry's grid is handed a build that only READS its snapshot (ADR-007 D6): Buckaroo is never given the aggregate to re-run, and no per-entry result.parquet is materialised for the viewer.""" @@ -330,7 +343,7 @@ def fake_post(url, json=None, timeout=None): assert not (entry_dir(project, res.content_hash) / "result.parquet").exists() -def test_unit_ensure_session_posts_a_build_over_files_that_exist_on_a_cold_cache(project, orders_parquet, monkeypatch): +def test_unit_ensure_session_posts_a_build_over_files_that_exist_on_a_cold_cache(project, orders_src, monkeypatch): """Everything the posted build reads exists by the time Buckaroo is called, on a cold compute cache. Historically the build ensure_session posted had to feed a cold grid with no pre-heal: under ADR-006 D4 the child's @@ -353,7 +366,8 @@ def test_unit_ensure_session_posts_a_build_over_files_that_exist_on_a_cold_cache child_h = catalog_create("chain", _chain_child_code("agg"))["hash"] # cheap child chained off it # Cold start, as a fresh clone / not-yet-warmed entry has: evict every file under compute_cache (the parent's - # snapshot and the ordered copy of the source under it) and clear the plan memo the in-process build warmed. + # snapshot and the source version's snapshot under it) and clear the plan memo the in-process build warmed. The + # clone of the imported bytes under data/.cas survives, which is what the source's snapshot is written again from. shutil.rmtree(compute_cache_dir(project), ignore_errors=True) cached_result_expr.cache_clear() @@ -395,7 +409,7 @@ def _capture_post(url, json=None, timeout=None): # --------------------------------------------------------------------------- -def test_unit_ensure_session_writes_stable_expanded_dir(project: str, orders_parquet: Path, monkeypatch): +def test_unit_ensure_session_writes_stable_expanded_dir(project: str, orders_src: str, monkeypatch): """A cheap entry is handed its own build, expanded into a stable per-entry .xorq_build_expanded dir. The expansion used to go to a random mkdtemp tracked in _expanded_dirs and @@ -434,7 +448,7 @@ def fake_post(url, json=None, timeout=None): assert expanded.is_dir() -def test_unit_ensure_session_heals_partial_expansion(project: str, orders_parquet: Path, monkeypatch): +def test_unit_ensure_session_heals_partial_expansion(project: str, orders_src: str, monkeypatch): """A crash mid-expansion must not poison the stable expanded dir forever. The expansion writes files into .xorq_build_expanded; a kill/OOM partway @@ -523,7 +537,7 @@ def test_unit_stop_cleans_many_expanded_dirs(project: str): @pytest.mark.parametrize("recipe", [_code, _cheap_code], ids=["worthy", "cheap"]) def test_unit_concurrent_opens_of_one_entry_share_one_intact_build_dir( - project: str, orders_parquet: Path, monkeypatch, recipe + project: str, orders_src: str, monkeypatch, recipe ): """N threads opening the same entry at once all get its derived session id and are all handed the SAME build directory, complete (ADR-007 D6). @@ -576,7 +590,7 @@ def worker(): assert (Path(build_dir) / "expr.yaml").is_file() -def test_unit_concurrent_different_hashes_each_load_once(project: str, orders_parquet: Path, monkeypatch): +def test_unit_concurrent_different_hashes_each_load_once(project: str, orders_src: str, monkeypatch): """Threads racing on N distinct content_hashes each get their own /load_expr POST, session id and build dir; no cross-talk. """ @@ -629,7 +643,7 @@ def worker(h): @pytest.mark.integration -def test_integration_spawn_and_load(project: str, orders_parquet: Path, isolated_home: Path): +def test_integration_spawn_and_load(project: str, orders_src: str, isolated_home: Path): # Build at least one catalog entry so there is a build to /load_expr. res = build_and_persist(project, _code(project)) @@ -652,7 +666,7 @@ def test_integration_spawn_and_load(project: str, orders_parquet: Path, isolated @pytest.mark.integration -def test_integration_load_expr_returns_nonzero_rows(project: str, orders_parquet: Path): +def test_integration_load_expr_returns_nonzero_rows(project: str, orders_src: str): """Smoking-gun integration test for the placeholder-expansion fix. Without expansion, /load_expr loads the schema fine (it reads YAML @@ -688,7 +702,7 @@ def test_integration_load_expr_returns_nonzero_rows(project: str, orders_parquet assert [c["name"] for c in body["columns"]] == ["region", "n", "__row_order"] # A cheap entry: the expansion lives in the entry's stable .xorq_build_expanded dir; expr.yaml must be - # placeholder-free so xorq can read the upstream ordered copy. + # placeholder-free so xorq can read the source version's snapshot upstream of it. expanded = entry_dir(project, cheap.content_hash) / ".xorq_build_expanded" assert "${TALLYMAN_PROJECT_ROOT}" not in (expanded / "expr.yaml").read_text() resp = httpx.post( @@ -703,7 +717,7 @@ def test_integration_load_expr_returns_nonzero_rows(project: str, orders_parquet @pytest.mark.integration -def test_integration_stop_cleans_up(project: str, orders_parquet: Path): +def test_integration_stop_cleans_up(project: str, orders_src: str): mgr = BuckarooManager(port=0, startup_timeout=15.0) mgr.start() assert mgr.proc is not None @@ -749,7 +763,7 @@ def load_session(self, content_hash: str, project: str, column_config_overrides= return {"status": "ok", "session_id": self.session, "detail": ""} -def test_entry_detail_decoupled_grid_session_via_session_endpoint(project: str, orders_parquet: Path): +def test_entry_detail_decoupled_grid_session_via_session_endpoint(project: str, orders_src: str): """The detail payload is metadata-only; the grid session loads lazily via /api/session so the detail request never blocks on Buckaroo (#133).""" from tallyman_companion import create_app @@ -770,7 +784,7 @@ def test_entry_detail_decoupled_grid_session_via_session_endpoint(project: str, assert sr["ws_url"] == "ws://127.0.0.1:8700/ws/abc123" -def test_entry_detail_falls_back_when_session_unavailable(project: str, orders_parquet: Path): +def test_entry_detail_falls_back_when_session_unavailable(project: str, orders_src: str): """When ensure_session returns None, API returns null for buckaroo fields.""" from tallyman_companion import create_app diff --git a/tests/test_buckaroo_handoff.py b/tests/test_buckaroo_handoff.py index 32f57c2b..603c0ecc 100644 --- a/tests/test_buckaroo_handoff.py +++ b/tests/test_buckaroo_handoff.py @@ -25,11 +25,11 @@ from tallyman_companion import create_app from tallyman_companion.buckaroo_lifecycle import BuckarooManager from tallyman_core import ( - data_dir, entry_dir, entry_expanded_build_dir, entry_manifest_path, entry_stat_cache_dir, + get_alias, read_manifest, ) from tallyman_core.paths import compute_cache_dir, tallyman_home @@ -40,24 +40,24 @@ def _agg_code(project: str) -> str: # an Aggregate: a worthy entry, materialized to a snapshot when it is created return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ def _second_agg_code(project: str) -> str: # a different worthy entry in the same project return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("category").aggregate(n=t.count()) """ -def _cheap_code(project: str) -> str: # a filter over one file: a cheap entry, re-run on every read +def _cheap_code(project: str) -> str: # a filter over one source alias: a cheap entry, re-run on every read return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.filter(t.price > 0) """ @@ -143,7 +143,7 @@ def _manager(fake: FakeBuckaroo) -> BuckarooManager: # --------------------------------------------------------------------------- -def test_a_session_buckaroo_forgot_is_posted_again(project, orders_parquet): +def test_a_session_buckaroo_forgot_is_posted_again(project, orders_src): """ADR-007 D6 (Buckaroo is handed something that already exists): Buckaroo drops a session that has had no browser attached for an hour, and tallyman used to answer the next open from its own session map, handing the grid an id Buckaroo no longer knew. Now every open posts ``/load_expr`` with the derived id; Buckaroo's own warm-hit @@ -165,7 +165,7 @@ def test_a_session_buckaroo_forgot_is_posted_again(project, orders_parquet): assert result["session_id"] == _sid(project, h) -def test_a_repeat_open_of_an_ordinary_entry_is_a_warm_hit_in_buckaroo(project, orders_parquet): +def test_a_repeat_open_of_an_ordinary_entry_is_a_warm_hit_in_buckaroo(project, orders_src): """ADR-007 D6: an ordinary entry posts none of the config-bearing fields and the same ``build_dir`` every time, so the second post is a no-op inside Buckaroo. This is what replaces tallyman's session map.""" h = build_and_persist(project, _agg_code(project)).content_hash @@ -188,7 +188,7 @@ def test_session_id_is_derived_from_the_project_and_the_hash(isolated_home): assert bk.session_id_for("one", "abc123") != bk.session_id_for("two", "abc123") -def test_tallyman_keeps_no_record_of_buckaroo_sessions(project, orders_parquet): +def test_tallyman_keeps_no_record_of_buckaroo_sessions(project, orders_src): """ADR-007 D6: ``_sessions``, ``evict_session``, ``~/.tallyman/buckaroo_sessions.json`` and the session count are retired, because nothing needs to know which grids are open.""" h = build_and_persist(project, _agg_code(project)).content_hash @@ -207,7 +207,7 @@ def test_tallyman_keeps_no_record_of_buckaroo_sessions(project, orders_parquet): # --------------------------------------------------------------------------- -def test_worthy_entry_is_handed_a_view_build_of_its_snapshot(project, orders_parquet): +def test_worthy_entry_is_handed_a_view_build_of_its_snapshot(project, orders_src): """ADR-007 D6: a worthy entry's grid is posted a *view build*: a build whose whole graph is one bare read of the entry's snapshot. Buckaroo then never runs the aggregate, join or sort, and never writes a snapshot.""" import xorq.vendor.ibis.expr.operations as ops @@ -233,7 +233,7 @@ def test_worthy_entry_is_handed_a_view_build_of_its_snapshot(project, orders_par assert Path(dict(reads[0].read_kwargs)["hash_path"]).resolve() == _snapshot_path(project, h).resolve() -def test_view_build_directory_is_stable_across_opens(project, orders_parquet): +def test_view_build_directory_is_stable_across_opens(project, orders_src): """ADR-007 D6: Buckaroo's stat-cache keys include the build directory's path, so the view build is written once to a stable per-entry directory and the same path is posted every time.""" h = build_and_persist(project, _agg_code(project)).content_hash @@ -249,16 +249,20 @@ def test_view_build_directory_is_stable_across_opens(project, orders_parquet): assert bodies[0]["build_dir"] == bodies[1]["build_dir"] -def test_cheap_entry_is_handed_its_own_expanded_build_over_files_that_exist(project, orders_parquet): +def test_cheap_entry_is_handed_its_own_expanded_build_over_files_that_exist(project, orders_src): """ADR-007 D6 and D13: a cheap entry is a stored plan over files that exist. ``load_session`` makes them exist - first (``ensure_materialized``), so with the clone and every cached copy of the source deleted, the build posted - to Buckaroo still reads files that are there at the moment it is posted.""" + first (``ensure_materialized``), so with every cached copy of the data deleted the build posted to Buckaroo + still reads files that are there at the moment it is posted. + + The clone of the imported bytes under ``data/.cas`` stays. It is what a source version's snapshot is written + again from (ADR-011 D1), so deleting it as well would make the rows unrecoverable instead of exercising the + heal. + """ from xorq.common.utils.graph_utils import walk_nodes from xorq.expr.relations import Read from xorq.ibis_yaml.compiler import load_expr h = build_and_persist(project, _cheap_code(project)).content_hash - shutil.rmtree(data_dir(project) / ".cas", ignore_errors=True) shutil.rmtree(compute_cache_dir(project), ignore_errors=True) reads_at_post: list[Path] = [] @@ -279,7 +283,7 @@ def at_post(body: dict) -> None: @pytest.mark.parametrize("recipe", [_agg_code, _cheap_code], ids=["worthy", "cheap"]) -def test_every_load_expr_body_names_the_row_order_column(project, orders_parquet, recipe): +def test_every_load_expr_body_names_the_row_order_column(project, orders_src, recipe): """ADR-008 D8 (Buckaroo's half is one hint): every ``/load_expr`` payload carries ``row_order_column`` so that Buckaroo can order and page by it (buckaroo-data/buckaroo#974). An ordinary entry sends none of the fields that would defeat Buckaroo's warm-hit short-circuit.""" @@ -300,7 +304,7 @@ def test_every_load_expr_body_names_the_row_order_column(project, orders_parquet # --------------------------------------------------------------------------- -def test_explicit_delete_then_open_heals_and_verifies_before_buckaroo_is_called(project, orders_parquet): +def test_explicit_delete_then_open_heals_and_verifies_before_buckaroo_is_called(project, orders_src): """ADR-007 D12 (files are deleted only by an explicit user action) and D5 (``ensure_materialized``): the Cache page's delete removes the snapshot, and the next open rewrites it and checks it against the recorded digest before the grid is posted. Buckaroo never has to repair a file.""" @@ -328,7 +332,7 @@ def at_post(body: dict) -> None: assert seen["digest"] == read_manifest(entry_dir(project, h)).result_digest -def test_unfaithful_heal_forces_a_reload_of_the_open_grid(project, orders_parquet): +def test_unfaithful_heal_forces_a_reload_of_the_open_grid(project, orders_src): """ADR-007 D6: an unfaithful heal is the one event that leaves an open session wrong, since the path now holds other rows than the stats Buckaroo computed. tallyman keeps no session record to drop, so the companion's hook wipes the entry's stat cache and posts ``/load_expr`` for the derived id with ``force_reload`` set.""" @@ -365,19 +369,20 @@ def test_unfaithful_heal_forces_a_reload_of_the_open_grid(project, orders_parque assert seen == [(True, False)], "the reload must be posted after the stale stats are wiped" -def test_klass_reload_reaches_every_entry_without_a_session_record(project, orders_parquet): +def test_klass_reload_reaches_every_entry_without_a_session_record(project, orders_src): """ADR-007 D6: ``reload_project_sessions`` used to walk tallyman's own record of open sessions, which the design deletes, and Buckaroo has no route that lists sessions. With derived ids it posts ``/reload_expr/`` for every entry of the project and treats Buckaroo's 404 as "not open". An entry that was never opened gets no session.""" + src = get_alias(project, orders_src) # the import's own entry is in the project too, and gets a post as well a = build_and_persist(project, _agg_code(project)).content_hash b = build_and_persist(project, _second_agg_code(project)).content_hash fake = FakeBuckaroo() bk = _manager(fake) - assert bk.load_session(a, project)["status"] == "ok" # A is open in a tab; B never was + assert bk.load_session(a, project)["status"] == "ok" # A is open in a tab; B and the source never were reloaded = bk.reload_project_sessions(project) - assert sorted(fake.reloads()) == sorted([f"/reload_expr/{_sid(project, a)}", f"/reload_expr/{_sid(project, b)}"]) + assert sorted(fake.reloads()) == sorted(f"/reload_expr/{_sid(project, h)}" for h in (src, a, b)) assert reloaded == 1, "only the open grid can have been reloaded" assert _sid(project, b) not in fake.sessions assert not hasattr(bk, "_sessions") diff --git a/tests/test_buckaroo_multi_project.py b/tests/test_buckaroo_multi_project.py index fe244c71..f62adf10 100644 --- a/tests/test_buckaroo_multi_project.py +++ b/tests/test_buckaroo_multi_project.py @@ -16,9 +16,10 @@ from tallyman_cli.fixtures import write_shoe_orders from tallyman_companion.buckaroo_lifecycle import BuckarooManager -from tallyman_core import data_dir, ensure_project +from tallyman_core import data_dir, ensure_project, set_active_project from tallyman_core.paths import artifacts_dir from tallyman_xorq import build_and_persist +from tallyman_xorq.source_import import update_and_depend # --------------------------------------------------------------------------- # Constructor shape @@ -38,10 +39,28 @@ def test_manager_constructor_no_longer_takes_project(isolated_home: Path): # --------------------------------------------------------------------------- +SRC_ALIAS = "orders_src" + + +def _seed_orders(project: str) -> None: + """Write the fixture into *project* and import it as a source alias. + + These tests stand up their own projects rather than using the shared ``project`` fixture, so they + do their own import. A recipe reads the alias, never the file (ADR-011 D1). + + The project is made active first because the recipe ``source_import._recipe`` generates calls + ``read_project_file(path)`` with no project, so the read resolves against whatever project is + active rather than the one passed to ``update_and_depend``. + """ + set_active_project(project) + path = write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=50, seed=1) + update_and_depend(path, SRC_ALIAS, project=project) + + def _entry_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({SRC_ALIAS!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ @@ -52,7 +71,7 @@ def test_ensure_session_takes_project_explicitly(isolated_home: Path, monkeypatc hashes = {} for proj in ("alpha", "beta"): ensure_project(proj) - write_shoe_orders(data_dir(proj) / "orders.parquet", n_rows=50, seed=1) + _seed_orders(proj) hashes[proj] = build_and_persist(proj, _entry_code(proj)).content_hash mgr = BuckarooManager() @@ -90,7 +109,7 @@ def test_no_session_file_is_written_anywhere(isolated_home: Path, monkeypatch): """Negative assertion: nothing gets written to ``~/.tallyman/buckaroo_sessions.json``, nor to a project's ``artifacts/catalog/buckaroo_sessions.json``, nor anywhere else, when sessions are opened (ADR-007 D6).""" ensure_project("alpha") - write_shoe_orders(data_dir("alpha") / "orders.parquet", n_rows=50, seed=1) + _seed_orders("alpha") h = build_and_persist("alpha", _entry_code("alpha")).content_hash mgr = BuckarooManager() diff --git a/tests/test_buckaroo_session.py b/tests/test_buckaroo_session.py index 37e749ff..6e38b610 100644 --- a/tests/test_buckaroo_session.py +++ b/tests/test_buckaroo_session.py @@ -4,8 +4,8 @@ from fastapi.testclient import TestClient from tallyman_companion.buckaroo_lifecycle import BuckarooManager +from tallyman_core import get_alias from tallyman_mcp.server import catalog_create -from tallyman_xorq.build import list_entries # #133: the catalog detail page loads the Buckaroo grid lazily via # GET /api/session/{hash}, which returns a typed status so the SPA can show a @@ -17,8 +17,8 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ @@ -36,10 +36,10 @@ def post(self, *a, **k): raise self._exc -def test_session_endpoint_unavailable_without_buckaroo(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_session_endpoint_unavailable_without_buckaroo(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) - h = list_entries(project)[0]["content_hash"] + h = get_alias(project, "shoe_sales") c = TestClient(fresh_companion_app) # built with no Buckaroo subprocess r = c.get(f"/{project}/api/session/{h}") assert r.status_code == 200 @@ -49,12 +49,12 @@ def test_session_endpoint_unavailable_without_buckaroo(fresh_companion_app, proj assert "Buckaroo" in body["detail"] -def test_entry_detail_does_not_carry_a_session(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_entry_detail_does_not_carry_a_session(fresh_companion_app, project, orders_src, monkeypatch): # The detail request stays metadata-only — the grid loads lazily — so it # never blocks on the snapshot bake + /load_expr POST (the #133 hang). monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) - h = list_entries(project)[0]["content_hash"] + h = get_alias(project, "shoe_sales") c = TestClient(fresh_companion_app) body = c.get(f"/{project}/api/entry/{h}").json() assert body["buckaroo_session"] is None @@ -79,10 +79,10 @@ def test_load_session_no_build_for_unknown_entry(project, monkeypatch): assert res["session_id"] is None -def test_load_session_classifies_timeout_then_error(project, orders_parquet, monkeypatch): +def test_load_session_classifies_timeout_then_error(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) - h = list_entries(project)[0]["content_hash"] + h = get_alias(project, "shoe_sales") bk = BuckarooManager() bk.proc = _AliveProc() diff --git a/tests/test_build.py b/tests/test_build.py index e860b56d..47518a04 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -9,24 +9,24 @@ from tallyman_xorq import BuildError, build_and_persist, list_entries -def _agg_code(parquet_path: Path) -> str: +def _agg_code(src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet_path.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ -def _cheap_code(parquet_path: Path) -> str: # parquet read + projection → cheap, bakes nothing +def _cheap_code(src: str) -> str: # source read + projection → cheap, bakes nothing return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet_path.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.select("region", "price", "__row_order") """ -def test_build_writes_entry(project: str, orders_parquet: Path): - code = _agg_code(orders_parquet) +def test_build_writes_entry(project: str, orders_src: str): + code = _agg_code(orders_src) res = build_and_persist(project, code, prompt="region totals") assert res.row_count == 4 # one row per region assert res.execute_seconds >= 0.0 @@ -69,7 +69,7 @@ def test_migrate_drop_result_parquet_sweeps_legacy(project: str): assert migrate_drop_result_parquet(project) == 0 -def test_build_records_admission_instrumentation(project: str, orders_parquet: Path, caplog): +def test_build_records_admission_instrumentation(project: str, orders_src: str, caplog): # #87: the build records the structural cache_worthy verdict alongside the # measured value-per-byte inputs (compile_seconds, execute_seconds, snapshot # bytes), so the structural-vs-measured cache decision (#30) is decidable @@ -79,7 +79,7 @@ def test_build_records_admission_instrumentation(project: str, orders_parquet: P import logging with caplog.at_level(logging.INFO, logger="tallyman.perf"): - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) m = json.loads((entry_dir(project, res.content_hash) / "manifest.json").read_text()) assert m["compile_seconds"] is not None and m["compile_seconds"] >= 0.0 @@ -96,11 +96,11 @@ def test_build_records_admission_instrumentation(project: str, orders_parquet: P assert any("admission" in r.getMessage() and res.content_hash in r.getMessage() for r in perf) -def test_build_cheap_entry_records_no_snapshot_bytes(project: str, orders_parquet: Path): +def test_build_cheap_entry_records_no_snapshot_bytes(project: str, orders_src: str): # A cheap (projection) entry is structurally not worthy and bakes no # snapshot, so cache_bytes is absent — it materialises nothing to size. The # compile/structural verdict is still recorded for the shadow comparison. - res = build_and_persist(project, _cheap_code(orders_parquet)) + res = build_and_persist(project, _cheap_code(orders_src)) m = json.loads((entry_dir(project, res.content_hash) / "manifest.json").read_text()) assert m["cache_worthy"] is False assert "cheap" in m["cache_worthy_why"] @@ -110,7 +110,7 @@ def test_build_cheap_entry_records_no_snapshot_bytes(project: str, orders_parque assert res.cache_bytes is None -def test_cheap_build_validates_row_projection(project: str): +def test_cheap_build_validates_row_projection(project: str, tmp_path: Path): # A cheap entry (pure projection — no Aggregate/Join/Sort/window/UDF) must # still evaluate its row-level projection at build time. count() is satisfied # from source metadata and prunes the projection, so a failing row-level op @@ -122,57 +122,60 @@ def test_cheap_build_validates_row_projection(project: str): import pyarrow as pa import pyarrow.parquet as pq - from tallyman_core.paths import data_dir + from tallyman_xorq.source_import import update_and_depend - p = data_dir(project) / "bad_dates.parquet" - p.parent.mkdir(parents=True, exist_ok=True) + p = tmp_path / "bad_dates.parquet" # month 20 is out of range — datafusion's date32 cast rejects it at row-eval # time, but count() prunes the cast so it never runs at build unless forced. + # The strings import fine; only the recipe's cast is bad. pq.write_table(pa.table({"d": ["2020-01-01", "2020-20-07", "2021-12-31"]}), p) + update_and_depend(p, "bad_dates", project=project) - def _cast_code(parquet: Path) -> str: + def _cast_code(src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.mutate(dd=t.d.cast("date")) """ with pytest.raises(BuildError): - build_and_persist(project, _cast_code(p)) + build_and_persist(project, _cast_code("bad_dates")) # The streaming validation is also the cheap path's row-count source, so a # valid cheap projection must still build and report the right count. - good = data_dir(project) / "good_dates.parquet" + good = tmp_path / "good_dates.parquet" pq.write_table(pa.table({"d": ["2020-01-01", "2021-12-31"]}), good) - res = build_and_persist(project, _cast_code(good)) + update_and_depend(good, "good_dates", project=project) + res = build_and_persist(project, _cast_code("good_dates")) assert res.cache_worthy is False # pure projection — the cheap path under test assert res.row_count == 2 -def test_build_idempotent_same_code(project: str, orders_parquet: Path): - code = _agg_code(orders_parquet) +def test_build_idempotent_same_code(project: str, orders_src: str): + code = _agg_code(orders_src) a = build_and_persist(project, code, prompt="first") b = build_and_persist(project, code, prompt="second") assert a.content_hash == b.content_hash assert a.entry_path == b.entry_path - assert len(list_entries(project)) == 1 + # Two entries: the source version the fixture imported, and this build over it. + assert len(list_entries(project)) == 2 -def test_build_distinct_code_distinct_hash(project: str, orders_parquet: Path): - code_a = _agg_code(orders_parquet) +def test_build_distinct_code_distinct_hash(project: str, orders_src: str): + code_a = _agg_code(orders_src) code_b = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({orders_parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}) filtered = t.filter(t.category == "boots") expr = filtered.group_by("region").aggregate(n=filtered.count()) """ a = build_and_persist(project, code_a) b = build_and_persist(project, code_b) assert a.content_hash != b.content_hash - assert len(list_entries(project)) == 2 + assert len(list_entries(project)) == 3 # the imported source plus the two builds -def test_build_missing_expr_variable(project: str, orders_parquet: Path): +def test_build_missing_expr_variable(project: str): code = "import xorq.api as xo\nx = 1\n" with pytest.raises(BuildError, match="not found"): build_and_persist(project, code) @@ -188,25 +191,25 @@ def test_list_entries_empty(project: str): assert list_entries(project) == [] -def test_build_hints_bare_ibis_import(project: str, orders_parquet: Path): +def test_build_hints_bare_ibis_import(project: str, orders_src: str): # `import ibis` triggers an AttributeError at user-code exec time because # the real ibis package doesn't expose all the methods xorq does. The # BuildError should include a hint pointing at xorq.vendor.ibis. code = f""" import ibis -from tallyman_xorq.io import read_project_file -t = read_project_file({orders_parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}) expr = t.mutate(flag=ibis.case().when(t.price > 50, "hi").else_("lo").end()) """ with pytest.raises(BuildError, match="import xorq.vendor.ibis as ibis"): build_and_persist(project, code) -def test_build_no_hint_on_unrelated_error(project: str, orders_parquet: Path): +def test_build_no_hint_on_unrelated_error(project: str, orders_src: str): code = f""" import xorq.vendor.ibis as ibis -from tallyman_xorq.io import read_project_file -t = read_project_file({orders_parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}) expr = t.nonexistent_column.sum() """ with pytest.raises(BuildError) as excinfo: @@ -232,7 +235,7 @@ def test_hint_bare_xorq_attribute_points_to_api(): def test_hint_ibis_read_parquet_points_to_loaders(): h = _ibis_import_hint("module 'ibis' has no attribute 'read_parquet'") - assert "read_project_file" in h + assert "catalog_import_source" in h and "tracked_expr_from_alias" in h def test_hint_ibis_math_func_is_column_method(): @@ -242,7 +245,7 @@ def test_hint_ibis_math_func_is_column_method(): def test_hint_io_import_typo_lists_real_exports(): h = _ibis_import_hint("cannot import name 'load_parquet_expr' from 'tallyman_xorq.io'") - assert "read_project_file" in h and "tracked_expr_from_alias" in h + assert "tracked_expr_from_alias" in h and "pinned_expr_from_alias" in h def test_hint_missing_table_column(): @@ -278,25 +281,25 @@ def test_build_hints_bare_xorq_namespace(project: str, orders_parquet: Path): # --------------------------------------------------------------------------- -def _nondeterministic_code(parquet_path: Path) -> str: +def _nondeterministic_code(src: str) -> str: return f""" import xorq.vendor.ibis as ibis -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet_path.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.mutate(built_at=ibis.now()) """ -def test_build_lints_nondeterministic_now(project: str, orders_parquet: Path): - res = build_and_persist(project, _nondeterministic_code(orders_parquet)) +def test_build_lints_nondeterministic_now(project: str, orders_src: str): + res = build_and_persist(project, _nondeterministic_code(orders_src)) joined = " ".join(res.lint_warnings) assert res.lint_warnings, "expected a nondeterminism lint warning" assert "now()" in joined assert "#88" in joined -def test_build_no_lint_for_deterministic_recipe(project: str, orders_parquet: Path): - res = build_and_persist(project, _agg_code(orders_parquet)) +def test_build_no_lint_for_deterministic_recipe(project: str, orders_src: str): + res = build_and_persist(project, _agg_code(orders_src)) assert res.lint_warnings == [] diff --git a/tests/test_cache_files.py b/tests/test_cache_files.py index 61605681..e0a04a61 100644 --- a/tests/test_cache_files.py +++ b/tests/test_cache_files.py @@ -26,16 +26,16 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ def _second_agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("category").aggregate(n=t.count()) """ @@ -44,12 +44,12 @@ def _nonreproducible_code(project: str) -> str: """A recipe whose UDF returns other values on every call: a worthy entry (a UDF always is) that cannot be reproduced, which ADR-009 D6 finds when the entry is created by running its query twice.""" return f""" -from tallyman_xorq.io import read_project_file +from tallyman_xorq.io import tracked_expr_from_alias from xorq.expr.udf import make_pandas_udf import xorq.vendor.ibis.expr.datatypes as dt from xorq.vendor.ibis import schema as ibis_schema -t = read_project_file("orders.parquet", project={project!r}) +t = tracked_expr_from_alias("orders_src", project={project!r}) def jitter(df): @@ -74,7 +74,7 @@ def _cache_rows(client: TestClient, project: str) -> dict[str, dict]: # --------------------------------------------------------------------------- -def test_startup_warm_up_writes_no_file(project, orders_parquet): +def test_startup_warm_up_writes_no_file(project, orders_src): """ADR-007 D12: the warm-up used to call ``cached_result_expr`` for every entry until a 3 s budget was spent, which heals a deleted snapshot. That undoes the Cache page's delete button, and one large heal blocks startup for as long as it takes. With ``compute_cache/`` emptied and no request made, starting the app leaves it empty.""" @@ -92,7 +92,7 @@ def test_startup_warm_up_writes_no_file(project, orders_parquet): assert not snap.exists() -def test_verify_sweep_reports_a_missing_snapshot_as_absent_and_writes_nothing(project, orders_parquet): +def test_verify_sweep_reports_a_missing_snapshot_as_absent_and_writes_nothing(project, orders_src): """ADR-007 D5 and D12: the verify sweep checks the files that exist and reports each entry that recorded a digest as faithful, unfaithful or absent. It is not a caller of ``ensure_materialized``: a sweep that rewrote every deleted snapshot in the project would undo the user's deletes. Every file that function writes is verified before @@ -120,7 +120,7 @@ def test_verify_sweep_reports_a_missing_snapshot_as_absent_and_writes_nothing(pr # --------------------------------------------------------------------------- -def test_cache_page_lists_a_reproducible_entry_as_not_pinned(project, orders_parquet): +def test_cache_page_lists_a_reproducible_entry_as_not_pinned(project, orders_src): """ADR-009 D6: an entry whose query gave the same content digest twice is reproducible, so its file may be deleted and made again. The Cache page says so for each row: ``pinned`` is False and there is no reason.""" h = build_and_persist(project, _agg_code(project)).content_hash @@ -133,7 +133,7 @@ def test_cache_page_lists_a_reproducible_entry_as_not_pinned(project, orders_par assert not row.get("orphan") -def test_a_non_reproducible_entry_is_pinned_and_its_delete_is_refused(project, orders_parquet): +def test_a_non_reproducible_entry_is_pinned_and_its_delete_is_refused(project, orders_src): """ADR-009 D6 and ADR-007 D12: an entry whose recipe is not reproducible is recorded as such when it is created, and its file is never deleted by tallyman, because deleting it would end the only copy of those rows. The Cache page's delete answers 409, says why, and leaves the file.""" @@ -154,7 +154,7 @@ def test_a_non_reproducible_entry_is_pinned_and_its_delete_is_refused(project, o assert snap.exists(), "the delete removed the file of an entry that cannot be recreated" -def test_an_entry_with_an_unfaithful_heal_record_is_pinned(project, orders_parquet): +def test_an_entry_with_an_unfaithful_heal_record_is_pinned(project, orders_src): """ADR-007 D12 and ADR-006 D12 (unfaithful entries are pinned and badged): a heal that wrote different rows than were built leaves an ``unfaithful_heal`` record in ``errors.jsonl``. That record is what pins the entry, so the Cache page's delete refuses it as it refuses an entry that was found not reproducible at creation.""" @@ -172,7 +172,7 @@ def test_an_entry_with_an_unfaithful_heal_record_is_pinned(project, orders_parqu assert snap.exists() -def test_a_snapshot_whose_entry_is_not_in_the_catalog_is_listed_and_can_be_deleted(project, orders_parquet): +def test_a_snapshot_whose_entry_is_not_in_the_catalog_is_listed_and_can_be_deleted(project, orders_src): """ADR-007 D14: a reset no longer prunes ``compute_cache/``, so the snapshot of an entry that a reset retired stays on disk until the user deletes it. The Cache page lists files by entry, so it needs a row for a file whose entry is not in the catalog, and the delete has to accept it.""" @@ -196,7 +196,7 @@ def test_a_snapshot_whose_entry_is_not_in_the_catalog_is_listed_and_can_be_delet assert not orphan.exists() -def test_a_normal_snapshot_can_still_be_deleted_and_the_next_read_re_creates_and_verifies_it(project, orders_parquet): +def test_a_normal_snapshot_can_still_be_deleted_and_the_next_read_re_creates_and_verifies_it(project, orders_src): """ADR-007 D12 and D5: the delete button keeps working for an ordinary entry. The next read rewrites the file and checks it against the recorded digest (no ``unfaithful_heal`` record), and the entry is still not pinned.""" h = build_and_persist(project, _agg_code(project)).content_hash @@ -214,3 +214,60 @@ def test_a_normal_snapshot_can_still_be_deleted_and_the_next_read_re_creates_and assert snapshot_file_digest(snap) == read_manifest(entry_dir(project, h)).result_digest assert not [e for e in list_errors(project) if e.get("code") == "unfaithful_heal"] assert _cache_rows(client, project)[h].get("pinned") is False + + +def test_a_source_snapshot_can_be_deleted_and_the_next_read_re_creates_it_from_the_clone(project, tmp_path): + """ADR-011 D1, revised: a source version's snapshot is cache too. + + The clone of the imported bytes under ``data/.cas`` and the reader options on the entry are everything + ``ensure_materialized`` needs to write the file again, which is ADR-007 D13's test for whether a file is cache. + So the Cache page lists a source row as unpinned, the delete takes it, and the next read makes it again. + """ + from tallyman_xorq import source_import + from tallyman_xorq.materialize import snapshot_path + + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + src = outside / "orders.parquet" + pq.write_table(pa.table({"region": ["e", "w", "e"], "price": [1.0, 2.0, 3.0]}), src) + out = source_import.update_and_depend(str(src), "orders", project=project) + snap = snapshot_path(project, out["hash"]) + assert snap.exists() + client = TestClient(create_app(project)) + + assert _cache_rows(client, project)[out["hash"]].get("pinned") is False + + response = client.delete(f"/{project}/api/result_cache/{out['hash']}") + assert response.status_code == 200, response.text + assert not snap.exists() + + cached_result_expr.cache_clear() + assert len(cached_result_expr(project, out["hash"]).execute()) == 3 + assert snap.exists() + assert snapshot_file_digest(snap) == read_manifest(entry_dir(project, out["hash"])).result_digest + assert not [e for e in list_errors(project) if e.get("code") == "unfaithful_heal"] + + +def test_a_source_snapshot_whose_clone_is_gone_is_pinned_and_its_delete_is_refused(project, tmp_path): + """The one case where a source snapshot cannot be made again: the clone of the imported bytes is gone too.""" + from tallyman_core.paths import data_dir + from tallyman_xorq import source_import + from tallyman_xorq.materialize import snapshot_path + + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + src = outside / "orders.parquet" + pq.write_table(pa.table({"region": ["e", "w", "e"], "price": [1.0, 2.0, 3.0]}), src) + out = source_import.update_and_depend(str(src), "orders", project=project) + (data_dir(project) / ".cas" / f"{out['digest']}.parquet").unlink() + client = TestClient(create_app(project)) + + row = _cache_rows(client, project)[out["hash"]] + assert row.get("pinned") is True, row + assert "orders-v1" in row.get("pinned_reason", ""), row + assert ".cas" in row.get("pinned_reason", ""), row # pinned by the lost clone, not by being a source + + response = client.delete(f"/{project}/api/result_cache/{out['hash']}") + + assert response.status_code == 409, response.text + assert snapshot_path(project, out["hash"]).exists() diff --git a/tests/test_cache_inspector.py b/tests/test_cache_inspector.py index e5c90847..a563f3a5 100644 --- a/tests/test_cache_inspector.py +++ b/tests/test_cache_inspector.py @@ -22,16 +22,16 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ def _filter_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) f = t.filter(t.category == "boots") expr = f.group_by("region").aggregate(total=f.price.sum(), n=f.count()) """ @@ -39,8 +39,8 @@ def _filter_code(project: str) -> str: def _cheap_code(project: str) -> str: # projection → cheap, bakes no snapshot return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ @@ -53,7 +53,7 @@ def _child_code(project: str, parent: str) -> str: # tracked_expr_from_alias """ -def test_cache_pane_lists_expensive_not_cheap(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_cache_pane_lists_expensive_not_cheap(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) # expensive → baked snapshot catalog_revise("shoe_sales", _filter_code(project)) # still expensive @@ -84,7 +84,7 @@ def test_cache_pane_lists_expensive_not_cheap(fresh_companion_app, project, orde assert cheap_h not in listed -def test_cache_pane_drops_evicted_snapshot(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_cache_pane_drops_evicted_snapshot(fresh_companion_app, project, orders_src, monkeypatch): # The page reflects what's on disk. After a snapshot is deleted — and not yet # re-viewed, so nothing re-baked it — the entry must drop out of the listing, # not linger as a phantom "cached" row. Keying the listing on manifest.cache_bytes @@ -112,7 +112,7 @@ def test_cache_pane_drops_evicted_snapshot(fresh_companion_app, project, orders_ assert after["total"] == before["total"] - row_size -def test_cache_delete_removes_only_snapshot(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_cache_delete_removes_only_snapshot(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) # expensive → baked snapshot h = list_entries(project)[0]["content_hash"] @@ -137,7 +137,7 @@ def test_cache_delete_removes_only_snapshot(fresh_companion_app, project, orders assert h in history_for(project, "shoe_sales") # alias history intact -def test_cache_delete_then_read_self_heals_warm_cache(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_cache_delete_then_read_self_heals_warm_cache(fresh_companion_app, project, orders_src, monkeypatch): # Deleting a snapshot must self-heal on the next read even when the read path's # cached_result_expr was already warm for that entry. Since ee0a90a the read # wrapper re-checks path.exists() on every call and self-heals regardless of the @@ -161,7 +161,7 @@ def test_cache_delete_then_read_self_heals_warm_cache(fresh_companion_app, proje assert baked_snapshot_path(project, h).exists() # re-baked -def test_cache_delete_missing_snapshot_404(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_cache_delete_missing_snapshot_404(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) h = list_entries(project)[0]["content_hash"] @@ -171,7 +171,7 @@ def test_cache_delete_missing_snapshot_404(fresh_companion_app, project, orders_ assert c.delete(f"/{project}/api/result_cache/{h}").status_code == 404 -def test_cache_delete_blocked_in_read_only(project, orders_parquet, monkeypatch): +def test_cache_delete_blocked_in_read_only(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) h = list_entries(project)[0]["content_hash"] @@ -184,7 +184,7 @@ def test_cache_delete_blocked_in_read_only(project, orders_parquet, monkeypatch) assert snap.exists() # untouched -def test_cache_delete_rejects_non_hex_hash(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_cache_delete_rejects_non_hex_hash(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) c = TestClient(fresh_companion_app) # A content hash is lowercase hex (it names an entry dir). A junk value can @@ -205,7 +205,7 @@ def test_cache_delete_rejects_non_hex_hash(fresh_companion_app, project, orders_ # (#132); the rendering is the SPA's job, this pins the JSON contract. -def test_entry_cache_expensive_breakdown(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_entry_cache_expensive_breakdown(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) # expensive → baked snapshot h = list_entries(project)[0]["content_hash"] @@ -235,7 +235,7 @@ def test_entry_cache_expensive_breakdown(fresh_companion_app, project, orders_pa assert body["total_formatted"] -def test_entry_cache_cheap_marks_snapshot_inapplicable(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_entry_cache_cheap_marks_snapshot_inapplicable(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) cheap = catalog_create("cheap_one", _cheap_code(project)) # projection → no snapshot h = cheap["hash"] @@ -251,7 +251,7 @@ def test_entry_cache_cheap_marks_snapshot_inapplicable(fresh_companion_app, proj assert any(comp["key"] == "build" and comp["bytes"] > 0 for comp in body["components"]) -def test_entry_cache_resolves_alias(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_entry_cache_resolves_alias(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) h = list_entries(project)[0]["content_hash"] @@ -262,7 +262,7 @@ def test_entry_cache_resolves_alias(fresh_companion_app, project, orders_parquet assert by_alias.json()["content_hash"] == h # alias resolved to its current head -def test_entry_cache_unknown_hash_404_malformed_400(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_entry_cache_unknown_hash_404_malformed_400(fresh_companion_app, project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) c = TestClient(fresh_companion_app) # Well-formed hex, no such entry → 404 (not the SPA index.html fallthrough). @@ -271,7 +271,7 @@ def test_entry_cache_unknown_hash_404_malformed_400(fresh_companion_app, project assert c.get(f"/{project}/api/entry_cache/NOPE-not-a-hash").status_code == 400 -def test_entry_cache_lineage_and_enrichment(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_entry_cache_lineage_and_enrichment(fresh_companion_app, project, orders_src, monkeypatch): # #134: the metadata tab links the entry's parents and dependent children and # carries source-size / last-modified / cheap-vs-expensive enrichment. monkeypatch.setenv("TALLYMAN_PROJECT", project) diff --git a/tests/test_cache_lab.py b/tests/test_cache_lab.py index e792a225..28049c16 100644 --- a/tests/test_cache_lab.py +++ b/tests/test_cache_lab.py @@ -65,9 +65,11 @@ from tallyman_core import catalog_state as cs from tallyman_core import paths +from tallyman_core.aliases import set_alias from tallyman_core.paths import data_dir, ensure_project, entry_dir, set_active_project from tallyman_xorq.build import build_and_persist from tallyman_xorq.result_cache import cached_result_expr +from tallyman_xorq.source_import import update_and_depend from tests.cache_lab_data import ( N_STATIONS, capacity_frame, @@ -82,7 +84,6 @@ TRIPS_ROWS = int(os.environ.get("CACHE_LAB_TRIPS", "150000")) CHAIN_DEPTH = int(os.environ.get("CACHE_LAB_CHAIN", "32")) -IDENTITY_MODE = os.environ.get("TALLYMAN_SOURCE_IDENTITY", "off") REPORT_DIR = Path(__file__).resolve().parent / "cache_lab_reports" @@ -115,12 +116,11 @@ def write(self) -> Path: "platform": platform.platform(), "xorq": version("xorq"), "xorq_dasher": version("xorq-dasher"), - "source_identity_mode": IDENTITY_MODE, }, "scenarios": self.scenarios, } REPORT_DIR.mkdir(exist_ok=True) - out = REPORT_DIR / f"{rev}-{IDENTITY_MODE}-{self.started.replace(':', '')}.json" + out = REPORT_DIR / f"{rev}-{self.started.replace(':', '')}.json" out.write_text(json.dumps(doc, indent=2, default=str)) print(f"\n=== cache lab report: {out} ===") for name, rec in self.scenarios.items(): @@ -171,10 +171,11 @@ def _warm_xorq_once(staged_data: Path) -> None: dd = data_dir("lab-warmup") dd.mkdir(parents=True, exist_ok=True) shutil.copy2(staged_data / "stations.parquet", dd / "stations.parquet") + update_and_depend(dd / "stations.parquet", "stations_src", project="lab-warmup") build_and_persist( "lab-warmup", _prelude("lab-warmup") - + 'expr = read_project_file("stations.parquet", project=_P).select("station_id", "__row_order")\n', + + 'expr = tracked_expr_from_alias("stations_src", project=_P).select("station_id", "__row_order")\n', ) _WARMED = True @@ -190,6 +191,9 @@ def lab(staged_data: Path, isolated_home: Path, request, lab_report): dd.mkdir(parents=True, exist_ok=True) for f in staged_data.iterdir(): shutil.copy2(f, dd / f.name) + # A recipe never opens a file (ADR-011 D2): each staged parquet enters the project as a + # source alias, "_src", and the scenario codes read that. + update_and_depend(dd / f.name, f"{f.stem}_src", project=name) rec = lab_report.scenario(request.node.name.removeprefix("test_")) class Lab: @@ -293,7 +297,7 @@ def md5_file(path: Path) -> str: def _prelude(project: str) -> str: return ( - "from tallyman_xorq.io import read_project_file, tracked_expr_from_alias, pinned_expr_from_alias\n" + "from tallyman_xorq.io import tracked_expr_from_alias, pinned_expr_from_alias\n" f"import xorq.vendor.ibis as ibis\n_P = {project!r}\n" ) @@ -308,8 +312,8 @@ def treadmill_code(project: str, edit: int) -> str: """ parts = [ _prelude(project), - 'trips = read_project_file("trips.parquet", project=_P)\n' - 'stations = read_project_file("stations.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' + 'stations = tracked_expr_from_alias("stations_src", project=_P)\n' "t = trips.filter(trips.start_station_id != trips.end_station_id)\n" "j = t.join(stations, t.start_station_id == stations.station_id)\n" "j = j.mutate(trip_minutes=(j.ended_at.cast('int64') - j.started_at.cast('int64')) / 60_000_000)\n", @@ -352,8 +356,8 @@ def flow_imbalance_code(project: str) -> str: lineage, tiny (~600 row) result. The high-value cache archetype. """ return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' - 'stations = read_project_file("stations.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' + 'stations = tracked_expr_from_alias("stations_src", project=_P)\n' "t = trips.mutate(day=trips.started_at.truncate('D'))\n" "dep = t.group_by(['start_station_id', 'day']).aggregate(departures=t.ride_id.count())\n" "arr = t.group_by(['end_station_id', 'day']).aggregate(arrivals=t.ride_id.count())\n" @@ -369,8 +373,8 @@ def flow_imbalance_code(project: str) -> str: def weather_mix_code(project: str) -> str: """Trips joined to hourly weather on the truncated hour, bucketed.""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' - 'wx = read_project_file("weather.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' + 'wx = tracked_expr_from_alias("weather_src", project=_P)\n' "t = trips.mutate(obs=trips.started_at.truncate('h'))\n" "j = t.join(wx, t.obs == wx.obs_hour)\n" "j = j.mutate(minutes=(j.ended_at.cast('int64') - j.started_at.cast('int64')) / 60_000_000)\n" @@ -389,7 +393,7 @@ def od_matrix_code(project: str, order: str = "name") -> str: "none": "expr = od\n", }[order] return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "t = trips.filter(trips.start_station_id != trips.end_station_id)\n" "t = t.mutate(minutes=(t.ended_at.cast('int64') - t.started_at.cast('int64')) / 60_000_000)\n" "od = t.group_by(['start_station_name', 'end_station_name']).aggregate(\n" @@ -400,7 +404,7 @@ def od_matrix_code(project: str, order: str = "name") -> str: def member_count_code(project: str) -> str: """Smallest honest aggregate; used where the scenario isn't about the expr.""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "expr = trips.group_by('member_casual').aggregate(n=trips.ride_id.count())\n" ) @@ -408,8 +412,8 @@ def member_count_code(project: str) -> str: def utilization_code(project: str) -> str: """Trips-per-dock against the fixed-width capacity table (swap scenario).""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' - 'cap = read_project_file("capacity.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' + 'cap = tracked_expr_from_alias("capacity_src", project=_P)\n' "g = trips.group_by('start_station_id').aggregate(n_starts=trips.ride_id.count())\n" "j = g.join(cap, g.start_station_id == cap.station_id)\n" "expr = j.select('start_station_id', 'n_starts', 'capacity',\n" @@ -420,7 +424,7 @@ def utilization_code(project: str) -> str: def cleaned_trips_code(project: str) -> str: """Parent of a tracked_expr_from_alias chain: filter + projection only (no join/agg/sort).""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "t = trips.filter((trips.start_station_id != trips.end_station_id)\n" " & (trips.ended_at > trips.started_at))\n" "expr = t.select('ride_id', 'rideable_type', 'started_at', 'ended_at',\n" @@ -428,9 +432,9 @@ def cleaned_trips_code(project: str) -> str: ) -def child_of_catalog_code(project: str, parent_hash: str) -> str: +def child_of_catalog_code(project: str, parent_ref: str) -> str: return _prelude(project) + ( - f"base = pinned_expr_from_alias({parent_hash!r}, project=_P)\n" + f"base = pinned_expr_from_alias({parent_ref!r}, project=_P)\n" "base = base.mutate(minutes=(base.ended_at.cast('int64') - base.started_at.cast('int64')) / 60_000_000)\n" "expr = base.group_by(['start_station_id', 'rideable_type']).aggregate(\n" " n=base.ride_id.count(), avg_minutes=base.minutes.mean())\n" @@ -444,7 +448,7 @@ def wide_compile_code(project: str, depth: int) -> str: in a loop without thinking about plan size. """ return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "f = trips.mutate(minutes=(trips.ended_at.cast('int64') - trips.started_at.cast('int64')) / 60_000_000,\n" " hour=trips.started_at.hour())\n" "for h in range(24):\n" @@ -468,7 +472,7 @@ def dock_contention_code(project: str) -> str: and the result is one row per station. """ return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "week = trips.filter(trips.started_at < ibis.timestamp('2025-09-08 00:00:00'))\n" "a = week.select(sid='start_station_id', ride='ride_id', ts='started_at')\n" "b = week.view().select(sid2='start_station_id', ride2='ride_id', ts2='started_at')\n" @@ -482,8 +486,8 @@ def event_annotate_code(project: str) -> str: """'Annotate each trip with its start station's maintenance events' — reads as an enrichment, is actually a row multiplier (~8 events/station).""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' - 'ev = read_project_file("events.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' + 'ev = tracked_expr_from_alias("events_src", project=_P)\n' "j = trips.join(ev, trips.start_station_id == ev.station_id)\n" "expr = j.select('ride_id', 'start_station_id', 'started_at', 'member_casual',\n" " 'event_type', 'event_date', 'crew')\n" @@ -493,7 +497,7 @@ def event_annotate_code(project: str) -> str: def groups_galore_code(project: str) -> str: """An Aggregate that barely reduces: group keys ≈ row identity.""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "t = trips.mutate(day=trips.started_at.truncate('D'), hour=trips.started_at.hour(),\n" " minutes=(trips.ended_at.cast('int64') - trips.started_at.cast('int64')) / 60_000_000)\n" "expr = t.group_by(['start_station_id', 'end_station_id', 'day', 'hour']).aggregate(\n" @@ -504,7 +508,7 @@ def groups_galore_code(project: str) -> str: def label_balloon_code(project: str) -> str: """Row-wise string assembly: output bytes balloon past the input's.""" return _prelude(project) + ( - 'trips = read_project_file("trips.parquet", project=_P)\n' + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' "t = trips.mutate(minutes=(trips.ended_at.cast('int64') - trips.started_at.cast('int64')) / 60_000_000)\n" "t = t.mutate(route_label=t.start_station_name.concat(' -> ').concat(t.end_station_name))\n" "t = t.mutate(trip_summary=t.member_casual.concat(' ').concat(t.rideable_type)\n" @@ -515,7 +519,8 @@ def label_balloon_code(project: str) -> str: def distinct_tiny_code(project: str) -> str: return _prelude(project) + ( - "trips = read_project_file(\"trips.parquet\", project=_P)\nexpr = trips.select('rideable_type').distinct()\n" + 'trips = tracked_expr_from_alias("trips_src", project=_P)\n' + "expr = trips.select('rideable_type').distinct()\n" ) @@ -570,8 +575,11 @@ def test_agg_views(lab): def test_touch_identity(lab): - """Byte-identical rewrite of the source (new inode + mtime). Strategy - differentiator: stat-keyed identity forks, content-keyed identity holds.""" + """A byte-identical rewrite of the outside file (new inode + mtime) must change nothing. + + Identity is the bytes and the reader options, so re-importing the same content is a no-op: the + alias stays where it is, no version is minted, and the entry over it keeps its hash. A stat-keyed + identity would fork here and manufacture a duplicate entry.""" code = od_matrix_code(lab.project) b1 = build(lab.project, code) @@ -579,18 +587,22 @@ def test_touch_identity(lab): tmp = src.with_suffix(".rewrite") shutil.copyfile(src, tmp) os.replace(tmp, src) # same bytes, fresh inode and mtime + again = update_and_depend(src, "trips_src", project=lab.project) b2 = build(lab.project, code) lab.record.update( { "hash_before": b1["hash"], "hash_after": b2["hash"], + "reimport_minted_a_version": again["created"], "identity_stable_on_identical_rewrite": b1["hash"] == b2["hash"], "duplicate_entries_created": int(b1["hash"] != b2["hash"]), "build_wall_s": [b1["wall_s"], b2["wall_s"]], "final_disk": disk(lab.project), } ) + assert again["created"] is False and again["version"] == 1 + assert b1["hash"] == b2["hash"], "an identical rewrite forked the entry" # Invariant either way: both entries hold the same logical result. a = cached_result_expr(lab.project, b1["hash"]).execute() b = cached_result_expr(lab.project, b2["hash"]).execute() @@ -603,8 +615,10 @@ def test_touch_identity(lab): def test_append_invalidates(lab, staged_data): - """A real content change MUST fork identity and refresh results — hard - invariant for any strategy.""" + """A real content change MUST mint the next source version and refresh results — a hard invariant. + + The edit reaches the catalog through the import, not through the build: the file on disk is + provenance, and re-importing it is what advances ``trips_src`` to v2.""" code = member_count_code(lab.project) b1 = build(lab.project, code) n_before = cached_result_expr(lab.project, b1["hash"]).execute()["n"].sum() @@ -614,6 +628,8 @@ def test_append_invalidates(lab, staged_data): max(TRIPS_ROWS // 20, 500), pd.read_parquet(lab.data / "stations.parquet"), np.random.default_rng(99) ) pd.concat([pd.read_parquet(src), extra], ignore_index=True).to_parquet(src) + v2 = update_and_depend(src, "trips_src", project=lab.project) + assert v2["created"] is True and v2["version"] == 2, v2 b2 = build(lab.project, code) n_after = cached_result_expr(lab.project, b2["hash"]).execute()["n"].sum() @@ -630,9 +646,11 @@ def test_append_invalidates(lab, staged_data): def test_stat_swap(lab): - """The nastiest invalidation case: replace content while preserving - (mtime, size, inode). A stat identity keeps the old hash and silently - serves stale bytes; a content identity forks. Recorded, not asserted.""" + """The nastiest invalidation case: replace content while preserving (mtime, size, inode). + + A stat-keyed identity keeps the old hash and silently serves stale bytes. There is no stat memo + left to defeat (ADR-011 D6 deleted it with ``source_digests.json``): an import digests the bytes + every time, so the swap is seen and mints v2. Asserted now, where it used to be recorded.""" code = utilization_code(lab.project) b1 = build(lab.project, code) stored_before = cached_result_expr(lab.project, b1["hash"]).execute() @@ -657,30 +675,40 @@ def test_stat_swap(lab): st_before.st_ino, ), "test rig failed to preserve the stat triple" + v2 = update_and_depend(cap_path, "capacity_src", project=lab.project) b2 = build(lab.project, code) detected = b2["hash"] != b1["hash"] - lab.record.update({"hash_before": b1["hash"], "hash_after": b2["hash"], "content_swap_detected": detected}) - if not detected: - # Same identity ⇒ the dedup path returned the OLD entry. Show whether - # its stored bytes now disagree with an honest recompute. - fresh = cached_result_expr(lab.project, b1["hash"]).execute() - merged = stored_before.merge(fresh, on="start_station_id", suffixes=("_stale", "_fresh")) - lab.record["stale_rows_served"] = int((merged.capacity_stale != merged.capacity_fresh).sum()) - else: - fresh = cached_result_expr(lab.project, b2["hash"]).execute() - assert ( - fresh.sort_values("start_station_id").capacity.to_numpy() - == swapped.sort_values("station_id").capacity.to_numpy() - ).all() + lab.record.update( + { + "hash_before": b1["hash"], + "hash_after": b2["hash"], + "content_swap_detected": detected, + "reimport_version": v2["version"], + } + ) + assert v2["created"] is True and v2["version"] == 2, "a stat-preserving content swap went unseen" + assert detected, "the entry over the swapped source kept its hash" + fresh = cached_result_expr(lab.project, b2["hash"]).execute() + assert ( + fresh.sort_values("start_station_id").capacity.to_numpy() + == swapped.sort_values("station_id").capacity.to_numpy() + ).all() + # The v1 entry still serves the bytes it was built from; that is what the clone is for. + kept = cached_result_expr(lab.project, b1["hash"]).execute() + pd.testing.assert_frame_equal( + kept.sort_values("start_station_id").reset_index(drop=True), + stored_before.sort_values("start_station_id").reset_index(drop=True), + check_dtype=False, + ) def test_selfheal_after_edit(lab): - """An entry's materializations are evicted AFTER the user edits the - source file in place. Self-heal recomputes from the persisted build: - does the old entry get back the data it was built from, or silently - absorb the new data under the old identity? A content-addressed - snapshot (cas) stays faithful; a plain data-path read recomputes over - the new bytes. Recorded, not asserted — it differentiates strategies.""" + """An entry's materializations are evicted AFTER the user edits the outside file in place. + + The heal must get back the data the entry was built from. It does, and not by being careful: the + edited file is provenance and is never opened again (ADR-011 D2), so the heal reads the source + entry's snapshot, or writes it again from the clone of the bytes that were imported. Asserted + now, where it used to be recorded as a strategy difference.""" code = od_matrix_code(lab.project) b1 = build(lab.project, code) before = cached_result_expr(lab.project, b1["hash"]).execute() @@ -705,7 +733,9 @@ def test_selfheal_after_edit(lab): "selfheal_faithful_after_source_edit": int(healed.n_trips.sum()) == int(before.n_trips.sum()), } ) - assert len(healed) > 0 # self-heal must at least produce a readable result + assert int(healed.n_trips.sum()) == int(before.n_trips.sum()), ( + "the heal absorbed the edited file; an imported version's rows do not move" + ) def test_reset_roundtrip(lab): @@ -764,7 +794,9 @@ def test_parent_regen(lab): import tempfile parent = build(lab.project, cleaned_trips_code(lab.project)) - child_code = child_of_catalog_code(lab.project, parent["hash"]) + # A recipe pins by version reference, never by bare hash (ADR-011 D5), so the parent needs a name first. + set_alias(lab.project, "lab_parent", parent["hash"]) + child_code = child_of_catalog_code(lab.project, "lab_parent-v1") c1 = build(lab.project, child_code) # The parent (cleaned_trips: filter+projection) is cheap, so it bakes no diff --git a/tests/test_charts.py b/tests/test_charts.py index 69d424f9..ba4e8c2b 100644 --- a/tests/test_charts.py +++ b/tests/test_charts.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -from pathlib import Path import pytest from fastapi.testclient import TestClient @@ -21,8 +20,8 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ @@ -75,7 +74,7 @@ def test_remove_chart(project: str): # --------------------------------------------------------------------------- -def test_catalog_chart_by_hash(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_chart_by_hash(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) res = build_and_persist(project, _agg_code(project)) out = catalog_chart(res.content_hash, SAMPLE_SPEC) @@ -84,7 +83,7 @@ def test_catalog_chart_by_hash(project: str, orders_parquet: Path, monkeypatch): assert get_chart(project, res.content_hash) is not None -def test_catalog_chart_by_alias(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_chart_by_alias(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) out = catalog_chart("shoe_sales", SAMPLE_SPEC) @@ -101,7 +100,7 @@ def test_catalog_chart_rejects_missing(project: str, monkeypatch): assert "error" in out -def test_catalog_chart_invalid_spec(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_chart_invalid_spec(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) res = build_and_persist(project, _agg_code(project)) out = catalog_chart(res.content_hash, "{not json}") @@ -113,7 +112,7 @@ def test_catalog_chart_invalid_spec(project: str, orders_parquet: Path, monkeypa # --------------------------------------------------------------------------- -def test_entry_detail_renders_chart_when_present(fresh_companion_app, project: str, orders_parquet: Path): +def test_entry_detail_renders_chart_when_present(fresh_companion_app, project: str, orders_src: str): """Entry API includes chart_spec when a chart has been attached.""" res = build_and_persist(project, _agg_code(project)) set_chart(project, res.content_hash, SAMPLE_SPEC) @@ -125,7 +124,7 @@ def test_entry_detail_renders_chart_when_present(fresh_companion_app, project: s assert body["chart_spec"]["mark"] == "bar" -def test_entry_detail_omits_chart_when_absent(fresh_companion_app, project: str, orders_parquet: Path): +def test_entry_detail_omits_chart_when_absent(fresh_companion_app, project: str, orders_src: str): """Entry API returns null chart_spec when no chart is attached.""" res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) @@ -134,7 +133,7 @@ def test_entry_detail_omits_chart_when_absent(fresh_companion_app, project: str, assert r.json()["chart_spec"] is None -def test_api_data_endpoint(fresh_companion_app, project: str, orders_parquet: Path): +def test_api_data_endpoint(fresh_companion_app, project: str, orders_src: str): res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/data/{res.content_hash}") @@ -151,7 +150,7 @@ def test_api_data_404_on_missing(fresh_companion_app, project: str): assert c.get(f"/{project}/api/data/deadbeef").status_code == 404 -def test_api_data_limit_truncates(fresh_companion_app, project: str, orders_parquet: Path): +def test_api_data_limit_truncates(fresh_companion_app, project: str, orders_src: str): """Sanity-check that limit is plumbed through.""" res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) @@ -162,7 +161,7 @@ def test_api_data_limit_truncates(fresh_companion_app, project: str, orders_parq assert body["limit"] == 2 -def test_api_data_offset_paginates(fresh_companion_app, project: str, orders_parquet: Path): +def test_api_data_offset_paginates(fresh_companion_app, project: str, orders_src: str): """T-20: offset+limit cursor pagination.""" res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) @@ -174,20 +173,20 @@ def test_api_data_offset_paginates(fresh_companion_app, project: str, orders_par assert a["offset"] == 0 and b["offset"] == 2 -def test_api_data_rejects_negative_limit(fresh_companion_app, project: str, orders_parquet: Path): +def test_api_data_rejects_negative_limit(fresh_companion_app, project: str, orders_src: str): res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/data/{res.content_hash}?limit=-1") assert r.status_code == 400 -def test_api_data_default_limit_is_200(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_api_data_default_limit_is_200(fresh_companion_app, project: str, orders_src: str, monkeypatch): """Defaults are demo-safe: aggregate fits, raw 2k-row loads cap at 200.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) # Use a passthrough expression (raw orders, 200 rows in the fixture). code = """ -from tallyman_xorq.io import read_project_file -expr = read_project_file("orders.parquet") +from tallyman_xorq.io import tracked_expr_from_alias +expr = tracked_expr_from_alias("orders_src") """ res = build_and_persist(project, code) c = TestClient(fresh_companion_app) @@ -205,7 +204,7 @@ def test_api_data_default_limit_is_200(fresh_companion_app, project: str, orders # --------------------------------------------------------------------------- -def test_api_chart_error_records_error(fresh_companion_app, project: str, orders_parquet: Path): +def test_api_chart_error_records_error(fresh_companion_app, project: str, orders_src: str): res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) r = c.post( @@ -229,13 +228,13 @@ def test_api_chart_error_rejects_malformed_hash(fresh_companion_app, project: st assert r.status_code == 400 -def test_catalog_chart_errors_empty_when_none_reported(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_chart_errors_empty_when_none_reported(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) res = build_and_persist(project, _agg_code(project)) assert catalog_chart_errors(res.content_hash)["items"] == [] -def test_catalog_chart_errors_filters_by_hash_and_tool(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_chart_errors_filters_by_hash_and_tool(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) res = build_and_persist(project, _agg_code(project)) from tallyman_core import record_error @@ -249,7 +248,7 @@ def test_catalog_chart_errors_filters_by_hash_and_tool(project: str, orders_parq assert out[0]["message"] == "render blew up" -def test_catalog_chart_errors_resolves_alias(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_chart_errors_resolves_alias(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) from tallyman_core import get_alias, record_error diff --git a/tests/test_code_edit.py b/tests/test_code_edit.py index b127a1f1..e0352b18 100644 --- a/tests/test_code_edit.py +++ b/tests/test_code_edit.py @@ -2,8 +2,6 @@ from __future__ import annotations -from pathlib import Path - from fastapi.testclient import TestClient from tallyman_core import get_alias, history_for @@ -12,16 +10,16 @@ def _agg(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ def _filter(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) f = t.filter(t.category == "boots") expr = f.group_by("region").aggregate(n=f.count()) """ @@ -29,14 +27,14 @@ def _filter(project: str) -> str: def _nondeterministic(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file +from tallyman_xorq.io import tracked_expr_from_alias import xorq.vendor.ibis as ibis -t = read_project_file("orders.parquet", project={project!r}) +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.mutate(built_at=ibis.now()) """ -def test_put_code_revises_alias(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_put_code_revises_alias(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg(project)) v1_hash = get_alias(project, "shoe_sales") @@ -54,7 +52,7 @@ def test_put_code_revises_alias(fresh_companion_app, project: str, orders_parque assert history_for(project, "shoe_sales") == [v1_hash, body["hash"]] -def test_put_code_carries_chart_and_display(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_put_code_carries_chart_and_display(fresh_companion_app, project: str, orders_src: str, monkeypatch): """A code-edit revise (PUT /api/code) seeds the new version's per-entry chart + display config from the previous one, same as catalog_revise.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -78,7 +76,7 @@ def test_put_code_carries_chart_and_display(fresh_companion_app, project: str, o assert set(r.json().get("carried_over", [])) == {"chart", "display_config"} -def test_put_code_surfaces_nondeterminism_lint(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_put_code_surfaces_nondeterminism_lint(fresh_companion_app, project: str, orders_src: str, monkeypatch): # An execution-nondeterministic recipe (now()) builds fine, but the PUT # /api/code reply must carry the advisory lint so the SPA can show it (#88). monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -97,7 +95,7 @@ def test_put_code_missing_alias_404(fresh_companion_app, project: str): assert r.status_code == 404 -def test_put_code_empty_400(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_put_code_empty_400(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg(project)) c = TestClient(fresh_companion_app) @@ -105,7 +103,7 @@ def test_put_code_empty_400(fresh_companion_app, project: str, orders_parquet: P assert r.status_code == 400 -def test_put_code_build_error_records_and_400s(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_put_code_build_error_records_and_400s(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg(project)) c = TestClient(fresh_companion_app) @@ -119,7 +117,7 @@ def test_put_code_build_error_records_and_400s(fresh_companion_app, project: str assert rec["tool"] == "api_code" -def test_put_code_serve_mode_403(project: str, orders_parquet: Path, monkeypatch): +def test_put_code_serve_mode_403(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg(project)) from tallyman_companion import create_app @@ -130,7 +128,7 @@ def test_put_code_serve_mode_403(project: str, orders_parquet: Path, monkeypatch assert r.status_code == 403 -def test_entry_detail_has_alias_for_named(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_entry_detail_has_alias_for_named(fresh_companion_app, project: str, orders_src: str, monkeypatch): """Named entries include alias and code in the JSON API response.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) out = catalog_create("shoe_sales", _agg(project)) @@ -142,7 +140,7 @@ def test_entry_detail_has_alias_for_named(fresh_companion_app, project: str, ord assert body["code"].strip() != "" -def test_entry_detail_no_alias_for_scratch(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_entry_detail_no_alias_for_scratch(fresh_companion_app, project: str, orders_src: str, monkeypatch): """Scratch entries have no alias in the JSON API response.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_run @@ -154,7 +152,7 @@ def test_entry_detail_no_alias_for_scratch(fresh_companion_app, project: str, or assert r.json()["alias"] is None -def test_entry_detail_omits_edit_button_in_serve_mode(project: str, orders_parquet: Path, monkeypatch): +def test_entry_detail_omits_edit_button_in_serve_mode(project: str, orders_src: str, monkeypatch): """read_only mode returns 403 on PUT /api/code/.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg(project)) @@ -164,3 +162,26 @@ def test_entry_detail_omits_edit_button_in_serve_mode(project: str, orders_parqu c = TestClient(app) r = c.put(f"/{project}/api/code/shoe_sales", json={"code": _filter(project)}) assert r.status_code == 403 + + +def test_put_code_refuses_a_source_alias(fresh_companion_app, project: str, orders_src: str, monkeypatch): + """A source alias's versions are imported files with no recipe to revise (ADR-011 D1), so the route refuses. + + It refuses before building, as ``catalog_revise`` does. It used to build the entry, fail to point the source + alias at it, and answer 500 with the entry left behind under no alias. + """ + from tallyman_core import entry_dir + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + catalog_create("shoe_sales", _agg(project)) + head = get_alias(project, orders_src) + entries = sorted(p.name for p in entry_dir(project, head).parent.iterdir()) + code = "from tallyman_xorq.io import tracked_expr_from_alias\nexpr = tracked_expr_from_alias('shoe_sales')\n" + + c = TestClient(fresh_companion_app, raise_server_exceptions=False) + r = c.put(f"/{project}/api/code/{orders_src}", json={"code": code}) + + assert r.status_code == 409, r.text + assert "catalog_import_source" in r.json()["detail"] + assert get_alias(project, orders_src) == head + assert sorted(p.name for p in entry_dir(project, head).parent.iterdir()) == entries diff --git a/tests/test_companion.py b/tests/test_companion.py index 2c67dc37..6f25bfac 100644 --- a/tests/test_companion.py +++ b/tests/test_companion.py @@ -1,18 +1,16 @@ from __future__ import annotations -from pathlib import Path - from fastapi.testclient import TestClient from tallyman_core import entry_dir from tallyman_xorq import build_and_persist -def _build_one(project: str, parquet: Path) -> str: - # The parquet sits in the project's data dir and enters a recipe through read_project_file (ADR-008 D12). +def _build_one(project: str, src_alias: str) -> str: + # The orders data entered the catalog as a source alias, and the recipe reads that alias (ADR-011 D1). code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet.name!r}, project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src_alias!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ return build_and_persist(project, code, prompt="by region").content_hash @@ -48,8 +46,8 @@ def test_api_entries_empty(fresh_companion_app, project: str): assert r.json() == {"project": project, "entries": []} -def test_catalog_renders_after_build(fresh_companion_app, project: str, orders_parquet: Path): - h = _build_one(project, orders_parquet) +def test_catalog_renders_after_build(fresh_companion_app, project: str, orders_src: str): + h = _build_one(project, orders_src) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/entries") assert r.status_code == 200 @@ -60,8 +58,8 @@ def test_catalog_renders_after_build(fresh_companion_app, project: str, orders_p assert "by region" in prompts -def test_entry_detail_renders_table(fresh_companion_app, project: str, orders_parquet: Path): - h = _build_one(project, orders_parquet) +def test_entry_detail_renders_table(fresh_companion_app, project: str, orders_src: str): + h = _build_one(project, orders_src) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/entry/{h}") assert r.status_code == 200 @@ -77,7 +75,7 @@ def test_entry_detail_renders_table(fresh_companion_app, project: str, orders_pa def test_api_data_cheap_entry_serves_page_without_materialising( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): """#90: a paginated read comes off cached_result_expr, not a full result.parquet. @@ -88,8 +86,8 @@ def test_api_data_cheap_entry_serves_page_without_materialising( """ monkeypatch.setenv("TALLYMAN_PROJECT", project) code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ h = build_and_persist(project, code, prompt="cols").content_hash @@ -108,7 +106,7 @@ def test_api_data_cheap_entry_serves_page_without_materialising( def test_api_data_expensive_entry_paginates_without_per_entry_parquet( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): """#90: pages off an expensive entry read the baked snapshot, not a duplicate. @@ -118,8 +116,8 @@ def test_api_data_expensive_entry_paginates_without_per_entry_parquet( """ monkeypatch.setenv("TALLYMAN_PROJECT", project) code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.order_by("order_id").select("order_id", "region", "price") """ h = build_and_persist(project, code, prompt="ordered").content_hash @@ -139,7 +137,7 @@ def test_api_data_expensive_entry_paginates_without_per_entry_parquet( def test_api_data_cheap_entry_paginates_consistently_across_pages( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): """#90: a cheap (unsorted) entry tiles consistently across pages. @@ -153,8 +151,8 @@ def test_api_data_cheap_entry_paginates_consistently_across_pages( """ monkeypatch.setenv("TALLYMAN_PROJECT", project) code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("order_id", "region", "price", "__row_order") """ h = build_and_persist(project, code, prompt="cheap").content_hash @@ -173,7 +171,7 @@ def test_api_data_cheap_entry_paginates_consistently_across_pages( def test_api_data_missing_manifest_serves_page_without_500( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): """#90: a missing manifest must not 500 the row read. @@ -185,8 +183,8 @@ def test_api_data_missing_manifest_serves_page_without_500( """ monkeypatch.setenv("TALLYMAN_PROJECT", project) code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ h = build_and_persist(project, code, prompt="cols").content_hash @@ -202,7 +200,7 @@ def test_api_data_missing_manifest_serves_page_without_500( def test_entry_detail_sidebar_lists_all_entries_with_current_highlighted( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): """Entries API returns named/forensic/scratch entries with correct flags.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -211,15 +209,15 @@ def test_entry_detail_sidebar_lists_all_entries_with_current_highlighted( catalog_create( "shoe_sales", f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """, ) scratch = catalog_run( f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.order_by("region") """, prompt="exploratory", @@ -235,9 +233,9 @@ def test_entry_detail_sidebar_lists_all_entries_with_current_highlighted( assert any(e["content_hash"] == scratch["hash"] for e in scratches) -def test_entry_detail_shows_build_artifacts(fresh_companion_app, project: str, orders_parquet: Path): +def test_entry_detail_shows_build_artifacts(fresh_companion_app, project: str, orders_src: str): """Build artifacts (expr.yaml etc) appear in the entry JSON.""" - h = _build_one(project, orders_parquet) + h = _build_one(project, orders_src) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/entry/{h}") assert r.status_code == 200 diff --git a/tests/test_dependents.py b/tests/test_dependents.py index 9bbc3e07..cd075018 100644 --- a/tests/test_dependents.py +++ b/tests/test_dependents.py @@ -10,23 +10,20 @@ dependents_index, descendant_cone, parents_of, - sources_of, ) -def _write_entry(project: str, content_hash: str, parents=None, sources=None) -> str: +def _write_entry(project: str, content_hash: str, parents=None) -> str: """Write a minimal entry (manifest.json only) with the given parent edges. The dependents reader walks ``manifest.parents`` and never executes a recipe, so a hand-written manifest is a faithful and fast graph fixture — - no xorq build required. ``parents`` is a list of ``(hash, ref, follow)``; - ``sources`` is the recorded ``{rel_path: digest}`` map (``None`` = built - under identity ``off``, the default). + no xorq build required. ``parents`` is a list of ``(hash, ref, follow)``. """ d = entry_dir(project, content_hash) d.mkdir(parents=True, exist_ok=True) refs = [ParentRef(hash=h, ref=r, follow=f) for (h, r, f) in (parents or [])] - write_manifest(d, Manifest(content_hash=content_hash, project=project, parents=refs or None, sources=sources)) + write_manifest(d, Manifest(content_hash=content_hash, project=project, parents=refs or None)) return content_hash @@ -42,24 +39,6 @@ def test_parents_of_returns_recorded_edges(project): assert [(e.hash, e.ref, e.follow) for e in edges] == [("root", "root", True)] -def test_sources_of_none_when_unrecorded(project): - # Built under identity off (sources never recorded) — None, not {}, so the - # staleness source axis reports "unknown" rather than silently "fresh". - _write_entry(project, "root") - assert sources_of(project, "root") is None - - -def test_sources_of_empty_dict_is_preserved(project): - # Identity on, but the recipe read no raw files: {} is distinct from None. - _write_entry(project, "noraw", sources={}) - assert sources_of(project, "noraw") == {} - - -def test_sources_of_returns_recorded_digests(project): - _write_entry(project, "leaf", sources={"orders.parquet": "abc123"}) - assert sources_of(project, "leaf") == {"orders.parquet": "abc123"} - - def test_build_dag_maps_every_live_entry(project): _write_entry(project, "a") _write_entry(project, "b", [("a", "a", True)]) diff --git a/tests/test_diff.py b/tests/test_diff.py index 893875b7..f977e1f7 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -27,16 +27,16 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ def _filter_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) filtered = t.filter(t.category == "boots") expr = filtered.group_by("region").aggregate(total=filtered.price.sum(), n=filtered.count()) """ @@ -117,7 +117,7 @@ def test_key_diff_detects_unique_numeric_key(): # --------------------------------------------------------------------------- -def test_full_diff_against_two_builds(project: str, orders_parquet: Path): +def test_full_diff_against_two_builds(project: str, orders_src: str): from tallyman_core import entry_dir from tallyman_xorq import build_and_persist from tallyman_xorq.result_cache import cached_result_expr @@ -141,7 +141,7 @@ def test_full_diff_against_two_builds(project: str, orders_parquet: Path): # --------------------------------------------------------------------------- -def test_catalog_diff_default_compares_n_minus_one_to_n(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_diff_default_compares_n_minus_one_to_n(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -151,7 +151,7 @@ def test_catalog_diff_default_compares_n_minus_one_to_n(project: str, orders_par assert out["after"]["version"] == 2 -def test_catalog_diff_explicit_versions(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_diff_explicit_versions(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -165,7 +165,7 @@ def test_catalog_diff_no_history(project: str, monkeypatch): assert "error" in out -def test_catalog_diff_out_of_range(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_diff_out_of_range(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) out = catalog_diff("shoe_sales", 1, 5) @@ -177,7 +177,7 @@ def test_catalog_diff_out_of_range(project: str, orders_parquet: Path, monkeypat # --------------------------------------------------------------------------- -def test_diff_route_default(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_diff_route_default(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -192,7 +192,7 @@ def test_diff_route_default(fresh_companion_app, project: str, orders_parquet: P assert "head" in body["diff"] -def test_diff_route_explicit(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_diff_route_explicit(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -203,7 +203,7 @@ def test_diff_route_explicit(fresh_companion_app, project: str, orders_parquet: assert body["va"] == 1 and body["vb"] == 2 -def test_diff_route_single_version_400(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_diff_route_single_version_400(fresh_companion_app, project: str, orders_src: str, monkeypatch): # Only 1 version: requesting vb=2 is out of range → 404. monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) @@ -212,7 +212,7 @@ def test_diff_route_single_version_400(fresh_companion_app, project: str, orders assert r.status_code == 404 -def test_diff_route_pk_search_timeout_504(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_diff_route_pk_search_timeout_504(fresh_companion_app, project: str, orders_src: str, monkeypatch): # A primary-key search that blows the budget → 504 with a detail the diff # page renders, instead of grinding for minutes. import itertools @@ -221,8 +221,8 @@ def test_diff_route_pk_search_timeout_504(fresh_companion_app, project: str, ord monkeypatch.setenv("TALLYMAN_PROJECT", project) dup = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) u = t.union(t, distinct=False) expr = u """ @@ -238,8 +238,8 @@ def test_diff_route_pk_search_timeout_504(fresh_companion_app, project: str, ord def _dup_versions(project: str) -> None: dup = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) u = t.union(t, distinct=False) expr = u """ @@ -259,7 +259,7 @@ def _boom(*args, **kwargs): def test_diff_route_keyless_entry_skips_keyed_diff( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): monkeypatch.setenv("TALLYMAN_PROJECT", project) _dup_versions(project) @@ -272,7 +272,7 @@ def test_diff_route_keyless_entry_skips_keyed_diff( assert "stats" in body["diff"] and "head" in body["diff"] -def test_catalog_diff_keyless_entry_skips_buckaroo_search(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_diff_keyless_entry_skips_buckaroo_search(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) _dup_versions(project) _forbid_buckaroo_pk_search(monkeypatch) @@ -286,7 +286,7 @@ def test_diff_route_no_alias_404(fresh_companion_app, project: str): assert c.get(f"/{project}/api/diff_data/missing/1/2").status_code == 404 -def test_diff_route_same_version_400(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_diff_route_same_version_400(fresh_companion_app, project: str, orders_src: str, monkeypatch): # Diffing a version against itself is rejected. monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) @@ -346,7 +346,7 @@ def test_key_diff_polars_detects_unique_numeric_key(): assert d["only_after"] == 3 -def test_head_diff_polars_reads_n_rows(project: str, orders_parquet: Path, tmp_path: Path): +def test_head_diff_polars_reads_n_rows(project: str, orders_src: str, tmp_path: Path): from tallyman_xorq import build_and_persist from tallyman_xorq.result_cache import cached_result_expr @@ -364,7 +364,7 @@ def test_head_diff_polars_reads_n_rows(project: str, orders_parquet: Path, tmp_p # --------------------------------------------------------------------------- -def test_stats_diff_xorq_matches_pandas(project: str, orders_parquet: Path, tmp_path: Path): +def test_stats_diff_xorq_matches_pandas(project: str, orders_src: str, tmp_path: Path): from tallyman_xorq import build_and_persist from tallyman_xorq.result_cache import cached_result_expr @@ -387,7 +387,7 @@ def test_stats_diff_xorq_matches_pandas(project: str, orders_parquet: Path, tmp_ assert abs(pd_result[col]["before"]["numeric"]["sum"] - xq_result[col]["before"]["numeric"]["sum"]) < 1e-6 -def test_head_diff_xorq_reads_n_rows(project: str, orders_parquet: Path, tmp_path: Path): +def test_head_diff_xorq_reads_n_rows(project: str, orders_src: str, tmp_path: Path): from tallyman_xorq import build_and_persist from tallyman_xorq.result_cache import cached_result_expr @@ -399,7 +399,7 @@ def test_head_diff_xorq_reads_n_rows(project: str, orders_parquet: Path, tmp_pat assert " str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ @@ -74,7 +74,7 @@ def test_remove_display_config(project: str): # --------------------------------------------------------------------------- -def test_entry_detail_includes_display_config(fresh_companion_app, project: str, orders_parquet): +def test_entry_detail_includes_display_config(fresh_companion_app, project: str, orders_src): res = build_and_persist(project, _agg_code(project)) set_display_config(project, res.content_hash, SAMPLE_CONFIG) c = TestClient(fresh_companion_app) @@ -85,7 +85,7 @@ def test_entry_detail_includes_display_config(fresh_companion_app, project: str, assert body["display_config"]["diff_provenance"]["source_alias"] == "sales" -def test_entry_detail_display_config_null_when_absent(fresh_companion_app, project: str, orders_parquet): +def test_entry_detail_display_config_null_when_absent(fresh_companion_app, project: str, orders_src): res = build_and_persist(project, _agg_code(project)) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/entry/{res.content_hash}") diff --git a/tests/test_errors.py b/tests/test_errors.py index 793f008c..6fb658ec 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -120,7 +120,7 @@ def test_error_detail_shows_tool_pill(fresh_companion_app, project: str): assert r.json()["error"]["tool"] == "catalog_run" -def test_error_detail_sidebar_has_full_catalog_list(fresh_companion_app, project: str, orders_parquet, monkeypatch): +def test_error_detail_sidebar_has_full_catalog_list(fresh_companion_app, project: str, orders_src: str, monkeypatch): """Entries and error APIs return independent data for the React sidebar.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create @@ -128,8 +128,8 @@ def test_error_detail_sidebar_has_full_catalog_list(fresh_companion_app, project catalog_create( "shoe_sales", f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """, ) diff --git a/tests/test_fouc.py b/tests/test_fouc.py index d5ff6cc2..bfdc0faf 100644 --- a/tests/test_fouc.py +++ b/tests/test_fouc.py @@ -7,17 +7,15 @@ from __future__ import annotations -from pathlib import Path - from fastapi.testclient import TestClient from tallyman_mcp.server import catalog_create -def _agg(project: str) -> str: +def _agg(project: str, src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ @@ -77,10 +75,10 @@ def test_unknown_api_path_404s(fresh_companion_app, project: str): assert '
' in r2.text -def test_entry_detail_api_returns_entry_data(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_entry_detail_api_returns_entry_data(fresh_companion_app, project: str, orders_src: str, monkeypatch): """Entry detail data is served via JSON API, not server-rendered HTML.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) - out = catalog_create("shoe_sales", _agg(project)) + out = catalog_create("shoe_sales", _agg(project, orders_src)) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/entry/{out['hash']}") assert r.status_code == 200 diff --git a/tests/test_git_state_guard.py b/tests/test_git_state_guard.py index 8dc8e233..9623a5a4 100644 --- a/tests/test_git_state_guard.py +++ b/tests/test_git_state_guard.py @@ -139,10 +139,10 @@ def segfault_git_repo(tmp_path: Path, monkeypatch) -> Path: return repo -def _agg_code(parquet_path: Path) -> str: +def _agg_code(src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet_path.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ @@ -164,10 +164,10 @@ def test_installed_guard_dispatches_git_fork_free(guard_env, spawn_spy, temp_git _assert_provenance_fork_free(spawn_spy, "guarded get_git_state") -def test_build_dispatches_git_provenance_fork_free(project, orders_parquet, spawn_spy): +def test_build_dispatches_git_provenance_fork_free(project, orders_src, spawn_spy): """End-to-end: a real catalog write, through real xorq build_expr -> compiler.py:507 -> lu.get_git_state, must capture provenance fork-free.""" - res = build_and_persist(project, _agg_code(orders_parquet), prompt="fork-safety") + res = build_and_persist(project, _agg_code(orders_src), prompt="fork-safety") assert res.content_hash _assert_provenance_fork_free(spawn_spy, "build_and_persist") diff --git a/tests/test_io.py b/tests/test_io.py index 7fd7679c..5cb4ab94 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -26,12 +26,15 @@ def test_project_path_traversal_blocked(project: str, monkeypatch): project_path("../../etc/passwd") -def test_read_project_file_returns_xorq_expr(orders_parquet: Path, project: str, monkeypatch): +def test_read_project_file_is_refused_outside_a_source_recipe(orders_parquet: Path, project: str, monkeypatch): + """Called anywhere but a generated source recipe it raises, and the error names the import (ADR-011 D2).""" + from tallyman_xorq.build import BuildError + monkeypatch.setenv("TALLYMAN_PROJECT", project) - expr = read_project_file("orders.parquet") - schema = expr.schema() - assert "region" in schema.names - assert "price" in schema.names + with pytest.raises(BuildError) as exc: + read_project_file("orders.parquet") + assert "catalog_import_source" in str(exc.value) + assert "tracked_expr_from_alias" in str(exc.value) # --------------------------------------------------------------------------- @@ -43,17 +46,25 @@ def test_read_project_file_returns_xorq_expr(orders_parquet: Path, project: str, def _parent_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "category", "price", "__row_order") """ def _parent_v2_code(project: str) -> str: return f""" +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) +expr = t.filter(t.category == "boots").select("region", "category", "price", "__row_order") +""" + + +def _raw_read_code(project: str) -> str: + return f""" from tallyman_xorq.io import read_project_file t = read_project_file("orders.parquet", project={project!r}) -expr = t.filter(t.category == "boots").select("region", "category", "price", "__row_order") +expr = t.select("region", "category", "price", "__row_order") """ @@ -78,7 +89,7 @@ def _two_versions(project: str, monkeypatch) -> tuple[str, str]: return v1, v2 -def test_pinned_bare_alias_rejected(orders_parquet: Path, project: str, monkeypatch): +def test_pinned_bare_alias_rejected(orders_src: str, project: str, monkeypatch): """A bare alias reads like a pin but resolves to the build-time head — the recipe text under-determines the entry. The error must name both accepted forms so the author can self-correct.""" @@ -90,7 +101,7 @@ def test_pinned_bare_alias_rejected(orders_parquet: Path, project: str, monkeypa assert "-v" in res["error"] and "hash" in res["error"], res["error"] -def test_pinned_version_ref_resolves_history(orders_parquet: Path, project: str, monkeypatch): +def test_pinned_version_ref_resolves_history(orders_src: str, project: str, monkeypatch): """"trips-v1" denotes the first revision forever, regardless of when the recipe is built — the pin is self-describing.""" from tallyman_core.manifest import read_manifest @@ -105,7 +116,7 @@ def test_pinned_version_ref_resolves_history(orders_parquet: Path, project: str, assert parents[0].hash != v2 -def test_pinned_version_ref_out_of_range_names_count(orders_parquet: Path, project: str, monkeypatch): +def test_pinned_version_ref_out_of_range_names_count(orders_src: str, project: str, monkeypatch): """A version past the history's end must say how many versions exist, not report a generic not-found.""" from tallyman_mcp.server import catalog_create @@ -116,7 +127,18 @@ def test_pinned_version_ref_out_of_range_names_count(orders_parquet: Path, proje assert "2 version" in res["error"], res["error"] -def test_alias_name_matching_version_syntax_rejected(orders_parquet: Path, project: str, monkeypatch): +def test_pinned_bare_content_hash_rejected(orders_src: str, project: str, monkeypatch): + """ADR-011 D5: a recipe names aliases. A bare content hash is opaque — the DAG is made of + aliases, so a pin must say which alias and which version of it.""" + from tallyman_mcp.server import catalog_create + + v1, _ = _two_versions(project, monkeypatch) + res = catalog_create("child", _pin_child_code(v1)) + assert "error" in res, res + assert "trips-v1" in res["error"], res["error"] + + +def test_alias_name_matching_version_syntax_rejected(orders_src: str, project: str, monkeypatch): """An alias literally named "foo-v2" would collide with version-reference syntax; reject it at creation and at rename (#166's cheapest closure), steering a parallel-take to the "-o" (option) convention.""" @@ -131,3 +153,93 @@ def test_alias_name_matching_version_syntax_rejected(orders_parquet: Path, proje renamed = catalog_rename("orders", "orders-v3") assert "error" in renamed, renamed assert "orders-o3" in renamed["error"], renamed["error"] + + +# --------------------------------------------------------------------------- +# ADR-011 D2 — files enter only by an explicit import. A raw read of a file +# tallyman does not own is a build error in an authored recipe, the same +# treatment ADR-008 D7 gives a raw xo.deferred_read_parquet. +# --------------------------------------------------------------------------- + + +def test_read_project_file_refused_in_an_authored_recipe(orders_parquet: Path, project: str, monkeypatch): + """The error names the import call, so the author can self-correct in one step.""" + from tallyman_mcp.server import catalog_create + + res = catalog_create("orders", _raw_read_code(project)) + assert "error" in res, res + assert "catalog_import_source" in res["error"], res["error"] + assert "read_project_file" in res["error"], res["error"] + + +def test_tallyman_read_csv_refused_in_an_authored_recipe(project: str, tmp_path: Path, monkeypatch): + """Same rule, same message: a CSV is a file outside the arena until it is imported.""" + from tallyman_mcp.server import catalog_create + + csv = tmp_path / "outside" / "orders.csv" + csv.parent.mkdir(parents=True, exist_ok=True) + csv.write_text("region,n\nnorth,1\nsouth,2\n") + code = f"from tallyman_xorq.io import tallyman_read_csv\nexpr = tallyman_read_csv({str(csv)!r})\n" + res = catalog_create("orders", code) + assert "error" in res, res + assert "catalog_import_source" in res["error"], res["error"] + + +def test_read_project_file_still_works_inside_a_generated_source_recipe(project: str, tmp_path: Path, monkeypatch): + """The importer's own recipe is the one place the raw read survives (D2).""" + import pandas as pd + + from tallyman_core import entry_dir + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = tmp_path / "outside" / "orders.parquet" + src.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame({"a": [1, 2, 3]}).to_parquet(src) + + out = source_import.update_and_depend(str(src), "orders") + + recipe = (entry_dir(project, out["hash"]) / "expr.py").read_text() + assert "read_project_file" in recipe + + +# --------------------------------------------------------------------------- +# ADR-011 stage 2 — the escape hatch that let the suite keep authoring raw +# reads is gone, and so is the source-record folding that fed manifest.sources. +# --------------------------------------------------------------------------- + + +def test_the_legacy_file_reads_escape_hatch_is_gone(orders_parquet: Path, project: str, monkeypatch): + """``TALLYMAN_LEGACY_FILE_READS=1`` no longer buys an authored recipe a raw read. + + It was scaffolding with an expiry: stage 1 shipped the refusal, the suite ran behind the hatch, and + the hatch goes with the call-site rewrite. Nothing in the repo may name it — a disabled refusal that + one environment variable turns off is not a refusal. + """ + import re + + from tallyman_mcp.server import catalog_create + + monkeypatch.setenv("TALLYMAN_LEGACY_FILE_READS", "1") + res = catalog_create("orders", _raw_read_code(project)) + assert "error" in res, res + assert "catalog_import_source" in res["error"], res["error"] + + root = Path(__file__).resolve().parent.parent + pattern = re.compile(r"TALLYMAN_LEGACY_FILE_READS") + offenders = [ + str(p.relative_to(root)) + for base in ("src", "tests", "scripts") + for p in sorted((root / base).rglob("*.py")) + if pattern.search(p.read_text()) and p != Path(__file__).resolve() + ] + assert offenders == [], f"the stage-1 escape hatch survives in {offenders}" + + +def test_the_parent_source_record_folding_is_gone(): + """``io._note_parent_records`` folded a parent's source digests into its child so ``gc_cas`` could + walk the closure. ADR-011 D6: the closure is the DAG, which ``manifest.parents`` already records.""" + from tallyman_xorq import io + + assert not hasattr(io, "_note_parent_records") + assert not hasattr(io, "_reconstructing_source_digest") diff --git a/tests/test_lineage_faithful_reads.py b/tests/test_lineage_faithful_reads.py index afc23dd5..b9511c1d 100644 --- a/tests/test_lineage_faithful_reads.py +++ b/tests/test_lineage_faithful_reads.py @@ -47,18 +47,18 @@ # --------------------------------------------------------------------------- -def _parent_v1_code(project: str) -> str: # full orders +def _parent_v1_code(project: str, src: str) -> str: # full orders return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.select("region", "category", "price", "__row_order") """ -def _parent_v2_code(project: str) -> str: # same source, boots only — fewer rows +def _parent_v2_code(project: str, src: str) -> str: # same source, boots only — fewer rows return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.filter(t.category == "boots").select("region", "category", "price", "__row_order") """ @@ -79,13 +79,12 @@ def _worthy_child_code() -> str: # Aggregate → bakes a result snapshot at bui """ -def _lineage(project: str, monkeypatch, child_code: str) -> tuple[str, str]: +def _lineage(project: str, src: str, monkeypatch, child_code: str) -> tuple[str, str]: """Build the #163 shape; return (child_v1_hash, child_v2_hash).""" monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) # default-ON cascade - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - a1 = catalog_create("trips", _parent_v1_code(project))["hash"] + a1 = catalog_create("trips", _parent_v1_code(project, src))["hash"] b1 = catalog_create("by_region", child_code)["hash"] - catalog_revise("trips", _parent_v2_code(project)) + catalog_revise("trips", _parent_v2_code(project, src)) b2 = get_alias(project, "by_region") # Sanity on the setup itself (not the invariant under test): the cascade # minted a distinct child version off the new head, and b1's binding to a1 @@ -102,34 +101,34 @@ def _lineage(project: str, monkeypatch, child_code: str) -> tuple[str, str]: # --------------------------------------------------------------------------- -def test_cheap_historical_read_uses_recorded_parent(project, orders_parquet, monkeypatch): +def test_cheap_historical_read_uses_recorded_parent(project, orders_src, monkeypatch): """I1/I4: a cheap entry rematerializes from its frozen lineage, not the live head. b1 was built over the full parent (recorded row_count). On main, reading b1 re-imports its recipe and ``tracked_expr_from_alias`` resolves ``trips`` to the *current* head, so the count comes back boots-only. """ - b1, _b2 = _lineage(project, monkeypatch, _cheap_child_code()) + b1, _b2 = _lineage(project, orders_src, monkeypatch, _cheap_child_code()) recorded_rows = read_manifest(entry_dir(project, b1)).row_count live_rows = int(cached_result_expr(project, b1).count().execute()) assert live_rows == recorded_rows -def test_worthy_revisions_resolve_distinct_snapshots(project, orders_parquet, monkeypatch): +def test_worthy_revisions_resolve_distinct_snapshots(project, orders_src, monkeypatch): """I1: two distinct entries must never resolve to one snapshot file. On main both revisions' recipes reconstruct against the same live head, so both derive the same snapshot key — the "both reconstruct onto the same file" collapse from #163. """ - b1, b2 = _lineage(project, monkeypatch, _worthy_child_code()) + b1, b2 = _lineage(project, orders_src, monkeypatch, _worthy_child_code()) p1 = baked_snapshot_path(project, b1) p2 = baked_snapshot_path(project, b2) assert p1 is not None and p2 is not None assert p1 != p2 -def test_untouched_historical_entry_verifies_faithful(project, orders_parquet, monkeypatch): +def test_untouched_historical_entry_verifies_faithful(project, orders_src, monkeypatch): """I4-verification: an entry nobody touched must reproduce its recorded digest. ``verify_result_faithful`` must mean "this entry's build reproduces its @@ -137,24 +136,24 @@ def test_untouched_historical_entry_verifies_faithful(project, orders_parquet, m state, so an ordinary parent advance makes every non-head revision report unfaithful (or resolve to a sibling's snapshot). """ - b1, _b2 = _lineage(project, monkeypatch, _worthy_child_code()) + b1, _b2 = _lineage(project, orders_src, monkeypatch, _worthy_child_code()) assert verify_result_faithful(project, b1) is True -def test_diff_of_two_revisions_is_not_a_self_join(project, orders_parquet, monkeypatch): +def test_diff_of_two_revisions_is_not_a_self_join(project, orders_src, monkeypatch): """I5: the diff consumer must see the two revisions' real, distinct results. v2's parent filters to boots, so per-region counts genuinely differ. On main both diff sides reconstruct against the same head and the outer join compares a result to itself — every delta exactly zero (#162's symptom). """ - b1, b2 = _lineage(project, monkeypatch, _worthy_child_code()) + b1, b2 = _lineage(project, orders_src, monkeypatch, _worthy_child_code()) df = build_diff_expr(b1, b2, keys=["region"]).execute() deltas = df["n_abs_delta"].fillna(0) assert (deltas != 0).any() -def test_cold_cache_read_reproduces_build_bytes(project, orders_parquet, monkeypatch): +def test_cold_cache_read_reproduces_build_bytes(project, orders_src, monkeypatch): """I2: wiping every cache may change latency only — never the bytes served. With the compute cache emptied, reading b1 self-heals its snapshot. The @@ -162,7 +161,7 @@ def test_cold_cache_read_reproduces_build_bytes(project, orders_parquet, monkeyp recorded digest. On main it executes the live-head reconstruction and manufactures today's bytes under b1's historical hash. """ - b1, _b2 = _lineage(project, monkeypatch, _worthy_child_code()) + b1, _b2 = _lineage(project, orders_src, monkeypatch, _worthy_child_code()) shutil.rmtree(compute_cache_dir(project), ignore_errors=True) cached_result_expr.cache_clear() cached_result_expr(project, b1) # cold read — forces the self-heal @@ -176,30 +175,29 @@ def test_cold_cache_read_reproduces_build_bytes(project, orders_parquet, monkeyp # --------------------------------------------------------------------------- -def _worthy_parent_code(project: str) -> str: # Aggregate at the parent → worthy parent +def _worthy_parent_code(project: str, src: str) -> str: # Aggregate at the parent → worthy parent return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count(), total=t.price.sum()) """ -def test_missing_build_is_hard_error(project, orders_parquet, monkeypatch): +def test_missing_build_is_hard_error(project, orders_src): """ADR D6: an entry whose xorq_build/ is gone must raise, not reconstruct. On main the read path never touches the build — it re-imports expr.py — so deleting xorq_build/ goes unnoticed and the read silently succeeds. The error must name the entry so the remedy (rebuild) is actionable. """ - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - h = catalog_create("plain", _parent_v1_code(project))["hash"] + h = catalog_create("plain", _parent_v1_code(project, orders_src))["hash"] shutil.rmtree(entry_build_dir(project, h)) cached_result_expr.cache_clear() with pytest.raises(Exception, match=h[:12]): cached_result_expr(project, h) -def test_chained_child_reads_after_every_file_is_evicted(project, orders_parquet, monkeypatch): +def test_chained_child_reads_after_every_file_is_evicted(project, orders_src): """ADR-007 D3 and D5 (replacing ADR-006 D4): a chained child reads correctly on a cold compute cache because the canonical read makes every file it needs exist first. @@ -208,11 +206,11 @@ def test_chained_child_reads_after_every_file_is_evicted(project, orders_parquet with no pre-heal, as Buckaroo does. ADR-007 retires that. The child's build holds a bare read of the parent's snapshot, and Buckaroo never replays anything cold, since tallyman runs every computation to completion before it asks Buckaroo to show anything. The guarantee that survives is the one a reader can observe: with every file - evicted, ``cached_result_expr`` (the one canonical read) makes the parent's snapshot, and the ordered copy of the - source under it, exist again and verified before the child's plan executes. + evicted, ``cached_result_expr`` (the one canonical read) makes the parent's snapshot exist and verify again, and + under it the imported source version's snapshot, which is re-created from the clone of the bytes (ADR-011 D1), + before the child's plan executes. """ - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - parent = catalog_create("trips", _worthy_parent_code(project))["hash"] + parent = catalog_create("trips", _worthy_parent_code(project, orders_src))["hash"] child_code = """ from tallyman_xorq.io import tracked_expr_from_alias t = tracked_expr_from_alias("trips") @@ -231,7 +229,7 @@ def test_chained_child_reads_after_every_file_is_evicted(project, orders_parquet assert verify_result_faithful(project, parent) is True # and checked against the recorded digest -def test_unfaithful_heal_wipes_stat_cache(project, orders_parquet, monkeypatch): +def test_unfaithful_heal_wipes_stat_cache(project, orders_src): """ADR D10: a heal that fails verification wipes the entry's stat cache. Buckaroo's summary-stats cache is keyed on expression structure and paths @@ -241,8 +239,7 @@ def test_unfaithful_heal_wipes_stat_cache(project, orders_parquet, monkeypatch): .buckaroo_stat_cache. The recorded digest is tampered here so the heal is unfaithful by construction. On main the mismatch only logs a warning. """ - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - h = catalog_create("by_region", _worthy_parent_code(project))["hash"] + h = catalog_create("by_region", _worthy_parent_code(project, orders_src))["hash"] mpath = entry_dir(project, h) / "manifest.json" doc = json.loads(mpath.read_text()) doc["result_digest"] = "arrow-sha256:" + "0" * 64 diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 334c7699..6c173e04 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -25,3 +25,15 @@ def test_manifest_optional_fields_default(): assert m.execute_seconds is None assert m.code_path == "expr.py" assert m.schema_path == "schema.json" + + +def test_a_manifest_records_no_sources_and_no_ordered_copies(): + """ADR-011 D6: both fields go. + + ``sources`` was written as a retention record and read as a freshness record — one field, two + meanings, and the cause of the permanently-stale child under the ADR's Problem. Once every input is + an entry the retention closure is the DAG, which ``parents`` already records. ``ordered_copies`` goes + with it: a source's ordered copy is now the source entry's own snapshot (D1). + """ + assert "sources" not in Manifest.model_fields + assert "ordered_copies" not in Manifest.model_fields diff --git a/tests/test_materialize.py b/tests/test_materialize.py index 095c46bb..260e6f6e 100644 --- a/tests/test_materialize.py +++ b/tests/test_materialize.py @@ -20,14 +20,15 @@ import pyarrow.parquet as pq import pytest -from tallyman_cli.fixtures import write_shoe_orders from tallyman_companion.diff import build_diff_expr from tallyman_core import data_dir, entry_dir +from tallyman_core.aliases import get_alias from tallyman_core.manifest import read_manifest -from tallyman_core.paths import compute_cache_dir, tallyman_home +from tallyman_core.paths import compute_cache_dir from tallyman_mcp.server import catalog_create, catalog_revise from tallyman_xorq import result_cache from tallyman_xorq.build import BuildError, build_and_persist +from tallyman_xorq.materialize import snapshot_path from tallyman_xorq.result_cache import baked_snapshot_path, cached_result_expr, snapshot_file_digest # --------------------------------------------------------------------------- # @@ -37,16 +38,16 @@ def _agg_code(project: str) -> str: # an Aggregate: worthy, so it has a snapshot return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ def _boots_agg_code(project: str) -> str: # a second version of the same aggregate, over fewer rows return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) b = t.filter(t.category == "boots") expr = b.group_by("region").aggregate(total=b.price.sum(), n=b.count()) """ @@ -68,10 +69,10 @@ def _worthy_grandchild_code(parent: str) -> str: # an Aggregate over the cheap """ -def _root_code(project: str) -> str: # a bare read of a source: a cheap root entry +def _root_code(project: str) -> str: # a bare read of an imported source: a cheap root entry return f""" -from tallyman_xorq.io import read_project_file -expr = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +expr = tracked_expr_from_alias("orders_src", project={project!r}) """ @@ -107,18 +108,18 @@ def _no_cascade(monkeypatch): # --------------------------------------------------------------------------- # -def test_a_recipe_that_calls_cache_is_a_build_error(project, orders_parquet): +def test_a_recipe_that_calls_cache_is_a_build_error(project, orders_src): """ADR-007 D1 (builds carry no cache nodes): a ``CachedNode`` in a recipe would write under ~/.cache/xorq.""" code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price").cache() """ with pytest.raises(BuildError, match="cache"): build_and_persist(project, code) -def test_a_build_carries_no_cache_node(project, orders_parquet, monkeypatch): +def test_a_build_carries_no_cache_node(project, orders_src, monkeypatch): """ADR-007 D1: neither a worthy entry nor its chained child has a ``CachedNode`` in its frozen build.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) parent = _hash(catalog_create("agg", _agg_code(project))) @@ -145,7 +146,7 @@ def test_the_xorq_cache_machinery_is_gone(): # --------------------------------------------------------------------------- # -def test_the_snapshot_path_is_derived_from_the_content_hash(project, orders_parquet, monkeypatch): +def test_the_snapshot_path_is_derived_from_the_content_hash(project, orders_src, monkeypatch): """ADR-007 D2 (snapshot location): ``compute_cache/result_cache/.parquet``, written at create.""" from tallyman_xorq.materialize import snapshot_path @@ -157,14 +158,14 @@ def test_the_snapshot_path_is_derived_from_the_content_hash(project, orders_parq assert baked_snapshot_path(project, h) == expected -def test_the_manifest_records_no_snapshot_key(project, orders_parquet, monkeypatch): +def test_the_manifest_records_no_snapshot_key(project, orders_src, monkeypatch): """ADR-007 D2: with the path computed from the hash there is no second derivation for a tripwire to compare.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) h = _hash(catalog_create("agg", _agg_code(project))) assert "snapshot_key" not in json.loads((entry_dir(project, h) / "manifest.json").read_text()) -def test_a_worthy_read_is_one_memoised_bare_read_of_the_snapshot(project, orders_parquet, monkeypatch): +def test_a_worthy_read_is_one_memoised_bare_read_of_the_snapshot(project, orders_src, monkeypatch): """ADR-007 D2: ``cached_result_expr`` is one bare read of the snapshot, memoised, and loads no build.""" from tallyman_xorq.materialize import snapshot_path @@ -187,7 +188,7 @@ def _no_build_load(*a, **k): # --------------------------------------------------------------------------- # -def test_a_child_of_a_worthy_parent_reads_the_parents_snapshot_and_is_cheap(project, orders_parquet, monkeypatch): +def test_a_child_of_a_worthy_parent_reads_the_parents_snapshot_and_is_cheap(project, orders_src, monkeypatch): """ADR-007 D3 (chaining is a bare read): a filter over an aggregate's snapshot is cheap, not a second copy.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) parent = _hash(catalog_create("agg", _agg_code(project))) @@ -200,7 +201,7 @@ def test_a_child_of_a_worthy_parent_reads_the_parents_snapshot_and_is_cheap(proj assert baked_snapshot_path(project, child) is None -def test_a_child_hash_follows_the_parents_snapshot_path_not_its_bytes(project, orders_parquet, monkeypatch): +def test_a_child_hash_follows_the_parents_snapshot_path_not_its_bytes(project, orders_src, monkeypatch): """ADR-007 D3: content identity goes in the path. Other bytes at the same path keep the child's hash; a new parent (a new path) changes it.""" from tallyman_xorq.materialize import snapshot_path @@ -225,7 +226,7 @@ def test_a_child_hash_follows_the_parents_snapshot_path_not_its_bytes(project, o # --------------------------------------------------------------------------- # -def test_a_create_replaces_a_snapshot_that_is_already_on_disk(project, orders_parquet): +def test_a_create_replaces_a_snapshot_that_is_already_on_disk(project, orders_src): """ADR-007 D4 (one writer) and D14: a create never looks for the file, it runs the query and replaces it.""" from tallyman_xorq.materialize import snapshot_path @@ -241,7 +242,17 @@ def test_a_create_replaces_a_snapshot_that_is_already_on_disk(project, orders_pa assert snapshot_file_digest(snap) == good -def test_materialize_leaves_one_file_and_no_temp_files(project, orders_parquet): +def _leftovers(snap: Path) -> list[str]: + """Anything in the snapshot directory that is not a finished snapshot. + + A project holds one snapshot per worthy entry, including the imported source these tests build + over, so the directory is never down to a single file. What must never be there is a temp name + from an interrupted write. + """ + return sorted(p.name for p in snap.parent.iterdir() if not p.name.endswith(".parquet")) + + +def test_materialize_leaves_one_file_and_no_temp_files(project, orders_src): """ADR-007 D4: a unique temp name in the destination directory, then ``os.replace``.""" from tallyman_xorq.materialize import materialize, snapshot_path @@ -250,10 +261,10 @@ def test_materialize_leaves_one_file_and_no_temp_files(project, orders_parquet): snap = snapshot_path(project, h) assert result.path == snap assert result.digest == _digest_of(project, h) - assert sorted(p.name for p in snap.parent.iterdir()) == [snap.name] + assert _leftovers(snap) == [] -def test_a_failed_materialization_keeps_the_previous_file(project, orders_parquet, monkeypatch): +def test_a_failed_materialization_keeps_the_previous_file(project, orders_src, monkeypatch): """ADR-007 D4: the destination only ever changes by an atomic replace of a complete file.""" from tallyman_xorq.materialize import materialize, snapshot_path @@ -268,7 +279,7 @@ def _boom(self, *args, **kwargs): with pytest.raises(OSError, match="disk full"): materialize(project, h) assert snap.read_bytes() == before - assert sorted(p.name for p in snap.parent.iterdir()) == [snap.name] + assert _leftovers(snap) == [] # --------------------------------------------------------------------------- # @@ -276,7 +287,7 @@ def _boom(self, *args, **kwargs): # --------------------------------------------------------------------------- # -def test_ensure_materialized_rewrites_a_missing_snapshot_and_verifies_it(project, orders_parquet): +def test_ensure_materialized_rewrites_a_missing_snapshot_and_verifies_it(project, orders_src): """ADR-007 D5 (``ensure_materialized``): a deleted snapshot is re-created and checked against the manifest.""" from tallyman_xorq.materialize import ensure_materialized, snapshot_path @@ -290,7 +301,7 @@ def test_ensure_materialized_rewrites_a_missing_snapshot_and_verifies_it(project assert snapshot_file_digest(snap) == recorded -def test_files_exist_before_anything_runs(project, orders_parquet, monkeypatch): +def test_files_exist_before_anything_runs(project, orders_src, monkeypatch): """ADR-007 D5: with an ancestor's snapshot deleted, opening a descendant rewrites the ancestor first, so no plan is ever executed over a file that is missing. This is #76's reproduction.""" from tallyman_xorq.materialize import snapshot_path @@ -307,7 +318,7 @@ def test_files_exist_before_anything_runs(project, orders_parquet, monkeypatch): assert len(expr.execute()) > 0 -def test_building_a_child_rewrites_a_deleted_parent_snapshot_first(project, orders_parquet, monkeypatch): +def test_building_a_child_rewrites_a_deleted_parent_snapshot_first(project, orders_src, monkeypatch): """ADR-007 D3 and D5: chaining at mint time makes the parent's file exist, since ``build_expr`` of a child fails while the parent's snapshot is absent.""" from tallyman_xorq.materialize import snapshot_path @@ -322,7 +333,7 @@ def test_building_a_child_rewrites_a_deleted_parent_snapshot_first(project, orde assert f"result_cache/{parent}.parquet" in _build_yaml(project, child) -def test_composing_a_diff_rewrites_both_deleted_snapshots_first(project, orders_parquet, monkeypatch): +def test_composing_a_diff_rewrites_both_deleted_snapshots_first(project, orders_src, monkeypatch): """ADR-007 D5 and D10 (diffs: what this set still does): both sides are read through ``cached_result_expr``, so both files exist before the join is composed, and the diff carries no row-order column from either side.""" from tallyman_xorq.materialize import snapshot_path @@ -350,7 +361,7 @@ def test_composing_a_diff_rewrites_both_deleted_snapshots_first(project, orders_ # --------------------------------------------------------------------------- # -def test_an_empty_compute_cache_reproduces_every_snapshot_an_entry_needs(project, orders_parquet, monkeypatch): +def test_an_empty_compute_cache_reproduces_every_snapshot_an_entry_needs(project, orders_src, monkeypatch): """ADR-007 D7 (the cold seam): with ``compute_cache/`` removed the canonical read reproduces every snapshot the entry needs, each with its recorded digest, ordered copies of sources included.""" from tallyman_xorq.materialize import snapshot_path @@ -404,7 +415,7 @@ def _xorq_cache_files() -> set[str]: return {str(p) for p in root.rglob("*") if p.is_file()} -def test_xorq_cache_directory_stays_untouched(project, orders_parquet, monkeypatch): +def test_xorq_cache_directory_stays_untouched(project, orders_src, monkeypatch): """ADR-007 D8 (the sentinel): a build, a chained child build, a view, a delete and a reopen write nothing under xorq's cache directory. Fails on main: a child build re-executes its parent into ``~/.cache/xorq``.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -424,13 +435,17 @@ def test_xorq_cache_directory_stays_untouched(project, orders_parquet, monkeypat # --------------------------------------------------------------------------- # # D13: a file is cache only if ensure_materialized can re-create it +# +# The ordered copies of sources used to be the second class of file here, with their own store, their +# own key and their own re-creation path. ADR-011 D1 folds them in: a raw input is an entry, its +# snapshot is the copy, and re-creating it from the clone of the imported bytes is +# ``materialize._heal_a_source``. ``tests/test_source_import.py`` owns those cases now — a deleted +# source snapshot, a deleted clone, a CSV's recorded reader options, a child reading through a +# source snapshot that is gone. What is left here is the part that is about an ENTRY's reach into +# ``compute_cache/``, not about a source. # --------------------------------------------------------------------------- # -def _ordered_copies(project: str) -> list[Path]: - return sorted((compute_cache_dir(project) / "ordered_sources").glob("*.parquet")) - - def _clones(project: str) -> list[Path]: cas = data_dir(project) / ".cas" return sorted(cas.iterdir()) if cas.is_dir() else [] @@ -441,73 +456,14 @@ def _rows(project: str, content_hash: str) -> list[dict]: return json.loads(df.to_json(orient="records")) -def test_the_ordered_copy_lives_under_the_compute_cache_and_is_recorded(project, orders_parquet, monkeypatch): - """ADR-007 D13 (ordered copies are cache): they live under ``compute_cache/``, ``csv_ordered/`` is retired, and - the manifest records what ``ensure_materialized`` needs to make one again.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - h = _hash(catalog_create("orders", _root_code(project))) - copies = _ordered_copies(project) - assert len(copies) == 1 - assert not (tallyman_home() / "csv_ordered").exists() - record = read_manifest(entry_dir(project, h)).ordered_copies[copies[0].stem] - assert record["source"] == "orders.parquet" - assert record["reader"]["kind"] == "parquet" - assert record["digest"] == read_manifest(entry_dir(project, h)).sources["orders.parquet"] - assert record["content_digest"] == snapshot_file_digest(copies[0]) - - -def test_a_deleted_ordered_copy_is_made_again_from_the_clone_and_checked(project, orders_parquet, monkeypatch): - """ADR-007 D13 and D5: opening a root entry re-creates its ordered copy from the clone.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - h = _hash(catalog_create("orders", _root_code(project))) - [copy] = _ordered_copies(project) - digest = snapshot_file_digest(copy) - rows = _rows(project, h) - copy.unlink() - cached_result_expr.cache_clear() - - assert _rows(project, h) == rows - assert snapshot_file_digest(copy) == digest - - -def test_a_deleted_clone_is_made_again_from_the_unchanged_live_source(project, orders_parquet, monkeypatch): - """ADR-007 D13: a clone can be copied again from the live source while the live bytes still hash to its name.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - h = _hash(catalog_create("orders", _root_code(project))) - [copy] = _ordered_copies(project) - [clone] = _clones(project) - rows = _rows(project, h) - copy.unlink() - clone.unlink() - cached_result_expr.cache_clear() - - assert _rows(project, h) == rows - assert clone.is_file() and copy.is_file() - +def test_an_empty_compute_cache_is_rebuilt_for_an_entry_that_records_no_parent(project, orders_src, monkeypatch): + """ADR-007 D13: a file is made again from the hash in its name, whoever reads it. -def test_a_child_that_inlines_a_cheap_parent_can_make_the_ordered_copy_again(project, orders_parquet, monkeypatch): - """ADR-007 D5 and D13: a child's build reads the parent's ordered copy directly, so the child's manifest carries - the record that lets it be made again.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - _hash(catalog_create("orders", _root_code(project))) - child = _hash(catalog_create("multi", _root_child_code("orders"))) - [copy] = _ordered_copies(project) - [clone] = _clones(project) - rows = _rows(project, child) - copy.unlink() - clone.unlink() - cached_result_expr.cache_clear() - - assert _rows(project, child) == rows - assert copy.is_file() - - -def test_an_ordered_copy_is_made_again_for_an_entry_whose_manifest_does_not_record_it( - project, orders_parquet, monkeypatch -): - """ADR-007 D13: a copy is made again from the record of whichever entry wrote it, so an entry that reaches a cheap - entry's graph by a path that records nothing (here ``cached_result_expr`` called from the recipe) still survives - the loss of ``compute_cache/``.""" + The child below reaches its parent's snapshot through ``cached_result_expr`` inside the recipe, so + it records no parent edge and its manifest says nothing about what it reads. Losing + ``compute_cache/`` entirely must still be survivable: every file under it is named by a content + hash, and the entry that owns that hash knows how to write it again. + """ monkeypatch.setenv("TALLYMAN_PROJECT", project) root = _hash(catalog_create("orders", _root_code(project))) code = f""" @@ -516,7 +472,7 @@ def test_an_ordered_copy_is_made_again_for_an_entry_whose_manifest_does_not_reco expr = t.filter(t.qty > 1) """ child = build_and_persist(project, code).content_hash - assert not read_manifest(entry_dir(project, child)).ordered_copies + assert read_manifest(entry_dir(project, child)).parents is None rows = _rows(project, child) shutil.rmtree(compute_cache_dir(project)) cached_result_expr.cache_clear() @@ -524,45 +480,23 @@ def test_an_ordered_copy_is_made_again_for_an_entry_whose_manifest_does_not_reco assert _rows(project, child) == rows -def test_when_nothing_can_make_a_file_again_the_error_names_the_source(project, orders_parquet, monkeypatch): - """ADR-007 D13: the clone is gone and the live source has changed, so the built rows are unrecoverable and the - error names the source file.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - h = _hash(catalog_create("orders", _root_code(project))) - [copy] = _ordered_copies(project) - [clone] = _clones(project) - copy.unlink() - clone.unlink() - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=37, seed=99) # the live source changed - cached_result_expr.cache_clear() - - with pytest.raises(Exception, match=r"orders\.parquet"): - cached_result_expr(project, h) - +def test_when_nothing_can_make_a_file_again_the_error_names_the_re_import(project, orders_src, monkeypatch): + """ADR-007 D13 through a child: the rows under it are unrecoverable, and the error says how to get them back. -def test_a_csv_ordered_copy_is_made_again_with_the_recorded_reader_options(project, monkeypatch): - """ADR-007 D13: the manifest records each source's reader options, so a CSV copy is re-created with the schema and - the ``scan_csv`` options it was first read with.""" + A source version's snapshot is cache while the clone of its imported bytes is on disk. With both + gone the rows exist nowhere — the outside file is provenance and is not consulted — so reading a + child of that source raises, naming the missing clone and the import that repairs it. + """ monkeypatch.setenv("TALLYMAN_PROJECT", project) - csv = data_dir(project) / "sample.csv" - csv.write_text("id;name;value\n3;charlie;30\n1;alice;10\n2;bob;20\n") - code = f""" -import xorq.vendor.ibis as ibis -from tallyman_xorq.io import tallyman_read_csv -schema = ibis.schema({{"id": "int64", "name": "string", "value": "int64"}}) -expr = tallyman_read_csv({str(csv)!r}, schema=schema, separator=";") -""" - h = build_and_persist(project, code).content_hash - [copy] = _ordered_copies(project) - record = read_manifest(entry_dir(project, h)).ordered_copies[copy.stem] - assert record["reader"]["kind"] == "csv" - assert record["reader"]["scan_kwargs"] == {"separator": ";"} - rows = _rows(project, h) - assert [r["id"] for r in rows] == [3, 1, 2] # file order, not sorted - - copy.unlink() - [clone] = _clones(project) - clone.unlink() + _hash(catalog_create("orders", _root_code(project))) + child = _hash(catalog_create("multi", _root_child_code("orders"))) + _rows(project, child) + + src_hash = get_alias(project, "orders_src") + snapshot_path(project, src_hash).unlink() + for clone in _clones(project): + clone.unlink() cached_result_expr.cache_clear() - assert _rows(project, h) == rows - assert snapshot_file_digest(copy) == record["content_digest"] + + with pytest.raises(Exception, match=r"catalog_import_source"): + _rows(project, child) diff --git a/tests/test_mcp_project_tools.py b/tests/test_mcp_project_tools.py index cb900bae..2ec9f90d 100644 --- a/tests/test_mcp_project_tools.py +++ b/tests/test_mcp_project_tools.py @@ -184,16 +184,16 @@ def handler(request: httpx.Request) -> httpx.Response: def test_existing_tool_response_includes_project(project: str, orders_parquet: Path): - from tallyman_mcp.server import catalog_load_parquet + from tallyman_mcp.server import catalog_import_source - out = catalog_load_parquet("orders.parquet", prompt="raw") + out = catalog_import_source(str(orders_parquet), "orders", prompt="raw") assert out.get("project") == project def test_list_returning_tool_wrapped_with_project_and_items(project: str, orders_parquet: Path): - from tallyman_mcp.server import catalog_list, catalog_load_parquet + from tallyman_mcp.server import catalog_import_source, catalog_list - catalog_load_parquet("orders.parquet", prompt="raw") + catalog_import_source(str(orders_parquet), "orders", prompt="raw") out = catalog_list() # catalog_list previously returned a bare list; the decorator now wraps it. assert isinstance(out, dict) @@ -206,9 +206,9 @@ def test_catalog_list_includes_compact_columns(project: str, orders_parquet: Pat # The model should be able to look up an entry's columns before writing an # expression against it, without a build. catalog_list carries a compact # "name:type, name:type" summary the LLM can scan at a glance. - from tallyman_mcp.server import catalog_list, catalog_load_parquet + from tallyman_mcp.server import catalog_import_source, catalog_list - catalog_load_parquet("orders.parquet", prompt="raw") + catalog_import_source(str(orders_parquet), "orders", prompt="raw") entry = catalog_list()["items"][0] cols = entry["columns"] assert isinstance(cols, str) diff --git a/tests/test_mcp_stdio.py b/tests/test_mcp_stdio.py index 4c1892c3..f091fb13 100644 --- a/tests/test_mcp_stdio.py +++ b/tests/test_mcp_stdio.py @@ -57,7 +57,7 @@ async def go(): # All tools from server.py should be present. for required in { "catalog_run", - "catalog_load_parquet", + "catalog_import_source", "catalog_create", "catalog_revise", "catalog_alias", @@ -74,12 +74,12 @@ async def go(): @pytest.mark.integration @needs_uv -def test_stdio_round_trips_catalog_load_parquet(isolated_home: Path, project: str, orders_parquet: Path): +def test_stdio_round_trips_catalog_import_source(isolated_home: Path, project: str, orders_parquet: Path): async def go(): async with _client(project, isolated_home) as client: result = await client.call_tool( - "catalog_load_parquet", - {"rel_path": "orders.parquet", "prompt": "raw"}, + "catalog_import_source", + {"outside_path": str(orders_parquet), "alias": "orders", "prompt": "raw"}, ) return result.data @@ -93,14 +93,20 @@ async def go(): def test_stdio_create_revise_diff_round_trip(isolated_home: Path, project: str, orders_parquet: Path): async def go(): async with _client(project, isolated_home) as client: - code1 = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) + # A recipe cannot open a file (ADR-011 D2): the data enters through an import, over the same + # stdio transport, and the recipes below read the source alias it creates. + await client.call_tool( + "catalog_import_source", + {"outside_path": str(orders_parquet), "alias": "orders_src", "prompt": "raw"}, + ) + code1 = """ +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src") expr = t.group_by("region").aggregate(n=t.count()) """ - code2 = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) + code2 = """ +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src") filtered = t.filter(t.category == "boots") expr = filtered.group_by("region").aggregate(n=filtered.count()) """ diff --git a/tests/test_mcp_tool.py b/tests/test_mcp_tool.py index 73b9bfa2..3efedcfa 100644 --- a/tests/test_mcp_tool.py +++ b/tests/test_mcp_tool.py @@ -2,21 +2,21 @@ from pathlib import Path -from tallyman_mcp.server import catalog_list, catalog_load_parquet, catalog_run +from tallyman_mcp.server import catalog_import_source, catalog_list, catalog_run -def _code(parquet: Path) -> str: - # The parquet sits in the project's data dir and enters a recipe through read_project_file (ADR-008 D12). +def _code(src: str) -> str: + # The orders data entered the catalog as a source alias, and a recipe reads the alias (ADR-011 D2). return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.group_by("region").aggregate(n=t.count()) """ -def test_catalog_run_success(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_run_success(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - out = catalog_run(_code(orders_parquet), prompt="by region") + out = catalog_run(_code(orders_src), prompt="by region") assert "error" not in out assert "hash" in out assert out["row_count"] == 4 @@ -31,36 +31,25 @@ def test_catalog_run_error_returns_dict(project: str, monkeypatch): assert "hash" not in out -def test_catalog_list_after_run(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_list_after_run(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_run(_code(orders_parquet), prompt="x") + catalog_run(_code(orders_src), prompt="x") rows = catalog_list()["items"] - assert len(rows) == 1 - assert rows[0]["row_count"] == 4 + # Two entries: the imported source version the fixture minted, and the aggregate built over it. + assert len(rows) == 2 + assert rows[0]["row_count"] == 4 # most recent first + assert {r["alias"] for r in rows} == {None, orders_src} -def test_catalog_load_parquet_success(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_import_source_success(project: str, orders_parquet: Path, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - out = catalog_load_parquet("orders.parquet", prompt="raw orders") + out = catalog_import_source(str(orders_parquet), "orders", prompt="raw orders") assert "error" not in out assert out["row_count"] == 200 # the test fixture has 200 rows assert any(f["name"] == "region" for f in out["schema"]["fields"]) - - -def test_catalog_load_parquet_missing(project: str, monkeypatch): - monkeypatch.setenv("TALLYMAN_PROJECT", project) - out = catalog_load_parquet("nope.parquet") - assert "error" in out - assert "nope.parquet" in out["error"] - - -def test_catalog_load_parquet_with_name_creates_alias(project: str, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_PROJECT", project) - out = catalog_load_parquet("orders.parquet", prompt="raw", name="orders") - assert "error" not in out assert out["alias"] == "orders" assert out["version"] == 1 - # Notebook auto-appended. + # Notebook auto-appended on the first version. from tallyman_core import notebook cells = notebook.load(project)["cells"] @@ -68,14 +57,21 @@ def test_catalog_load_parquet_with_name_creates_alias(project: str, orders_parqu assert cells[0]["alias"] == "orders" -def test_catalog_run_surfaces_nondeterminism_lint(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_import_source_missing(project: str, tmp_path: Path, monkeypatch): + monkeypatch.setenv("TALLYMAN_PROJECT", project) + out = catalog_import_source(str(tmp_path / "nope.parquet"), "orders") + assert "error" in out + assert "nope.parquet" in out["error"] + + +def test_catalog_run_surfaces_nondeterminism_lint(project: str, orders_src: str, monkeypatch): # An execution-nondeterministic recipe (now()) builds fine but must carry an # advisory lint in the tool reply so the model/user sees it (#88). monkeypatch.setenv("TALLYMAN_PROJECT", project) code = f""" import xorq.vendor.ibis as ibis -from tallyman_xorq.io import read_project_file -t = read_project_file({orders_parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}) expr = t.mutate(built_at=ibis.now()) """ out = catalog_run(code, prompt="nondeterministic") @@ -84,21 +80,23 @@ def test_catalog_run_surfaces_nondeterminism_lint(project: str, orders_parquet: assert any("now()" in w for w in out["lint_warnings"]) -def test_catalog_load_parquet_name_collision_rejected(project: str, orders_parquet, monkeypatch): +def test_catalog_import_source_repeated_on_unchanged_bytes_is_a_noop(project: str, orders_parquet, monkeypatch): + """The old catalog_load_parquet errored on an existing alias; an import of the same bytes is idempotent.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_load_parquet("orders.parquet", name="orders") - out = catalog_load_parquet("orders.parquet", name="orders") - assert "error" in out - assert "already exists" in out["error"] + first = catalog_import_source(str(orders_parquet), "orders") + again = catalog_import_source(str(orders_parquet), "orders") + assert "error" not in again + assert again["hash"] == first["hash"] + assert again["version"] == 1 + assert again["created"] is False -def test_catalog_load_parquet_records_relative_path_only(project: str, orders_parquet, monkeypatch): - """The synthesized code uses `read_project_file(rel_path)` without an explicit - project= arg, so a project rename wouldn't invalidate the build. (T-24.)""" +def test_catalog_import_source_records_no_project_argument(project: str, orders_parquet, monkeypatch): + """The generated recipe carries no explicit project=, so a project rename does not invalidate the build. (T-24.)""" monkeypatch.setenv("TALLYMAN_PROJECT", project) - out = catalog_load_parquet("orders.parquet") + out = catalog_import_source(str(orders_parquet), "orders") from tallyman_core import entry_dir code = (entry_dir(project, out["hash"]) / "expr.py").read_text() assert "project=" not in code - assert "read_project_file('orders.parquet')" in code + assert "read_project_file(" in code diff --git a/tests/test_native_catalog_store.py b/tests/test_native_catalog_store.py index f4cc63d6..1360a67a 100644 --- a/tests/test_native_catalog_store.py +++ b/tests/test_native_catalog_store.py @@ -26,19 +26,19 @@ from tallyman_xorq import build_and_persist -def _agg_code(parquet_path: Path) -> str: - # A parquet file enters a recipe through read_project_file (ADR-008 D12); the file sits in the project's data dir. +def _agg_code(src: str) -> str: + # Data enters a recipe as a source alias (ADR-011 D1); the file behind it was imported by the fixture. return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file({parquet_path.name!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({src!r})\n" "expr = t.group_by('region').aggregate(total=t.price.sum(), n=t.count())\n" ) -def _agg_code_n(parquet_path: Path, agg: str) -> str: +def _agg_code_n(src: str, agg: str) -> str: return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file({parquet_path.name!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({src!r})\n" f"expr = t.group_by('region').aggregate({agg})\n" ) @@ -50,6 +50,18 @@ def _tracked_zip_hashes(project: str) -> set[str]: return catalog.tracked_recipe_hashes(project) +def _src_hash(project: str, src: str) -> str: + """The entry the source alias points at. + + An import mints an ordinary entry (ADR-011 D1), so every project that uses the ``orders_src`` + fixture carries one more entry dir, one more durable recipe zip and one more pointer than it + did when a recipe read the file directly. + """ + from tallyman_core import aliases as al + + return al.load_aliases(project)[src] + + def _build_checkpoint(project: str, code: str, message: str) -> str: res = build_and_persist(project, code) cs.checkpoint_catalog(project, message) @@ -60,7 +72,7 @@ class _XorqSpy: """Wrap ``subprocess.run`` and count only ``uv run xorq catalog ...`` calls. Scoped to the xorq argv so it never trips on the ``cp -c`` CoW clone - (source_identity.py:111) or any other subprocess. Delegates everything to + (source_identity.py:73) or any other subprocess. Delegates everything to the real ``subprocess.run`` so behaviour is unchanged. """ @@ -92,13 +104,13 @@ def _boom(*a, **k): al.set_alias(project, "myalias", "deadbeefdead", expect_exists=False) -def test_uses_no_subprocess_build(project, orders_parquet, monkeypatch): +def test_uses_no_subprocess_build(project, orders_src, monkeypatch): """A full build invokes zero ``xorq catalog`` subprocesses (call-count spy, not "did not raise" — the swallowed-registration path would false-green).""" cs.genesis(project) spy = _XorqSpy(subprocess.run) monkeypatch.setattr(subprocess, "run", spy) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) assert spy.xorq_calls == 0, f"build spawned {spy.xorq_calls} xorq catalog subprocess(es)" @@ -130,13 +142,13 @@ def test_src_has_no_xorq_catalog_argv(): # --------------------------------------------------------------------------- -def test_recipe_zip_members_are_allowlist(project, orders_parquet, tmp_path): +def test_recipe_zip_members_are_allowlist(project, orders_src, tmp_path): """write_recipe_zip archives exactly the allowlist — never a stray. Allowlist-not-denylist: a denylist skipping only today's known strays would pass existence checks yet leak a future artifact (here ``foo.bin``). """ - res = build_and_persist(project, _agg_code(orders_parquet), prompt="seed") + res = build_and_persist(project, _agg_code(orders_src), prompt="seed") ed = paths.entry_dir(project, res.content_hash) # strays that must NOT leak into the durable zip (prompts.jsonl is relocated # to a tracked prompts/.jsonl now; drop one here to prove the zip @@ -160,7 +172,7 @@ def test_recipe_zip_members_are_allowlist(project, orders_parquet, tmp_path): assert not any(s in r for r in rels for s in ("foo.bin", "result.parquet", "prompts.jsonl", "xorq_build_expanded")) -def test_recipe_zip_byte_stable_across_rebuilds(project, orders_parquet, tmp_path): +def test_recipe_zip_byte_stable_across_rebuilds(project, orders_src, tmp_path): """The archive is byte-identical across rebuilds — a defaulted ZipInfo mtime would embed the wall clock and silently re-churn the content-addressed git blob on every rebuild. Bumping file mtimes between the two zips is what a @@ -168,7 +180,7 @@ def test_recipe_zip_byte_stable_across_rebuilds(project, orders_parquet, tmp_pat import os import time - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) ed = paths.entry_dir(project, res.content_hash) z1 = tmp_path / "a.zip" catalog.write_recipe_zip(ed, z1, res.content_hash) @@ -183,41 +195,18 @@ def test_recipe_zip_byte_stable_across_rebuilds(project, orders_parquet, tmp_pat assert z1.read_bytes() == z2.read_bytes() -def test_recipe_zip_contains_expr_py(project, orders_parquet, tmp_path): +def test_recipe_zip_contains_expr_py(project, orders_src, tmp_path): """expr.py is a durable zip member (D3): the tracked_expr_from_alias chaining path re-execs it so a cheap entry's recompute roots on the shared in-process backend, and two entries union/join on one backend (#75).""" - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) ed = paths.entry_dir(project, res.content_hash) dest = tmp_path / "recipe.zip" catalog.write_recipe_zip(ed, dest, res.content_hash) assert f"{res.content_hash}/expr.py" in zipfile.ZipFile(dest).namelist() -def test_salt_zip_named_by_salted_hash(project, monkeypatch): - """Under source-identity salt mode the durable zip is named by the salted - content_hash (= the entry dir name), not xorq's path-identity build hash, so - the tracked zip set and the entry pointers don't diverge (Risk #5).""" - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "salt") - from tallyman_cli.fixtures import write_shoe_orders - - write_shoe_orders(paths.data_dir(project) / "orders.parquet", n_rows=120, seed=1) - code = ( - "from tallyman_xorq.io import read_project_file\n" - "t = read_project_file('orders.parquet')\n" - "expr = t.group_by('region').aggregate(n=t.count())\n" - ) - cs.genesis(project) - res = build_and_persist(project, code) - import json - - manifest = json.loads((paths.entry_dir(project, res.content_hash) / "manifest.json").read_text()) - assert manifest.get("sources"), "salt mode should have captured a source digest" - cs.checkpoint_catalog(project, "create") - assert res.content_hash in _tracked_zip_hashes(project) - - -def test_failed_build_no_orphan_pointer(project, orders_parquet, monkeypatch): +def test_failed_build_no_orphan_pointer(project, orders_src, monkeypatch): """A build that fails after creating its entry dir leaves no dir behind, so the next checkpoint records no pointer without a durable recipe (D1/Risk #7). """ @@ -230,14 +219,15 @@ def _boom_load(*a, **k): monkeypatch.setattr(compiler, "load_expr", _boom_load) with pytest.raises(Exception): - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) # No monkeypatch.undo(): it shares the instance that isolated_home used to set # TALLYMAN_HOME, so undo() would point entries_dir at the real home and make # this check vacuous. monkeypatch auto-reverts at teardown. ed = paths.entries_dir(project) + # The import's own entry is the only dir that may be there: it was minted before the build ran. leftover = [c.name for c in ed.iterdir() if c.is_dir()] if ed.exists() else [] - assert not leftover, f"failed build left orphan entry dir(s): {leftover}" + assert leftover == [_src_hash(project, orders_src)], f"failed build left orphan entry dir(s): {leftover}" # --------------------------------------------------------------------------- @@ -245,43 +235,43 @@ def _boom_load(*a, **k): # --------------------------------------------------------------------------- -def test_zip_staged_not_committed_until_checkpoint(project, orders_parquet): +def test_zip_staged_not_committed_until_checkpoint(project, orders_src): """The durable ``entries/.zip`` is written by the checkpoint, not the build (D1).""" cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) assert res.content_hash not in _tracked_zip_hashes(project), "zip must not exist before a checkpoint" cs.checkpoint_catalog(project, "create") assert res.content_hash in _tracked_zip_hashes(project), "checkpoint did not commit the recipe zip" -def test_n_builds_track_n_zips(project, orders_parquet): +def test_n_builds_track_n_zips(project, orders_src): """N distinct builds, each checkpointed, leave N durable tracked zips (the #48 failure mode was N build dirs but fewer tracked recipes).""" cs.genesis(project) - hashes = [_build_checkpoint(project, _agg_code_n(orders_parquet, _AGGS[i]), f"c{i}") for i in range(4)] + hashes = [_build_checkpoint(project, _agg_code_n(orders_src, _AGGS[i]), f"c{i}") for i in range(4)] tracked = _tracked_zip_hashes(project) missing = [h for h in hashes if h not in tracked] assert not missing, f"entries with no durable tracked zip: {missing} (tracked: {tracked})" -def test_build_dir_files_not_staged(project, orders_parquet): +def test_build_dir_files_not_staged(project, orders_src): """The untracked build dir (``entries//xorq_build/...``) is never git-tracked under ``git add -A`` — the .gitignore + allowlist keep it out (Risk #1).""" cs.genesis(project) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) cs.checkpoint_catalog(project, "create") rc, out, _ = cs.run_git(["ls-files"], cwd=paths.catalog_dir(project)) staged = [p for p in out.split() if "/xorq_build/" in p or p.endswith("/expr.py") or p.endswith("/manifest.json")] assert not staged, f"build-dir files were staged: {staged}" -def test_unlisted_path_not_tracked(project, orders_parquet): +def test_unlisted_path_not_tracked(project, orders_src): """A tracked path outside the allowed surface fails the consistency check — the durable guard against ``git add -A`` silently committing a stray.""" cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) cs.checkpoint_catalog(project, "create") cd = paths.catalog_dir(project) (cd / "stray.txt").write_text("not part of the tracked surface\n") @@ -295,21 +285,21 @@ def test_unlisted_path_not_tracked(project, orders_parquet): # --------------------------------------------------------------------------- -def test_reset_keeps_pointer_zip_agreement(project, orders_parquet): +def test_reset_keeps_pointer_zip_agreement(project, orders_src): """A reset to an earlier step leaves the tracked recipe set equal to the entry_hashes pointers (the consistency check passes, #52).""" cs.genesis(project) - hashes = [_build_checkpoint(project, _agg_code_n(orders_parquet, _AGGS[i]), f"c{i}") for i in range(2)] - cs.reset_to(project, 1) # back to the one-entry step + hashes = [_build_checkpoint(project, _agg_code_n(orders_src, _AGGS[i]), f"c{i}") for i in range(2)] + cs.reset_to(project, 1) # back to the step with only the first build on top of the import pointers = set(cs.read_tallyman_state(project)["entry_hashes"]) - assert pointers == _tracked_zip_hashes(project) == {hashes[0]} + assert pointers == _tracked_zip_hashes(project) == {hashes[0], _src_hash(project, orders_src)} -def test_back_then_forward_restores_recipe(project, orders_parquet): +def test_back_then_forward_restores_recipe(project, orders_src): """Reset back then forward restores entry 2's durable recipe zip, not merely its bullpen dir copy.""" cs.genesis(project) - hashes = [_build_checkpoint(project, _agg_code_n(orders_parquet, _AGGS[i]), f"c{i}") for i in range(2)] + hashes = [_build_checkpoint(project, _agg_code_n(orders_src, _AGGS[i]), f"c{i}") for i in range(2)] cs.reset_to(project, 1) cs.reset_to(project, 2) assert hashes[1] in _tracked_zip_hashes(project) @@ -320,7 +310,7 @@ def test_back_then_forward_restores_recipe(project, orders_parquet): # --------------------------------------------------------------------------- -def test_connect_shim_matches_xorq_api_connect(project, orders_parquet, monkeypatch): +def test_connect_shim_matches_xorq_api_connect(project, orders_src, monkeypatch): """The catalog-free connect shim is equivalent to xorq.api.connect(): it builds the same datafusion backend, so a materialization run through the shim (``single_partition_backend`` connects through it) writes the SAME @@ -348,7 +338,7 @@ def test_connect_shim_matches_xorq_api_connect(project, orders_parquet, monkeypa assert type(shim) is type(api) assert _profile_content_token(shim) == _profile_content_token(api), "the shim must build the same kind of backend" - res = build_and_persist(project, _agg_code(orders_parquet)) # materialized through the shim + res = build_and_persist(project, _agg_code(orders_src)) # materialized through the shim snap = snapshot_path(project, res.content_hash) assert snap.exists(), "the shim must write a real snapshot" shim_df = pd.read_parquet(snap).sort_values("region").reset_index(drop=True) @@ -366,17 +356,17 @@ def test_connect_shim_matches_xorq_api_connect(project, orders_parquet, monkeypa # --------------------------------------------------------------------------- -def test_prompt_history_survives_reset(project, orders_parquet): +def test_prompt_history_survives_reset(project, orders_src): """The per-entry prompt history is a tracked ``prompts/.jsonl`` now, so a re-run's appended prompt rolls back with the tree on reset (it used to be a loose, untracked ``entries//prompts.jsonl`` git reset never touched).""" from tallyman_xorq import read_prompts cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet), prompt="first") + res = build_and_persist(project, _agg_code(orders_src), prompt="first") h = res.content_hash cs.checkpoint_catalog(project, "c1") - build_and_persist(project, _agg_code(orders_parquet), prompt="second") # re-run, same hash, appends + build_and_persist(project, _agg_code(orders_src), prompt="second") # re-run, same hash, appends cs.checkpoint_catalog(project, "c2") assert [p["prompt"] for p in read_prompts(project, h)] == ["first", "second"] cs.reset_to(project, 1) # only "first" had been recorded at step 1 @@ -385,14 +375,14 @@ def test_prompt_history_survives_reset(project, orders_parquet): assert [p["prompt"] for p in read_prompts(project, h)] == ["first", "second"] -def test_display_config_survives_reset(project, orders_parquet): +def test_display_config_survives_reset(project, orders_src): """A display config (tracked ``display_configs/.json``) rolls back with git reset — the one decomposed section with no survives-reset coverage before (its old capture/materialize round-trip tests are deleted with this cut).""" from tallyman_core import display_configs as dc cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) h = res.content_hash cs.checkpoint_catalog(project, "c1") # step 1: entry, no display config dc.set_display_config(project, h, {"pinned_rows": [1, 2]}) @@ -404,7 +394,7 @@ def test_display_config_survives_reset(project, orders_parquet): assert dc.get_display_config(project, h) == {"pinned_rows": [1, 2]} -def test_tracked_tree_is_the_decomposed_surface(project, orders_parquet): +def test_tracked_tree_is_the_decomposed_surface(project, orders_src): """End to end: after building an entry and touching every decomposed section, the git-tracked tree is *exactly* the native decomposed surface — no catalog.yaml, no build dirs, no xorq metadata/symlinks. Pins the whole cut.""" @@ -415,7 +405,7 @@ def test_tracked_tree_is_the_decomposed_surface(project, orders_parquet): from tallyman_core import summary_stats as ss cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet), prompt="first") + res = build_and_persist(project, _agg_code(orders_src), prompt="first") h = res.content_hash al.set_alias(project, "by_region", h, expect_exists=False) charts.set_chart(project, h, {"mark": "bar"}) @@ -433,6 +423,8 @@ def test_tracked_tree_is_the_decomposed_surface(project, orders_parquet): "notebook.jsonl", "entries.jsonl", # no compute_cache.jsonl: a reset leaves compute_cache/ alone (ADR-007 D14) f"entries/{h}.zip", + # The imported source is a version of a source alias, so it has a recipe zip of its own (ADR-011 D1). + f"entries/{_src_hash(project, orders_src)}.zip", f"chart_specs/{h}.vl.json", f"display_configs/{h}.json", "post_processing/pp1.py", @@ -457,7 +449,7 @@ def test_tracked_tree_is_the_decomposed_surface(project, orders_parquet): "display_configs/nested/leak.json", ], ) -def test_nested_stray_under_allowlisted_prefix_is_rejected(project, orders_parquet, rel): +def test_nested_stray_under_allowlisted_prefix_is_rejected(project, orders_src, rel): """A stray nested *under* an allowlisted dir prefix must be rejected. ``fnmatch``'s ``*`` spans ``/``, so ``post_processing/sub/leak.py`` wrongly @@ -466,7 +458,7 @@ def test_nested_stray_under_allowlisted_prefix_is_rejected(project, orders_parqu allowlist must be path-segment-aware. """ cs.genesis(project) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) cs.checkpoint_catalog(project, "create") cd = paths.catalog_dir(project) p = cd / rel @@ -480,11 +472,11 @@ def test_nested_stray_under_allowlisted_prefix_is_rejected(project, orders_parqu catalog.assert_catalog_consistent(project, pointers) -def test_legit_disabled_subdir_member_still_passes(project, orders_parquet): +def test_legit_disabled_subdir_member_still_passes(project, orders_src): """The segment-aware allowlist must not over-reject the legitimate two-level members it already tracks (``post_processing/_disabled/*.py``, &c.).""" cs.genesis(project) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) cs.checkpoint_catalog(project, "create") cd = paths.catalog_dir(project) for rel in ("post_processing/_disabled/old.py", "stats/_disabled/old.py"): @@ -496,7 +488,7 @@ def test_legit_disabled_subdir_member_still_passes(project, orders_parquet): catalog.assert_catalog_consistent(project, pointers) # must NOT raise -def test_interrupted_atomic_write_tmp_not_committed(project, orders_parquet): +def test_interrupted_atomic_write_tmp_not_committed(project, orders_src): """A ``*.tmp`` left by a crash mid atomic-write (``aliases._write`` / ``write_recipe_zip``) must never be staged by the checkpoint's ``git add -A``, so it cannot durably wedge a later ``reset_to`` via the tracked-surface @@ -504,7 +496,7 @@ def test_interrupted_atomic_write_tmp_not_committed(project, orders_parquet): from tallyman_core import aliases as al cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) al.set_alias(project, "regions", res.content_hash) cs.checkpoint_catalog(project, "cp1") cd = paths.catalog_dir(project) @@ -540,7 +532,7 @@ def test_checkpoint_skips_manifestless_entry_dir(project): # --------------------------------------------------------------------------- -def test_consistency_guard_surfaces_dangling_alias_hash(project, orders_parquet): +def test_consistency_guard_surfaces_dangling_alias_hash(project, orders_src): """A committed ``aliases.jsonl`` whose latest/history hash names no durable recipe must fail the consistency guard, not reset clean (M1). A dangling alias head later blows up the self-chaining ``tracked_expr_from_alias`` build, so @@ -548,23 +540,24 @@ def test_consistency_guard_surfaces_dangling_alias_hash(project, orders_parquet) from tallyman_core import aliases as al cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) al.set_alias(project, "ghost", "ffffffffffff", expect_exists=False) # no such entry cs.checkpoint_catalog(project, "dangling alias") pointers = set(cs.read_tallyman_state(project)["entry_hashes"]) - assert pointers == {res.content_hash} # the real entry's pointer/recipe agree (guard 2 passes) + # The real entries — the build and the import behind it — have pointer/recipe agreement (guard 2 passes). + assert pointers == {res.content_hash, _src_hash(project, orders_src)} with pytest.raises(RuntimeError, match="ffffffffffff"): catalog.assert_catalog_consistent(project, pointers) -def test_consistency_guard_surfaces_dangling_alias_history_hash(project, orders_parquet): +def test_consistency_guard_surfaces_dangling_alias_history_hash(project, orders_src): """The alias check covers history, not only the latest head — a revised alias whose *older* version's recipe is missing is just as dangling (M1: 'at minimum cover aliases.jsonl history').""" from tallyman_core import aliases as al cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet)) + res = build_and_persist(project, _agg_code(orders_src)) al.set_alias(project, "regions", "ffffffffffff", expect_exists=False) # v1: no recipe al.set_alias(project, "regions", res.content_hash) # v2: real head; history = [ffff…, res] cs.checkpoint_catalog(project, "alias with a dangling history entry") @@ -573,14 +566,14 @@ def test_consistency_guard_surfaces_dangling_alias_history_hash(project, orders_ catalog.assert_catalog_consistent(project, pointers) -def test_consistency_guard_surfaces_orphan_chart_hash(project, orders_parquet): +def test_consistency_guard_surfaces_orphan_chart_hash(project, orders_src): """A committed ``chart_specs/.vl.json`` whose hash names no durable recipe must fail the guard (L1) — the guard validates every content-addressed sidecar against the live recipe set, not just the recipe↔pointer pair.""" from tallyman_core import charts cs.genesis(project) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) charts.set_chart(project, "ffffffffffff", {"mark": "bar"}) # no such entry cs.checkpoint_catalog(project, "orphan chart") pointers = set(cs.read_tallyman_state(project)["entry_hashes"]) @@ -588,14 +581,14 @@ def test_consistency_guard_surfaces_orphan_chart_hash(project, orders_parquet): catalog.assert_catalog_consistent(project, pointers) -def test_consistency_guard_surfaces_orphan_display_config_hash(project, orders_parquet): +def test_consistency_guard_surfaces_orphan_display_config_hash(project, orders_src): """A committed ``display_configs/.json`` whose hash names no durable recipe must fail the guard (L1), the display-config twin of the orphan-chart case.""" from tallyman_core import display_configs as dc cs.genesis(project) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) dc.set_display_config(project, "ffffffffffff", {"pinned_rows": [1]}) # no such entry cs.checkpoint_catalog(project, "orphan display config") pointers = set(cs.read_tallyman_state(project)["entry_hashes"]) @@ -603,7 +596,7 @@ def test_consistency_guard_surfaces_orphan_display_config_hash(project, orders_p catalog.assert_catalog_consistent(project, pointers) -def test_failed_build_after_copy_leaves_no_orphan_dir(project, orders_parquet, monkeypatch): +def test_failed_build_after_copy_leaves_no_orphan_dir(project, orders_src, monkeypatch): """A build that fails AFTER the entry dir is populated but past the load_expr/execute paths (here: ``make_portable_inplace``) must also leave no orphan entry dir (L4). The D1 'no partial dirs' invariant covers every @@ -617,23 +610,24 @@ def _boom(*a, **k): monkeypatch.setattr(build_mod, "make_portable_inplace", _boom) with pytest.raises(Exception): - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) # No monkeypatch.undo(): it would also revert isolated_home's TALLYMAN_HOME # setenv (same monkeypatch instance), pointing entries_dir at the real home # and making this check vacuous. monkeypatch auto-reverts at teardown. ed = paths.entries_dir(project) + # The import's own entry is the only dir that may be there: it was minted before the build ran. leftover = [c.name for c in ed.iterdir() if c.is_dir()] if ed.exists() else [] - assert not leftover, f"failed build left orphan entry dir(s): {leftover}" + assert leftover == [_src_hash(project, orders_src)], f"failed build left orphan entry dir(s): {leftover}" -def test_consistency_guard_surfaces_orphan_recipe(project, orders_parquet): +def test_consistency_guard_surfaces_orphan_recipe(project, orders_src): """The orphan branch of guard 2 (a tracked ``entries/.zip`` with no pointer) must raise — the untested half of the pointer/recipe agreement check (L8). The missing-pointer half is covered by ``test_reset_to_revision.py::test_reset_surfaces_pointer_without_durable_recipe``.""" cs.genesis(project) - build_and_persist(project, _agg_code(orders_parquet)) + build_and_persist(project, _agg_code(orders_src)) cs.checkpoint_catalog(project, "create") # tracks entries/.zip with pytest.raises(RuntimeError, match="tracked recipes with no pointer"): catalog.assert_catalog_consistent(project, set()) # pointers empty, recipe tracked @@ -644,7 +638,7 @@ def test_consistency_guard_surfaces_orphan_recipe(project, orders_parquet): # --------------------------------------------------------------------------- -def test_alias_state_survives_reset(project, orders_parquet): +def test_alias_state_survives_reset(project, orders_src): """The alias map + history (the tracked ``aliases.jsonl`` this cut introduced) rolls back with ``git reset`` — the carrier whose only coverage was the deleted ``test_reset_rolls_back_alias_state`` (M2). Mirrors the display-config / @@ -652,24 +646,26 @@ def test_alias_state_survives_reset(project, orders_parquet): from tallyman_core import aliases as al cs.genesis(project) - h0 = _build_checkpoint(project, _agg_code_n(orders_parquet, _AGGS[0]), "c0") # step 1 + # The source alias was set by the import, before step 0, so it is in every step's tree. + src = {orders_src: _src_hash(project, orders_src)} + h0 = _build_checkpoint(project, _agg_code_n(orders_src, _AGGS[0]), "c0") # step 1 al.set_alias(project, "regions", h0, expect_exists=False) cs.checkpoint_catalog(project, "c1") # step 2: regions -> h0 (history [h0]) - h1 = build_and_persist(project, _agg_code_n(orders_parquet, _AGGS[1])).content_hash + h1 = build_and_persist(project, _agg_code_n(orders_src, _AGGS[1])).content_hash al.set_alias(project, "regions", h1) # revise: history [h0, h1] al.set_alias(project, "avg", h1, expect_exists=False) cs.checkpoint_catalog(project, "c2") # step 3: regions -> h1 (history [h0, h1]) + avg -> h1 - assert al.load_aliases(project) == {"regions": h1, "avg": h1} + assert al.load_aliases(project) == {"regions": h1, "avg": h1, **src} cs.reset_to(project, 2) - assert al.load_aliases(project) == {"regions": h0} + assert al.load_aliases(project) == {"regions": h0, **src} assert al.history_for(project, "regions") == [h0] cs.reset_to(project, 3) - assert al.load_aliases(project) == {"regions": h1, "avg": h1} + assert al.load_aliases(project) == {"regions": h1, "avg": h1, **src} assert al.history_for(project, "regions") == [h0, h1] -def test_append_prompt_is_atomic_and_complete(project, orders_parquet): +def test_append_prompt_is_atomic_and_complete(project, orders_src): """``_append_prompt`` rewrites ``prompts/.jsonl`` via tmp + replace (C2): it runs outside the project lock (only the checkpoint holds it), so the file must stay whole JSONL across appends with no leftover ``*.tmp`` the @@ -677,8 +673,8 @@ def test_append_prompt_is_atomic_and_complete(project, orders_parquet): from tallyman_xorq import read_prompts cs.genesis(project) - res = build_and_persist(project, _agg_code(orders_parquet), prompt="first") + res = build_and_persist(project, _agg_code(orders_src), prompt="first") h = res.content_hash - build_and_persist(project, _agg_code(orders_parquet), prompt="second") # re-run, same hash, appends + build_and_persist(project, _agg_code(orders_src), prompt="second") # re-run, same hash, appends assert [r["prompt"] for r in read_prompts(project, h)] == ["first", "second"] assert not list(paths.prompts_path(project, h).parent.glob("*.tmp")) # no interrupted temp left behind diff --git a/tests/test_notebook.py b/tests/test_notebook.py index 686865da..a62c6957 100644 --- a/tests/test_notebook.py +++ b/tests/test_notebook.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -from pathlib import Path import pytest from fastapi.testclient import TestClient @@ -23,16 +22,16 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ def _filter_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) filtered = t.filter(t.category == "boots") expr = filtered.group_by("region").aggregate(n=filtered.count()) """ @@ -130,7 +129,7 @@ def test_load_raises_on_malformed_line(project: str): # --------------------------------------------------------------------------- -def test_catalog_create_appends_cell(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_create_appends_cell(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project), prompt="seed text") cells = notebook.load(project)["cells"] @@ -139,7 +138,7 @@ def test_catalog_create_appends_cell(project: str, orders_parquet: Path, monkeyp assert cells[0]["markdown"] == "seed text" -def test_catalog_revise_does_not_duplicate_cell(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_revise_does_not_duplicate_cell(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project), prompt="seed") catalog_revise("shoe_sales", _filter_code(project), prompt="revised") @@ -149,7 +148,7 @@ def test_catalog_revise_does_not_duplicate_cell(project: str, orders_parquet: Pa assert cells[0]["markdown"] == "seed" -def test_catalog_alias_appends_cell(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_alias_appends_cell(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) scratch = catalog_run(_agg_code(project), prompt="exploratory") catalog_alias(scratch["hash"], "shoe_sales") @@ -158,7 +157,7 @@ def test_catalog_alias_appends_cell(project: str, orders_parquet: Path, monkeypa assert cells[0]["alias"] == "shoe_sales" -def test_catalog_rename_renames_cell_in_place(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_rename_renames_cell_in_place(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("old", _agg_code(project)) catalog_rename("old", "new") @@ -167,7 +166,7 @@ def test_catalog_rename_renames_cell_in_place(project: str, orders_parquet: Path assert cells[0]["alias"] == "new" -def test_catalog_unalias_drops_cell(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_unalias_drops_cell(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("dropme", _agg_code(project)) out = catalog_unalias("dropme") @@ -175,7 +174,7 @@ def test_catalog_unalias_drops_cell(project: str, orders_parquet: Path, monkeypa assert notebook.load(project) == {"cells": []} -def test_notebook_reorder_tool(project: str, orders_parquet: Path, monkeypatch): +def test_notebook_reorder_tool(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("a", _agg_code(project)) catalog_create("b", _filter_code(project)) @@ -185,7 +184,7 @@ def test_notebook_reorder_tool(project: str, orders_parquet: Path, monkeypatch): assert after == ["b", "a"] -def test_notebook_remove_tool(project: str, orders_parquet: Path, monkeypatch): +def test_notebook_remove_tool(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("a", _agg_code(project)) cells = notebook.load(project)["cells"] @@ -193,7 +192,7 @@ def test_notebook_remove_tool(project: str, orders_parquet: Path, monkeypatch): assert notebook.load(project) == {"cells": []} -def test_notebook_edit_markdown_tool(project: str, orders_parquet: Path, monkeypatch): +def test_notebook_edit_markdown_tool(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("a", _agg_code(project), prompt="original") cell = notebook.load(project)["cells"][0] @@ -223,7 +222,7 @@ def test_notebook_route_empty(fresh_companion_app, project: str): assert r.json()["cells"] == [] -def test_notebook_route_renders_cells(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_notebook_route_renders_cells(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project), prompt="region totals") c = TestClient(fresh_companion_app) @@ -235,7 +234,7 @@ def test_notebook_route_renders_cells(fresh_companion_app, project: str, orders_ assert cells[0]["entry_meta"]["prompt"] == "region totals" -def test_api_notebook_reorder(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_api_notebook_reorder(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("a", _agg_code(project)) catalog_create("b", _filter_code(project)) @@ -249,7 +248,7 @@ def test_api_notebook_reorder(fresh_companion_app, project: str, orders_parquet: assert [x["alias"] for x in notebook.load(project)["cells"]] == ["b", "a"] -def test_api_notebook_remove(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_api_notebook_remove(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("a", _agg_code(project)) cells = notebook.load(project)["cells"] @@ -262,7 +261,7 @@ def test_api_notebook_remove(fresh_companion_app, project: str, orders_parquet: assert notebook.load(project) == {"cells": []} -def test_api_markdown_put(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_api_markdown_put(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("a", _agg_code(project), prompt="seed") cell = notebook.load(project)["cells"][0] @@ -287,7 +286,7 @@ def test_api_notebook_unknown_action(fresh_companion_app, project: str): assert r.status_code == 400 -def test_notebook_renders_markdown_as_html(fresh_companion_app, project: str, orders_parquet, monkeypatch): +def test_notebook_renders_markdown_as_html(fresh_companion_app, project: str, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create as _cc from tallyman_mcp.server import notebook_edit_markdown as _em @@ -305,7 +304,7 @@ def test_notebook_renders_markdown_as_html(fresh_companion_app, project: str, or def test_notebook_empty_markdown_shows_placeholder_in_edit_mode( - fresh_companion_app, project: str, orders_parquet, monkeypatch + fresh_companion_app, project: str, orders_src, monkeypatch ): monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create as _cc @@ -319,7 +318,7 @@ def test_notebook_empty_markdown_shows_placeholder_in_edit_mode( assert not cell_data["markdown"] -def test_put_markdown_returns_rendered_html(fresh_companion_app, project: str, orders_parquet, monkeypatch): +def test_put_markdown_returns_rendered_html(fresh_companion_app, project: str, orders_src, monkeypatch): """T-10: in-place updates need the server to send back rendered HTML so the client can swap it in without a reload.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -335,7 +334,7 @@ def test_put_markdown_returns_rendered_html(fresh_companion_app, project: str, o assert body["markdown"] == "## Hello\n\nworld" -def test_notebook_page_includes_sortable_js(fresh_companion_app, project: str, orders_parquet, monkeypatch): +def test_notebook_page_includes_sortable_js(fresh_companion_app, project: str, orders_src, monkeypatch): """T-09: drag-reorder is handled by dnd-kit in the React SPA; API must accept reorder.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create as _cc @@ -348,7 +347,7 @@ def test_notebook_page_includes_sortable_js(fresh_companion_app, project: str, o assert any(cell["alias"] == "shoe_sales" for cell in cells) -def test_serve_mode_omits_sortable_js(project: str, orders_parquet, monkeypatch): +def test_serve_mode_omits_sortable_js(project: str, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_companion import create_app from tallyman_mcp.server import catalog_create as _cc @@ -362,7 +361,7 @@ def test_serve_mode_omits_sortable_js(project: str, orders_parquet, monkeypatch) assert r.status_code == 403 -def test_notebook_uses_buckaroo_embed_when_session_available(project: str, orders_parquet, monkeypatch): +def test_notebook_uses_buckaroo_embed_when_session_available(project: str, orders_src, monkeypatch): """notebook_full API returns buckaroo_session when Buckaroo is running.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_companion import create_app @@ -408,7 +407,7 @@ def load_session(self, content_hash, project, column_config_overrides=None): assert sr.json()["ws_url"].startswith("ws://127.0.0.1:8700/ws/sess-") -def test_notebook_falls_back_to_html_when_no_buckaroo(fresh_companion_app, project: str, orders_parquet, monkeypatch): +def test_notebook_falls_back_to_html_when_no_buckaroo(fresh_companion_app, project: str, orders_src, monkeypatch): """notebook_full API signals buckaroo unavailable when no manager is configured.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create as _cc diff --git a/tests/test_pack.py b/tests/test_pack.py index faaf608f..c79d1780 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -14,18 +14,18 @@ from tallyman_xorq import build_and_persist -def _build_one(project: str, orders_parquet: Path) -> str: +def _build_one(project: str, src: str) -> str: code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ res = build_and_persist(project, code, prompt="by region") return res.content_hash -def test_pack_creates_tgz(project: str, orders_parquet: Path, isolated_home: Path, tmp_path: Path): - h = _build_one(project, orders_parquet) +def test_pack_creates_tgz(project: str, orders_src: str, isolated_home: Path, tmp_path: Path): + h = _build_one(project, orders_src) set_alias(project, "by_region", h) out_path = tmp_path / "spike.tgz" @@ -41,13 +41,16 @@ def test_pack_creates_tgz(project: str, orders_parquet: Path, isolated_home: Pat assert f"{project}/artifacts/catalog/aliases.jsonl" in names assert f"{project}/data/orders.parquet" in names assert any(f"{project}/artifacts/catalog/entries/{h}/" in n for n in names) + # The clone of the imported bytes ships too. A source version's snapshot is cache re-created from + # the clone (ADR-011 D1), so a bundle without it could not make the rows it serves again. + assert any(n.startswith(f"{project}/data/.cas/") and n.endswith(".parquet") for n in names) -def test_pack_skips_buckaroo_sessions(project: str, orders_parquet: Path, isolated_home: Path, tmp_path: Path): +def test_pack_skips_buckaroo_sessions(project: str, orders_src: str, isolated_home: Path, tmp_path: Path): # Write a stale sessions file that mustn't ship. from tallyman_core import catalog_dir - _build_one(project, orders_parquet) + _build_one(project, orders_src) sessions_file = catalog_dir(project) / "buckaroo_sessions.json" sessions_file.write_text("{}") assert sessions_file.exists() @@ -69,11 +72,11 @@ def test_pack_unknown_project(isolated_home: Path, tmp_path: Path): assert "not found" in result.output -def test_pack_round_trip_serves(project: str, orders_parquet: Path, isolated_home: Path, tmp_path: Path, monkeypatch): +def test_pack_round_trip_serves(project: str, orders_src: str, isolated_home: Path, tmp_path: Path, monkeypatch): """The full handoff loop: pack → extract to a random location → point TALLYMAN_PROJECT_PATH at the extract → serve read-only and verify every alias resolves.""" - h = _build_one(project, orders_parquet) + h = _build_one(project, orders_src) set_alias(project, "by_region", h) out_path = tmp_path / "bundle.tgz" diff --git a/tests/test_parquet_digest_order_probe.py b/tests/test_parquet_digest_order_probe.py index 0e48c3f2..b153afd7 100644 --- a/tests/test_parquet_digest_order_probe.py +++ b/tests/test_parquet_digest_order_probe.py @@ -32,24 +32,26 @@ snapshot_file_digest, verify_result_faithful, ) +from tallyman_xorq.source_import import update_and_depend _HEALS = 3 +_SOURCE = "orders_big_src" def _big_agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders_big.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({_SOURCE!r}, project={project!r}) expr = t.group_by("order_id").aggregate(total=t.price.sum(), n=t.count()) """ -def test_parquet_bake_digest_stable_across_heals(project, monkeypatch, caplog): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_parquet_bake_digest_stable_across_heals(project, caplog): src = write_shoe_orders(data_dir(project) / "orders_big.parquet", n_rows=250_000, seed=7) # A sanity floor only: the file is well below the 10 MiB scan-split threshold (see the module docstring), and the # group-by over 250,000 order ids is what exercises the aggregate's repartitioning. assert src.stat().st_size > 1_048_576, "fixture too small to be a meaningful aggregate input" + update_and_depend(src, _SOURCE, project=project) h = catalog_create("big_by_order", _big_agg_code(project))["hash"] recorded = read_manifest(entry_dir(project, h)).result_digest diff --git a/tests/test_post_processing.py b/tests/test_post_processing.py index ad02a67a..d3320728 100644 --- a/tests/test_post_processing.py +++ b/tests/test_post_processing.py @@ -159,28 +159,28 @@ def test_mcp_remove_nonexistent_returns_error(project: str): # --------------------------------------------------------------------------- -def _agg_code(project: str) -> str: +def _agg_code(project: str, src: str) -> str: return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file('orders.parquet', project={project!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({src!r}, project={project!r})\n" "expr = t.group_by('region').aggregate(n=t.count())\n" ) -def _cheap_code(project: str) -> str: # parquet read + projection → cheap, bakes no snapshot +def _cheap_code(project: str, src: str) -> str: # source read + projection → cheap, bakes no snapshot return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file('orders.parquet', project={project!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({src!r}, project={project!r})\n" "expr = t.select('region', 'price', '__row_order')\n" ) -def test_run_post_processing_by_alias(project: str, orders_parquet, monkeypatch): +def test_run_post_processing_by_alias(project: str, orders_src: str, monkeypatch): """run_post_processing resolves an alias and returns filtered rows.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create, catalog_run_post_processing - catalog_create("agg", _agg_code(project)) + catalog_create("agg", _agg_code(project, orders_src)) resp = catalog_run_post_processing( code="def process(expr):\n return expr\n", entry="agg", @@ -191,12 +191,12 @@ def test_run_post_processing_by_alias(project: str, orders_parquet, monkeypatch) assert isinstance(resp["preview"], list) -def test_run_post_processing_by_hash(project: str, orders_parquet, monkeypatch): +def test_run_post_processing_by_hash(project: str, orders_src: str, monkeypatch): """run_post_processing accepts a raw content hash.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create, catalog_run_post_processing - out = catalog_create("agg2", _agg_code(project)) + out = catalog_create("agg2", _agg_code(project, orders_src)) resp = catalog_run_post_processing( code="def process(expr):\n return expr\n", entry=out["hash"], @@ -205,12 +205,12 @@ def test_run_post_processing_by_hash(project: str, orders_parquet, monkeypatch): assert resp["entry"] == out["hash"] -def test_run_post_processing_filter_reduces_rows(project: str, orders_parquet, monkeypatch): +def test_run_post_processing_filter_reduces_rows(project: str, orders_src: str, monkeypatch): """A filter in process() produces fewer rows than the original.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create, catalog_run_post_processing - catalog_create("agg3", _agg_code(project)) + catalog_create("agg3", _agg_code(project, orders_src)) all_rows = catalog_run_post_processing("def process(expr):\n return expr\n", "agg3") filtered = catalog_run_post_processing("def process(expr):\n return expr.filter(expr.n > 9999)\n", "agg3") assert "error" not in all_rows @@ -218,12 +218,12 @@ def test_run_post_processing_filter_reduces_rows(project: str, orders_parquet, m assert filtered["row_count"] < all_rows["row_count"] -def test_run_post_processing_bad_code_returns_error(project: str, orders_parquet, monkeypatch): +def test_run_post_processing_bad_code_returns_error(project: str, orders_src: str, monkeypatch): """Syntax errors in the process function surface as error dict.""" monkeypatch.setenv("TALLYMAN_PROJECT", project) from tallyman_mcp.server import catalog_create, catalog_run_post_processing - catalog_create("agg4", _agg_code(project)) + catalog_create("agg4", _agg_code(project, orders_src)) resp = catalog_run_post_processing("def process(expr):\n return 42\n", "agg4") assert "error" in resp @@ -237,7 +237,7 @@ def test_run_post_processing_unknown_entry_returns_error(project: str, monkeypat assert "error" in resp -def test_run_post_processing_writes_no_result_parquet(project: str, orders_parquet, monkeypatch): +def test_run_post_processing_writes_no_result_parquet(project: str, orders_src: str, monkeypatch): """Post-processing reads the entry's cache-resolving expression, not an on-demand ``result.parquet``. A cheap entry recomputes and an expensive one reads its baked snapshot — neither materialises a per-entry parquet on disk @@ -248,7 +248,7 @@ def test_run_post_processing_writes_no_result_parquet(project: str, orders_parqu from tallyman_mcp.server import catalog_create, catalog_run_post_processing noop = "def process(expr):\n return expr\n" - for alias, code in (("cheap_pp", _cheap_code(project)), ("agg_pp", _agg_code(project))): + for alias, code in (("cheap_pp", _cheap_code(project, orders_src)), ("agg_pp", _agg_code(project, orders_src))): out = catalog_create(alias, code) assert "error" not in out, out h = out["hash"] @@ -256,23 +256,3 @@ def test_run_post_processing_writes_no_result_parquet(project: str, orders_parqu assert "error" not in resp, resp assert resp["row_count"] > 0 assert not (entry_dir(project, h) / "result.parquet").exists() - - -def test_run_post_processing_under_salt_writes_no_result_parquet(project: str, orders_parquet, monkeypatch): - """salt source-identity mode used to route reads through an on-demand - ``result.parquet`` (path-only snapshot keys collide across salted entries). - With the on-demand layer gone, a salted entry recomputes through the same - cheap/expensive path as off-mode and writes no per-entry parquet. - """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "salt") - from tallyman_core import entry_dir - from tallyman_mcp.server import catalog_create, catalog_run_post_processing - - out = catalog_create("salted_pp", _agg_code(project)) - assert "error" not in out, out - h = out["hash"] - resp = catalog_run_post_processing(code="def process(expr):\n return expr\n", entry="salted_pp") - assert "error" not in resp, resp - assert resp["row_count"] > 0 - assert not (entry_dir(project, h) / "result.parquet").exists() diff --git a/tests/test_primary_key.py b/tests/test_primary_key.py index 5ff780b1..f5fe4353 100644 --- a/tests/test_primary_key.py +++ b/tests/test_primary_key.py @@ -11,20 +11,20 @@ from tallyman_xorq.primary_key import _parent_hash, diff_keys, resolve_primary_key -def _select(project: str, cols: str) -> str: +def _select(project: str, src: str, cols: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.select({cols}) """ -def _dup_rows(project: str) -> str: +def _dup_rows(project: str, src: str) -> str: # Every order twice: high-cardinality columns, but no column set is more # than 50% distinct, so the search can never stop early. return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.union(t, distinct=False) """ @@ -40,9 +40,9 @@ def _current_hash(project: str) -> str: return list_entries(project)[0]["content_hash"] -def test_resolve_detects_and_caches(project, orders_parquet, monkeypatch): +def test_resolve_detects_and_caches(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("rides", _select(project, '"order_id", "region", "price", "__row_order"')) + catalog_create("rides", _select(project, orders_src, '"order_id", "region", "price", "__row_order"')) h = _current_hash(project) assert resolve_primary_key(project, h) == ["order_id"] # order_id is unique assert (entry_dir(project, h) / "primary_key.json").exists() # cached @@ -50,11 +50,11 @@ def test_resolve_detects_and_caches(project, orders_parquet, monkeypatch): assert resolve_primary_key(project, h) == ["order_id"] -def test_row_preserving_revision_inherits_key(project, orders_parquet, monkeypatch): +def test_row_preserving_revision_inherits_key(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("rides", _select(project, '"order_id", "region", "price", "__row_order"')) + catalog_create("rides", _select(project, orders_src, '"order_id", "region", "price", "__row_order"')) base = _current_hash(project) - catalog_revise("rides", _select(project, '"price", "region", "order_id", "__row_order"')) # reorder + catalog_revise("rides", _select(project, orders_src, '"price", "region", "order_id", "__row_order"')) # reorder child = _current_hash(project) assert child != base assert _parent_hash(project, child) == base @@ -64,19 +64,19 @@ def test_row_preserving_revision_inherits_key(project, orders_parquet, monkeypat assert diff_keys(project, base, child) == ["order_id"] -def test_dropping_key_column_breaks_inheritance(project, orders_parquet, monkeypatch): +def test_dropping_key_column_breaks_inheritance(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("rides", _select(project, '"order_id", "region", "price", "__row_order"')) + catalog_create("rides", _select(project, orders_src, '"order_id", "region", "price", "__row_order"')) base = _current_hash(project) - catalog_revise("rides", _select(project, '"region", "price", "__row_order"')) # drops order_id + catalog_revise("rides", _select(project, orders_src, '"region", "price", "__row_order"')) # drops order_id child = _current_hash(project) # order_id no longer present → can't be the join key for the pair assert diff_keys(project, base, child) != ["order_id"] -def test_search_times_out_without_caching(project, orders_parquet, monkeypatch): +def test_search_times_out_without_caching(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("dups", _dup_rows(project)) + catalog_create("dups", _dup_rows(project, orders_src)) h = _current_hash(project) # Each budget check costs the whole 1s budget, so the search stops before # its first query. @@ -87,18 +87,18 @@ def test_search_times_out_without_caching(project, orders_parquet, monkeypatch): assert not (entry_dir(project, h) / "primary_key.json").exists() -def test_search_within_budget_still_finds_key(project, orders_parquet, monkeypatch): +def test_search_within_budget_still_finds_key(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("rides", _select(project, '"order_id", "region", "price", "__row_order"')) + catalog_create("rides", _select(project, orders_src, '"order_id", "region", "price", "__row_order"')) h = _current_hash(project) # A unique single column is found in a couple of queries, well inside budget. monkeypatch.setattr(pk, "_clock", _ticking_clock(0.3)) assert resolve_primary_key(project, h) == ["order_id"] -def test_keyless_table_resolves_empty_without_timing_out(project, orders_parquet, monkeypatch): +def test_keyless_table_resolves_empty_without_timing_out(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("dups", _dup_rows(project)) + catalog_create("dups", _dup_rows(project, orders_src)) h = _current_hash(project) # Same few-query budget as the timeout test: one distinct count over every # candidate column shows no subset can reach the threshold, so the search diff --git a/tests/test_profile_pageload.py b/tests/test_profile_pageload.py index a6272e92..3403e6d3 100644 --- a/tests/test_profile_pageload.py +++ b/tests/test_profile_pageload.py @@ -26,16 +26,16 @@ def _hash(res: dict) -> str: @pytest.fixture -def corpus(project, orders_parquet, isolated_home, monkeypatch) -> dict[str, tuple[str, int]]: - """A cheap root entry and a worthy aggregate over the same source: alias -> (hash, rows).""" +def corpus(project, orders_src, isolated_home, monkeypatch) -> dict[str, tuple[str, int]]: + """A cheap root entry and a worthy aggregate over the same source alias: alias -> (hash, rows).""" monkeypatch.setenv("TALLYMAN_PROJECT", project) monkeypatch.setenv("TALLYMAN_AUTO_RECALC", "0") orders = _hash( catalog_create( "orders", f""" -from tallyman_xorq.io import read_project_file -expr = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +expr = tracked_expr_from_alias({orders_src!r}) """, ) ) @@ -43,15 +43,19 @@ def corpus(project, orders_parquet, isolated_home, monkeypatch) -> dict[str, tup catalog_create( "agg", f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """, ) ) + # The source version is an ordinary entry (ADR-011 D1), so the corpus is three entries, not two: + # the profiler measures the imported rows the same way it measures anything built from them. + from tallyman_core.aliases import get_alias + return { alias: (h, int(cached_result_expr(project, h).count().execute())) - for alias, h in (("orders", orders), ("agg", agg)) + for alias, h in (("orders", orders), ("agg", agg), (orders_src, get_alias(project, orders_src))) } diff --git a/tests/test_project_lock.py b/tests/test_project_lock.py index 376373e1..0d2955b5 100644 --- a/tests/test_project_lock.py +++ b/tests/test_project_lock.py @@ -19,18 +19,18 @@ from tallyman_xorq import build_and_persist -def _agg_code(project: str) -> str: +def _agg_code(project: str, src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ -def _cheap_code(project: str) -> str: +def _cheap_code(project: str, src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.filter(t.price > 0) """ @@ -111,7 +111,7 @@ def test_project_lock_is_public_and_the_private_name_is_an_alias(): # --------------------------------------------------------------------------- -def test_two_builds_in_one_project_never_run_at_once(project, orders_parquet, monkeypatch): +def test_two_builds_in_one_project_never_run_at_once(project, orders_src, monkeypatch): """ADR-007 D11: a build is a write, so two builds of two different entries in one project queue. ``build_expr`` is the first heavy step of a build; it is wrapped to count how many threads are inside it. Each waits (on an event, not a sleep) for the other to be inside too. Unserialized, they meet at once and the peak is 2. Serialized, the @@ -151,7 +151,7 @@ def run() -> None: return run - threads = _run_in_threads([builder(_agg_code(project)), builder(_cheap_code(project))]) + threads = _run_in_threads([builder(_agg_code(project, orders_src)), builder(_cheap_code(project, orders_src))]) assert not any(t.is_alive() for t in threads), "a build never finished" assert state["peak"] == 1, f"{state['peak']} builds of one project ran at the same time" @@ -159,7 +159,7 @@ def run() -> None: assert len(results) == 2 and results[0].content_hash != results[1].content_hash -def test_a_failing_concurrent_build_of_one_entry_cannot_delete_the_winners_entry(project, orders_parquet, monkeypatch): +def test_a_failing_concurrent_build_of_one_entry_cannot_delete_the_winners_entry(project, orders_src, monkeypatch): """ADR-007 D11 (the audit's finding): two builds of one entry, and one of them fails after laying down the entry directory. Its cleanup removes the directory, which the other build is still using, so the winner then fails too. @@ -189,10 +189,8 @@ def instrumented(*args, **kwargs): second_arrived.wait(timeout=1.5) return real_load_expr(*args, **kwargs) - # Two cold builders also race for the content-addressed clone of the source (``ensure_cas_path`` writes a fixed - # ``.parquet.tmp``), which is a second instance of the same defect but not the one injected below. - # Building any other entry over the same source first leaves the clone in place, so that only the injection differs. - build_and_persist(project, _cheap_code(project)) + # The source's clone and snapshot are written once by the import (ADR-011 D1), before either builder starts, so + # nothing but the injection below distinguishes the two threads. monkeypatch.setattr(compiler, "load_expr", instrumented) start = threading.Barrier(2) results: list = [] @@ -201,7 +199,7 @@ def instrumented(*args, **kwargs): def builder() -> None: try: start.wait(timeout=10) - results.append(build_and_persist(project, _agg_code(project))) + results.append(build_and_persist(project, _agg_code(project, orders_src))) except BaseException as exc: # noqa: BLE001 - reported by the assertion below errors.append(exc) @@ -229,7 +227,7 @@ def _snapshot_path(project: str, content_hash: str): return snapshot_path(project, content_hash) -def test_concurrent_materializations_of_one_entry_all_succeed(project, orders_parquet): +def test_concurrent_materializations_of_one_entry_all_succeed(project, orders_src): """ADR-007 D11 and D4 (one writer, used by the build and by every heal): four threads materialize the same entry. The writer takes a unique temp name in the destination directory and renames it into place, under the project lock, so each call succeeds and the file is whole afterwards. xorq's writer used a fixed ``.parquet.tmp``: @@ -239,7 +237,7 @@ def test_concurrent_materializations_of_one_entry_all_succeed(project, orders_pa from tallyman_xorq.result_cache import snapshot_file_digest - h = build_and_persist(project, _agg_code(project)).content_hash + h = build_and_persist(project, _agg_code(project, orders_src)).content_hash manifest = read_manifest(entry_dir(project, h)) start = threading.Barrier(4) digests: list[str] = [] diff --git a/tests/test_promote_diff.py b/tests/test_promote_diff.py index 22a1e55e..59672e87 100644 --- a/tests/test_promote_diff.py +++ b/tests/test_promote_diff.py @@ -21,16 +21,16 @@ def _agg_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ def _filter_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) filtered = t.filter(t.category == "boots") expr = filtered.group_by("region").aggregate(total=filtered.price.sum(), n=filtered.count()) """ @@ -41,7 +41,7 @@ def _filter_code(project: str) -> str: # --------------------------------------------------------------------------- -def test_build_compare_expr_has_sentinel_columns(project: str, orders_parquet: Path): +def test_build_compare_expr_has_sentinel_columns(project: str, orders_src: str): from tallyman_companion.diff import build_compare_expr a = build_and_persist(project, _agg_code(project)) @@ -62,7 +62,7 @@ def test_build_compare_expr_has_sentinel_columns(project: str, orders_parquet: P assert "n_abs_delta" in schema -def test_build_compare_expr_delta_columns_follow_v2(project: str, orders_parquet: Path): +def test_build_compare_expr_delta_columns_follow_v2(project: str, orders_src: str): from tallyman_companion.diff import build_compare_expr a = build_and_persist(project, _agg_code(project)) @@ -79,7 +79,7 @@ def test_build_compare_expr_delta_columns_follow_v2(project: str, orders_parquet assert cols[v2_idx + 2] == f"{prefix}_abs_delta" -def test_build_compare_expr_overrides_hide_raw_cols(project: str, orders_parquet: Path): +def test_build_compare_expr_overrides_hide_raw_cols(project: str, orders_src: str): from tallyman_companion.diff import build_compare_expr a = build_and_persist(project, _agg_code(project)) @@ -95,7 +95,7 @@ def test_build_compare_expr_overrides_hide_raw_cols(project: str, orders_parquet assert "total_abs_delta" not in overrides -def test_build_compare_expr_numeric_col_uses_diverging_colormap(project: str, orders_parquet: Path): +def test_build_compare_expr_numeric_col_uses_diverging_colormap(project: str, orders_src: str): # The base overrides (used by promoted diff entries, which render without # the diff display klasses) color numeric value columns by pct_delta. The # live /diff route strips this via strip_live_diff_color so the per-view @@ -145,7 +145,7 @@ def test_strip_live_diff_color_drops_numeric_keeps_categorical(): assert stripped["region"]["color_map_config"]["color_rule"] == "color_categorical" -def test_build_compare_expr_key_col_uses_categorical_colormap(project: str, orders_parquet: Path): +def test_build_compare_expr_key_col_uses_categorical_colormap(project: str, orders_src: str): from tallyman_companion.diff import build_compare_expr a = build_and_persist(project, _agg_code(project)) @@ -163,7 +163,7 @@ def test_build_compare_expr_key_col_uses_categorical_colormap(project: str, orde # --------------------------------------------------------------------------- -def test_build_diff_expr_returns_ibis_expr(project: str, orders_parquet: Path): +def test_build_diff_expr_returns_ibis_expr(project: str, orders_src: str): from tallyman_companion.diff import build_diff_expr a = build_and_persist(project, _agg_code(project)) @@ -172,7 +172,7 @@ def test_build_diff_expr_returns_ibis_expr(project: str, orders_parquet: Path): assert "membership" in expr.schema() -def test_build_diff_expr_writes_no_result_parquet(project: str, orders_parquet: Path, monkeypatch): +def test_build_diff_expr_writes_no_result_parquet(project: str, orders_src: str, monkeypatch): """build_diff_expr composes the two entries' cache-resolving expressions directly. It must not materialise an on-demand ``result.parquet`` for either side — those parquets are the layer #73 began retiring and are never @@ -195,7 +195,7 @@ def test_build_diff_expr_writes_no_result_parquet(project: str, orders_parquet: # --------------------------------------------------------------------------- -def test_catalog_promote_diff_creates_entry(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_creates_entry(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -207,7 +207,7 @@ def test_catalog_promote_diff_creates_entry(project: str, orders_parquet: Path, assert out["row_count"] > 0 -def test_catalog_promote_diff_auto_alias(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_auto_alias(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -215,7 +215,7 @@ def test_catalog_promote_diff_auto_alias(project: str, orders_parquet: Path, mon assert out["alias"] == "diff_shoe_sales_v1_v2" -def test_catalog_promote_diff_custom_alias(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_custom_alias(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -223,7 +223,7 @@ def test_catalog_promote_diff_custom_alias(project: str, orders_parquet: Path, m assert out["alias"] == "my_diff" -def test_catalog_promote_diff_stores_display_config(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_stores_display_config(project: str, orders_src: str, monkeypatch): from tallyman_core.display_configs import get_display_config monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -240,7 +240,7 @@ def test_catalog_promote_diff_stores_display_config(project: str, orders_parquet assert cfg["column_config_overrides"]["membership"]["merge_rule"] == "hidden" -def test_catalog_promote_diff_repoint_cascades_to_follower_atomically(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_repoint_cascades_to_follower_atomically(project: str, orders_src: str, monkeypatch): # D5: catalog_promote_diff re-pointing an EXISTING alias is an auto-recalc # trigger. Re-promoting onto the same target advances it, so its followers must # cascade-recompute in the SAME revision (the walk runs before promote_diff's @@ -291,14 +291,14 @@ def test_catalog_promote_diff_no_history_returns_error(project: str, monkeypatch assert "error" in out -def test_catalog_promote_diff_out_of_range_returns_error(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_out_of_range_returns_error(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) out = catalog_promote_diff("shoe_sales", va=1, vb=5) assert "error" in out -def test_catalog_promote_diff_negative_indices(project: str, orders_parquet: Path, monkeypatch): +def test_catalog_promote_diff_negative_indices(project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -312,7 +312,7 @@ def test_catalog_promote_diff_negative_indices(project: str, orders_parquet: Pat # --------------------------------------------------------------------------- -def test_http_promote_diff_creates_entry(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_http_promote_diff_creates_entry(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) catalog_revise("shoe_sales", _filter_code(project)) @@ -328,7 +328,38 @@ def test_http_promote_diff_creates_entry(fresh_companion_app, project: str, orde assert body["row_count"] > 0 -def test_http_promote_diff_sets_display_config(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_http_promote_diff_refuses_a_source_alias_as_its_target( + fresh_companion_app, project: str, orders_src: str, tmp_path: Path, monkeypatch +): + """The generated target name can already be a source alias; the route refuses before building, as the tool does. + + It used to build the diff entry, fail to point the source alias at it, and answer 500 with the entry left behind. + """ + import pandas as pd + + from tallyman_core import entry_dir, get_alias + from tallyman_xorq.source_import import update_and_depend + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + catalog_create("shoe_sales", _agg_code(project)) + catalog_revise("shoe_sales", _filter_code(project)) + other = tmp_path / "outside" / "other.parquet" + other.parent.mkdir(parents=True) + pd.DataFrame({"x": [1, 2, 3]}).to_parquet(other) + update_and_depend(str(other), "diff_shoe_sales_v1_v2", project=project) + head = get_alias(project, "diff_shoe_sales_v1_v2") + entries = sorted(p.name for p in entry_dir(project, head).parent.iterdir()) + + c = TestClient(fresh_companion_app, raise_server_exceptions=False) + r = c.post(f"/{project}/api/promote_diff/shoe_sales/1/2") + + assert r.status_code == 409, r.text + assert "source alias" in r.json()["detail"] + assert get_alias(project, "diff_shoe_sales_v1_v2") == head + assert sorted(p.name for p in entry_dir(project, head).parent.iterdir()) == entries + + +def test_http_promote_diff_sets_display_config(fresh_companion_app, project: str, orders_src: str, monkeypatch): from tallyman_core.display_configs import get_display_config monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -350,7 +381,7 @@ def test_http_promote_diff_unknown_alias_404(fresh_companion_app, project: str): assert r.status_code == 404 -def test_http_promote_diff_out_of_range_400(fresh_companion_app, project: str, orders_parquet: Path, monkeypatch): +def test_http_promote_diff_out_of_range_400(fresh_companion_app, project: str, orders_src: str, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) @@ -360,7 +391,7 @@ def test_http_promote_diff_out_of_range_400(fresh_companion_app, project: str, o def test_http_entry_detail_diff_has_display_config( - fresh_companion_app, project: str, orders_parquet: Path, monkeypatch + fresh_companion_app, project: str, orders_src: str, monkeypatch ): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("shoe_sales", _agg_code(project)) @@ -383,7 +414,7 @@ def test_http_entry_detail_diff_has_display_config( # --------------------------------------------------------------------------- -def test_marimo_export_diff_entry_inlines_source_code(project: str, orders_parquet: Path, monkeypatch): +def test_marimo_export_diff_entry_inlines_source_code(project: str, orders_src: str, monkeypatch): from tallyman_core import notebook as nb_mod from tallyman_core.display_configs import set_display_config from tallyman_core.marimo_export import notebook_to_marimo @@ -420,7 +451,7 @@ def test_marimo_export_diff_entry_inlines_source_code(project: str, orders_parqu assert "b_expr = expr" in nb_source -def test_marimo_export_diff_entry_includes_overrides(project: str, orders_parquet: Path, monkeypatch): +def test_marimo_export_diff_entry_includes_overrides(project: str, orders_src: str, monkeypatch): from tallyman_core import notebook as nb_mod from tallyman_core.display_configs import set_display_config from tallyman_core.marimo_export import notebook_to_marimo @@ -457,12 +488,12 @@ def test_marimo_export_diff_entry_includes_overrides(project: str, orders_parque def test_marimo_export_loads_in_marimo_without_collisions( - project: str, orders_parquet: Path, monkeypatch, tmp_path: Path + project: str, orders_src: str, monkeypatch, tmp_path: Path ): """The exported notebook must load in marimo without a MultipleDefinitionError. Each code cell inlines an ``expr.py`` that binds ``expr`` and imports - ``os`` / ``xo`` / ``read_project_file``. With more than one entry those names + ``os`` / ``xo`` / ``tracked_expr_from_alias``. With more than one entry those names used to leak into marimo's global dataflow graph and collide; the closure wrapper in ``marimo_export`` keeps them cell-local. String assertions alone never caught this — we have to actually build the marimo graph. diff --git a/tests/test_prompts.py b/tests/test_prompts.py index ca40da35..6dc774fe 100644 --- a/tests/test_prompts.py +++ b/tests/test_prompts.py @@ -1,52 +1,50 @@ from __future__ import annotations -from pathlib import Path - from fastapi.testclient import TestClient from tallyman_xorq import build_and_persist, read_prompts -def _code(parquet: Path) -> str: - # The parquet sits in the project's data dir and enters a recipe through read_project_file (ADR-008 D12). +def _code(src: str) -> str: + # The orders data entered the catalog as a source alias, and a recipe reads the alias (ADR-011 D2). return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file({parquet.name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}) expr = t.group_by("region").aggregate(n=t.count()) """ -def test_first_build_records_one_prompt(project: str, orders_parquet: Path): - res = build_and_persist(project, _code(orders_parquet), prompt="first") +def test_first_build_records_one_prompt(project: str, orders_src: str): + res = build_and_persist(project, _code(orders_src), prompt="first") history = read_prompts(project, res.content_hash) assert len(history) == 1 assert history[0]["prompt"] == "first" -def test_re_run_appends_prompt(project: str, orders_parquet: Path): - a = build_and_persist(project, _code(orders_parquet), prompt="first") - build_and_persist(project, _code(orders_parquet), prompt="second") +def test_re_run_appends_prompt(project: str, orders_src: str): + a = build_and_persist(project, _code(orders_src), prompt="first") + build_and_persist(project, _code(orders_src), prompt="second") history = read_prompts(project, a.content_hash) assert [p["prompt"] for p in history] == ["first", "second"] -def test_manifest_prompt_remains_first(project: str, orders_parquet: Path): +def test_manifest_prompt_remains_first(project: str, orders_src: str): import json - a = build_and_persist(project, _code(orders_parquet), prompt="first") - build_and_persist(project, _code(orders_parquet), prompt="second") + a = build_and_persist(project, _code(orders_src), prompt="first") + build_and_persist(project, _code(orders_src), prompt="second") manifest = json.loads((a.entry_path / "manifest.json").read_text()) assert manifest["prompt"] == "first" -def test_no_prompt_does_not_write_history(project: str, orders_parquet: Path): - res = build_and_persist(project, _code(orders_parquet), prompt=None) +def test_no_prompt_does_not_write_history(project: str, orders_src: str): + res = build_and_persist(project, _code(orders_src), prompt=None) assert read_prompts(project, res.content_hash) == [] -def test_entry_detail_renders_prompt_history(fresh_companion_app, project: str, orders_parquet: Path): - a = build_and_persist(project, _code(orders_parquet), prompt="alpha attempt") - build_and_persist(project, _code(orders_parquet), prompt="beta attempt") +def test_entry_detail_renders_prompt_history(fresh_companion_app, project: str, orders_src: str): + a = build_and_persist(project, _code(orders_src), prompt="alpha attempt") + build_and_persist(project, _code(orders_src), prompt="beta attempt") c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/entry/{a.content_hash}") assert r.status_code == 200 diff --git a/tests/test_rebuild_native_catalog.py b/tests/test_rebuild_native_catalog.py index a8878daa..96948899 100644 --- a/tests/test_rebuild_native_catalog.py +++ b/tests/test_rebuild_native_catalog.py @@ -37,11 +37,11 @@ def _load_rebuild(): return mod -def _agg(parquet: Path, *, avg: bool = False) -> str: +def _agg(source: str, *, avg: bool = False) -> str: extra = ", avg=t.price.mean()" if avg else "" return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file({parquet.name!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({source!r})\n" f"expr = t.group_by('region').aggregate(n=t.count(){extra})\n" ) @@ -53,11 +53,11 @@ def _child(alias: str, project: str) -> str: ) -def _build_corpus(project: str, parquet: Path) -> dict: +def _build_corpus(project: str, source: str) -> dict: cs.genesis(project) - a = build_and_persist(project, _agg(parquet), prompt="agg by region") + a = build_and_persist(project, _agg(source), prompt="agg by region") al.set_alias(project, "regions", a.content_hash) - a2 = build_and_persist(project, _agg(parquet, avg=True), prompt="add avg") + a2 = build_and_persist(project, _agg(source, avg=True), prompt="add avg") al.set_alias(project, "regions", a2.content_hash) # revision: history [a, a2] b = build_and_persist(project, _child("regions", project), prompt="filter regions") from tallyman_core.charts import set_chart @@ -67,14 +67,16 @@ def _build_corpus(project: str, parquet: Path) -> dict: return {"a": a.content_hash, "a2": a2.content_hash, "b": b.content_hash} -def test_rebuild_preserves_entries_aliases_charts_prompts(project, orders_parquet): +def test_rebuild_preserves_entries_aliases_charts_prompts(project, orders_src): rb = _load_rebuild() - before = _build_corpus(project, orders_parquet) + before = _build_corpus(project, orders_src) + source_hash = al.get_alias(project, orders_src) remap = rb.rebuild_project(project, log=lambda *a: None) - # Every original entry was rebuilt. - assert set(remap) == set(before.values()) + # Every original entry was rebuilt, the imported source among them. + assert set(remap) == set(before.values()) | {source_hash} + assert remap[source_hash] == source_hash # A same-path rebuild is hash-stable for the recipe-deterministic roots. The # tracked_expr_from_alias child bakes its aggregate parent's snapshot, whose bytes are # not datafusion-order-deterministic, so its hash may drift — the rebuild @@ -101,16 +103,17 @@ def test_rebuild_preserves_entries_aliases_charts_prompts(project, orders_parque assert len(cached_result_expr(project, h).execute()) >= 0 -def test_rebuild_dry_run_writes_nothing(project, orders_parquet): +def test_rebuild_dry_run_writes_nothing(project, orders_src): rb = _load_rebuild() - before = _build_corpus(project, orders_parquet) + before = _build_corpus(project, orders_src) + expected = set(before.values()) | {al.get_alias(project, orders_src)} head_before = cs.list_revisions(project) remap = rb.rebuild_project(project, dry_run=True, log=lambda *a: None) assert remap == {} # nothing rebuilt # Catalog untouched: same entries, same revision timeline. - assert set(cs.read_tallyman_state(project)["entry_hashes"]) == set(before.values()) + assert set(cs.read_tallyman_state(project)["entry_hashes"]) == expected assert cs.list_revisions(project) == head_before @@ -178,3 +181,123 @@ def test_read_old_catalog_reads_catalog_yaml_era(project): assert oc.post_processing.get("pp1") assert oc.stats.get("st1") assert [c["alias"] for c in oc.notebook_cells] == ["thing"] + + +def test_rebuild_replays_a_source_entry_as_an_import(project, tmp_path): + """A source entry has no recipe to re-exec, so the rebuild imports it again (ADR-011). + + ``build_and_persist`` cannot rebuild one: the generated recipe reads the entry's own snapshot, which + the rebuild has just wiped, and a raw read is a build error anywhere else. The replay therefore goes + through ``update_and_depend``, from the provenance path when it still holds the imported bytes and + from the clone in ``data/.cas`` when it does not. The hash is a function of the bytes and the reader + options, so it survives the round-trip unchanged even though the outside file is gone. + """ + import pandas as pd + + from tallyman_core.aliases import SOURCE_KIND, alias_kind, history_for + from tallyman_core.manifest import read_manifest + from tallyman_core.paths import entry_dir + from tallyman_xorq.source_import import update_and_depend + + rb = _load_rebuild() + cs.genesis(project) + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + src = outside / "orders.parquet" + pd.DataFrame({"region": ["n", "s", "n"], "price": [1.0, 2.0, 3.0]}).to_parquet(src) + + v1 = update_and_depend(src, "orders", project=project) + child = build_and_persist( + project, + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias('orders', {project!r})\n" + "expr = t.group_by('region').aggregate(n=t.count())\n", + prompt="count by region", + ) + al.set_alias(project, "regions", child.content_hash) + cs.checkpoint_catalog(project, "corpus") + src.unlink() # the outside file is provenance; the clone is what the rebuild reads + + remap = rb.rebuild_project(project, log=lambda *a: None) + + assert set(remap) == {v1["hash"], child.content_hash} + assert remap[v1["hash"]] == v1["hash"], "a source entry's hash is its bytes and its reader options" + assert alias_kind(project, "orders") == SOURCE_KIND + assert history_for(project, "orders") == [v1["hash"]] + provenance = read_manifest(entry_dir(project, v1["hash"])).provenance + assert provenance is not None and provenance.alias == "orders" and provenance.version == 1 + assert len(cached_result_expr(project, remap[child.content_hash]).execute()) == 2 + + pointers = set(cs.read_tallyman_state(project)["entry_hashes"]) + assert pointers == set(remap.values()) + catalog.assert_catalog_consistent(project, pointers) + + +def test_toposort_orders_a_renamed_sources_versions_by_the_alias_that_holds_them(project): + """A source version follows the previous version of the alias that holds it now (ADR-011). + + ``provenance["alias"]`` is the name the version was imported under. After a rename no alias has that name, so + ordering by it gave v1 and v2 no edge and left them in hash order. Here v2's hash sorts first, and hash order + would replay v2's bytes as the renamed alias's v1. + """ + rb = _load_rebuild() + v1, v2 = "bbbbbbbbbbbb", "aaaaaaaaaaaa" + recipes = {v1: "# a source version\n", v2: "# a source version\n"} + provenance = {v1: {"alias": "a_src", "version": 1}, v2: {"alias": "a_src", "version": 2}} + history = {"renamed_src": [v1, v2]} + + assert rb.toposort(recipes, {"renamed_src": v2}, history, provenance) == [v1, v2] + + +def test_rebuild_replays_a_renamed_source_under_the_name_it_has_now(project, tmp_path): + """A renamed source is re-imported under its alias now, so a child reading that alias rebuilds (ADR-011). + + Replaying under ``provenance["alias"]`` minted the import-time name, ``a_src``, and never made ``renamed_src`` + until the final alias write, after every child had tried to build. + """ + import pandas as pd + import pytest + + from tallyman_core.aliases import SOURCE_KIND, load_kinds, rename_alias + from tallyman_core.manifest import read_manifest + from tallyman_core.paths import entry_dir + from tallyman_xorq.source_import import update_and_depend + + rb = _load_rebuild() + cs.genesis(project) + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + src = outside / "orders.parquet" + pd.DataFrame({"region": ["n", "s", "n"], "price": [1.0, 2.0, 3.0]}).to_parquet(src) + v1 = update_and_depend(src, "a_src", project=project) + pd.DataFrame({"region": ["n", "s", "e", "e"], "price": [1.0, 2.0, 3.0, 4.0]}).to_parquet(src) + v2 = update_and_depend(src, "a_src", project=project) + rename_alias(project, "a_src", "renamed_src") + child = build_and_persist( + project, + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias('renamed_src', {project!r})\n" + "expr = t.group_by('region').aggregate(n=t.count())\n", + prompt="count by region", + ) + al.set_alias(project, "regions", child.content_hash) + cs.checkpoint_catalog(project, "corpus") + logged: list[str] = [] + + try: + remap = rb.rebuild_project(project, log=logged.append) + except Exception as exc: # noqa: BLE001 + pytest.fail(f"the rebuild failed after a rename: {exc!r}") + + assert remap[v1["hash"]] == v1["hash"] and remap[v2["hash"]] == v2["hash"] + assert load_kinds(project) == {"renamed_src": SOURCE_KIND, "regions": "catalog"} + assert al.history_for(project, "renamed_src") == [v1["hash"], v2["hash"]] + provenance = read_manifest(entry_dir(project, v1["hash"])).provenance + assert (provenance.alias, provenance.version) == ("renamed_src", 1) + # v1's outside path now holds v2's bytes, so v1 came from its clone, and the log says under which name. + assert any("renamed_src-v1" in line for line in logged), logged + assert len(cached_result_expr(project, remap[child.content_hash]).execute()) == 3 + + pointers = set(cs.read_tallyman_state(project)["entry_hashes"]) + assert pointers == set(remap.values()) + catalog.assert_catalog_consistent(project, pointers) diff --git a/tests/test_recalc.py b/tests/test_recalc.py index 41c6b46b..b0004fb4 100644 --- a/tests/test_recalc.py +++ b/tests/test_recalc.py @@ -7,18 +7,18 @@ replays to the same hash (a no-op), and a build failure stops the walk with the prefix left committed. -Note on staleness axes: a *cheap* child inlines its parent's recipe, so the -parent's source is collected into the child's own ``manifest.sources`` (the -documented cheap-chain leak) — a source edit makes such a child *directly* stale, -not merely transitively. A pure transitive/cascade case therefore needs the -followed-alias axis (advance the alias with ``catalog_revise``), which is what -``test_alias_advance_cascades_to_the_follower`` exercises. +Note on staleness: there is one axis (ADR-011 D6, the source axis is gone). A raw +input is a source alias, so re-importing the file advances that alias and the root +that reads it goes stale exactly the way a child of any other alias does. A child +of the root is then carried transitively. ``test_alias_advance_cascades_to_the_follower`` +advances a catalog alias instead, which makes its follower *directly* stale. """ from __future__ import annotations +from pathlib import Path + from tallyman_cli.fixtures import write_shoe_orders -from tallyman_core import data_dir from tallyman_core.aliases import get_alias from tallyman_core.catalog_state import current_step, reset_to from tallyman_core.manifest import Manifest, ParentRef, write_manifest @@ -26,21 +26,22 @@ from tallyman_mcp.server import catalog_create, catalog_revise from tallyman_xorq.build import list_entries from tallyman_xorq.recalc import recalc +from tallyman_xorq.source_import import update_and_depend from tallyman_xorq.staleness import scan -def _base_code(project: str) -> str: # root over orders.parquet +def _base_code(project: str) -> str: # root over the imported orders source return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ def _base_code_v2(project: str) -> str: # a different graph → a new content hash, same source return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order").mutate(extra=1) """ @@ -53,10 +54,10 @@ def _child_code(alias: str) -> str: # tracked child off an alias """ -def _pinned_child_code(content_hash: str) -> str: # pinned child off a specific hash +def _pinned_child_code(ref: str) -> str: # pinned child off one version of an alias return f""" from tallyman_xorq.io import pinned_expr_from_alias -t = pinned_expr_from_alias({content_hash!r}) +t = pinned_expr_from_alias({ref!r}) expr = t.mutate(doubled=t.price * 2) """ @@ -66,9 +67,15 @@ def _hash(result: dict) -> str: return result["hash"] -def _edit_source(project: str) -> None: - """Rewrite orders.parquet in place with different content (new digest).""" - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=250, seed=7) +def _advance_source(project: str, path: Path) -> None: + """Re-import the orders file with different rows, advancing the ``orders_src`` alias. + + A build never opens a file (ADR-011 D2), so editing one on disk changes nothing. New raw data + reaches the catalog by importing it again: that mints the next source version and points the + alias at it, which is what makes the entries reading that alias stale. + """ + write_shoe_orders(path, n_rows=250, seed=7) + update_and_depend(path, "orders_src", project=project) # --------------------------------------------------------------------------- @@ -76,11 +83,10 @@ def _edit_source(project: str) -> None: # --------------------------------------------------------------------------- -def test_dry_run_previews_the_ordered_cone_without_building(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_dry_run_previews_the_ordered_cone_without_building(project, orders_parquet, orders_src): a = _hash(catalog_create("a", _base_code(project))) b = _hash(catalog_create("b", _child_code("a"))) - _edit_source(project) + _advance_source(project, orders_parquet) before = {e["content_hash"] for e in list_entries(project)} report = recalc(project, [a], dry_run=True) @@ -105,13 +111,12 @@ def test_dry_run_previews_the_ordered_cone_without_building(project, orders_parq # --------------------------------------------------------------------------- -def test_recalc_rebuilds_drifted_root_and_repoints_the_cone(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_recalc_rebuilds_drifted_root_and_repoints_the_cone(project, orders_parquet, orders_src): a1 = _hash(catalog_create("a", _base_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) - _edit_source(project) - assert scan(project)[a1].stale is True # source axis + _advance_source(project, orders_parquet) + assert scan(project)[a1].stale is True # the alias it reads, orders_src, advanced report = recalc(project, [a1], dry_run=False) @@ -129,11 +134,10 @@ def test_recalc_rebuilds_drifted_root_and_repoints_the_cone(project, orders_parq assert parents[0]["hash"] == a2 and parents[0]["follow"] is True -def test_alias_advance_cascades_to_the_follower(project, orders_parquet, monkeypatch): - # The pure followed-alias axis: advancing a's alias (not its source) makes its - # follower b *directly* stale on the alias axis; recalc re-resolves b to the - # advanced head. b's source is unchanged, so this isolates the alias cascade. - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_alias_advance_cascades_to_the_follower(project, orders_src, monkeypatch): + # Advancing a's alias (not re-importing its source) makes its follower b + # *directly* stale; recalc re-resolves b to the advanced head. orders_src is + # unchanged, so this isolates the catalog-alias cascade from the import. # Opt out of auto-recalc-on-revise so the revise advances a WITHOUT recomputing # b: otherwise the cascade re-points "b" to a fresh head and b1 becomes a # superseded husk (never actionably stale under #154), not the *live* directly- @@ -157,13 +161,12 @@ def test_alias_advance_cascades_to_the_follower(project, orders_parquet, monkeyp assert parents[0]["hash"] == a2 # now bound to the advanced a -def test_recalc_is_one_undoable_transaction(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_recalc_is_one_undoable_transaction(project, orders_parquet, orders_src): a1 = _hash(catalog_create("a", _base_code(project))) _hash(catalog_create("b", _child_code("a"))) step_before = current_step(project) - _edit_source(project) + _advance_source(project, orders_parquet) report = recalc(project, [a1], dry_run=False) assert report.checkpoint_step == current_step(project) assert report.checkpoint_step > step_before @@ -178,12 +181,11 @@ def test_recalc_is_one_undoable_transaction(project, orders_parquet, monkeypatch # --------------------------------------------------------------------------- -def test_hash_pinned_child_is_left_on_its_pin(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_hash_pinned_child_is_left_on_its_pin(project, orders_src): a1 = _hash(catalog_create("a", _base_code(project))) - # A literal-hash argument records follow=False — the child asked for THIS + # A version reference records follow=False — the child asked for THIS # revision, so a recalc that advances the alias must not advance the child. - b1 = _hash(catalog_create("b", _pinned_child_code(a1))) + b1 = _hash(catalog_create("b", _pinned_child_code("a-v1"))) a2 = _hash(catalog_revise("a", _base_code_v2(project))) assert a2 != a1 @@ -195,8 +197,7 @@ def test_hash_pinned_child_is_left_on_its_pin(project, orders_parquet, monkeypat assert next(e for e in report.entries if e.content_hash == b1).action == "noop" -def test_recalc_with_nothing_stale_is_a_noop(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_recalc_with_nothing_stale_is_a_noop(project, orders_src): a1 = _hash(catalog_create("a", _base_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) step_before = current_step(project) @@ -218,8 +219,7 @@ def test_recalc_with_nothing_stale_is_a_noop(project, orders_parquet, monkeypatc # --------------------------------------------------------------------------- -def test_build_failure_stops_the_walk_and_keeps_the_prefix(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_build_failure_stops_the_walk_and_keeps_the_prefix(project, orders_parquet, orders_src): a1 = _hash(catalog_create("a", _base_code(project))) b1 = _hash(catalog_create("b", _child_code("a"))) @@ -228,7 +228,7 @@ def test_build_failure_stops_the_walk_and_keeps_the_prefix(project, orders_parqu # is the deterministic stand-in. (entry_dir(project, b1) / "expr.py").write_text("this is not valid python (((\n") - _edit_source(project) + _advance_source(project, orders_parquet) report = recalc(project, [a1], dry_run=False) assert report.status == "failed" diff --git a/tests/test_recalc_surfaces.py b/tests/test_recalc_surfaces.py index 9eef470d..da6b741f 100644 --- a/tests/test_recalc_surfaces.py +++ b/tests/test_recalc_surfaces.py @@ -10,24 +10,26 @@ from __future__ import annotations import subprocess +from pathlib import Path from tallyman_cli.fixtures import write_shoe_orders -from tallyman_core import data_dir, paths +from tallyman_core import paths from tallyman_core.aliases import get_alias +from tallyman_xorq.source_import import update_and_depend def _base_code(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ def _base_code_v2(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order").mutate(extra=1) """ @@ -46,8 +48,13 @@ def _steps(project: str) -> set[str]: return set(out.split()) -def _edit_source(project: str) -> None: - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=250, seed=7) +def _advance_source(project: str, path: Path) -> None: + """Re-import the orders file with different rows: the ``orders_src`` alias advances (ADR-011 D3). + + Editing the file alone is invisible to the catalog — a build reads an alias, never a path. + """ + write_shoe_orders(path, n_rows=250, seed=7) + update_and_depend(path, "orders_src", project=project) # --------------------------------------------------------------------------- @@ -55,35 +62,34 @@ def _edit_source(project: str) -> None: # --------------------------------------------------------------------------- -def test_mcp_scan_staleness_lists_drifted_entries(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_mcp_scan_staleness_lists_drifted_entries(project, orders_parquet, orders_src): from tallyman_mcp.server import catalog_create, catalog_scan_staleness a = catalog_create("a", _base_code(project))["hash"] b = catalog_create("b", _child_code("a"))["hash"] - _edit_source(project) + _advance_source(project, orders_parquet) out = catalog_scan_staleness() - # a is directly stale on the source axis; b is a cheap child that inlines a's - # recipe, so a's source leaks into b's manifest.sources — b is directly stale - # too (see test_recalc on the cheap-chain leak). - assert a in out["stale"] and b in out["stale"] + # a reads orders_src, which the re-import advanced: a is directly stale. b reads + # the alias "a", which has not moved, so b is only carried — transitively stale. + assert a in out["stale"] + assert b not in out["stale"] and b in out["transitively_stale"] assert out["entries"][a]["stale"] is True - assert out["entries"][a]["reasons"][0]["axis"] == "source" + assert out["entries"][a]["reasons"][0]["axis"] == "alias" + assert out["entries"][a]["reasons"][0]["ref"] == "orders_src" -def test_mcp_scan_staleness_classifies_orphans_when_auto_recalc_on(project, orders_parquet, monkeypatch): - # D5 scan-on-load backstop: bypasses (and source drift) leave entries stale +def test_mcp_scan_staleness_classifies_orphans_when_auto_recalc_on(project, orders_parquet, orders_src, monkeypatch): + # D5 scan-on-load backstop: bypasses (and re-imports) leave entries stale # without a triggering revise, so no auto-recalc ever classifies them. The # scan-on-load surface runs the same orphan classification so a project load # surfaces them even when no revise has happened since. - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) # default ON from tallyman_mcp.server import catalog_create, catalog_scan_staleness a = catalog_create("a", _base_code(project))["hash"] catalog_create("b", _child_code("a")) - _edit_source(project) # a (and its cheap child) directly source-stale; no revise fired + _advance_source(project, orders_parquet) # a directly stale; no revise fired out = catalog_scan_staleness() assert a in out["stale"] @@ -93,16 +99,15 @@ def test_mcp_scan_staleness_classifies_orphans_when_auto_recalc_on(project, orde assert "UNEXPLAINED" in orphans[a]["explanation"] -def test_mcp_scan_staleness_no_orphans_when_auto_recalc_off(project, orders_parquet, monkeypatch): +def test_mcp_scan_staleness_no_orphans_when_auto_recalc_off(project, orders_parquet, orders_src): # Manual mode (flag off) owns its own staleness: the explicit scan→recalc # workflow expects directly-stale followers as an intermediate state, so the # scan surface must NOT flag them as orphan/UNEXPLAINED ("file a bug"). - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") from tallyman_core.config import set_auto_recalc from tallyman_mcp.server import catalog_create, catalog_scan_staleness a = catalog_create("a", _base_code(project))["hash"] - _edit_source(project) + _advance_source(project, orders_parquet) set_auto_recalc(project, False) out = catalog_scan_staleness() @@ -110,13 +115,12 @@ def test_mcp_scan_staleness_no_orphans_when_auto_recalc_off(project, orders_parq assert out["orphan_stale"] == [] # but no orphan classification in manual mode -def test_mcp_recalc_dry_run_previews_and_checkpoints_nothing(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_mcp_recalc_dry_run_previews_and_checkpoints_nothing(project, orders_parquet, orders_src): from tallyman_mcp.server import catalog_create, catalog_recalc a = catalog_create("a", _base_code(project))["hash"] catalog_create("b", _child_code("a")) - _edit_source(project) + _advance_source(project, orders_parquet) base = _steps(project) out = catalog_recalc(dry_run=True) # roots default to the stale set @@ -128,13 +132,12 @@ def test_mcp_recalc_dry_run_previews_and_checkpoints_nothing(project, orders_par assert get_alias(project, "a") == a # nothing rebuilt -def test_mcp_recalc_commit_is_exactly_one_revision(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_mcp_recalc_commit_is_exactly_one_revision(project, orders_parquet, orders_src): from tallyman_mcp.server import catalog_create, catalog_recalc a = catalog_create("a", _base_code(project))["hash"] catalog_create("b", _child_code("a")) - _edit_source(project) + _advance_source(project, orders_parquet) base = _steps(project) out = catalog_recalc(dry_run=False) @@ -144,8 +147,7 @@ def test_mcp_recalc_commit_is_exactly_one_revision(project, orders_parquet, monk assert len(_steps(project)) == len(base) + 1 # one revision for the whole walk -def test_mcp_recalc_no_stale_reports_nothing(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_mcp_recalc_no_stale_reports_nothing(project, orders_src): from tallyman_mcp.server import catalog_create, catalog_recalc catalog_create("a", _base_code(project)) @@ -156,11 +158,10 @@ def test_mcp_recalc_no_stale_reports_nothing(project, orders_parquet, monkeypatc assert _steps(project) == base -def test_mcp_recalc_notify_forwards_remap_and_step(project, orders_parquet, monkeypatch): +def test_mcp_recalc_notify_forwards_remap_and_step(project, orders_parquet, orders_src, monkeypatch): # A committed recalc must hand the companion BOTH the remap and the checkpoint # `step`, so the republished SSE event carries the normalized {kind, remap, # step} shape the in-process /api/recalc route emits (not just remap). - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") import tallyman_mcp.server as server from tallyman_mcp.server import catalog_create, catalog_recalc @@ -169,7 +170,7 @@ def test_mcp_recalc_notify_forwards_remap_and_step(project, orders_parquet, monk catalog_create("a", _base_code(project)) catalog_create("b", _child_code("a")) - _edit_source(project) + _advance_source(project, orders_parquet) out = catalog_recalc(dry_run=False) assert out["status"] == "ok" @@ -181,7 +182,7 @@ def test_mcp_recalc_notify_forwards_remap_and_step(project, orders_parquet, monk assert captured["step"] == out["checkpoint_step"] -def test_recalc_notify_wire_delivers_remap_to_the_real_handler(project, orders_parquet, monkeypatch): +def test_recalc_notify_wire_delivers_remap_to_the_real_handler(project, orders_parquet, orders_src, monkeypatch): # End-to-end across the seam the mocked tests miss: the exact JSON # tallyman_mcp._notify puts on the wire, fed to the REAL /internal/notify # handler, must deliver remap + step to the republished SSE event. Previously @@ -194,7 +195,6 @@ def test_recalc_notify_wire_delivers_remap_to_the_real_handler(project, orders_p from tallyman_companion import create_app from tallyman_mcp.server import catalog_create, catalog_recalc - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") sent: dict = {} @@ -216,7 +216,7 @@ def post(self, _url, json=None): catalog_create("a", _base_code(project)) catalog_create("b", _child_code("a")) - _edit_source(project) + _advance_source(project, orders_parquet) out = catalog_recalc(dry_run=False) # the real call site fires _notify("recalc", ...) assert out["remap"] and sent.get("kind") == "recalc" @@ -241,14 +241,13 @@ def post(self, _url, json=None): # --------------------------------------------------------------------------- -def test_companion_staleness_endpoint(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_companion_staleness_endpoint(fresh_companion_app, project, orders_parquet, orders_src): from fastapi.testclient import TestClient from tallyman_mcp.server import catalog_create - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") a = catalog_create("a", _base_code(project))["hash"] - _edit_source(project) + _advance_source(project, orders_parquet) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/staleness") @@ -258,17 +257,18 @@ def test_companion_staleness_endpoint(fresh_companion_app, project, orders_parqu assert body["entries"][a]["stale"] is True -def test_companion_staleness_classifies_orphans(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_companion_staleness_classifies_orphans( + fresh_companion_app, project, orders_parquet, orders_src, monkeypatch +): # Parity with the MCP scan-on-load surface: /api/staleness returns the same # classified orphan_stale so the SPA can surface unexplained staleness on load. from fastapi.testclient import TestClient from tallyman_mcp.server import catalog_create - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) # default ON a = catalog_create("a", _base_code(project))["hash"] - _edit_source(project) + _advance_source(project, orders_parquet) c = TestClient(fresh_companion_app) r = c.get(f"/{project}/api/staleness") @@ -281,16 +281,15 @@ def test_companion_staleness_classifies_orphans(fresh_companion_app, project, or def test_companion_recalc_commit_is_one_revision_and_repoints( - fresh_companion_app, project, orders_parquet, monkeypatch + fresh_companion_app, project, orders_parquet, orders_src ): from fastapi.testclient import TestClient from tallyman_mcp.server import catalog_create - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") a = catalog_create("a", _base_code(project))["hash"] catalog_create("b", _child_code("a")) - _edit_source(project) + _advance_source(project, orders_parquet) c = TestClient(fresh_companion_app) base = _steps(project) @@ -315,12 +314,11 @@ def test_companion_recalc_commit_is_one_revision_and_repoints( # --------------------------------------------------------------------------- -def test_companion_revise_cascades_and_is_one_revision(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_companion_revise_cascades_and_is_one_revision(fresh_companion_app, project, orders_src, monkeypatch): from fastapi.testclient import TestClient from tallyman_mcp.server import catalog_create - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) a = catalog_create("a", _base_code(project))["hash"] b = catalog_create("b", _child_code("a"))["hash"] @@ -339,14 +337,13 @@ def test_companion_revise_cascades_and_is_one_revision(fresh_companion_app, proj assert len(_steps(project)) == len(base) + 1 -def test_companion_revise_publishes_recalc_event(project, orders_parquet, monkeypatch): +def test_companion_revise_publishes_recalc_event(project, orders_src, monkeypatch): from fastapi.testclient import TestClient import tallyman_companion.app as appmod from tallyman_companion import create_app from tallyman_mcp.server import catalog_create - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) catalog_create("a", _base_code(project)) b = catalog_create("b", _child_code("a"))["hash"] @@ -372,7 +369,7 @@ def test_companion_revise_publishes_recalc_event(project, orders_parquet, monkey assert step is None -def test_companion_promote_diff_is_one_revision(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_companion_promote_diff_is_one_revision(fresh_companion_app, project, orders_src, monkeypatch): # Finding #1: the companion POST /api/promote_diff route self-checkpoints # (app.py) AND was not in _checkpoint_exempt, so the dispatch middleware # checkpointed it a second time — a double-commit (trailing empty revision), @@ -382,7 +379,6 @@ def test_companion_promote_diff_is_one_revision(fresh_companion_app, project, or from tallyman_mcp.server import catalog_create, catalog_revise - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") monkeypatch.delenv("TALLYMAN_AUTO_RECALC", raising=False) catalog_create("ss", _base_code(project)) # v1 catalog_revise("ss", _base_code_v2(project)) # v2 → a diff exists between v1 and v2 diff --git a/tests/test_replay.py b/tests/test_replay.py index 17b9258f..d2bfdb34 100644 --- a/tests/test_replay.py +++ b/tests/test_replay.py @@ -16,15 +16,15 @@ def _storyboard(project: str, parquet: Path) -> dict: "project": project, "steps": [ { - "tool": "catalog_load_parquet", - "args": {"rel_path": "orders.parquet", "name": "orders", "prompt": "raw"}, - "narration": "load the parquet", + "tool": "catalog_import_source", + "args": {"outside_path": str(parquet), "alias": "orders", "prompt": "raw"}, + "narration": "import the parquet", }, { "tool": "catalog_create", "args": { "name": "by_region", - "code": "from tallyman_xorq.io import read_project_file\nt = read_project_file('orders.parquet')\nexpr = t.group_by('region').aggregate(n=t.count())\n", # noqa: E501 + "code": "from tallyman_xorq.io import tracked_expr_from_alias\nt = tracked_expr_from_alias('orders')\nexpr = t.group_by('region').aggregate(n=t.count())\n", # noqa: E501 "prompt": "by region", }, }, @@ -32,7 +32,7 @@ def _storyboard(project: str, parquet: Path) -> dict: "tool": "catalog_revise", "args": { "name": "by_region", - "code": "from tallyman_xorq.io import read_project_file\nt = read_project_file('orders.parquet')\nf = t.filter(t.category == 'boots')\nexpr = f.group_by('region').aggregate(n=f.count())\n", # noqa: E501 + "code": "from tallyman_xorq.io import tracked_expr_from_alias\nt = tracked_expr_from_alias('orders')\nf = t.filter(t.category == 'boots')\nexpr = f.group_by('region').aggregate(n=f.count())\n", # noqa: E501 "prompt": "boots only", }, }, @@ -63,7 +63,7 @@ def test_replay_stops_on_error_by_default(project: str, orders_parquet: Path, tm "project": project, "steps": [ {"tool": "catalog_run", "args": {"code": "expr = nope", "prompt": "broken"}}, - {"tool": "catalog_load_parquet", "args": {"rel_path": "orders.parquet"}}, + {"tool": "catalog_import_source", "args": {"outside_path": str(orders_parquet), "alias": "orders"}}, ], } sb_file = tmp_path / "demo.json" @@ -79,7 +79,7 @@ def test_replay_continue_on_error(project: str, orders_parquet: Path, tmp_path: "project": project, "steps": [ {"tool": "catalog_run", "args": {"code": "expr = nope"}}, - {"tool": "catalog_load_parquet", "args": {"rel_path": "orders.parquet"}}, + {"tool": "catalog_import_source", "args": {"outside_path": str(orders_parquet), "alias": "orders"}}, ], } sb_file = tmp_path / "demo.json" @@ -94,7 +94,7 @@ def test_replay_skip_field(project: str, orders_parquet: Path, tmp_path: Path, m sb = { "project": project, "steps": [ - {"tool": "catalog_load_parquet", "args": {"rel_path": "orders.parquet"}}, + {"tool": "catalog_import_source", "args": {"outside_path": str(orders_parquet), "alias": "orders"}}, {"tool": "catalog_run", "args": {"code": "won't run"}, "skip": True}, ], } diff --git a/tests/test_reset_keeps_compute_cache.py b/tests/test_reset_keeps_compute_cache.py index b18de430..42548631 100644 --- a/tests/test_reset_keeps_compute_cache.py +++ b/tests/test_reset_keeps_compute_cache.py @@ -31,14 +31,23 @@ from tallyman_xorq.result_cache import cached_result_expr -def _recipe(project: str, source: str, tail: str = "") -> str: +def _recipe(alias: str, tail: str = "") -> str: + """A recipe over an imported source alias — a recipe cannot open a file (ADR-011 D2).""" return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file({source!r}, project={project!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({alias!r})\n" f"expr = t{tail}\n" ) +def _import(project: str, path: Path, alias: str) -> str: + """Import *path* as a source alias and return the digest of the bytes, which names its clone.""" + from tallyman_xorq.source_import import update_and_depend + + out = update_and_depend(path, alias, project=project) + return read_manifest(entry_dir(project, out["hash"])).provenance.digest + + def _stat_map(root: Path) -> dict[str, tuple[int, int, int]]: """Every file under *root* with the fields a move, a copy or a rewrite would change.""" out = {} @@ -55,12 +64,16 @@ def two_steps(project: str, orders_parquet: Path) -> SimpleNamespace: extra = data_dir(project) / "extra.parquet" pd.DataFrame({"k": ["x", "y", "x", "z"], "v": [1.0, 2.0, 3.0, 4.0]}).to_parquet(extra) - base = build_and_persist(project, _recipe(project, "orders.parquet")).content_hash + orders_digest = _import(project, orders_parquet, "orders_base_src") + base = build_and_persist(project, _recipe("orders_base_src")).content_hash s1 = cs.checkpoint_catalog(project, "s1") - cheap = build_and_persist(project, _recipe(project, "extra.parquet", ".filter(t.v > 1)")).content_hash - worthy = build_and_persist( - project, _recipe(project, "extra.parquet", ".group_by('k').aggregate(s=t.v.sum())") - ).content_hash + + # extra is imported AFTER s1, so the reset back retires its source entry too (ADR-011 D1 makes a + # source version an ordinary entry). Its clone is then the one no surviving entry refers to, which + # is the case D13 is about: park it in the bullpen, never unlink it. + extra_digest = _import(project, extra, "extra_src") + cheap = build_and_persist(project, _recipe("extra_src", ".filter(t.v > 1)")).content_hash + worthy = build_and_persist(project, _recipe("extra_src", ".group_by('k').aggregate(s=t.v.sum())")).content_hash s2 = cs.checkpoint_catalog(project, "s2") assert s1 is not None and s2 is not None and s1 != s2 @@ -71,8 +84,8 @@ def two_steps(project: str, orders_parquet: Path) -> SimpleNamespace: worthy=worthy, s1=s1, s2=s2, - extra_digest=read_manifest(entry_dir(project, cheap)).sources["extra.parquet"], - orders_digest=read_manifest(entry_dir(project, base)).sources["orders.parquet"], + extra_digest=extra_digest, + orders_digest=orders_digest, ) @@ -141,7 +154,8 @@ def test_no_compute_cache_pointer_file_is_written(project: str, orders_parquet: """ADR-007 D14: ``compute_cache.jsonl``, the git-tracked list of every file that was under the directory at each checkpoint, is retired. Capturing it was a cost every checkpoint paid, and it grew with the cache (#22).""" cs.ensure_catalog_repo(project) - build_and_persist(project, _recipe(project, "orders.parquet", ".group_by('region').aggregate(n=t.count())")) + _import(project, orders_parquet, "orders_base_src") + build_and_persist(project, _recipe("orders_base_src", ".group_by('region').aggregate(n=t.count())")) assert cs.checkpoint_catalog(project, "one") is not None diff --git a/tests/test_reset_to_revision.py b/tests/test_reset_to_revision.py index 9c5f16ee..a4104526 100644 --- a/tests/test_reset_to_revision.py +++ b/tests/test_reset_to_revision.py @@ -32,8 +32,8 @@ def _agg_code(project: str) -> str: return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file('orders.parquet', project={project!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias('orders_src', project={project!r})\n" "expr = t.group_by('region').aggregate(total=t.price.sum(), n=t.count())\n" ) @@ -191,10 +191,10 @@ def test_reset_roundtrip_over_every_kind(project): # --------------------------------------------------------------------------- -def test_compute_cache_is_per_project(project, orders_parquet): +def test_compute_cache_is_per_project(project, orders_src): # Tallyman's materialized files live in a per-project dir, not the global - # ~/.cache/xorq: a build of a worthy entry writes its snapshot (and the ordered - # copy of its source) under compute_cache_dir inside the project. The guarantee + # ~/.cache/xorq: a build of a worthy entry writes its snapshot under + # compute_cache_dir inside the project. The guarantee # is the *location*. A reset does not manage this directory (ADR-007 D14), which # tests/test_reset_keeps_compute_cache.py pins. from tallyman_xorq.result_cache import baked_snapshot_path @@ -208,34 +208,29 @@ def test_compute_cache_is_per_project(project, orders_parquet): assert cdir in snap.parents -def _read_project_file_code(project: str, rel: str) -> str: - return f"from tallyman_xorq.io import read_project_file\nexpr = read_project_file({rel!r}, project={project!r})\n" - - -def test_reset_reclaims_orphaned_cas_clones(project, orders_parquet): - # #86: under the cas default a read_project_file build clones its source into - # data/.cas/. That dir lives outside the catalog git repo, so the - # reset's `git reset --hard` can't roll it back — reset_to's _retire_cas_clones - # step retires clones no *surviving* entry references (into the bullpen, never - # deleted: ADR-007 D14), keeping those a surviving entry still does. This - # guards the wiring (the gc_cas unit test only covers the leaf); the - # post-prune live set is what matters here. +def test_reset_reclaims_orphaned_cas_clones(project, orders_src): + # An import clones its bytes into data/.cas/. That dir lives outside the catalog git + # repo, so the reset's `git reset --hard` can't roll it back — reset_to's _retire_cas_clones + # step retires clones no *surviving* entry references (into the bullpen, never deleted: + # ADR-007 D14), keeping those a surviving entry still does. This guards the wiring (the gc_cas + # unit test only covers the leaf); the post-prune live set is what matters here. import pandas as pd from tallyman_core import data_dir + from tallyman_xorq.source_import import update_and_depend cs.ensure_catalog_repo(project) cas = data_dir(project) / ".cas" - # Baseline: an entry over orders.parquet, so its clone is live at step s1. - build_and_persist(project, _read_project_file_code(project, "orders.parquet")) + # Baseline: the orders source is imported, so its clone is live at step s1. s1 = cs.checkpoint_catalog(project, "baseline") base_clones = {p.name for p in cas.iterdir()} - assert base_clones, "the cas-default build should have cloned orders.parquet into .cas" + assert base_clones, "the import should have cloned orders.parquet into .cas" - # A second entry over a *different* source — its clone is only live after s1. - pd.DataFrame({"a": [1, 2, 3]}).to_parquet(data_dir(project) / "extra.parquet") - build_and_persist(project, _read_project_file_code(project, "extra.parquet")) + # A second source — its clone is only live after s1. + extra = data_dir(project) / "extra.parquet" + pd.DataFrame({"a": [1, 2, 3]}).to_parquet(extra) + update_and_depend(extra, "extra_src", project=project) cs.checkpoint_catalog(project, "added later") assert {p.name for p in cas.iterdir()} > base_clones # extra's clone is present now @@ -244,21 +239,22 @@ def test_reset_reclaims_orphaned_cas_clones(project, orders_parquet): assert {p.name for p in cas.iterdir()} == base_clones -def test_reset_keeps_cas_clones_when_a_manifest_is_unreadable(project, orders_parquet, monkeypatch): +def test_reset_keeps_cas_clones_when_a_manifest_is_unreadable(project, orders_src, monkeypatch): # _retire_cas_clones is conservative: if any surviving entry's manifest can't be # read it skips the whole sweep rather than retire a clone on partial info. # An unreadable manifest must therefore leave every .cas clone in place. import pandas as pd from tallyman_core import data_dir + from tallyman_xorq.source_import import update_and_depend cs.ensure_catalog_repo(project) cas = data_dir(project) / ".cas" - build_and_persist(project, _read_project_file_code(project, "orders.parquet")) s1 = cs.checkpoint_catalog(project, "baseline") - pd.DataFrame({"a": [1, 2, 3]}).to_parquet(data_dir(project) / "extra.parquet") - build_and_persist(project, _read_project_file_code(project, "extra.parquet")) + extra = data_dir(project) / "extra.parquet" + pd.DataFrame({"a": [1, 2, 3]}).to_parquet(extra) + update_and_depend(extra, "extra_src", project=project) cs.checkpoint_catalog(project, "added later") before = {p.name for p in cas.iterdir()} @@ -272,6 +268,50 @@ def _boom(*_a, **_k): assert {p.name for p in cas.iterdir()} == before # nothing reclaimed +def test_a_reset_keeps_the_clone_of_a_live_source_entry(project, tmp_path): + """The clone of an imported source is retained by its entry, not by a ``manifest.sources`` record. + + ADR-011 D6 deletes that map, so the retention closure ``_retire_cas_clones`` walks is the DAG. + Retiring a live source version's clone is the one failure here that destroys data quietly: the + snapshot would still be on disk, so nothing would look wrong until it was deleted and could not be + made again. The reset below keeps v1's clone (its entry survives) and retires v2's (its entry does + not), and v1's snapshot is then re-created from the clone it kept. + """ + import pandas as pd + + from tallyman_core import data_dir, read_manifest + from tallyman_core import paths as _paths + from tallyman_xorq.materialize import ensure_materialized, snapshot_path + from tallyman_xorq.source_import import update_and_depend + + cs.ensure_catalog_repo(project) + cas = data_dir(project) / ".cas" + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + + src = outside / "orders.parquet" + pd.DataFrame({"region": ["n", "s"], "price": [1.0, 2.0]}).to_parquet(src) + v1 = update_and_depend(src, "orders", project=project) + s1 = cs.checkpoint_catalog(project, "orders v1") + + pd.DataFrame({"region": ["n", "s", "e"], "price": [1.0, 2.0, 3.0]}).to_parquet(src) + v2 = update_and_depend(src, "orders", project=project) + cs.checkpoint_catalog(project, "orders v2") + assert v2["hash"] != v1["hash"] + + cs.reset_to(project, s1) + + prov = read_manifest(_paths.entry_dir(project, v1["hash"])).provenance + assert prov is not None + kept = cas / f"{prov.digest}{prov.suffix}" + assert kept.is_file(), f"the live source version's clone was retired: {sorted(p.name for p in cas.iterdir())}" + assert v2["digest"] not in {p.stem for p in cas.iterdir()}, "the rewound version's clone should be parked" + + snapshot_path(project, v1["hash"]).unlink() + ensure_materialized(project, v1["hash"]) + assert snapshot_path(project, v1["hash"]).is_file() + + # --------------------------------------------------------------------------- # W5 — dispatch-boundary checkpoint hook (opt-out) # --------------------------------------------------------------------------- @@ -290,7 +330,7 @@ def test_every_mcp_tool_is_checkpoint_wrapped(): assert server.mcp.tool.__name__ == "_checkpointing_tool" # registration is wrapped mutating = { "catalog_run", - "catalog_load_parquet", + "catalog_import_source", "catalog_create", "catalog_revise", "catalog_alias", @@ -322,7 +362,7 @@ def test_mcp_mutating_tool_checkpoints_once(project): assert _steps(project) == after -def test_mcp_multi_mutator_tool_is_one_step(project, orders_parquet): +def test_mcp_multi_mutator_tool_is_one_step(project, orders_src): from tallyman_mcp.server import catalog_create base = _steps(project) @@ -616,7 +656,7 @@ def test_reset_surfaces_pointer_without_durable_recipe(project): # and test_n_builds_track_n_zips. -def test_deleted_snapshot_of_a_warmed_entry_self_heals_without_cache_clear(project, orders_parquet, monkeypatch): +def test_deleted_snapshot_of_a_warmed_entry_self_heals_without_cache_clear(project, orders_src, monkeypatch): """A warmed worthy entry whose snapshot file goes away must be made again on the next read, never dangle on the memoised bare read of a file that no longer exists. diff --git a/tests/test_result_cache.py b/tests/test_result_cache.py index 7d32f56e..1bf68be1 100644 --- a/tests/test_result_cache.py +++ b/tests/test_result_cache.py @@ -7,27 +7,29 @@ from tallyman_xorq.build import list_entries from tallyman_xorq.result_cache import cache_worthy +ORDERS_SRC = "orders_src" # the source alias conftest's ``orders_src`` fixture imports the shoe-orders file under + def _agg_code(project: str) -> str: # Aggregate → expensive → cache-worthy return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ -def _project_code(project: str) -> str: # parquet read + projection → cheap (keeps __row_order, ADR-008 D3) +def _project_code(project: str) -> str: # source read + projection → cheap (keeps __row_order, ADR-008 D3) return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.select("region", "price", "__row_order") """ def _parent_code(project: str, col: str) -> str: # single-column rename → distinct, unionable return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.select("__row_order", k=t.{col}) """ @@ -59,8 +61,8 @@ def _chain_off_expensive_code(parent: str) -> str: # cheap child chaining off O def _agg_avg_code(project: str) -> str: # a second Aggregate → expensive, joins on region return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.group_by("region").aggregate(avg_price=t.price.mean()) """ @@ -76,12 +78,12 @@ def _join_two_expensive_code(parent_a: str, parent_b: str) -> str: # multi-pare def _scalar_udf_code(project: str) -> str: # scalar UDF only (no Aggregate/Join/Sort) → worthy via UDF (#81) return f""" -from tallyman_xorq.io import read_project_file +from tallyman_xorq.io import tracked_expr_from_alias from xorq.expr.udf import make_pandas_udf import xorq.vendor.ibis.expr.datatypes as dt from xorq.vendor.ibis import schema as ibis_schema -t = read_project_file("orders.parquet", project={project!r}) +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) def plusone(df): @@ -118,10 +120,18 @@ def _call_under_tight_recursion_guard(fn, *args): def _hash_of(project: str) -> str: - return list_entries(project)[0]["content_hash"] + """The most recently written built entry. + + An import is an entry too (ADR-011 D1), so skip the ones whose manifest records one: every project + here carries the ``orders_src`` version, and a test always means the entry it just created. + """ + for manifest in list_entries(project): + if not manifest.get("provenance"): + return manifest["content_hash"] + raise AssertionError(f"{project!r} has no built entry, only imported sources") -def test_classifier_skips_cheap_caches_expensive(project, orders_parquet, monkeypatch): +def test_classifier_skips_cheap_caches_expensive(project, orders_src, monkeypatch): # The verdict is decided once, at build, on the live expression (ADR-008 D4) and read from the manifest ever after. from tallyman_core import read_manifest @@ -137,7 +147,7 @@ def test_classifier_skips_cheap_caches_expensive(project, orders_parquet, monkey assert "cheap" in read_manifest(entry_dir(project, proj_h)).cache_worthy_why -def test_cheap_entry_writes_no_result_parquet(project, orders_parquet, monkeypatch): +def test_cheap_entry_writes_no_result_parquet(project, orders_src, monkeypatch): # A cheap entry materialises nothing — no result.parquet, ever (not at build, # not on read), and nothing under the (retired) result_cache dir. from tallyman_xorq.result_cache import cached_result_expr @@ -155,7 +165,7 @@ def test_cheap_entry_writes_no_result_parquet(project, orders_parquet, monkeypat assert not rp.exists() -def test_expensive_entry_bakes_result_cache(project, orders_parquet, monkeypatch): +def test_expensive_entry_bakes_result_cache(project, orders_src, monkeypatch): # An expensive entry is materialised when it is created (ADR-007 D4): its snapshot is written to the compute cache # — not a build-time entry result.parquet, not the retired catalog-level result_cache dir. from tallyman_core.paths import compute_cache_dir @@ -180,7 +190,7 @@ def test_expensive_entry_bakes_result_cache(project, orders_parquet, monkeypatch assert not (entry_dir(project, h) / "result.parquet").exists() -def test_scalar_udf_is_worthy(project, orders_parquet, monkeypatch): +def test_scalar_udf_is_worthy(project, orders_src, monkeypatch): # #81: a scalar UDF used without any accompanying expensive op must be judged # worthy. A make_pandas_udf node is classed after the user's function (here # `plusone`), so its live leaf name carries no "UDF"; it is matched by base class @@ -199,7 +209,7 @@ def test_scalar_udf_is_worthy(project, orders_parquet, monkeypatch): assert classify_expr(_recipe_expr(project, h)).worthy is True -def test_scalar_udf_only_entry_bakes_result_cache(project, orders_parquet, monkeypatch): +def test_scalar_udf_only_entry_bakes_result_cache(project, orders_src, monkeypatch): # #81 net effect: a scalar-UDF-only expression that was judged cheap wrote no snapshot, so every viewer / diff / # tracked_expr_from_alias read recomputed the UDF over the whole DAG (a silent perf regression). With the UDF # judged worthy the entry is materialised like any other expensive entry and cached_result_expr reads a @@ -223,7 +233,7 @@ def test_scalar_udf_only_entry_bakes_result_cache(project, orders_parquet, monke assert any(compute_cache_dir(project).rglob("result_cache/*.parquet")) -def test_cheap_entry_cached_result_expr_is_not_a_cache_node(project, orders_parquet, monkeypatch): +def test_cheap_entry_cached_result_expr_is_not_a_cache_node(project, orders_src, monkeypatch): # A cheap entry recomputes on read — its expression is not a CachedNode. from tallyman_xorq.result_cache import cached_result_expr @@ -233,7 +243,7 @@ def test_cheap_entry_cached_result_expr_is_not_a_cache_node(project, orders_parq assert type(cached_result_expr(project, h).op()).__name__ != "CachedNode" -def test_cold_read_logs_path_and_wall_time(project, orders_parquet, monkeypatch, caplog): +def test_cold_read_logs_path_and_wall_time(project, orders_src, monkeypatch, caplog): # #87: each cold read is visible on tallyman.perf with wall-clock, so a Join-descendant that is structurally worthy # but cost-cheap is visible from the read path, not just lifecycle counts. A worthy entry whose snapshot exists is # served without loading its build (ADR-007 D2), so it logs no cold read; a cheap entry's cold read loads its @@ -271,7 +281,7 @@ def perf_tags(call): assert any("healed" in m and agg_h in m for m in msgs) -def test_multi_parent_tracked_expr_from_alias_shares_one_backend(project, orders_parquet, monkeypatch): +def test_multi_parent_tracked_expr_from_alias_shares_one_backend(project, orders_src, monkeypatch): # #75: combining two tracked_expr_from_alias parents in one expression must resolve to a # single backend. #73's expression-level tracked_expr_from_alias loaded each parent into # its own backend, so a union/join raised "Multiple backends found". @@ -289,7 +299,7 @@ def test_multi_parent_tracked_expr_from_alias_shares_one_backend(project, orders ibis.union(a, b)._find_backend() -def test_multi_parent_tracked_expr_from_alias_union_builds(project, orders_parquet, monkeypatch): +def test_multi_parent_tracked_expr_from_alias_union_builds(project, orders_src, monkeypatch): # #75: an entry that unions two tracked_expr_from_alias parents must build end-to-end. # On the #73 branch this aborted with BuildError: Multiple backends found. monkeypatch.setenv("TALLYMAN_PROJECT", project) @@ -302,64 +312,64 @@ def test_multi_parent_tracked_expr_from_alias_union_builds(project, orders_parqu assert res["row_count"] == 400 -def _mix_code(project: str, parent: str) -> str: # read_project_file + tracked_expr_from_alias in one expression +def _mix_code(project: str, parent: str) -> str: # a source parent + a cheap parent in one expression return f""" -from tallyman_xorq.io import read_project_file, tracked_expr_from_alias -fp = read_project_file("orders.parquet", project={project!r}).select(k="region") +from tallyman_xorq.io import tracked_expr_from_alias +fp = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}).select(k="region") fc = tracked_expr_from_alias({parent!r}).select(k="k") expr = fp.union(fc) """ -def test_read_project_file_and_tracked_expr_from_alias_mix_shares_one_backend(project, orders_parquet, monkeypatch): - # #75: a read_project_file read combined with a tracked_expr_from_alias parent in one - # expression must resolve to a single backend. #73 rooted read_project_file on the - # default backend but loaded tracked_expr_from_alias into its own, so the mix raised - # "Multiple backends found". +def test_source_and_cheap_parent_mix_shares_one_backend(project, orders_src, monkeypatch): + # #75: the two ways a parent resolves must meet on one backend. A source version is worthy, so it + # comes back as a bare read of its snapshot; a cheap parent comes back as its own frozen graph. + # #73 rooted the raw read on the default backend but loaded a parent into its own, so the mix + # raised "Multiple backends found". import xorq.vendor.ibis as ibis - from tallyman_xorq.io import read_project_file, tracked_expr_from_alias + from tallyman_xorq.io import tracked_expr_from_alias monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("pa", _parent_code(project, "region")) - fp = read_project_file("orders.parquet", project=project).select(k="region") + fp = tracked_expr_from_alias(orders_src, project=project).select(k="region") fc = tracked_expr_from_alias("pa").select(k="k") ibis.union(fp, fc)._find_backend() # must not raise -def test_read_project_file_and_tracked_expr_from_alias_mix_builds(project, orders_parquet, monkeypatch): - # #75: an entry mixing read_project_file and tracked_expr_from_alias must build end-to-end. +def test_source_and_cheap_parent_mix_builds(project, orders_src, monkeypatch): + # #75: an entry mixing a source parent and a cheap parent must build end-to-end. monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("pa", _parent_code(project, "region")) res = catalog_create("mix", _mix_code(project, "pa")) assert "error" not in res, res - # 200 read_project_file rows + 200 tracked_expr_from_alias rows. + # 200 rows from the source version + 200 from the cheap parent over it. assert res["row_count"] == 400 -def test_expensive_tracked_expr_from_alias_mix_shares_one_backend(project, orders_parquet, monkeypatch): +def test_expensive_tracked_expr_from_alias_mix_shares_one_backend(project, orders_src, monkeypatch): # #75: an expensive parent resolves to a deferred read of its baked snapshot — - # a bare Read on the default backend — so mixing it with a read_project_file read - # also stays on one backend. A baked CachedNode (the #73 form) would have + # a bare Read on the default backend — so mixing it with a fresh aggregate over + # the source stays on one backend. A baked CachedNode (the #73 form) would have # dragged its own storage backend in and re-raised "Multiple backends found". import xorq.vendor.ibis as ibis - from tallyman_xorq.io import read_project_file, tracked_expr_from_alias + from tallyman_xorq.io import tracked_expr_from_alias monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("agg", _agg_code(project)) # Aggregate → expensive → baked snapshot - ft = read_project_file("orders.parquet", project=project) + ft = tracked_expr_from_alias(orders_src, project=project) fp = ft.group_by("region").aggregate(total=ft.price.sum(), n=ft.count()) fc = tracked_expr_from_alias("agg").drop("__row_order") # region, total, n — same schema as fp ibis.union(fp, fc)._find_backend() # must not raise -def test_diff_route_survives_cold_cache(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_diff_route_survives_cold_cache(fresh_companion_app, project, orders_src, monkeypatch): # The diff route composes both entries' reads; on a cold compute cache it must make the files they read exist (the - # ordered copy of the source) rather than blank the diff (there is no per-entry result.parquet to evict any more). + # source version's snapshot) rather than blank the diff (there is no per-entry result.parquet to evict any more). import shutil from tallyman_core.paths import compute_cache_dir @@ -378,7 +388,7 @@ def test_diff_route_survives_cold_cache(fresh_companion_app, project, orders_par assert "stats" in diff or "keyed" in diff -def test_cached_result_expr_self_heals_expensive_parent_chain_on_cold_cache(project, orders_parquet, monkeypatch): +def test_cached_result_expr_self_heals_expensive_parent_chain_on_cold_cache(project, orders_src, monkeypatch): """#73/#74: reading an entry that ``tracked_expr_from_alias``s an expensive parent must make an evicted parent snapshot exist again, not error. @@ -413,7 +423,7 @@ def test_cached_result_expr_self_heals_expensive_parent_chain_on_cold_cache(proj assert len(df) == expected_rows -def test_cached_result_expr_self_heals_multi_parent_expensive_join_on_cold_cache(project, orders_parquet, monkeypatch): +def test_cached_result_expr_self_heals_multi_parent_expensive_join_on_cold_cache(project, orders_src, monkeypatch): """#73/#75: a multi-parent entry joining two expensive ``tracked_expr_from_alias`` parents must make *both* evicted parent snapshots exist again on a cold cache. @@ -446,7 +456,7 @@ def test_cached_result_expr_self_heals_multi_parent_expensive_join_on_cold_cache assert len(df) == expected_rows -def test_revise_in_place_self_reference_terminates(project, orders_parquet, monkeypatch): +def test_revise_in_place_self_reference_terminates(project, orders_src, monkeypatch): """A revise-in-place recipe that reads ``tracked_expr_from_alias`` of its OWN alias must not self-recurse forever when later reconstructed (the recipe re-import that remains). @@ -554,7 +564,7 @@ def test_resolve_noncyclic_hash_multi_hop_stays_in_requested_lineage(project): _RECONSTRUCTING.reset(token) -def test_recipe_reconstruction_does_not_leak_sys_modules(project, orders_parquet, monkeypatch): +def test_recipe_reconstruction_does_not_leak_sys_modules(project, orders_src, monkeypatch): # Recipe reconstruction (_recipe_expr -> _import_script) survives as a diagnostic (the structural-nondeterminism # check, which re-imports the recipe twice); it was a per-READ operation under #73. _import_script registers the # recipe module in sys.modules under a unique uuid name; left there, every reconstruction leaks a module object @@ -587,27 +597,13 @@ def test_ensure_result_is_removed(): assert not hasattr(rc, "ensure_viewer_expanded_build") -def _overwrite_orders(project: str, seed: int) -> None: - """Rewrite the orders source with different values (same schema, same rows). - - A read_project_file recompute under ``off`` identity reads the live path, so this - makes a previously-built entry's recompute drift from its build-time bytes — - the execution-nondeterminism stand-in #83's faithfulness check must catch. - """ - from tallyman_cli.fixtures import write_shoe_orders - from tallyman_core import data_dir - - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=200, seed=seed) - - -def test_build_records_stable_result_digest(project, orders_parquet, monkeypatch): +def test_build_records_stable_result_digest(project, orders_src, monkeypatch): # ADR-004-result-digest-canonical-ordering, as redefined by ADR-009 D2: worthy entries record a result_digest, the # content digest (arrow-sha256) of the snapshot read back, stable run-to-run because the snapshot is written in a # canonical order on a single-partition connection. Cheap entries record no digest — they have no snapshot to hash. from tallyman_core import entry_dir, read_manifest from tallyman_xorq.result_cache import baked_snapshot_path, snapshot_file_digest - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) # Worthy entry (Aggregate): must record a digest equal to the snapshot's content digest. @@ -627,14 +623,13 @@ def test_build_records_stable_result_digest(project, orders_parquet, monkeypatch assert not cheap_digest, f"cheap entry unexpectedly recorded result_digest: {cheap_digest!r}" -def test_verify_result_faithful_true_for_deterministic_entry(project, orders_parquet, monkeypatch): +def test_verify_result_faithful_true_for_deterministic_entry(project, orders_src, monkeypatch): # ADR-004-result-digest-canonical-ordering: for a worthy (snapshot-writing) entry, # verify_result_faithful compares the snapshot's content digest to the recorded # digest and returns True for a clean, deterministic build. A cheap entry records # no digest, so verify_result_faithful returns None for it. from tallyman_xorq.result_cache import verify_result_faithful - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) # Worthy entry: digest recorded at build; verify_result_faithful checks the file. @@ -648,7 +643,7 @@ def test_verify_result_faithful_true_for_deterministic_entry(project, orders_par assert verify_result_faithful(project, h2) is None -def test_verify_result_faithful_detects_snapshot_drift(project, orders_parquet, monkeypatch): +def test_verify_result_faithful_detects_snapshot_drift(project, orders_src, monkeypatch): # ADR-004-result-digest-canonical-ordering: verify_result_faithful compares the # snapshot's content digest against the recorded digest. For a worthy entry # whose snapshot no longer holds the rows it was built with (one value differs), @@ -658,7 +653,6 @@ def test_verify_result_faithful_detects_snapshot_drift(project, orders_parquet, from tallyman_xorq.result_cache import baked_snapshot_path, verify_result_faithful - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) # Worthy entry: starts faithful. @@ -676,15 +670,13 @@ def test_verify_result_faithful_detects_snapshot_drift(project, orders_parquet, pq.write_table(drifted, snap) assert verify_result_faithful(project, h) is False - # Cheap entry: always None, even after source change. + # Cheap entry: always None — it has no snapshot, so there is nothing to compare. catalog_create("proj", _project_code(project)) h2 = _hash_of(project) assert verify_result_faithful(project, h2) is None - _overwrite_orders(project, seed=1) - assert verify_result_faithful(project, h2) is None -def test_self_heal_warns_on_unfaithful_recompute(project, orders_parquet, monkeypatch, caplog): +def test_self_heal_warns_on_unfaithful_recompute(project, orders_src, monkeypatch, caplog): # #83: eviction's load-bearing assumption is that an evicted snapshot recomputes # to what was evicted. For an entry whose recompute differs from the digest # recorded at build, that is false — the snapshot self-heals to DIFFERENT rows, @@ -692,17 +684,15 @@ def test_self_heal_warns_on_unfaithful_recompute(project, orders_parquet, monkey # snapshot's digest != the recorded one, and the read still succeeds (the check # is advisory, never fatal). # - # Source drift can no longer provoke this on its own: the ordered copy an entry - # reads is named by the source's content (ADR-008 D2), so a heal reads the rows the - # entry was built from, and an edited source is a new entry - # (test_cas_expensive_self_heal_after_edit_serves_built_bytes). The mismatch is - # provoked directly, by recording a digest the frozen build cannot reproduce. + # Source drift cannot provoke this at all: an entry reads a source version's snapshot, editing the + # imported file changes nothing, and importing the edited bytes mints a new version (ADR-011 D1, + # tests/test_source_import.py::test_editing_the_outside_file_after_import_changes_nothing). The + # mismatch is provoked directly, by recording a digest the frozen build cannot reproduce. import json import logging from tallyman_xorq.result_cache import baked_snapshot_path, cached_result_expr - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("agg", _agg_code(project)) # expensive → snapshot h = _hash_of(project) @@ -727,7 +717,7 @@ def test_self_heal_warns_on_unfaithful_recompute(project, orders_parquet, monkey def test_structural_attribution_degrades_to_not_structural_when_the_manifest_is_unreadable( - project, orders_parquet, monkeypatch + project, orders_src, monkeypatch ): # #88 part 2, best-effort contract: the structural-vs-execution attribution must # never break the read it only annotates. recipe_is_structurally_nondeterministic @@ -739,7 +729,6 @@ def test_structural_attribution_degrades_to_not_structural_when_the_manifest_is_ # labeled execution (#83) and the read still succeeds. from tallyman_xorq.result_cache import recipe_is_structurally_nondeterministic - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("agg", _agg_code(project)) h = _hash_of(project) @@ -749,7 +738,7 @@ def test_structural_attribution_degrades_to_not_structural_when_the_manifest_is_ assert recipe_is_structurally_nondeterministic(project, h) is False -def test_cached_result_expr_self_heals_after_warm_then_evict(project, orders_parquet, monkeypatch): +def test_cached_result_expr_self_heals_after_warm_then_evict(project, orders_src, monkeypatch): """An expensive entry's baked snapshot evicted *after* a warm read must still self-heal on the next read, not dangle on a stale ``deferred_read_parquet``. @@ -778,7 +767,7 @@ def test_cached_result_expr_self_heals_after_warm_then_evict(project, orders_par assert len(df) == expected -def test_concurrent_cold_heal_is_single_flighted(project, orders_parquet, monkeypatch): +def test_concurrent_cold_heal_is_single_flighted(project, orders_src, monkeypatch): """Concurrent cold reads of the same expensive entry must single-flight the snapshot heal (#79). @@ -851,257 +840,7 @@ def worker(): assert len(cached_result_expr(project, agg_h).execute()) == expected -# --------------------------------------------------------------------------- -# #115: a cold read must serve the rows the entry was BUILT from, not the live -# source after it is edited in place. #86 made *build* identity content-aware -# (a rebuild forks the hash). Under ADR-008 D2 a read never touches the live source at -# all: the frozen build reads an ordered copy named by the source's content, and a copy -# that is missing is made again from the frozen data/.cas/ clone (ADR-007 D13). -# These pin the closure for both #74 hazards: a cheap entry's cold read, and an -# expensive entry self-healing an evicted snapshot. -# --------------------------------------------------------------------------- - - -def _write_ints(path, values) -> None: - import pandas as pd - - pd.DataFrame({"x": list(values)}).to_parquet(path) - - -def _src_read_code(project: str, rel: str) -> str: # cheap: a bare project read - return f"from tallyman_xorq.io import read_project_file\nexpr = read_project_file({rel!r}, project={project!r})\n" - - -def _src_group_code(project: str, rel: str) -> str: # expensive: group_by → Aggregate → baked snapshot - # order_by('x') pins a deterministic row order so the #83 result_digest is stable - # build-to-self-heal (an unordered aggregate would reshuffle and read as drift). - return ( - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file({rel!r}, project={project!r})\n" - "expr = t.group_by('x').aggregate(c=t.count()).order_by('x')\n" - ) - - -def test_cas_cold_read_after_inplace_edit_serves_built_bytes(project, monkeypatch): - """#115: a cheap entry's cold read serves the rows it was built from, even after - the source is edited in place. - - Pre-#115 the cold read re-imported the recipe and re-ran read_project_file, which under - cas re-digested the now-edited live file and cloned it under a NEW digest — serving - the edited rows under the entry's ORIGINAL content_hash. The read now loads the frozen build, which - reads the ordered copy of the bytes the entry was built from. - """ - from tallyman_core import data_dir - from tallyman_xorq import build_and_persist - from tallyman_xorq import source_identity as si - from tallyman_xorq.result_cache import cached_result_expr - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") # several sibling tests force off - assert si.mode() == "cas" - - src = data_dir(project) / "src.parquet" - _write_ints(src, [0, 1, 2]) - h = build_and_persist(project, _src_read_code(project, "src.parquet")).content_hash - - # Edit the source in place AFTER building. The 3→5 row edit changes the file - # size, so the stat-keyed digest memo invalidates without TALLYMAN_SOURCE_REHASH. - _write_ints(src, [0, 1, 2, 3, 4]) - cached_result_expr.cache_clear() # force a genuinely COLD read (load the frozen build again) - - df = cached_result_expr(project, h).execute() - # Assert on VALUES, not just the count, so a fix that read some other 3-row file - # can't false-green: only the frozen clone holds exactly {0,1,2} now. - assert sorted(df["x"].tolist()) == [0, 1, 2] - - -def test_cas_cold_read_after_source_deleted_serves_built_bytes(project, monkeypatch): - """#115 / #148-review: a cold read serves the frozen bytes even when the live - source has been DELETED (or moved), not just edited in place. - - Pre-fix read_project_file called project_path first, whose is_file() guard raised - ProjectDataNotFound before the pinned reconstruction could run — so a deleted source - defeated it entirely. A read no longer needs the live source: the frozen build reads - its ordered copy. - """ - from tallyman_core import data_dir - from tallyman_xorq import build_and_persist - from tallyman_xorq import source_identity as si - from tallyman_xorq.result_cache import cached_result_expr - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - assert si.mode() == "cas" - - src = data_dir(project) / "src.parquet" - _write_ints(src, [0, 1, 2]) - h = build_and_persist(project, _src_read_code(project, "src.parquet")).content_hash - - src.unlink() # source gone entirely — the frozen .cas clone is the only copy left - cached_result_expr.cache_clear() # force a genuinely COLD read (load the frozen build again) - - df = cached_result_expr(project, h).execute() - # Only the frozen bytes hold exactly {0,1,2}; assert on values, not just the count. - assert sorted(df["x"].tolist()) == [0, 1, 2] - - -def test_cas_expensive_self_heal_after_edit_serves_built_bytes(project, monkeypatch, caplog): - """#115: an expensive entry self-healing an evicted snapshot recomputes from the - FROZEN source, not the edited live file. - - Pre-#115 the self-heal re-imported the recipe and re-executed it, and that recompute read the edited source, so - the snapshot self-healed to different rows under the original content_hash — exactly the #83 UNFAITHFUL signal. - A heal now runs the entry's frozen build, which reads the ordered copy of the bytes it was built from, so the heal - is faithful and the warning does not fire. - """ - import logging - - from tallyman_core import data_dir - from tallyman_xorq import build_and_persist - from tallyman_xorq import source_identity as si - from tallyman_xorq.result_cache import baked_snapshot_path, cached_result_expr - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - assert si.mode() == "cas" - - src = data_dir(project) / "src.parquet" - _write_ints(src, [0, 1, 2]) - h = build_and_persist(project, _src_group_code(project, "src.parquet")).content_hash - - # The snapshot's path is a function of the content hash (ADR-007 D2), so it does not move when the source is edited. - p = baked_snapshot_path(project, h) - assert p is not None and p.exists() - - _write_ints(src, [0, 1, 2, 3, 4]) # edit the source in place - p.unlink() # evict the baked snapshot → the next read must self-heal - cached_result_expr.cache_clear() - - caplog.clear() - with caplog.at_level(logging.WARNING, logger="tallyman.perf"): - df = cached_result_expr(project, h).execute() - - # group_by('x') over the frozen {0,1,2} → 3 groups; the edited {0,1,2,3,4} would - # give 5. The self-heal must recompute from the frozen clone. - assert sorted(df["x"].tolist()) == [0, 1, 2] - assert p.exists() # the heal rewrote the ORIGINAL-keyed snapshot, not a drifted one - # The #83 faithfulness check must NOT fire: the recompute matches the build digest - # because it read the frozen rows (pre-fix it self-healed to edited rows and - # logged UNFAITHFUL). - msgs = [r.getMessage() for r in caplog.records if r.name == "tallyman.perf"] - assert not any("UNFAITHFUL" in m for m in msgs), msgs - - -def test_cas_nested_chain_reconstruction_pins_each_entrys_sources(project, monkeypatch): - """#115: a nested chain remakes each entry's own frozen source. - - B unions tracked_expr_from_alias(A) with its own read_project_file(b). B's plan reads two ordered copies: the - parent A's, because A is cheap and its graph is inlined into B's build, and B's own. With BOTH sources edited in - place and every file under compute_cache gone, reading B remakes each ordered copy from the clone of ITS OWN - built bytes (each record travels in B's manifest: A's was folded in from A's), so the rows are the frozen ones. - Pre-ADR-007 this pinned a contextvar that had to be overwritten for A's reconstruction and restored for B's. - """ - import shutil - - from tallyman_core import data_dir - from tallyman_core.paths import compute_cache_dir - from tallyman_xorq import source_identity as si - from tallyman_xorq.result_cache import cached_result_expr - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - assert si.mode() == "cas" - - srca = data_dir(project) / "a.parquet" - srcb = data_dir(project) / "b.parquet" - _write_ints(srca, [0, 1, 2]) - _write_ints(srcb, [10, 11]) - - catalog_create("pa", _src_read_code(project, "a.parquet")) # cheap parent over a.parquet - b_code = ( - "from tallyman_xorq.io import read_project_file, tracked_expr_from_alias\n" - "a = tracked_expr_from_alias('pa')\n" - f"b = read_project_file('b.parquet', project={project!r})\n" - "expr = a.union(b)\n" - ) - res = catalog_create("pb", b_code) - assert "error" not in res, res - b_h = _hash_of(project) - - _write_ints(srca, [0, 1, 2, 3, 4]) # edit BOTH sources in place after building - _write_ints(srcb, [10, 11, 12]) - shutil.rmtree(compute_cache_dir(project), ignore_errors=True) # only the clones and the manifests are left - cached_result_expr.cache_clear() - - df = cached_result_expr(project, b_h).execute() - # Frozen a = {0,1,2} ∪ frozen b = {10,11}. A mix-up would serve the edited {0,1,2,3,4} and/or {10,11,12}. - assert sorted(df["x"].tolist()) == [0, 1, 2, 10, 11] - - -def test_cas_child_records_reconstructed_parent_frozen_digest(project, monkeypatch): - """#115: building a child records the parent's FROZEN source digest, keeping the - parent's .cas clone alive for the reset's clone sweep. - - Chaining a child off the parent inlines the parent's frozen graph. The child must record the parent's frozen - digest (the bytes actually read) into its manifest.sources — folded in from the parent's manifest, not a - re-digest of the edited live file — so the clone the child's ordered copy is made from stays referenced. - """ - from tallyman_core import data_dir, entry_dir, read_manifest - from tallyman_xorq import source_identity as si - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - assert si.mode() == "cas" - - src = data_dir(project) / "src.parquet" - _write_ints(src, [0, 1, 2]) - catalog_create("pp", _src_read_code(project, "src.parquet")) - p_h = _hash_of(project) - p_digest = read_manifest(entry_dir(project, p_h)).sources["src.parquet"] - - _write_ints(src, [0, 1, 2, 3, 4]) # edit in place, then build a child off the parent - cc_code = ( - "from tallyman_xorq.io import tracked_expr_from_alias\n" - "t = tracked_expr_from_alias('pp')\nexpr = t.mutate(y=t.x * 2)\n" - ) - res = catalog_create("cc", cc_code) - assert "error" not in res, res - c_h = _hash_of(project) - - child_sources = read_manifest(entry_dir(project, c_h)).sources or {} - assert child_sources.get("src.parquet") == p_digest - # Sanity: the frozen digest the child pinned is NOT the edited live digest. - assert si.digest_for(project, src) != p_digest - - -def test_off_built_entry_reads_under_cas_default_without_crash(project, monkeypatch): - """#115 graceful degradation: an entry built under mode=off (no recorded sources) - read under the cas default reads its ordered copy, not a crash. - - Under mode=off no clone is taken and manifest.sources is None, but the ordered copy is still written (named by - the source's content) and recorded, so a read after flipping to cas serves the built rows without any live read. - (Per the single-user rebuild-always policy, rebuilding such an entry under cas is the path to a frozen clone; - this only pins the no-crash contract.) - """ - from tallyman_core import data_dir, entry_dir, read_manifest - from tallyman_xorq import build_and_persist - from tallyman_xorq.result_cache import cached_result_expr - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - src = data_dir(project) / "src.parquet" - _write_ints(src, [0, 1, 2]) - - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") - h = build_and_persist(project, _src_read_code(project, "src.parquet")).content_hash - assert read_manifest(entry_dir(project, h)).sources is None # off records no sources - - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") # flip to the default - cached_result_expr.cache_clear() - df = cached_result_expr(project, h).execute() # must not raise; reads the ordered copy - assert sorted(df["x"].tolist()) == [0, 1, 2] - - -def test_reset_to_clears_result_plan_memo(project, orders_parquet, monkeypatch): +def test_reset_to_clears_result_plan_memo(project, orders_src, monkeypatch): """reset_to changes which entries exist (it retires and restores entry dirs, and leaves compute_cache alone, ADR-007 D14), so it must also invalidate the in-process ``_resolve_result_plan`` memo, which holds loaded builds (#80 / #96). A reactive reset layer needs this cache invalidated alongside the entry sweep, not left to @@ -1126,7 +865,7 @@ def test_reset_to_clears_result_plan_memo(project, orders_parquet, monkeypatch): assert cached_result_expr.cache_info().currsize == 0 -def test_reset_endpoint_clears_compare_expr_memo(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_reset_endpoint_clears_compare_expr_memo(fresh_companion_app, project, orders_src, monkeypatch): """The diff compare-expr LRU (``_build_compare_expr``) caches a serialized build that bakes in each entry's snapshot path and — unlike ``cached_result_expr`` — never re-checks ``path.exists()`` on a hit. A reset @@ -1155,7 +894,7 @@ def test_reset_endpoint_clears_compare_expr_memo(fresh_companion_app, project, o assert _build_compare_expr.cache_info().currsize == 0 -def test_notify_project_reset_evicts_the_compare_build(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_notify_project_reset_evicts_the_compare_build(fresh_companion_app, project, orders_src, monkeypatch): """The cross-process ``project_reset`` notify evicts the companion's compare build. ``reset-to`` is normally run out of process (the ``tallyman reset-to`` CLI): @@ -1207,7 +946,7 @@ def test_notify_project_reset_evicts_the_compare_build(fresh_companion_app, proj assert _build_compare_expr.cache_info().currsize == 0 -def test_notify_project_reset_clears_result_plan_memo(fresh_companion_app, project, orders_parquet, monkeypatch): +def test_notify_project_reset_clears_result_plan_memo(fresh_companion_app, project, orders_src, monkeypatch): """Parity with the in-server reset: the cross-process ``project_reset`` notify clears the companion's ``cached_result_expr`` memo too. Hygiene rather than correctness — ``cached_result_expr`` makes its files exist on every read — but both reset @@ -1231,7 +970,7 @@ def test_notify_project_reset_clears_result_plan_memo(fresh_companion_app, proje assert cached_result_expr.cache_info().currsize == 0 -def test_result_digest_is_snapshot_content_digest(project, orders_parquet, monkeypatch): +def test_result_digest_is_snapshot_content_digest(project, orders_src, monkeypatch): # ADR-009 D2 (amending ADR-004-result-digest-canonical-ordering): the result_digest for a worthy entry is the # content digest of the snapshot read back (arrow-sha256:), not a hash of the file's bytes (which moves with # the writer's version, the codec and the row-group size) and not a per-row repr() hash. It equals @@ -1240,7 +979,6 @@ def test_result_digest_is_snapshot_content_digest(project, orders_parquet, monke from tallyman_core import entry_dir, read_manifest from tallyman_xorq.result_cache import baked_snapshot_path, snapshot_file_digest - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("agg", _agg_code(project)) h = _hash_of(project) @@ -1263,40 +1001,38 @@ def _literal_nondeterministic_code(project: str) -> str: # frozen build twice, ADR-009 D6, does not flag it either.) return ( "import random\n" - "from tallyman_xorq.io import read_project_file\n" - f"t = read_project_file('orders.parquet', project={project!r})\n" + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r})\n" "expr = t.group_by('region').aggregate(total=t.price.sum()).mutate(nonce=random.random())\n" ) -def test_structural_nondeterminism_detected_for_baked_literal(project, orders_parquet, monkeypatch): +def test_structural_nondeterminism_detected_for_baked_literal(project, orders_src, monkeypatch): # #88 part 2: the structural case the op lint is blind to. Two reconstructions of # a recipe that bakes a random literal at author time hash differently, so the # entry's structural content_hash is itself unstable — distinct from an # execution-nondeterministic recipe whose graph is fixed. from tallyman_xorq.result_cache import recipe_is_structurally_nondeterministic - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("nd", _literal_nondeterministic_code(project)) h = _hash_of(project) assert recipe_is_structurally_nondeterministic(project, h) is True -def test_deterministic_recipe_is_not_structurally_nondeterministic(project, orders_parquet, monkeypatch): +def test_deterministic_recipe_is_not_structurally_nondeterministic(project, orders_src, monkeypatch): # #88 part 2: a deterministic recipe reconstructs to the same graph hash both # times, so the structural detector does not false-positive on a clean entry — # its drift, if any, is execution-level (#83), not structural. from tallyman_xorq.result_cache import recipe_is_structurally_nondeterministic - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("agg", _agg_code(project)) h = _hash_of(project) assert recipe_is_structurally_nondeterministic(project, h) is False -def test_deterministic_udf_entry_is_not_structurally_nondeterministic(project, orders_parquet, monkeypatch): +def test_deterministic_udf_entry_is_not_structurally_nondeterministic(project, orders_src, monkeypatch): # #88 part 2: a deterministic UDF recipe must NOT be flagged structural. xorq's # make_pandas_udf mints a fresh class per reconstruction (see # test_scalar_udf_is_worthy), so two reconstructions @@ -1306,14 +1042,13 @@ def test_deterministic_udf_entry_is_not_structurally_nondeterministic(project, o # exclude UDF entries so a UDF self-heal drift is labeled execution, not structural. from tallyman_xorq.result_cache import recipe_is_structurally_nondeterministic - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("udf", _scalar_udf_code(project)) h = _hash_of(project) assert recipe_is_structurally_nondeterministic(project, h) is False -def test_baked_literal_entry_reads_faithfully_from_frozen_build(project, orders_parquet, monkeypatch, caplog): +def test_baked_literal_entry_reads_faithfully_from_frozen_build(project, orders_src, monkeypatch, caplog): # Inversion of the pre-#163 behavior this test used to pin. On main, reading # an entry whose recipe bakes a random literal re-imported expr.py, re-rolled # the literal, derived a *different* snapshot key, and self-healed to bytes @@ -1328,7 +1063,6 @@ def test_baked_literal_entry_reads_faithfully_from_frozen_build(project, orders_ from tallyman_xorq.result_cache import cached_result_expr, verify_result_faithful - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("nd", _literal_nondeterministic_code(project)) # expensive (Aggregate) → baked h = _hash_of(project) @@ -1348,13 +1082,12 @@ def test_baked_literal_entry_reads_faithfully_from_frozen_build(project, orders_ # --------------------------------------------------------------------------- -def test_cheap_entry_records_no_digest(project, orders_parquet, monkeypatch): +def test_cheap_entry_records_no_digest(project, orders_src, monkeypatch): # ADR-004-result-digest-canonical-ordering: a cheap entry (row-preserving over one file) # writes no snapshot, so there is nothing to hash. The manifest must record no # result_digest (None / falsy). from tallyman_core import entry_dir, read_manifest - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("proj", _project_code(project)) h = _hash_of(project) @@ -1371,13 +1104,13 @@ def _ordered_agg_code(project: str) -> str: # author's own key, which is kept (ADR-008 D10) — Sort adds a Sort op making it # expensive/worthy regardless. return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()).order_by("region") """ -def test_result_digest_stable_run_to_run(project, orders_parquet, monkeypatch): +def test_result_digest_stable_run_to_run(project, orders_src, monkeypatch): # ADR-004-result-digest-canonical-ordering: the result_digest for a worthy entry # with a deterministic (explicitly ordered) result is stable across two independent # materialisations. Uses an explicitly-ordered aggregate (Sort op) so the rows are @@ -1385,7 +1118,6 @@ def test_result_digest_stable_run_to_run(project, orders_parquet, monkeypatch): from tallyman_core import entry_dir, read_manifest from tallyman_xorq.result_cache import baked_snapshot_path, snapshot_file_digest - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("agg_ord", _ordered_agg_code(project)) diff --git a/tests/test_revise_carry_over.py b/tests/test_revise_carry_over.py index b8ea10e9..89df18a1 100644 --- a/tests/test_revise_carry_over.py +++ b/tests/test_revise_carry_over.py @@ -8,8 +8,8 @@ def _select(project: str, cols: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select({cols}) """ @@ -19,7 +19,7 @@ def _current_hash(project: str) -> str: return list_entries(project)[0]["content_hash"] -def test_revise_carries_chart_and_display_config(project, orders_parquet, monkeypatch): +def test_revise_carries_chart_and_display_config(project, orders_src, monkeypatch): """A revise must seed the new version's per-entry config (chart + display config) from the previous version. Both are keyed by content hash, so the new version's fresh hash would otherwise orphan them ("chart didn't carry diff --git a/tests/test_revise_self_reference.py b/tests/test_revise_self_reference.py index f502eec6..32c01560 100644 --- a/tests/test_revise_self_reference.py +++ b/tests/test_revise_self_reference.py @@ -6,15 +6,15 @@ # (tracked_expr_from_alias("X") / pinned_expr_from_alias("X")) — that makes the # code an opaque self-reference up a chain of identically-named versions instead # of a self-contained recipe you can read in one view. catalog_revise rejects it -# and points at the two good shapes: inline the source (source-shaped) or pin the -# previous version by its content hash (expensive). A by-hash reference stays -# legal. +# and points at the two good shapes: start from the source alias (source-shaped) +# or pin the previous version by its content hash (expensive). A by-hash +# reference stays legal. def _src(project: str, cols: str = '"region", "price", "__row_order"') -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select({cols}) """ @@ -27,15 +27,15 @@ def _self_alias(project: str, alias: str) -> str: """ -def _pin_by_hash(project: str, prev_hash: str) -> str: +def _pin_by_version(project: str, ref: str) -> str: return f""" from tallyman_xorq.io import pinned_expr_from_alias -t = pinned_expr_from_alias({prev_hash!r}, project={project!r}) +t = pinned_expr_from_alias({ref!r}, project={project!r}) expr = t.mutate(region2=t.region) """ -def test_revise_rejects_self_alias_reference(project, orders_parquet, monkeypatch): +def test_revise_rejects_self_alias_reference(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) v1 = catalog_create("parking", _src(project)) out = catalog_revise("parking", _self_alias(project, "parking")) @@ -47,17 +47,17 @@ def test_revise_rejects_self_alias_reference(project, orders_parquet, monkeypatc assert get_alias(project, "parking") == v1["hash"] -def test_revise_allows_pin_by_hash_of_prior_version(project, orders_parquet, monkeypatch): +def test_revise_allows_pin_by_version_of_prior_version(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) - v1 = catalog_create("parking", _src(project)) - out = catalog_revise("parking", _pin_by_hash(project, v1["hash"])) + catalog_create("parking", _src(project)) + out = catalog_revise("parking", _pin_by_version(project, "parking-v1")) assert "error" not in out, out assert out.get("version") == 2 -def test_revise_allows_inlined_source(project, orders_parquet, monkeypatch): +def test_revise_allows_source_shaped_revision(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) catalog_create("parking", _src(project)) - out = catalog_revise("parking", _src(project, cols='"region", "__row_order"')) # inline, no alias ref + out = catalog_revise("parking", _src(project, cols='"region", "__row_order"')) # a source alias, not its own assert "error" not in out, out assert out.get("version") == 2 diff --git a/tests/test_row_order.py b/tests/test_row_order.py index d8403143..a1bb91dd 100644 --- a/tests/test_row_order.py +++ b/tests/test_row_order.py @@ -6,6 +6,9 @@ ``tests/test_row_order_pages.py``; sort grafting and hoisting (ADR-008 D10 and D11) in ``tests/test_row_order_sorts.py``. +Every recipe here reads an alias, because a file enters the catalog only through an import (ADR-011 D1). The ordered +copy of ADR-008 D2 is the source entry's own snapshot, so the claims made about it are asserted there. + The new API these tests touch (``tallyman_xorq.materialize.snapshot_path`` and ``tallyman_xorq.worthiness.classify_expr``) is imported inside helpers, not at module top: a top-level import of a module that does not exist yet makes ruff mis-group the block, the CI lint job fails, and the tests never run. @@ -13,7 +16,6 @@ from __future__ import annotations -import os import re from pathlib import Path @@ -25,18 +27,19 @@ from tallyman_companion.diff import build_compare_expr, build_diff_expr from tallyman_core import data_dir, read_manifest, set_alias -from tallyman_core.paths import compute_cache_dir, entry_build_dir, entry_dir +from tallyman_core.paths import entry_build_dir, entry_dir from tallyman_xorq.build import BuildError, build_and_persist from tallyman_xorq.primary_key import resolve_primary_key from tallyman_xorq.result_cache import cache_worthy, cached_result_expr +from tallyman_xorq.source_import import update_and_depend ROW_ORDER = "__row_order" ROW_ORDER_RIGHT = "__row_order_right" +ORDERS_SRC = "orders_src" # the shoe-orders source alias the conftest fixture imports _PRELUDE = """ -import xorq.api as xo import xorq.vendor.ibis as ibis -from tallyman_xorq.io import pinned_expr_from_alias, read_project_file, tallyman_read_csv, tracked_expr_from_alias +from tallyman_xorq.io import tracked_expr_from_alias """ # A select of these columns is written the way the ADR-008 D3 error shows it: t.select("g", "n", "__row_order"). @@ -46,19 +49,17 @@ # --------------------------------------------------------------------------- # # helpers # --------------------------------------------------------------------------- # -def _over(project: str, source: str, body: str) -> str: - """A recipe over one source file: ``t`` is ``read_project_file(source)`` and ``expr`` is *body*.""" - return f"{_PRELUDE}t = read_project_file({source!r}, project={project!r})\nexpr = {body}\n" - +def _over(project: str, alias: str, body: str) -> str: + """A recipe over the entry named *alias*: ``t`` is that entry and ``expr`` is *body*. -def _orders(project: str, body: str) -> str: - """A recipe over the shoe-orders source: ``t`` is a read of orders.parquet and ``expr`` is *body*.""" - return _over(project, "orders.parquet", body) + One helper for a source alias and a catalog alias, because a recipe reads both the same way (ADR-011 D1). + """ + return f"{_PRELUDE}t = tracked_expr_from_alias({alias!r}, project={project!r})\nexpr = {body}\n" -def _chained(alias: str, body: str) -> str: - """A recipe that builds on the entry named *alias*: ``t`` is that entry and ``expr`` is *body*.""" - return f"{_PRELUDE}t = tracked_expr_from_alias({alias!r})\nexpr = {body}\n" +def _orders(project: str, body: str) -> str: + """A recipe over the imported shoe-orders source: ``t`` is ``orders_src`` and ``expr`` is *body*.""" + return _over(project, ORDERS_SRC, body) def _create(project: str, alias: str, code: str) -> str: @@ -69,12 +70,18 @@ def _create(project: str, alias: str, code: str) -> str: def _write(project: str, name: str, columns: dict) -> Path: - """A small parquet source under the project's data dir.""" + """A small parquet file under the project's data dir, before it is imported.""" path = data_dir(project) / name pq.write_table(pa.table(columns), path) return path +def _imported(project: str, name: str, columns: dict, alias: str) -> str: + """Write that parquet file and import it under the source alias *alias*; returns the alias name.""" + update_and_depend(_write(project, name, columns), alias, project=project) + return alias + + def _names(res) -> list[str]: """The column names a build recorded in its schema.""" return [f["name"] for f in res.schema["fields"]] @@ -92,36 +99,34 @@ def _classify(expr): return classify_expr(expr) -def _ordered_copies(project: str) -> list[Path]: - """Ordered copies of sources: ADR-007 D13 puts them under the project's compute cache.""" - return sorted((compute_cache_dir(project) / "ordered_sources").glob("*.parquet")) +def _import_csv(project: str, csv: Path, alias: str) -> str: + """Import a two-int-column CSV under *alias*, with the schema pinned; returns the alias name. - -def _csv_recipe(csv: Path) -> str: - return ( - "import xorq.vendor.ibis as ibis\n" - "from tallyman_xorq.io import tallyman_read_csv\n" - f"expr = tallyman_read_csv({str(csv)!r}, schema=ibis.schema({{'k': 'int64', 'v': 'int64'}}))\n" - ) + The schema, the delimiter and everything else about how a CSV is read are named in the import call and + recorded on the entry (ADR-011 D12), so the recipe that reads the alias says nothing about them. + """ + update_and_depend(csv, alias, project=project, schema=ibis.schema({"k": "int64", "v": "int64"})) + return alias # --------------------------------------------------------------------------- # # ADR-008 D2: every file tallyman reads carries __row_order # --------------------------------------------------------------------------- # -def test_editing_a_csv_forks_the_content_hash(project): - """ADR-008 D2 (ordered copy built from the content-addressed clone), #168: an edit is a new entry. +def test_editing_a_csv_and_importing_it_again_forks_the_content_hash(project): + """ADR-011 D1 and #168: re-importing edited bytes mints a new version, and a recipe over it is a new entry. - Today the intermediate is keyed by the CSV's path and overwritten in place, so the recipe hashes to the same - entry before and after the edit and the new rows are never seen. + The old intermediate was keyed by the CSV's path and overwritten in place, so the recipe hashed to the same + entry before and after the edit and the new rows were never seen. A source version is content-addressed, so + the second import is a different parent and the same recipe text is a different entry. """ csv = data_dir(project) / "edited.csv" csv.write_text("k,v\n1,10\n2,20\n3,30\n") - first = build_and_persist(project, _csv_recipe(csv)).content_hash + alias = _import_csv(project, csv, "edited_src") + first = build_and_persist(project, _over(project, alias, "t")).content_hash csv.write_text("k,v\n1,10\n2,999\n3,30\n4,40\n") - st = csv.stat() - os.utime(csv, ns=(st.st_atime_ns, st.st_mtime_ns + 5_000_000_000)) # newer than any filesystem clock granularity - second = build_and_persist(project, _csv_recipe(csv)).content_hash + _import_csv(project, csv, alias) + second = build_and_persist(project, _over(project, alias, "t")).content_hash assert second != first, "editing the CSV and re-running the same recipe must create a new entry" cached_result_expr.cache_clear() @@ -129,7 +134,7 @@ def test_editing_a_csv_forks_the_content_hash(project): assert cached_result_expr(project, second).execute()["v"].tolist() == [10, 999, 30, 40] -def test_a_worthy_snapshot_ends_in_row_order(project, orders_parquet): +def test_a_worthy_snapshot_ends_in_row_order(project, orders_src): """ADR-008 D2: a snapshot ends in an int64 ``__row_order`` holding ``0..N-1`` in the file's row order.""" res = build_and_persist(project, _orders(project, "t.order_by(t.price.desc())")) assert _names(res)[-1] == ROW_ORDER, f"the entry's schema must end in __row_order, got {_names(res)}" @@ -142,39 +147,38 @@ def test_a_worthy_snapshot_ends_in_row_order(project, orders_parquet): assert prices == sorted(prices, reverse=True), "numbered in the order the file is written in" -def test_the_ordered_copy_of_a_parquet_source_ends_in_row_order(project, orders_parquet): - """ADR-008 D2: a parquet source enters tallyman through an ordered copy with ``__row_order`` last.""" - build_and_persist(project, _orders(project, "t")) +def test_the_snapshot_of_a_parquet_source_ends_in_row_order(project, orders_parquet): + """ADR-008 D2 over ADR-011 D1: the import writes the file's rows, in file order, with ``__row_order`` last. - copies = _ordered_copies(project) - assert len(copies) == 1, f"expected one ordered copy under compute_cache/ordered_sources, found {copies}" - copy, source = pq.read_table(copies[0]), pq.read_table(orders_parquet) - assert copy.column_names == [*source.column_names, ROW_ORDER] - assert copy.schema.field(ROW_ORDER).type == pa.int64() - assert copy[ROW_ORDER].to_pylist() == list(range(source.num_rows)) + The ordered copy of ADR-008 D2 is the source entry's own snapshot now, so the claim is asserted there. + """ + imported = update_and_depend(orders_parquet, ORDERS_SRC, project=project) + + snapshot = pq.read_table(_snapshot_path(project, imported["hash"])) + source = pq.read_table(orders_parquet) + assert snapshot.column_names == [*source.column_names, ROW_ORDER] + assert snapshot.schema.field(ROW_ORDER).type == pa.int64() + assert snapshot[ROW_ORDER].to_pylist() == list(range(source.num_rows)) for name in source.column_names: - assert copy[name].to_pylist() == source[name].to_pylist(), f"{name} must be copied in file order" + assert snapshot[name].to_pylist() == source[name].to_pylist(), f"{name} must be copied in file order" -def test_the_ordered_copy_of_a_csv_source_ends_in_row_order(project): - """ADR-008 D2: a CSV source enters tallyman through an ordered copy with ``__row_order`` last.""" +def test_the_snapshot_of_a_csv_source_ends_in_row_order(project): + """ADR-008 D2 over ADR-011 D1: a CSV is parsed at import and its snapshot keeps the file's row order.""" csv = data_dir(project) / "ordered.csv" csv.write_text("k,v\n3,30\n1,10\n2,20\n") - build_and_persist(project, _csv_recipe(csv)) + imported = update_and_depend(csv, "ordered_src", project=project) - copies = _ordered_copies(project) - assert len(copies) == 1, f"expected one ordered copy under compute_cache/ordered_sources, found {copies}" - copy = pq.read_table(copies[0]) - assert copy.column_names == ["k", "v", ROW_ORDER] - assert copy["k"].to_pylist() == [3, 1, 2] # file order, not sorted order - assert copy[ROW_ORDER].to_pylist() == [0, 1, 2] + snapshot = pq.read_table(_snapshot_path(project, imported["hash"])) + assert snapshot.column_names == ["k", "v", ROW_ORDER] + assert snapshot["k"].to_pylist() == [3, 1, 2] # file order, not sorted order + assert snapshot[ROW_ORDER].to_pylist() == [0, 1, 2] def test_a_parquet_source_with_its_own_row_order_column_has_it_overwritten(project): """ADR-008 D2: a source that already has ``__row_order`` (a file tallyman exported) has it overwritten.""" - _write(project, "exported.parquet", {"k": [10, 20, 30, 40], ROW_ORDER: [5, 3, 9, 1]}) - code = f"{_PRELUDE}expr = read_project_file('exported.parquet', project={project!r})\n" - res = build_and_persist(project, code) + alias = _imported(project, "exported.parquet", {"k": [10, 20, 30, 40], ROW_ORDER: [5, 3, 9, 1]}, "exported_src") + res = build_and_persist(project, _over(project, alias, "t")) df = cached_result_expr(project, res.content_hash).execute() assert list(df.columns) == ["k", ROW_ORDER] @@ -182,14 +186,14 @@ def test_a_parquet_source_with_its_own_row_order_column_has_it_overwritten(proje assert df[ROW_ORDER].tolist() == [0, 1, 2, 3], "the source's own values must be overwritten with 0..N-1" -def test_a_worthy_entry_that_keeps_its_parents_rows_renumbers_them(project, orders_parquet): +def test_a_worthy_entry_that_keeps_its_parents_rows_renumbers_them(project, orders_src): """ADR-008 D2: ``materialize`` replaces an inherited ``__row_order`` with positions in its own file. A filter leaves gaps in the parent's positions. The entry is worthy because of the window function, and its file numbers its own rows ``0..M-1`` with no gaps. """ _create(project, "orders", _orders(project, "t")) - child = build_and_persist(project, _chained("orders", "t.filter(t.qty > 2).mutate(rn=ibis.row_number())")) + child = build_and_persist(project, _over(project, "orders", "t.filter(t.qty > 2).mutate(rn=ibis.row_number())")) assert _names(child)[-1] == ROW_ORDER, f"the entry's schema must end in __row_order, got {_names(child)}" table = pq.read_table(_snapshot_path(project, child.content_hash)) @@ -201,19 +205,19 @@ def test_a_worthy_entry_that_keeps_its_parents_rows_renumbers_them(project, orde # --------------------------------------------------------------------------- # # ADR-008 D3: a cheap entry that drops __row_order is a build error # --------------------------------------------------------------------------- # -def test_a_cheap_select_that_omits_row_order_names_its_parent_and_the_fix(project, orders_parquet): +def test_a_cheap_select_that_omits_row_order_names_its_parent_and_the_fix(project, orders_src): """ADR-008 D3: the error names the parent entry and shows the corrected select.""" parent = _create(project, "orders", _orders(project, "t")) with pytest.raises(BuildError) as exc: - build_and_persist(project, _chained("orders", "t.select('region', 'price')")) + build_and_persist(project, _over(project, "orders", "t.select('region', 'price')")) msg = str(exc.value) assert ROW_ORDER in msg, msg assert _CORRECTED_SELECT.search(msg), f"the message must show the corrected select: {msg}" assert parent in msg or parent[:12] in msg or "orders" in msg, f"the message must name the parent entry: {msg}" -def test_a_cheap_select_over_a_source_read_omits_row_order_with_the_fix(project, orders_parquet): - """ADR-008 D3: the same error when the cheap entry reads a source file directly.""" +def test_a_cheap_select_over_a_source_read_omits_row_order_with_the_fix(project, orders_src): + """ADR-008 D3: the same error when the cheap entry reads a source alias directly.""" with pytest.raises(BuildError) as exc: build_and_persist(project, _orders(project, "t.select('region', 'price')")) msg = str(exc.value) @@ -221,7 +225,7 @@ def test_a_cheap_select_over_a_source_read_omits_row_order_with_the_fix(project, assert _CORRECTED_SELECT.search(msg), f"the message must show the corrected select: {msg}" -def test_the_same_select_over_a_worthy_recipe_builds_and_is_numbered(project, orders_parquet): +def test_the_same_select_over_a_worthy_recipe_builds_and_is_numbered(project, orders_src): """ADR-008 D3: a worthy entry is exempt, because the writer numbers its rows.""" res = build_and_persist( project, _orders(project, "t.group_by('region').aggregate(n=t.count()).select('region', 'n')") @@ -229,7 +233,7 @@ def test_the_same_select_over_a_worthy_recipe_builds_and_is_numbered(project, or assert _names(res) == ["region", "n", ROW_ORDER] -def test_a_computed_column_added_after_row_order_leaves_it_last(project, orders_parquet): +def test_a_computed_column_added_after_row_order_leaves_it_last(project, orders_src): """ADR-008 D3: tallyman moves ``__row_order`` to the last position, at the top of the expression only.""" res = build_and_persist(project, _orders(project, "t.mutate(double=t.price * 2)")) names = _names(res) @@ -243,8 +247,9 @@ def test_a_computed_column_added_after_row_order_leaves_it_last(project, orders_ # --------------------------------------------------------------------------- # def test_asking_for_an_order_numbers_the_file_in_that_order(project): """ADR-008 D3: an ``order_by`` makes the entry worthy and the writer numbers the rows in the requested order.""" - _write(project, "amounts.parquet", {"name": list("abcdef"), "amount": [40, 10, 60, 20, 50, 30]}) - res = build_and_persist(project, _over(project, "amounts.parquet", "t.order_by(t.amount.desc())")) + columns = {"name": list("abcdef"), "amount": [40, 10, 60, 20, 50, 30]} + alias = _imported(project, "amounts.parquet", columns, "amounts_src") + res = build_and_persist(project, _over(project, alias, "t.order_by(t.amount.desc())")) df = cached_result_expr(project, res.content_hash).execute() assert ROW_ORDER in df.columns, f"the entry must carry __row_order, got {list(df.columns)}" @@ -260,18 +265,18 @@ def test_asking_for_an_order_numbers_the_file_in_that_order(project): pytest.param("t.mutate(__row_order=t.order_id).order_by('region')", id="worthy entry"), ], ) -def test_assigning_to_row_order_is_a_build_error(project, orders_parquet, body): +def test_assigning_to_row_order_is_a_build_error(project, orders_src, body): """ADR-008 D6: arbitrary values could contain ties or gaps, so a recipe may not assign to the column.""" with pytest.raises(BuildError) as exc: build_and_persist(project, _orders(project, body)) assert ROW_ORDER in str(exc.value) -def test_a_debugging_copy_of_row_order_survives_materialization(project, orders_parquet): +def test_a_debugging_copy_of_row_order_survives_materialization(project, orders_src): """ADR-008 D6: ``__row_order_v1`` is ordinary data: it keeps the parent's positions after the child is written.""" parent = _create(project, "foo_v1", _orders(project, "t.order_by(t.price.desc())")) child_code = ( - f"{_PRELUDE}t = tracked_expr_from_alias('foo_v1')\n" + f"{_PRELUDE}t = tracked_expr_from_alias('foo_v1', project={project!r})\n" "c = t.mutate(__row_order_v1=t['__row_order'])\n" "expr = c.order_by(c.qty)\n" ) @@ -343,7 +348,7 @@ def test_the_cheap_test_classifies_these_shapes_as_cheap(tmp_path, label): assert verdict.worthy is False, f"{label} must be cheap: {verdict}" -def test_the_verdict_is_read_from_the_manifest_with_no_expr_yaml_parsed(project, orders_parquet, monkeypatch): +def test_the_verdict_is_read_from_the_manifest_with_no_expr_yaml_parsed(project, orders_src, monkeypatch): """ADR-008 D4: computed once at build and recorded; ``classify_build`` and its regex over expr.yaml are retired.""" cheap = build_and_persist(project, _orders(project, "t.filter(t.qty > 1).mutate(double=t.price * 2)")).content_hash worthy = build_and_persist( @@ -366,8 +371,9 @@ def guarded(self, *args, **kwargs): def test_a_value_level_unnest_makes_the_entry_worthy(project): """ADR-008 D4: an ``unnest`` inside a select multiplies rows, which a list of relation operations cannot see.""" - _write(project, "lists.parquet", {"k": [1, 2, 3], "tags": [["x", "y"], ["z"], ["x", "y", "z"]]}) - res = build_and_persist(project, _over(project, "lists.parquet", "t.select('k', tag=t.tags.unnest())")) + columns = {"k": [1, 2, 3], "tags": [["x", "y"], ["z"], ["x", "y", "z"]]} + alias = _imported(project, "lists.parquet", columns, "lists_src") + res = build_and_persist(project, _over(project, alias, "t.select('k', tag=t.tags.unnest())")) assert res.cache_worthy is True assert read_manifest(entry_dir(project, res.content_hash)).cache_worthy is True @@ -379,7 +385,7 @@ def test_a_value_level_unnest_makes_the_entry_worthy(project): pytest.param("t.select('region', 'category').distinct()", id="distinct"), ], ) -def test_a_union_and_a_distinct_are_materialized(project, orders_parquet, body): +def test_a_union_and_a_distinct_are_materialized(project, orders_src, body): """ADR-008 D4: neither can carry one parent's row order, and today's deny-list classes both as cheap.""" res = build_and_persist(project, _orders(project, body)) assert res.cache_worthy is True @@ -389,7 +395,7 @@ def test_a_union_and_a_distinct_are_materialized(project, orders_parquet, body): # --------------------------------------------------------------------------- # # ADR-008 D6: the reserved name # --------------------------------------------------------------------------- # -def test_the_primary_key_search_never_returns_row_order(project, orders_parquet): +def test_the_primary_key_search_never_returns_row_order(project, orders_src): """ADR-008 D6: ``__row_order`` is unique in every table, so it would win the search for any table without a key.""" h = build_and_persist(project, _orders(project, "t.select('region', 'category').order_by('region')")).content_hash assert ROW_ORDER in cached_result_expr(project, h).columns, "the entry must carry __row_order for this test" @@ -399,10 +405,10 @@ def test_the_primary_key_search_never_returns_row_order(project, orders_parquet) assert key == [], "region and category alone are not unique, and __row_order does not count" -def test_a_diff_carries_no_row_order_column_from_either_side(project, orders_parquet): +def test_a_diff_carries_no_row_order_column_from_either_side(project, orders_src): """ADR-008 D6: ``build_compare_expr`` and ``build_diff_expr`` drop it from both sides before joining.""" a = _create(project, "orders", _orders(project, "t")) - b = build_and_persist(project, _chained("orders", "t.filter(t.qty > 1)")).content_hash + b = build_and_persist(project, _over(project, "orders", "t.filter(t.qty > 1)")).content_hash a_expr, b_expr = cached_result_expr(project, a), cached_result_expr(project, b) assert ROW_ORDER in a_expr.columns and ROW_ORDER in b_expr.columns, "both inputs must carry __row_order" @@ -416,17 +422,19 @@ def test_a_diff_carries_no_row_order_column_from_either_side(project, orders_par # ADR-008 D6: joins # --------------------------------------------------------------------------- # def _keyed_sources(project: str) -> None: + """Three keyed files, imported as the source aliases ``a_src``, ``b_src`` and ``c_src``.""" for name in ("a", "b", "c"): - _write(project, f"{name}.parquet", {"k": list(range(5)), name: [f"{name}{i}" for i in range(5)]}) + columns = {"k": list(range(5)), name: [f"{name}{i}" for i in range(5)]} + _imported(project, f"{name}.parquet", columns, f"{name}_src") def _three_way(project: str, right_side: str = "{}") -> str: """Three sources joined in one recipe; *right_side* is a template for how the right-hand inputs are written.""" return ( f"{_PRELUDE}" - f"a = read_project_file('a.parquet', project={project!r})\n" - f"b = read_project_file('b.parquet', project={project!r})\n" - f"c = read_project_file('c.parquet', project={project!r})\n" + f"a = tracked_expr_from_alias('a_src', project={project!r})\n" + f"b = tracked_expr_from_alias('b_src', project={project!r})\n" + f"c = tracked_expr_from_alias('c_src', project={project!r})\n" f"expr = a.join({right_side.format('b')}, 'k').join({right_side.format('c')}, 'k')\n" ) @@ -435,8 +443,8 @@ def test_a_join_entrys_file_has_no_right_hand_row_order(project): """ADR-008 D6: ibis renames the right side's copy to ``__row_order_right``; the writer drops it.""" _keyed_sources(project) code = ( - f"{_PRELUDE}a = read_project_file('a.parquet', project={project!r})\n" - f"b = read_project_file('b.parquet', project={project!r})\nexpr = a.join(b, 'k')\n" + f"{_PRELUDE}a = tracked_expr_from_alias('a_src', project={project!r})\n" + f"b = tracked_expr_from_alias('b_src', project={project!r})\nexpr = a.join(b, 'k')\n" ) res = build_and_persist(project, code) assert _names(res)[-1] == ROW_ORDER, _names(res) @@ -451,13 +459,13 @@ def test_a_join_entry_can_be_joined_to_a_third_entry(project): """ADR-008 D6: with the right-hand copy dropped from its file, the join entry joins to a third entry.""" _keyed_sources(project) ab = ( - f"{_PRELUDE}a = read_project_file('a.parquet', project={project!r})\n" - f"b = read_project_file('b.parquet', project={project!r})\nexpr = a.join(b, 'k')\n" + f"{_PRELUDE}a = tracked_expr_from_alias('a_src', project={project!r})\n" + f"b = tracked_expr_from_alias('b_src', project={project!r})\nexpr = a.join(b, 'k')\n" ) _create(project, "ab", ab) third = ( - f"{_PRELUDE}ab = tracked_expr_from_alias('ab')\n" - f"c = read_project_file('c.parquet', project={project!r})\nexpr = ab.join(c, 'k')\n" + f"{_PRELUDE}ab = tracked_expr_from_alias('ab', project={project!r})\n" + f"c = tracked_expr_from_alias('c_src', project={project!r})\nexpr = ab.join(c, 'k')\n" ) res = build_and_persist(project, third) @@ -491,37 +499,51 @@ def test_a_three_way_join_builds_when_the_right_hand_inputs_drop_row_order(proje # ADR-008 D7: CSV roots # --------------------------------------------------------------------------- # def test_a_csv_root_is_a_cheap_read_with_exactly_one_row_order_column(project): - """ADR-008 D7: ``tallyman_read_csv`` loses its trailing ``order_by`` and its column becomes ``__row_order``.""" + """ADR-008 D7: the CSV's own order is in ``__row_order``, and the recipe that reads it adds no sort. + + The parse happens at import now (ADR-011 D12), so the root of the recipe is a plain read of the source + entry's snapshot: no trailing ``order_by`` to strip, and no ``original_row_order`` column left over. + """ csv = data_dir(project) / "root.csv" csv.write_text("k,v\n3,30\n1,10\n2,20\n") - res = build_and_persist(project, _csv_recipe(csv)) + alias = _import_csv(project, csv, "root_src") + res = build_and_persist(project, _over(project, alias, "t")) yaml_text = (entry_build_dir(project, res.content_hash) / "expr.yaml").read_text() - assert not re.search(r"op:\s*Sort\b", yaml_text), "a CSV root is a plain read of its ordered copy: no Sort" + assert not re.search(r"op:\s*Sort\b", yaml_text), "a CSV root is a plain read of the source snapshot: no Sort" assert cache_worthy(project, res.content_hash) is False assert read_manifest(entry_dir(project, res.content_hash)).cache_worthy is False names = _names(res) assert names == ["k", "v", ROW_ORDER], names assert "original_row_order" not in names + df = cached_result_expr(project, res.content_hash).execute() + assert df["k"].tolist() == [3, 1, 2] # the CSV's own order # --------------------------------------------------------------------------- # -# ADR-008 D12: a raw parquet read is a build error +# ADR-008 D12 / ADR-011 D2: a raw parquet read is a build error # --------------------------------------------------------------------------- # +def _assert_names_the_import(msg: str, path: Path) -> None: + """The refusal names the import to run and the alias read that replaces the raw read.""" + assert "catalog_import_source" in msg, msg + assert "tracked_expr_from_alias" in msg, msg + assert str(path) in msg, msg + + def test_a_raw_parquet_read_of_a_source_file_is_a_build_error(project, orders_parquet): - """ADR-008 D12: such a read has no digest, no clone and no ordered copy, so no ``__row_order``.""" + """ADR-008 D12: such a read has no digest and no clone, so the entry has no ``__row_order`` and no provenance.""" code = f"import xorq.api as xo\nexpr = xo.deferred_read_parquet({str(orders_parquet)!r})\n" with pytest.raises(BuildError) as exc: build_and_persist(project, code) - assert "read_project_file" in str(exc.value) + _assert_names_the_import(str(exc.value), orders_parquet) def test_a_raw_parquet_read_of_a_file_outside_the_project_is_a_build_error(project, tmp_path): - """ADR-008 D12: only tallyman's own files (snapshots and ordered copies under compute_cache) may be read raw.""" + """ADR-008 D12: only tallyman's own files (the snapshots under compute_cache) may be read raw.""" outside = tmp_path / "elsewhere.parquet" pq.write_table(pa.table({"k": [1, 2, 3]}), outside) code = f"import xorq.api as xo\nt = xo.deferred_read_parquet({str(outside)!r})\nexpr = t.filter(t.k > 1)\n" with pytest.raises(BuildError) as exc: build_and_persist(project, code) - assert "read_project_file" in str(exc.value) + _assert_names_the_import(str(exc.value), outside) diff --git a/tests/test_row_order_pages.py b/tests/test_row_order_pages.py index 2190b349..6dd28e0d 100644 --- a/tests/test_row_order_pages.py +++ b/tests/test_row_order_pages.py @@ -12,21 +12,21 @@ from __future__ import annotations -import shutil from pathlib import Path import pyarrow.parquet as pq import pytest from fastapi.testclient import TestClient -from tallyman_core import data_dir from tallyman_xorq.build import build_and_persist +from tallyman_xorq.source_import import update_and_depend from tests.big_parquet import write_big_parquet ROW_ORDER = "__row_order" OFFSET, LIMIT, REQUESTS = 1_000_000, 50, 8 +BIG_SRC = "big_src" -_PRELUDE = "from tallyman_xorq.io import read_project_file\n" +_PRELUDE = "from tallyman_xorq.io import tracked_expr_from_alias\n" @pytest.fixture(scope="module") @@ -41,14 +41,14 @@ def big_frame(big_source): @pytest.fixture -def big(project, big_source) -> Path: - dest = data_dir(project) / "big.parquet" - shutil.copyfile(big_source, dest) - return dest +def big_src(project, big_source) -> str: + """The 26 MB file imported as a source alias (ADR-011 D1), and the alias name a recipe reads.""" + update_and_depend(big_source, BIG_SRC, project=project) + return BIG_SRC def _recipe(project: str, body: str) -> str: - return f"{_PRELUDE}t = read_project_file('big.parquet', project={project!r})\nexpr = {body}\n" + return f"{_PRELUDE}t = tracked_expr_from_alias({BIG_SRC!r}, project={project!r})\nexpr = {body}\n" def _requests(client: TestClient, project: str, content_hash: str) -> list[list[dict]]: @@ -72,7 +72,9 @@ def _requests(client: TestClient, project: str, content_hash: str) -> list[list[ pytest.param("t.filter(t.g < 150)", 150, id="cheap entry that drops rows"), ], ) -def test_a_deep_page_is_exactly_the_rows_at_those_positions(fresh_companion_app, project, big, big_frame, body, max_g): +def test_a_deep_page_is_exactly_the_rows_at_those_positions( + fresh_companion_app, project, big_src, big_frame, body, max_g +): """ADR-008 D5: eight identical requests each return the rows at positions offset..offset+limit-1. ``id`` is the file position, so for the entries that keep every row ``__row_order`` equals ``id``. A cheap entry diff --git a/tests/test_row_order_sorts.py b/tests/test_row_order_sorts.py index 2c46f951..19959373 100644 --- a/tests/test_row_order_sorts.py +++ b/tests/test_row_order_sorts.py @@ -13,7 +13,6 @@ from __future__ import annotations -import shutil from pathlib import Path import pyarrow as pa @@ -23,21 +22,26 @@ from tallyman_core import data_dir from tallyman_xorq.build import BuildError, build_and_persist from tallyman_xorq.result_cache import cached_result_expr +from tallyman_xorq.source_import import update_and_depend from tests.big_parquet import write_big_parquet ROW_ORDER = "__row_order" +AMOUNTS_SRC = "amounts_src" +BIG_SRC = "big_src" -_PRELUDE = "import xorq.vendor.ibis as ibis\nfrom tallyman_xorq.io import read_project_file\n" +_PRELUDE = "import xorq.vendor.ibis as ibis\nfrom tallyman_xorq.io import tracked_expr_from_alias\n" # The parent's rows are in this order, so an entry that ignores the author's sort is written 40, 10, 60, ... AMOUNTS = [40, 10, 60, 20, 50, 30] @pytest.fixture -def amounts(project: str) -> Path: +def amounts(project: str) -> str: + """Six rows out of amount order, imported as a source alias (ADR-011 D1); the alias name a recipe reads.""" path = data_dir(project) / "amounts.parquet" pq.write_table(pa.table({"name": list("abcdef"), "amount": AMOUNTS}), path) - return path + update_and_depend(path, AMOUNTS_SRC, project=project) + return AMOUNTS_SRC @pytest.fixture(scope="module") @@ -45,10 +49,10 @@ def big_source(tmp_path_factory) -> Path: return write_big_parquet(tmp_path_factory.mktemp("big_source") / "big.parquet") -def _sorted_then(project: str, body: str, source: str = "amounts.parquet") -> str: +def _sorted_then(project: str, body: str, source: str = AMOUNTS_SRC) -> str: """A recipe whose ``by`` is the source ordered by amount, descending, followed by *body*.""" return ( - f"{_PRELUDE}t = read_project_file({source!r}, project={project!r})\n" + f"{_PRELUDE}t = tracked_expr_from_alias({source!r}, project={project!r})\n" "by = t.order_by(t.amount.desc())\n" f"expr = {body}\n" ) @@ -75,8 +79,11 @@ def test_a_sort_that_feeds_a_limit_keeps_the_rows_the_natural_order_picks(projec decided by the tie-break of the sort that feeds the limit, not by the sort above it. ``id`` is the file position, so ``(g, id)`` is ``(g, __row_order)``. """ - shutil.copyfile(big_source, data_dir(project) / "big.parquet") - code = f"{_PRELUDE}t = read_project_file('big.parquet', project={project!r})\nexpr = t.order_by('g').limit(1000)\n" + update_and_depend(big_source, BIG_SRC, project=project) + code = ( + f"{_PRELUDE}t = tracked_expr_from_alias({BIG_SRC!r}, project={project!r})\n" + "expr = t.order_by('g').limit(1000)\n" + ) res = build_and_persist(project, code) frame = pq.read_table(big_source).to_pandas() @@ -151,7 +158,7 @@ def test_a_sort_key_that_did_not_survive_is_a_build_error_naming_it(project, amo def test_a_sort_key_that_was_an_expression_is_a_build_error_when_a_step_follows(project, amounts): """ADR-008 D11: an expression is not an output column, so there is nothing to lead the top-level sort with.""" code = ( - f"{_PRELUDE}t = read_project_file('amounts.parquet', project={project!r})\n" + f"{_PRELUDE}t = tracked_expr_from_alias({AMOUNTS_SRC!r}, project={project!r})\n" "by = t.order_by(t.amount + 1)\n" "expr = by.mutate(double=by.amount * 2)\n" ) diff --git a/tests/test_serve.py b/tests/test_serve.py index 7f31a59d..32773344 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -10,10 +10,10 @@ from tallyman_xorq import build_and_persist -def _code(project_name: str) -> str: +def _code(project_name: str, src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project_name!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project_name!r}) expr = t.group_by("region").aggregate(n=t.count()) """ @@ -26,8 +26,8 @@ def test_read_only_mode_rejects_notify(fresh_companion_app, project: str): assert r.status_code == 403 -def test_read_only_mode_serves_get_routes(built_spa, project: str, orders_parquet: Path): - res = build_and_persist(project, _code(project)) +def test_read_only_mode_serves_get_routes(built_spa, project: str, orders_src: str): + res = build_and_persist(project, _code(project, orders_src)) set_alias(project, "shoe_sales", res.content_hash) app = create_app(project, read_only=True) c = TestClient(app) @@ -146,13 +146,13 @@ def test_notify_recalc_publishes_normalized_event(project: str, monkeypatch): def test_project_path_override_relocates_project( - project: str, orders_parquet: Path, isolated_home: Path, tmp_path: Path, monkeypatch + project: str, orders_src: str, isolated_home: Path, tmp_path: Path, monkeypatch ): """The full hand-off scenario: copy the project to a random location, point TALLYMAN_PROJECT_PATH at it, and verify the catalog still loads and entries still execute through the portability layer. """ - res = build_and_persist(project, _code(project), prompt="region totals") + res = build_and_persist(project, _code(project, orders_src), prompt="region totals") set_alias(project, "shoe_sales", res.content_hash) # Simulate untarring the project dir somewhere arbitrary (NOT under TALLYMAN_HOME). diff --git a/tests/test_snapshot_format.py b/tests/test_snapshot_format.py index 886405f4..c52720ec 100644 --- a/tests/test_snapshot_format.py +++ b/tests/test_snapshot_format.py @@ -5,8 +5,8 @@ - D1 (materialization runs single-partition): float aggregates heal to the digest they were built with, and the plan runs on a connection with ``target_partitions = 1`` that is not the process default. - D3 (the snapshot's format): one row group is 1,048,576 rows, zstd, format 2.6, statistics and a page index, and - ``__row_order`` is the last column. The ordered copy of a source has 122,880-row groups. ``schema.json`` is read from - the file that was written. + ``__row_order`` is the last column. A source version's own snapshot has 122,880-row groups instead. ``schema.json`` + is read from the file that was written. - D4 (a mismatch record names its likely cause): the manifest records engine versions and the snapshot format version, and an unfaithful heal says "the engine changed" when it did. - D6 (create runs the query twice and compares): a recipe that differs run to run is recorded as not reproducible and @@ -24,7 +24,6 @@ import json import os -import shutil from dataclasses import dataclass from importlib.metadata import version from pathlib import Path @@ -37,11 +36,12 @@ from fastapi.testclient import TestClient from xorq.config import default_backend -from tallyman_core import data_dir, ensure_project +from tallyman_core import data_dir, ensure_project, set_active_project from tallyman_core.errors import list_errors from tallyman_core.manifest import read_manifest -from tallyman_core.paths import compute_cache_dir, entry_dir, entry_schema_path +from tallyman_core.paths import entry_dir, entry_schema_path from tallyman_xorq.build import build_and_persist +from tallyman_xorq.ordered_copy import ORDERED_COPY_ROW_GROUP_ROWS from tallyman_xorq.result_cache import ( UNFAITHFUL_HEAL_HOOKS, baked_snapshot_path, @@ -49,12 +49,15 @@ snapshot_file_digest, verify_result_faithful, ) +from tallyman_xorq.source_import import update_and_depend from tests.big_parquet import write_big_parquet PREFIX = "arrow-sha256:" SNAPSHOT_ROW_GROUP = 1_048_576 # ADR-009 D3 -ORDERED_COPY_ROW_GROUP = 122_880 # ADR-009 D3, io._CSV_PARQUET_WRITE +SOURCE_ROW_GROUP = 122_880 # ADR-009 D3, ordered_copy.ORDERED_COPY_ROW_GROUP_ROWS BIG_ROWS = 1_500_000 # tests/big_parquet.py default +ORDERS_SRC = "orders_src" # the shoe-orders source alias the conftest fixture imports +BIG_SRC = "big_src" # the 1.5M-row fixture, imported per project # --------------------------------------------------------------------------- @@ -92,8 +95,10 @@ def _evict(project: str, content_hash: str) -> Path: return snap -def _install(project: str, big_source: Path, name: str = "big.parquet") -> None: - shutil.copy(big_source, data_dir(project) / name) +def _import_big(project: str, big_source: Path) -> str: + """Import the 1.5M-row fixture as a source alias (ADR-011 D1); returns the alias name a recipe reads.""" + update_and_depend(big_source, BIG_SRC, project=project) + return BIG_SRC # --------------------------------------------------------------------------- @@ -103,16 +108,16 @@ def _install(project: str, big_source: Path, name: str = "big.parquet") -> None: def _agg_code(project: str) -> str: # Aggregate: worthy, deterministic return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.group_by("region").aggregate(total=t.price.sum(), n=t.count()) """ -def _order_by_code(project: str) -> str: # a Sort over the 1.5M-row file: worthy, and more than one row group +def _order_by_code(project: str) -> str: # a Sort over the 1.5M-row source: worthy, and more than one row group return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("big.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({BIG_SRC!r}, project={project!r}) expr = t.order_by("id") """ @@ -120,8 +125,8 @@ def _order_by_code(project: str) -> str: # a Sort over the 1.5M-row file: worth def _float_agg_code(project: str, *, grouped: bool) -> str: aggregate = "t.group_by('g').aggregate" if grouped else "t.aggregate" return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("big.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({BIG_SRC!r}, project={project!r}) expr = {aggregate}(s=t.v.sum(), m=t.v.mean()) """ @@ -129,12 +134,12 @@ def _float_agg_code(project: str, *, grouped: bool) -> str: def _noisy_udf_code(project: str) -> str: """A scalar UDF that returns a different value on every call: the entry is worthy and not reproducible.""" return f""" -from tallyman_xorq.io import read_project_file +from tallyman_xorq.io import tracked_expr_from_alias from xorq.expr.udf import make_pandas_udf import xorq.vendor.ibis.expr.datatypes as dt from xorq.vendor.ibis import schema as ibis_schema -t = read_project_file("orders.parquet", project={project!r}) +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) def noisy(df): @@ -154,13 +159,13 @@ def _counting_udf_code(project: str, counter: Path) -> str: xorq pickles a UDF into the build, so a module-level counter would be a copy; a file is visible from outside. """ return f""" -from tallyman_xorq.io import read_project_file +from tallyman_xorq.io import tracked_expr_from_alias from xorq.expr.udf import make_pandas_udf import xorq.vendor.ibis.expr.datatypes as dt from xorq.vendor.ibis import schema as ibis_schema COUNTER = {str(counter)!r} -t = read_project_file("orders.parquet", project={project!r}) +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) def bump(df): @@ -177,10 +182,10 @@ def bump(df): def _timestamp_code(project: str) -> str: """An expression whose type is ``timestamp[s]``, which parquet stores as ``timestamp[ms]``.""" return f""" -from tallyman_xorq.io import read_project_file +from tallyman_xorq.io import tracked_expr_from_alias import xorq.vendor.ibis.expr.datatypes as dt -t = read_project_file("events.parquet", project={project!r}) +t = tracked_expr_from_alias("events_src", project={project!r}) expr = t.mutate(ts0=t.ts.cast(dt.Timestamp(scale=0))).order_by("id") """ @@ -200,6 +205,7 @@ class _Built: home: Path project: str content_hash: str + source_hash: str # the entry the import minted, whose snapshot the recipe reads @pytest.fixture(scope="module") @@ -210,9 +216,13 @@ def _big_built(tmp_path_factory, big_source): patch.setenv("TALLYMAN_HOME", str(home)) try: ensure_project("fmt") - _install("fmt", big_source) + # An import resolves the read in its generated recipe through the ACTIVE project + # (``source_import._recipe`` writes ``read_project_file(path)`` with no project), so a project built in a + # home of its own has to be marked active before anything can be imported into it. + set_active_project("fmt") + imported = update_and_depend(big_source, BIG_SRC, project="fmt") result = build_and_persist("fmt", _order_by_code("fmt")) - yield _Built(home, "fmt", result.content_hash) + yield _Built(home, "fmt", result.content_hash, imported["hash"]) finally: patch.undo() @@ -248,7 +258,7 @@ def test_float_aggregate_heals_to_the_digest_it_was_built_with(project, big_sour different file, an ``unfaithful_heal`` record and a stat-cache wipe each time. Both shapes are checked because the ungrouped accumulator and the grouped one merge differently. """ - _install(project, big_source) + _import_big(project, big_source) result = build_and_persist(project, _float_agg_code(project, grouped=grouped)) h = result.content_hash recorded = read_manifest(entry_dir(project, h)).result_digest @@ -284,7 +294,7 @@ def test_single_partition_backend_is_one_partition_with_a_pinned_batch_size(): assert _show(default_backend(), "datafusion.execution.target_partitions") != "1" -def test_materialize_runs_the_plan_on_the_single_partition_backend(project, orders_parquet, monkeypatch): +def test_materialize_runs_the_plan_on_the_single_partition_backend(project, orders_src, monkeypatch): """ADR-009 D1: the loaded build is rebound onto the single-partition connection, not left on its own backends. ``load_expr`` mints its own backend objects, so a loaded build ignores a single-partition connection it was never @@ -364,21 +374,25 @@ def test_snapshot_ends_in_row_order_numbered_from_zero_in_file_order(big_built): assert (table["id"].to_numpy() == np.arange(n)).all() -def test_ordered_copy_of_a_source_has_row_groups_of_122880_rows(big_built): - """ADR-009 D3 (the format version covers the ordered copies): polars writes them, in pinned 122,880-row groups. +def test_the_snapshot_of_a_source_has_row_groups_of_122880_rows(big_built): + """ADR-009 D3 (the format version covers a source version's snapshot too): pinned 122,880-row groups. - An ungrouped float total over a source reads that layout (#187), so it is as pinned as the snapshot's. + A source version is an entry and its snapshot is the ordered copy of ADR-008 D2, so it is written by the same + writer as any other snapshot but in the smaller groups the import pins. An ungrouped float total straight off a + source reads that layout (#187), so it is as fixed as the 1,048,576 rows of a computed snapshot. """ - copies = sorted((compute_cache_dir(big_built.project) / "ordered_sources").glob("*.parquet")) - assert copies, "reading a source should have written an ordered copy under compute_cache/ordered_sources/" - for copy in copies: - sizes = _row_group_sizes(copy) - assert sizes[:-1] == [ORDERED_COPY_ROW_GROUP] * (len(sizes) - 1), (copy.name, len(sizes), sizes[:3]) - assert 0 < sizes[-1] <= ORDERED_COPY_ROW_GROUP - assert pq.read_schema(copy).names[-1] == "__row_order" + assert ORDERED_COPY_ROW_GROUP_ROWS == SOURCE_ROW_GROUP + snapshot = _materialize_module().snapshot_path(big_built.project, big_built.source_hash) + assert snapshot.exists(), "the import should have written the source entry's snapshot" + + sizes = _row_group_sizes(snapshot) + assert sizes[:-1] == [SOURCE_ROW_GROUP] * (len(sizes) - 1), (len(sizes), sizes[:3]) + assert 0 < sizes[-1] <= SOURCE_ROW_GROUP + assert sum(sizes) == BIG_ROWS + assert pq.read_schema(snapshot).names[-1] == "__row_order" -def test_manifest_records_the_snapshot_format_version(project, orders_parquet): +def test_manifest_records_the_snapshot_format_version(project, orders_src): """ADR-009 D3: row-group size and ``batch_size`` are contract; the manifest records a version for both.""" materialize_module = _materialize_module() result = build_and_persist(project, _agg_code(project)) @@ -386,7 +400,7 @@ def test_manifest_records_the_snapshot_format_version(project, orders_parquet): assert manifest.snapshot_format == materialize_module.SNAPSHOT_FORMAT_VERSION -def test_materialize_returns_the_content_digest_of_the_file_it_wrote(project, orders_parquet): +def test_materialize_returns_the_content_digest_of_the_file_it_wrote(project, orders_src): """ADR-009 D2 and ADR-007 D4 (one writer): the digest is computed by one function, from the file read back.""" materialize_module = _materialize_module() h = build_and_persist(project, _agg_code(project)).content_hash @@ -411,6 +425,7 @@ def test_recorded_schema_is_read_from_the_written_file(project): The expression's type is ``timestamp[s]``; parquet has no seconds unit, so the file holds ``timestamp[ms]``. """ + events = data_dir(project) / "events.parquet" pq.write_table( pa.table( { @@ -419,8 +434,9 @@ def test_recorded_schema_is_read_from_the_written_file(project): "v": np.random.default_rng(1).random(100), } ), - data_dir(project) / "events.parquet", + events, ) + update_and_depend(events, "events_src", project=project) h = build_and_persist(project, _timestamp_code(project)).content_hash recorded = {f["name"]: f["type"] for f in json.loads(entry_schema_path(project, h).read_text())["fields"]} @@ -442,7 +458,7 @@ def test_recorded_schema_is_read_from_the_written_file(project): ], ids=["cheap computed column", "aggregate", "sort", "window function"], ) -def test_every_recorded_schema_ends_in_row_order(project, orders_parquet, body, expected): +def test_every_recorded_schema_ends_in_row_order(project, orders_src, body, expected): """ADR-009 D3 and ADR-008 D2: cheap entries carry ``__row_order`` last, worthy ones get it from the writer. A worthy entry that keeps its parent's rows inherits ``__row_order`` mid-table and the writer replaces it with a @@ -450,8 +466,8 @@ def test_every_recorded_schema_ends_in_row_order(project, orders_parquet, body, """ code = f""" import xorq.vendor.ibis as ibis -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) {body} """ h = build_and_persist(project, code).content_hash @@ -464,7 +480,7 @@ def test_every_recorded_schema_ends_in_row_order(project, orders_parquet, body, # --------------------------------------------------------------------------- -def test_manifest_records_the_engine_versions(project, orders_parquet): +def test_manifest_records_the_engine_versions(project, orders_src): """ADR-009 D4 (a mismatch record names its likely cause): xorq, xorq-datafusion and pyarrow at build.""" result = build_and_persist(project, _agg_code(project)) recorded = read_manifest(entry_dir(project, result.content_hash)).engine_versions @@ -475,7 +491,7 @@ def test_manifest_records_the_engine_versions(project, orders_parquet): @pytest.mark.parametrize("engine_changed", [True, False], ids=["engine version changed", "engine versions match"]) def test_an_unfaithful_heal_names_an_engine_change_instead_of_blaming_the_recipe( - project, orders_parquet, engine_changed + project, orders_src, engine_changed ): """ADR-009 D4: with an upgrade in play the record says so, and the loud response of ADR-006 D7 still happens. @@ -519,7 +535,7 @@ def hook(hook_project: str, hook_hash: str) -> None: # --------------------------------------------------------------------------- -def test_create_records_a_non_reproducible_recipe_and_names_the_column(project, orders_parquet): +def test_create_records_a_non_reproducible_recipe_and_names_the_column(project, orders_src): """ADR-009 D6 (create runs the query twice and compares): the build succeeds, the entry says it is not reproducible. A recipe that calls ``sample()`` is legitimate, so the build still succeeds. The columns whose per-column digests @@ -537,7 +553,7 @@ def test_create_records_a_non_reproducible_recipe_and_names_the_column(project, assert snap is not None and snap.exists() -def test_create_records_a_deterministic_recipe_as_reproducible(project, orders_parquet): +def test_create_records_a_deterministic_recipe_as_reproducible(project, orders_src): """ADR-009 D6: a recipe that runs the same both times is recorded as reproducible, with no offending columns.""" result = build_and_persist(project, _agg_code(project)) assert result.reproducible is True @@ -547,11 +563,11 @@ def test_create_records_a_deterministic_recipe_as_reproducible(project, orders_p assert not manifest.nonreproducible_columns -def test_a_cheap_entry_is_not_checked_for_reproducibility(project, orders_parquet): +def test_a_cheap_entry_is_not_checked_for_reproducibility(project, orders_src): """ADR-009 D6: a cheap entry is not run twice here; it records no digest either (ADR-006 D9, no cheap digests).""" code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({ORDERS_SRC!r}, project={project!r}) expr = t.mutate(total=t.price * t.qty) """ result = build_and_persist(project, code) @@ -561,7 +577,7 @@ def test_a_cheap_entry_is_not_checked_for_reproducibility(project, orders_parque assert manifest.result_digest is None -def test_create_runs_the_query_twice_and_a_heal_runs_it_once(project, orders_parquet, tmp_path): +def test_create_runs_the_query_twice_and_a_heal_runs_it_once(project, orders_src, tmp_path): """ADR-009 D6: only a create runs the query twice, since a create has nothing recorded to compare against. A heal is compared against the recorded digest, so once is enough. The UDF appends a byte to a file on every call. @@ -581,7 +597,7 @@ def test_create_runs_the_query_twice_and_a_heal_runs_it_once(project, orders_par assert at_create == 2 * at_heal, f"create called the UDF {at_create} times, a heal {at_heal}" -def test_materialize_runs_once_and_runs_twice_only_when_asked_to_check(project, orders_parquet, tmp_path): +def test_materialize_runs_once_and_runs_twice_only_when_asked_to_check(project, orders_src, tmp_path): """ADR-009 D6 and ADR-007 D4: the checking run is an option of the one writer, not a second code path.""" materialize_module = _materialize_module() counter = tmp_path / "calls.bin" @@ -600,7 +616,7 @@ def test_materialize_runs_once_and_runs_twice_only_when_asked_to_check(project, assert checked.differing_columns == [] -def test_materialize_names_the_columns_that_differ_between_the_two_runs(project, orders_parquet): +def test_materialize_names_the_columns_that_differ_between_the_two_runs(project, orders_src): """ADR-009 D6: ``differing_columns`` is what the build result reports to the author.""" materialize_module = _materialize_module() h = build_and_persist(project, _noisy_udf_code(project)).content_hash @@ -614,7 +630,7 @@ def test_materialize_names_the_columns_that_differ_between_the_two_runs(project, # --------------------------------------------------------------------------- -def test_verify_result_faithful_follows_the_content_not_the_bytes(project, orders_parquet): +def test_verify_result_faithful_follows_the_content_not_the_bytes(project, orders_src): """ADR-009 D2 in the verify path: the same rows in another format are faithful, a changed value is not. ``verify_result_faithful`` runs on every heal and in the corpus sweep. Under a byte hash, rewriting the snapshot @@ -638,7 +654,7 @@ def test_verify_result_faithful_follows_the_content_not_the_bytes(project, order assert verify_result_faithful(project, h) is False -def test_verify_result_faithful_is_false_after_the_recorded_digest_is_tampered(project, orders_parquet): +def test_verify_result_faithful_is_false_after_the_recorded_digest_is_tampered(project, orders_src): """ADR-009 D2: the recorded value is an ``arrow-sha256:`` digest, and a wrong one fails verification.""" h = build_and_persist(project, _agg_code(project)).content_hash assert read_manifest(entry_dir(project, h)).result_digest.startswith(PREFIX) @@ -653,7 +669,7 @@ def test_verify_result_faithful_is_false_after_the_recorded_digest_is_tampered(p # --------------------------------------------------------------------------- -def test_a_non_reproducible_entrys_snapshot_survives_an_explicit_delete(fresh_companion_app, project, orders_parquet): +def test_a_non_reproducible_entrys_snapshot_survives_an_explicit_delete(fresh_companion_app, project, orders_src): """ADR-009 D6: the Cache page's delete skips a file that cannot be recreated, and says why. A snapshot whose recipe is not reproducible would come back as different rows, and everything built on the @@ -674,7 +690,7 @@ def test_a_non_reproducible_entrys_snapshot_survives_an_explicit_delete(fresh_co def test_an_entry_whose_heal_was_unfaithful_is_pinned_against_an_explicit_delete( - fresh_companion_app, project, orders_parquet + fresh_companion_app, project, orders_src ): """ADR-009 D6 and ADR-006 D12 (unfaithful entries are pinned and badged): the same pin, reached after the fact.""" h = build_and_persist(project, _agg_code(project)).content_hash diff --git a/tests/test_source_cache.py b/tests/test_source_cache.py index 1145187a..6b85edee 100644 --- a/tests/test_source_cache.py +++ b/tests/test_source_cache.py @@ -8,8 +8,8 @@ from tallyman_xorq.build import BuildError, build_and_persist from tallyman_xorq.result_cache import cache_worthy -# The rewrite step before a build (ADR-007 D1): no xorq cache node is ever created, raw CSV reads are banned (every -# CSV goes through tallyman_read_csv), and in-memory reads are rejected. +# The rewrite step before a build (ADR-007 D1): no xorq cache node is ever created, raw CSV reads are banned (a CSV +# enters the catalog by an import, ADR-011 D2), and in-memory reads are rejected. @pytest.fixture @@ -43,8 +43,8 @@ def _csv_aggregate(project: str) -> str: def _parquet_projection(project: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ @@ -54,12 +54,12 @@ def _build_yaml(project: str, content_hash: str) -> str: def test_deferred_read_csv_is_banned(project, sales_csv, monkeypatch): - # xo.deferred_read_csv is banned — tallyman_read_csv is the only CSV ingest path. + # xo.deferred_read_csv is banned — a CSV enters by catalog_import_source and a recipe reads the alias. monkeypatch.setenv("TALLYMAN_PROJECT", project) with pytest.raises(BuildError) as exc: build_and_persist(project, _csv_projection(project)) assert "deferred_read_csv" in str(exc.value) - assert "tallyman_read_csv" in str(exc.value) + assert "catalog_import_source" in str(exc.value) def test_deferred_read_csv_banned_for_aggregate(project, sales_csv, monkeypatch): @@ -68,10 +68,10 @@ def test_deferred_read_csv_banned_for_aggregate(project, sales_csv, monkeypatch) with pytest.raises(BuildError) as exc: build_and_persist(project, _csv_aggregate(project)) assert "deferred_read_csv" in str(exc.value) - assert "tallyman_read_csv" in str(exc.value) + assert "catalog_import_source" in str(exc.value) -def test_parquet_read_has_no_source_cache(project, orders_parquet, monkeypatch): +def test_parquet_read_has_no_source_cache(project, orders_src, monkeypatch): monkeypatch.setenv("TALLYMAN_PROJECT", project) res = build_and_persist(project, _parquet_projection(project)) # No build carries a cache node (ADR-007 D1), and a projection over a parquet source is cheap: nothing is diff --git a/tests/test_source_identity.py b/tests/test_source_identity.py index ce746dcc..cff26ce0 100644 --- a/tests/test_source_identity.py +++ b/tests/test_source_identity.py @@ -1,17 +1,13 @@ -"""Source-content identity (``tallyman_xorq.source_identity``) under the cas default. - -The cache lab (``tests/test_cache_lab.py``) is the heavyweight, on-demand home for -identity scenarios, but it is marked ``cache_lab`` and never runs in CI. These are -the fast-suite pins for the parts of #86 that CI must guard: that ``cas`` is the -default, that a rebuild over an edited source forks the content hash (the -``test_append_invalidates`` contract, lifted into the fast suite), and that the -``.cas`` GC reclaims clones no live entry references. - -Scope note (#86): flipping the default to cas makes *build* identity content-aware -(a rebuild forks the hash). It does NOT make cold *reconstruction* content-faithful -— a cheap entry's recipe is re-run on read and re-digests the live source — so there -is deliberately no "cold read after edit returns the original bytes" assertion here. -That guarantee needs digest-pinned reconstruction, tracked separately. +"""The clone store (``tallyman_xorq.source_identity``) — the frozen copy of every imported byte. + +An import digests the file it is given, clones it to ``data/.cas/`` and verifies the +copy before publishing it. The clone is what makes a source version durable once the outside file is +edited or deleted, and what re-creates its snapshot. These are the fast-suite pins for the parts CI +must guard: that the clone is a frozen snapshot rather than a hardlink, that the ``.cas`` GC reclaims +only what no live entry references, and that there is exactly one identity mode (ADR-011 D8). + +The cache lab (``tests/test_cache_lab.py``) is the heavyweight, on-demand home for identity scenarios, +but it is marked ``cache_lab`` and never runs in CI. """ from __future__ import annotations @@ -19,7 +15,6 @@ import pandas as pd from tallyman_core import data_dir -from tallyman_xorq import build_and_persist from tallyman_xorq import source_identity as si @@ -27,40 +22,6 @@ def _write_parquet(path, n_rows: int) -> None: pd.DataFrame({"a": list(range(n_rows))}).to_parquet(path) -def _read_code(project: str, rel: str) -> str: - return f"from tallyman_xorq.io import read_project_file\nexpr = read_project_file({rel!r}, project={project!r})\n" - - -def test_cas_is_default(monkeypatch): - """With TALLYMAN_SOURCE_IDENTITY unset, identity defaults to content-addressed.""" - monkeypatch.delenv("TALLYMAN_SOURCE_IDENTITY", raising=False) - assert si.mode() == "cas" - - -def test_cas_rebuild_over_edited_source_forks_hash(project, monkeypatch): - """Editing a source in place and rebuilding must fork the content hash. - - Under the old ``off`` default the build hash keys on the path string only - (xorq's SnapshotStrategy), so an in-place edit dedups to the stale entry — - this is the documented ``test_append_invalidates`` failure. Under ``cas`` the - read goes through ``data/.cas/``, so the path embeds the content and - the hash forks. - """ - monkeypatch.delenv("TALLYMAN_SOURCE_IDENTITY", raising=False) - monkeypatch.setenv("TALLYMAN_SOURCE_REHASH", "1") # never serve a stale digest - assert si.mode() == "cas" - - src = data_dir(project) / "src.parquet" - _write_parquet(src, 3) - code = _read_code(project, "src.parquet") - h1 = build_and_persist(project, code).content_hash - - _write_parquet(src, 5) # user edits the source in place - h2 = build_and_persist(project, code).content_hash - - assert h1 != h2, "cas: an in-place source edit must fork the content hash on rebuild" - - def test_gc_cas_removes_unreferenced_digests(project): """gc_cas deletes only ``.cas`` clones no live entry references.""" cas_dir = data_dir(project) / ".cas" @@ -87,10 +48,33 @@ def test_cas_clone_is_a_snapshot_not_a_hardlink(project): """ src = data_dir(project) / "frozen.parquet" _write_parquet(src, 3) - digest = si.digest_for(project, src) + digest = si._digest_file(src) clone = si.ensure_cas_path(project, src, digest) _write_parquet(src, 5) # edit the source in place, after cloning assert pd.read_parquet(clone).shape[0] == 3 assert clone.stat().st_ino != src.stat().st_ino # distinct inode — not a hardlink + + +def test_source_identity_has_one_mode(): + """ADR-011 D8: import always digests and always clones. + + There is no configuration under which a raw input is unversioned, so ``TALLYMAN_SOURCE_IDENTITY``, + the ``off`` and ``salt`` modes and the salted hash they needed are gone — checked against the source + tree, because a mode nothing reads is still a mode someone can set and be misled by. + """ + import re + from pathlib import Path + + for name in ("mode", "salted_hash", "digest_for", "begin_collect", "note_source", "end_collect"): + assert not hasattr(si, name), f"source_identity.{name} is deleted by ADR-011 D6/D8" + + src = Path(__file__).resolve().parent.parent / "src" + pattern = re.compile(r"TALLYMAN_SOURCE_IDENTITY|TALLYMAN_SOURCE_REHASH|source_digests\.json") + offenders = [ + f"{p.relative_to(src)}:{p.read_text().count(chr(10), 0, m.start()) + 1}" + for p in sorted(src.rglob("*.py")) + for m in pattern.finditer(p.read_text()) + ] + assert not offenders, f"the source-identity modes and their digest memo are gone: {offenders}" diff --git a/tests/test_source_import.py b/tests/test_source_import.py new file mode 100644 index 00000000..de912b78 --- /dev/null +++ b/tests/test_source_import.py @@ -0,0 +1,1546 @@ +"""Importing a file is how data enters the catalog (ADR-011). + +A raw input is an alias whose versions are ordinary entries. ``update_and_depend`` copies the bytes into the arena +(the ``.cas`` clone store), writes ONE parquet snapshot carrying ``__row_order`` named by the source entry's content +hash, and points a source alias at that entry. A recipe then reads ``tracked_expr_from_alias("orders")`` like any +other parent, and the file the bytes came from is provenance: never read again. + +These pin stage 1 of the ADR — the import path and the refusals (D1, D2, D3, D5, D9, D10, D12). The staleness source +axis (D6) and the source-identity modes (D8) are stage 2. +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from tallyman_core import data_dir, entry_dir, get_alias, history_for, read_manifest +from tallyman_xorq.materialize import snapshot_path + +ROW_ORDER = "__row_order" + + +# --------------------------------------------------------------------------- +# helpers +# --------------------------------------------------------------------------- + + +def _outside(tmp_path: Path) -> Path: + """A directory outside the project: an import takes any path (data/ is not special).""" + d = tmp_path / "outside" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _write_parquet(path: Path, n_rows: int) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + frame = pd.DataFrame( + {"region": [f"r{i % 3}" for i in range(n_rows)], "price": [float(i) for i in range(n_rows)]} + ) + frame.to_parquet(path) + return path + + +def _write_csv(path: Path, rows: list[tuple[str, int]], sep: str = ",") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + body = sep.join(("region", "n")) + "\n" + "\n".join(sep.join((r, str(n))) for r, n in rows) + "\n" + path.write_text(body) + return path + + +def _snapshot_frame(project: str, content_hash: str): + import polars as pl + + return pl.read_parquet(str(snapshot_path(project, content_hash))) + + +def _child_code(alias: str, project: str) -> str: + return ( + "from tallyman_xorq.io import tracked_expr_from_alias\n" + f"t = tracked_expr_from_alias({alias!r}, project={project!r})\n" + "expr = t.group_by('region').aggregate(total=t.price.sum())\n" + ) + + +# --------------------------------------------------------------------------- +# D3 — the full case table of update_and_depend +# --------------------------------------------------------------------------- + + +def test_absent_alias_mints_v1(project: str, tmp_path: Path, monkeypatch): + """Row 1: the alias does not exist, so the import mints v1 and points the alias at it.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + out = source_import.update_and_depend(str(src), "orders") + + assert out["alias"] == "orders" + assert out["version"] == 1 + assert out["created"] is True + assert get_alias(project, "orders") == out["hash"] + assert history_for(project, "orders") == [out["hash"]] + assert entry_dir(project, out["hash"]).is_dir() + + +def test_same_bytes_is_a_noop(project: str, tmp_path: Path, monkeypatch): + """Row 3: no pin and the bytes equal the head, so the head does not move and no entry is minted.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + first = source_import.update_and_depend(str(src), "orders") + + again = source_import.update_and_depend(str(src), "orders") + + assert again["hash"] == first["hash"] + assert again["version"] == 1 + assert again["created"] is False + assert history_for(project, "orders") == [first["hash"]] + + +def test_changed_bytes_mint_the_next_version(project: str, tmp_path: Path, monkeypatch): + """Row 2: no pin and the bytes differ from the head, so the next version is minted and the head advances.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + v1 = source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + + v2 = source_import.update_and_depend(str(src), "orders") + + assert v2["version"] == 2 + assert v2["created"] is True + assert v2["hash"] != v1["hash"] + assert history_for(project, "orders") == [v1["hash"], v2["hash"]] + assert get_alias(project, "orders") == v2["hash"] + + +def test_pinned_version_with_matching_digest_is_a_noop(project: str, tmp_path: Path, monkeypatch): + """Row 4: the pin names a version that exists and the bytes are that version's, so nothing happens.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + v1 = source_import.update_and_depend(str(src), "orders") + + again = source_import.update_and_depend(str(src), "orders", pinned_version=1) + + assert again["hash"] == v1["hash"] + assert again["version"] == 1 + assert again["created"] is False + + +def test_pinned_version_with_a_different_digest_errors(project: str, tmp_path: Path, monkeypatch): + """Row 5: the pin claims a version the file's bytes are not.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(src), "orders", pinned_version=1) + + assert "orders-v1" in str(exc.value) + + +def test_pinned_next_version_mints_it(project: str, tmp_path: Path, monkeypatch): + """Row 6: the pin names head+1 and the bytes are new, so that version is minted.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + + v2 = source_import.update_and_depend(str(src), "orders", pinned_version=2) + + assert v2["version"] == 2 + assert v2["created"] is True + assert get_alias(project, "orders") == v2["hash"] + + +def test_pinned_version_beyond_next_errors(project: str, tmp_path: Path, monkeypatch): + """Row 7: version numbers are consecutive, so a pin past head+1 is refused rather than leaving a gap.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(src), "orders", pinned_version=5) + + assert "skip" in str(exc.value).lower() + + +def test_pinned_older_version_returns_it_without_moving_the_head(project: str, tmp_path: Path, monkeypatch): + """Row 8: the caller explicitly claims an older version and its bytes match, so that version is returned.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + v1 = source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + v2 = source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 10) # back to v1's bytes + + out = source_import.update_and_depend(str(src), "orders", pinned_version=1) + + assert out["hash"] == v1["hash"] + assert out["version"] == 1 + assert out["created"] is False + assert get_alias(project, "orders") == v2["hash"], "an older pin must not move the head" + + +def test_bytes_matching_an_older_version_error_names_reset(project: str, tmp_path: Path, monkeypatch): + """Row 9 / D11: history is append-only and monotonic, so restoring v1's bytes cannot mint a v3.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 10) # back to v1's bytes + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(src), "orders") + + message = str(exc.value) + assert "orders-v1" in message + assert "reset" in message.lower() + + +def test_the_append_only_refusal_does_not_advise_a_second_alias(project: str, tmp_path: Path, monkeypatch): + """D11's refusal offers ways back that work: a reset, or reading the old version pinned. + + It used to offer "import these bytes under a different alias", which is refused too, because bytes an alias + already holds cannot be imported under another (ADR-011 D1). + """ + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 10) # back to v1's bytes + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(src), "orders") + + message = str(exc.value) + assert "different alias" not in message, message + assert "pinned_expr_from_alias('orders-v1')" in message, message + with pytest.raises(source_import.SourceImportError, match="orders-v1"): + source_import.update_and_depend(str(src), "orders_again") + + +def test_absent_alias_with_a_skipped_pin_errors(project: str, tmp_path: Path, monkeypatch): + """A brand-new source alias starts at v1; a pin of v2 would leave version 1 undefined forever.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(src), "orders", pinned_version=2) + + assert "skip" in str(exc.value).lower() + assert get_alias(project, "orders") is None + + +def test_an_import_names_its_own_project(project: str, tmp_path: Path, monkeypatch): + """``project=`` is an override, so an import must not depend on which project happens to be active. + + The generated recipe's ``read_project_file`` is legal because a contextvar says which source entry + is being minted, and the read has to resolve to the same project that contextvar names. Resolving + the ambient active project instead makes the importer's own recipe hit the D2 refusal written for + authored recipes, and the import fails with an error about a file tallyman does not own. + """ + from tallyman_core import ensure_project, set_active_project + from tallyman_xorq import source_import + + other = "other_project" + ensure_project(other) + set_active_project(project) # the active project is NOT the one being imported into + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 6) + + out = source_import.update_and_depend(str(src), "orders", project=other) + + assert out["version"] == 1 + assert get_alias(other, "orders") == out["hash"] + assert snapshot_path(other, out["hash"]).is_file() + assert not snapshot_path(project, out["hash"]).exists() + + +def test_import_refuses_a_directory(project: str, tmp_path: Path, monkeypatch): + """Multi-file datasets are open question 2 of the ADR; until one is designed a directory is refused clearly.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + d = _outside(tmp_path) / "parts" + d.mkdir(parents=True, exist_ok=True) + _write_parquet(d / "part-0000.parquet", 5) + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(d), "orders") + + assert "director" in str(exc.value).lower() + + +def test_import_refuses_a_missing_path(project: str, tmp_path: Path, monkeypatch): + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + missing = _outside(tmp_path) / "nope.parquet" + + with pytest.raises(source_import.SourceImportError) as exc: + source_import.update_and_depend(str(missing), "orders") + + assert "nope.parquet" in str(exc.value) + + +# --------------------------------------------------------------------------- +# D1 — a source entry is worthy and its snapshot IS the ordered copy +# --------------------------------------------------------------------------- + + +def test_the_snapshot_is_the_ordered_copy(project: str, tmp_path: Path, monkeypatch): + """One parquet file, in ``result_cache/`` under the entry's content hash, carrying ``__row_order`` 0..N-1. + + ``compute_cache/ordered_sources/`` and the copy key go away for an import: the entry hash names the file. + """ + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 7) + + out = source_import.update_and_depend(str(src), "orders") + + snap = snapshot_path(project, out["hash"]) + assert snap.is_file() + frame = _snapshot_frame(project, out["hash"]) + assert frame.columns[-1] == ROW_ORDER + assert frame[ROW_ORDER].to_list() == list(range(7)) + + from tallyman_core.paths import compute_cache_dir + from tallyman_xorq.materialize import RESULT_CACHE_DIRNAME + + subdirs = [d.name for d in compute_cache_dir(project).iterdir()] + assert subdirs == [RESULT_CACHE_DIRNAME], f"an import writes no separate ordered copy: {subdirs}" + assert read_manifest(entry_dir(project, out["hash"])).cache_worthy is True + + +def test_identical_bytes_under_a_second_alias_is_an_error(project: str, tmp_path: Path, monkeypatch): + """One set of bytes is one source version under one alias; importing it again under another name is refused. + + The likeliest way to get here is not knowing the bytes are already in the project, so the error names the alias + and version that hold them and the way to give them a second name. The first alias's entry is untouched: a second + import used to rewrite its manifest and recipe, and a failure part-way deleted the entry outright. + """ + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + a = source_import.update_and_depend(str(src), "orders") + entry = entry_dir(project, a["hash"]) + manifest_before = (entry / "manifest.json").read_bytes() + recipe_before = (entry / "expr.py").read_bytes() + copy = _write_parquet(_outside(tmp_path) / "copy_of_orders.parquet", 10) + assert copy.read_bytes() == src.read_bytes() + + with pytest.raises(source_import.SourceImportError) as info: + source_import.update_and_depend(str(copy), "orders_again") + + message = str(info.value) + assert "orders-v1" in message + assert "tracked_expr_from_alias('orders')" in message + assert "catalog_create" in message + assert get_alias(project, "orders_again") is None + assert (entry / "manifest.json").read_bytes() == manifest_before + assert (entry / "expr.py").read_bytes() == recipe_before + + +def test_bytes_of_an_older_version_of_another_alias_are_refused(project: str, tmp_path: Path, monkeypatch): + """The bytes need not be another alias's head: any version of any other alias holds them.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + first_bytes = src.read_bytes() + source_import.update_and_depend(str(src), "orders") + _write_parquet(src, 20) + source_import.update_and_depend(str(src), "orders") + old = _outside(tmp_path) / "old_orders.parquet" + old.write_bytes(first_bytes) + + with pytest.raises(source_import.SourceImportError, match="orders-v1"): + source_import.update_and_depend(str(old), "archive") + + assert get_alias(project, "archive") is None + + +def test_a_repair_import_writes_the_snapshot_and_nothing_else(project: str, tmp_path: Path, monkeypatch): + """Re-importing a version whose snapshot is gone restores the snapshot and leaves the entry alone. + + The entry's recipe, build and manifest are the record of the import, and a repair is not an import: it goes + through the heal ``ensure_materialized`` uses. So nothing is rebuilt (a failing ``build_expr`` does not matter), + and repairing from another path does not move the recorded provenance path or its time. + """ + import xorq.ibis_yaml.compiler as compiler + + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + a = source_import.update_and_depend(str(src), "orders") + entry = entry_dir(project, a["hash"]) + record = {name: (entry / name).read_bytes() for name in ("manifest.json", "expr.py")} + snapshot = snapshot_path(project, a["hash"]) + before = snapshot.read_bytes() + snapshot.unlink() + moved = _outside(tmp_path) / "moved" / "orders.parquet" + moved.parent.mkdir() + moved.write_bytes(src.read_bytes()) + + def rebuild(*args, **kwargs): + raise RuntimeError("a repair rebuilt the entry") + + monkeypatch.setattr(compiler, "build_expr", rebuild) + out = source_import.update_and_depend(str(moved), "orders") + + assert out["created"] is False + assert snapshot.read_bytes() == before + assert {name: (entry / name).read_bytes() for name in record} == record + + +def test_a_repair_import_is_verified_like_any_heal(project: str, tmp_path: Path, monkeypatch): + """A repair that writes other rows under the recorded hash is an unfaithful heal, and is recorded as one. + + The recorded digest and row count stay the build-time ones. A repair used to take whatever it wrote as the new + truth, so a child built on the old rows read the new ones with nothing to say they had changed. + """ + from tallyman_core.errors import list_errors + from tallyman_xorq import source_import + from tallyman_xorq.digest import content_digest + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + a = source_import.update_and_depend(str(src), "orders") + recorded = read_manifest(entry_dir(project, a["hash"])) + snapshot_path(project, a["hash"]).unlink() + write = source_import._write_snapshot + + def one_row_short(clone, reader, dest): # a reader that now parses the same bytes into other rows + write(clone, reader, dest) + table = pq.read_table(dest) + pq.write_table(table.slice(0, table.num_rows - 1), dest) + return content_digest(dest) + + monkeypatch.setattr(source_import, "_write_snapshot", one_row_short) + source_import.update_and_depend(str(src), "orders") + + after = read_manifest(entry_dir(project, a["hash"])) + assert (after.result_digest, after.row_count) == (recorded.result_digest, recorded.row_count) + codes = [r.get("code") for r in list_errors(project, limit=1000) if r.get("hash") == a["hash"]] + assert "unfaithful_heal" in codes, codes + + +def test_an_entry_whose_manifest_is_gone_is_written_again_by_a_re_import(project: str, tmp_path: Path, monkeypatch): + """An entry directory with no readable manifest, which a crash part-way through an import leaves, is not an entry. + + A re-import of its bytes writes it again rather than failing to read it. + """ + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + a = source_import.update_and_depend(str(src), "orders") + (entry_dir(project, a["hash"]) / "manifest.json").unlink() + + out = source_import.update_and_depend(str(src), "orders") + + assert (out["hash"], out["created"]) == (a["hash"], False) + assert read_manifest(entry_dir(project, a["hash"])).provenance.alias == "orders" + + +def test_a_source_version_keeps_its_raw_bytes_in_the_clone_store(project: str, tmp_path: Path, monkeypatch): + """Open question 1, answered for stage 1: the imported bytes stay in ``.cas`` beside the snapshot. + + ADR-005's CSV suggestion-and-retry contract needs to re-read the file as imported, and the outside path may be + gone by then. + """ + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + out = source_import.update_and_depend(str(src), "orders") + + clone = data_dir(project) / ".cas" / f"{out['digest']}.parquet" + assert clone.is_file() + assert clone.read_bytes() == src.read_bytes() + + +def test_the_provenance_path_is_recorded_and_never_read_again(project: str, tmp_path: Path, monkeypatch): + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + out = source_import.update_and_depend(str(src), "orders") + + provenance = read_manifest(entry_dir(project, out["hash"])).provenance + assert provenance is not None + assert provenance.path == str(src) + assert provenance.alias == "orders" + assert provenance.version == 1 + assert provenance.digest == out["digest"] + + +def test_editing_the_outside_file_after_import_changes_nothing(project: str, tmp_path: Path, monkeypatch): + """The build reads the arena, so the file the bytes came from can change under it with no effect.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq import build_and_persist + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + source_import.update_and_depend(str(src), "orders") + code = _child_code("orders", project) + before = build_and_persist(project, code) + + _write_parquet(src, 999) + after = build_and_persist(project, code) + + assert after.content_hash == before.content_hash + assert after.row_count == before.row_count + + +def test_deleting_the_outside_file_after_import_changes_nothing(project: str, tmp_path: Path, monkeypatch): + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.result_cache import cached_result_expr + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + src.unlink() + + rows = cached_result_expr(project, out["hash"]).to_pyarrow().num_rows + assert rows == 10 + + +def test_a_child_of_a_source_alias_goes_stale_and_recalc_clears_it(project: str, tmp_path: Path, monkeypatch): + """An import is a catalog operation (D7): it advances an alias, so followers go stale like any other revise.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + monkeypatch.setenv("TALLYMAN_AUTO_RECALC", "0") + from tallyman_mcp.server import catalog_create + from tallyman_xorq.recalc import recalc + from tallyman_xorq.staleness import scan + + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + source_import.update_and_depend(str(src), "orders") + child = catalog_create("totals", _child_code("orders", project)) + assert "error" not in child, child + + _write_parquet(src, 20) + source_import.update_and_depend(str(src), "orders") + + verdicts = scan(project) + child_hash = get_alias(project, "totals") + assert verdicts[child_hash].stale is True + assert [r.axis for r in verdicts[child_hash].reasons] == ["alias"] + + recalc(project, [child_hash]) + assert scan(project)[get_alias(project, "totals")].stale is False + + +def test_a_source_entry_reports_no_source_axis(project: str, tmp_path: Path, monkeypatch): + """A source entry has no recorded source file, so the source axis is not applicable rather than unknown.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.staleness import entry_staleness + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + + verdict = entry_staleness(project, out["hash"]) + assert verdict.stale is False + assert verdict.unknown_axes == [] + + +# --------------------------------------------------------------------------- +# D1 — pyarrow writes the snapshot, in the pinned layout +# --------------------------------------------------------------------------- +# +# ``compute_cache/result_cache/`` holds one shape of parquet file. polars cannot write it — it has no option for the +# parquet format version and none for the page index (pola-rs/polars#12752, open against 1.44.2) — so polars stays +# where it is genuinely needed, parsing a CSV in file order under the schema DSL of ADR-005, and its Arrow data goes +# to the writer every computed snapshot already uses. + + +def _layout(project: str, content_hash: str) -> dict: + md = pq.ParquetFile(snapshot_path(project, content_hash)).metadata + return { + "format_version": md.format_version, + "created_by": md.created_by, + "row_group_rows": [md.row_group(i).num_rows for i in range(md.num_row_groups)], + "page_index": all( + md.row_group(i).column(c).has_offset_index and md.row_group(i).column(c).has_column_index + for i in range(md.num_row_groups) + for c in range(md.row_group(i).num_columns) + ), + } + + +def test_a_source_snapshot_is_written_by_pyarrow_in_the_pinned_layout(project: str, tmp_path: Path, monkeypatch): + """The parquet format version, the page index and the row-group size of every other file in ``result_cache/``.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.ordered_copy import ORDERED_COPY_ROW_GROUP_ROWS + + rows = ORDERED_COPY_ROW_GROUP_ROWS + 5_000 + src = _write_parquet(_outside(tmp_path) / "orders.parquet", rows) + + out = source_import.update_and_depend(str(src), "orders") + + layout = _layout(project, out["hash"]) + assert layout["format_version"] == "2.6", layout + assert layout["created_by"].startswith("parquet-cpp-arrow"), layout + assert layout["page_index"] is True, layout + assert layout["row_group_rows"] == [ORDERED_COPY_ROW_GROUP_ROWS, 5_000], layout + + +def test_a_csv_source_snapshot_is_written_by_pyarrow_too(project: str, tmp_path: Path, monkeypatch): + """polars parses the CSV; the bytes on disk are written by the same writer as a parquet source's.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_csv(_outside(tmp_path) / "orders.csv", [("east", 1), ("west", 2), ("east", 3)]) + + out = source_import.update_and_depend(str(src), "orders") + + layout = _layout(project, out["hash"]) + assert layout["format_version"] == "2.6", layout + assert layout["created_by"].startswith("parquet-cpp-arrow"), layout + assert layout["page_index"] is True, layout + frame = _snapshot_frame(project, out["hash"]) + assert frame["region"].to_list() == ["east", "west", "east"] + assert frame[ROW_ORDER].to_list() == [0, 1, 2] + + +def test_a_source_snapshot_has_the_layout_of_a_computed_one(project: str, tmp_path: Path, monkeypatch): + """Two files in one directory, written by one writer: the format version and the page index agree.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + monkeypatch.setenv("TALLYMAN_AUTO_RECALC", "0") + from tallyman_mcp.server import catalog_create + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 20) + out = source_import.update_and_depend(str(src), "orders") + assert "error" not in catalog_create("totals", _child_code("orders", project)) + + source = _layout(project, out["hash"]) + computed = _layout(project, get_alias(project, "totals")) + assert source["format_version"] == computed["format_version"] + assert source["created_by"] == computed["created_by"] + assert source["page_index"] == computed["page_index"] is True + + +def test_importing_a_parquet_needs_no_polars(project: str, tmp_path: Path, monkeypatch): + """A parquet source has its rows and its types in the file, so pyarrow reads it and nothing else is involved.""" + import polars as pl + + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + def boom(*a, **k): + raise AssertionError("a parquet import read the file with polars") + + monkeypatch.setattr(pl, "scan_parquet", boom) + monkeypatch.setattr(pl, "read_parquet", boom) + + out = source_import.update_and_depend(str(src), "orders") + + # Read back with pyarrow: polars is still patched out, and the point of the test is that it is not needed. + written = pq.read_table(snapshot_path(project, out["hash"])) + assert written[ROW_ORDER].to_pylist() == list(range(10)) + + +def test_a_parquet_source_keeps_the_types_the_file_has(project: str, tmp_path: Path, monkeypatch): + """polars rewrote a date as a timestamp, a time32 as a time64 and a map as a list of structs (#197).""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _outside(tmp_path) / "typed.parquet" + table = pa.table( + { + "d": pa.array([19_723, 19_724], type=pa.date32()), + "tm": pa.array([1_000, 2_000], type=pa.time32("ms")), + "m": pa.array([[("a", 1)], [("b", 2)]], type=pa.map_(pa.string(), pa.int64())), + "x": pa.array([1.5, 2.5], type=pa.float64()), + } + ) + pq.write_table(table, src) + + out = source_import.update_and_depend(str(src), "typed") + + written = pq.read_schema(snapshot_path(project, out["hash"])) + assert [(f.name, str(f.type)) for f in written] == [ + *[(f.name, str(f.type)) for f in pq.read_schema(src)], + (ROW_ORDER, "int64"), + ] + + +def test_a_csv_import_streams_into_the_pyarrow_writer(project: str, tmp_path: Path, monkeypatch): + """A big source is the point of the row-group size, so the CSV goes batch by batch into pyarrow's writer. + + polars never collects the frame — patching ``collect`` out proves the rows arrive through ``collect_batches``, + and the layout of the file proves they arrive at the writer this branch introduces rather than at polars' sink. + """ + import polars as pl + + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_csv(_outside(tmp_path) / "orders.csv", [(f"r{i % 3}", i) for i in range(500)]) + + def boom(*a, **k): + raise AssertionError("the CSV import collected the whole frame into memory") + + monkeypatch.setattr(pl.LazyFrame, "collect", boom) + + out = source_import.update_and_depend(str(src), "orders") + + # Read back with pyarrow: polars' own reader collects, and the test has just patched that out. + written = pq.read_table(snapshot_path(project, out["hash"])) + assert written[ROW_ORDER].to_pylist() == list(range(500)) + assert _layout(project, out["hash"])["created_by"].startswith("parquet-cpp-arrow") + + +# --------------------------------------------------------------------------- +# D1 / ADR-007 D13 — the snapshot is CACHE, re-created from the clone +# --------------------------------------------------------------------------- +# +# The clone under ``data/.cas`` holds the bytes as imported and the entry records the reader options, so +# ``ensure_materialized`` CAN make the snapshot again — which is ADR-007 D13's whole test for whether a file is +# cache. Only when the clone is gone too is the version unrecoverable. + + +def _clone_of(project: str, out: dict) -> Path: + return data_dir(project) / ".cas" / f"{out['digest']}{Path(out['path']).suffix}" + + +def test_ensure_materialized_recreates_a_source_snapshot_from_the_clone(project: str, tmp_path: Path, monkeypatch): + """A deleted source snapshot comes back from the clone, with the rows it had.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.errors import list_errors + from tallyman_xorq.materialize import ensure_materialized + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + before = snapshot_path(project, out["hash"]).read_bytes() + src.unlink() # the outside file is gone: the clone is what re-creates the snapshot + snapshot_path(project, out["hash"]).unlink() + + ensure_materialized(project, out["hash"]) + + assert snapshot_path(project, out["hash"]).is_file() + assert snapshot_path(project, out["hash"]).read_bytes() == before + frame = _snapshot_frame(project, out["hash"]) + assert frame[ROW_ORDER].to_list() == list(range(10)) + assert [r for r in list_errors(project, limit=1000) if r.get("hash") == out["hash"]] == [] + + +def test_a_csv_source_snapshot_is_recreated_under_the_reader_options_it_was_imported_with( + project: str, tmp_path: Path, monkeypatch +): + """The re-creation has only the manifest's record of the reader, which went through JSON (#198).""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.materialize import ensure_materialized + + src = _write_csv(_outside(tmp_path) / "orders.csv", [("east", 1), ("west", 2)], sep=";") + schema = (("region", "string"), ("n", "float64")) + out = source_import.update_and_depend(str(src), "orders", separator=";", schema=schema) + before = snapshot_path(project, out["hash"]).read_bytes() + src.unlink() + snapshot_path(project, out["hash"]).unlink() + + ensure_materialized(project, out["hash"]) + + assert snapshot_path(project, out["hash"]).read_bytes() == before + frame = _snapshot_frame(project, out["hash"]) + assert frame.columns == ["region", "n", ROW_ORDER] + assert frame["n"].to_list() == [1.0, 2.0] + + +def test_a_recreated_source_snapshot_is_verified_against_its_recorded_digest( + project: str, tmp_path: Path, monkeypatch +): + """Verified exactly as any other re-created snapshot is: a digest that is not the recorded one is recorded.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.errors import list_errors + from tallyman_xorq.materialize import ensure_materialized + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + # The clone is overwritten with other rows under the same digest name, so the re-creation cannot be faithful. + other = _write_parquet(_outside(tmp_path) / "other.parquet", 4) + _clone_of(project, out).write_bytes(other.read_bytes()) + snapshot_path(project, out["hash"]).unlink() + + ensure_materialized(project, out["hash"]) + + codes = [r.get("code") for r in list_errors(project, limit=1000) if r.get("hash") == out["hash"]] + assert "unfaithful_heal" in codes, codes + + +def test_a_source_snapshot_whose_clone_is_gone_names_the_re_import(project: str, tmp_path: Path, monkeypatch): + """Only when the bytes are gone from the arena as well is a source version unrecoverable, and it says so.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.build import BuildError + from tallyman_xorq.materialize import ensure_materialized + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + snapshot_path(project, out["hash"]).unlink() + _clone_of(project, out).unlink() + + with pytest.raises(BuildError) as exc: + ensure_materialized(project, out["hash"]) + + message = str(exc.value) + assert "orders-v1" in message + assert "catalog_import_source" in message + # It has to say WHICH file is missing: the snapshot alone is a cache miss, the clone as well is the loss. + assert ".cas" in message, message + assert out["digest"] in message, message + assert not snapshot_path(project, out["hash"]).exists() + + +def test_a_child_reads_after_its_sources_snapshot_is_deleted(project: str, tmp_path: Path, monkeypatch): + """The payoff: deleting ``result_cache/`` no longer leaves a source entry unreadable with the data right there.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + monkeypatch.setenv("TALLYMAN_AUTO_RECALC", "0") + from tallyman_mcp.server import catalog_create + from tallyman_xorq.result_cache import cached_result_expr + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + out = source_import.update_and_depend(str(src), "orders") + child = catalog_create("totals", _child_code("orders", project)) + assert "error" not in child, child + child_hash = get_alias(project, "totals") + expected = cached_result_expr(project, child_hash).execute() + + for snap in snapshot_path(project, out["hash"]).parent.glob("*.parquet"): + snap.unlink() + cached_result_expr.cache_clear() + + assert len(cached_result_expr(project, child_hash).execute()) == len(expected) + assert snapshot_path(project, out["hash"]).is_file() + + +def test_the_cache_page_offers_to_delete_a_source_snapshot(project: str, tmp_path: Path, monkeypatch): + """A source snapshot is an ordinary deletable snapshot while its clone is on disk to make it again from.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.materialize import pinned_reason + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + + assert pinned_reason(project, out["hash"]) is None + + +def test_a_source_snapshot_whose_clone_is_gone_is_pinned(project: str, tmp_path: Path, monkeypatch): + """Without the clone the snapshot is the last copy of those rows, so the delete leaves it alone and says why.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.materialize import pinned_reason + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + _clone_of(project, out).unlink() + + reason = pinned_reason(project, out["hash"]) + assert reason is not None + assert "orders-v1" in reason + assert ".cas" in reason, reason # the pin is the missing clone, not the fact that this is a source + + +# ``manifest.provenance.alias`` and ``.version`` are the name the import was made under, recorded once. A rename +# carries an alias's history and kind to the new name (``aliases.rename_alias``) and an unalias drops it, so a +# message that names a source version, or tells the user which import repairs it, has to ask the alias store who +# holds the entry now. + + +def _advised_import(message: str) -> tuple[list, dict]: + """The arguments of the ``catalog_import_source(...)`` call that ends *message*, parsed as Python literals.""" + import ast + + call = ast.parse(message[message.index("catalog_import_source(") :].rstrip("."), mode="eval").body + assert isinstance(call, ast.Call), message + return [ast.literal_eval(a) for a in call.args], {k.arg: ast.literal_eval(k.value) for k in call.keywords} + + +def test_a_pinned_source_snapshot_is_named_by_the_alias_it_was_renamed_to(project: str, tmp_path: Path, monkeypatch): + """The pin names the version as the catalog knows it now; the name it was imported under is history.""" + from tallyman_core.aliases import rename_alias + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.materialize import pinned_reason + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "a_src") + rename_alias(project, "a_src", "renamed_src") + _clone_of(project, out).unlink() + + reason = pinned_reason(project, out["hash"]) + assert reason is not None + assert "renamed_src-v1" in reason, reason + assert "source version a_src-v1" not in reason, reason + + +def test_the_re_import_advice_names_the_alias_a_source_was_renamed_to(project: str, tmp_path: Path, monkeypatch): + """The heal error names the version by its alias now, and so does the import it advises.""" + from tallyman_core.aliases import rename_alias + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.build import BuildError + from tallyman_xorq.materialize import ensure_materialized + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "a_src") + rename_alias(project, "a_src", "renamed_src") + snapshot_path(project, out["hash"]).unlink() + _clone_of(project, out).unlink() + + with pytest.raises(BuildError) as exc: + ensure_materialized(project, out["hash"]) + + message = str(exc.value) + assert "renamed_src-v1" in message, message + assert f"catalog_import_source({str(src)!r}, 'renamed_src', pinned_version=1)" in message, message + assert "'a_src'" not in message, message + + +def test_following_the_re_import_advice_after_a_rename_repairs_the_version(project: str, tmp_path: Path, monkeypatch): + """The advice is a call the user runs as written, so it has to repair the renamed alias's version. + + Advice that named the import-time alias minted a second source alias, ``a_src``, over the same entry, and the + renamed one's snapshot came back only by accident. + """ + from tallyman_core.aliases import load_kinds, rename_alias + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.build import BuildError + from tallyman_xorq.materialize import ensure_materialized + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "a_src") + before = snapshot_path(project, out["hash"]).read_bytes() + rename_alias(project, "a_src", "renamed_src") + snapshot_path(project, out["hash"]).unlink() + _clone_of(project, out).unlink() + with pytest.raises(BuildError) as exc: + ensure_materialized(project, out["hash"]) + args, kwargs = _advised_import(str(exc.value)) + + repaired = source_import.update_and_depend(*args, **kwargs) + + assert load_kinds(project) == {"renamed_src": "source"} + assert (repaired["alias"], repaired["version"], repaired["created"]) == ("renamed_src", 1, False) + assert repaired["hash"] == out["hash"] + assert history_for(project, "renamed_src") == [out["hash"]] + assert snapshot_path(project, out["hash"]).read_bytes() == before + + +def test_catalog_rename_then_the_advised_catalog_import_source_repairs_the_version( + project: str, tmp_path: Path, monkeypatch +): + """The same, through the tools a user has: the advice is an MCP call, run against the MCP tool.""" + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.aliases import load_kinds + from tallyman_mcp.server import catalog_import_source, catalog_rename + from tallyman_xorq.build import BuildError + from tallyman_xorq.materialize import ensure_materialized + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = catalog_import_source(str(src), "a_src") + assert "error" not in catalog_rename("a_src", "renamed_src") + snapshot_path(project, out["hash"]).unlink() + _clone_of(project, out).unlink() + with pytest.raises(BuildError) as exc: + ensure_materialized(project, out["hash"]) + args, kwargs = _advised_import(str(exc.value)) + + repaired = catalog_import_source(*args, **kwargs) + + assert "error" not in repaired, repaired + assert load_kinds(project) == {"renamed_src": "source"} + assert snapshot_path(project, out["hash"]).is_file() + + +def test_a_source_version_no_alias_holds_is_not_advised_back_under_its_old_name( + project: str, tmp_path: Path, monkeypatch +): + """After an unalias no alias holds the version, and the messages say so instead of naming the dead alias. + + Re-importing under the old name would not restore anything that exists: it would mint a new alias of that name. + """ + from tallyman_core.aliases import remove_alias + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_xorq.build import BuildError + from tallyman_xorq.materialize import ensure_materialized, pinned_reason + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "a_src") + remove_alias(project, "a_src") + _clone_of(project, out).unlink() + + reason = pinned_reason(project, out["hash"]) + assert reason is not None + assert "no source alias" in reason, reason + assert "imported as a_src-v1" in reason, reason + + snapshot_path(project, out["hash"]).unlink() + with pytest.raises(BuildError) as exc: + ensure_materialized(project, out["hash"]) + + message = str(exc.value) + assert "no source alias" in message, message + assert "'a_src'" not in message, message + assert "pinned_version" not in message, message + + +def test_following_the_re_import_advice_for_a_csv_repairs_the_version(project: str, tmp_path: Path, monkeypatch): + """A CSV's entry hash covers its reader options (D12), so the advised import has to carry them. + + Without them the same file hashes to another entry, and the advised call is refused as "not orders-v1". + """ + from tallyman_mcp.server import catalog_import_source + from tallyman_xorq.build import BuildError + from tallyman_xorq.materialize import ensure_materialized + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_csv(_outside(tmp_path) / "orders.csv", [("north", 1), ("south", 2)], sep=";") + out = catalog_import_source( + str(src), "orders", schema={"region": "string", "n": "int64"}, reader_options={"separator": ";"} + ) + assert "error" not in out, out + snapshot_path(project, out["hash"]).unlink() + _clone_of(project, out).unlink() + with pytest.raises(BuildError) as exc: + ensure_materialized(project, out["hash"]) + args, kwargs = _advised_import(str(exc.value)) + + repaired = catalog_import_source(*args, **kwargs) + + assert "error" not in repaired, repaired + assert (repaired["hash"], repaired["created"]) == (out["hash"], False) + assert snapshot_path(project, out["hash"]).is_file() + + +def test_a_pinned_import_read_another_way_says_the_reader_options_differ(project: str, tmp_path: Path, monkeypatch): + """The same bytes under other reader options are another entry, and the refusal has to say so. + + "This file is not orders-v1" is false about the file; what differs is how it is read. + """ + from tallyman_mcp.server import catalog_import_source + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_csv(_outside(tmp_path) / "orders.csv", [("north", 1), ("south", 2)], sep=";") + assert "error" not in catalog_import_source(str(src), "orders", reader_options={"separator": ";"}) + + out = catalog_import_source(str(src), "orders", pinned_version=1) + + assert "reader options" in out.get("error", ""), out + + +def test_a_source_recipe_names_its_alias_as_the_one_it_was_imported_under(project: str, tmp_path: Path, monkeypatch): + """The generated recipe is written once, at import, so the Code tab must not present that name as the entry's. + + After a rename the entry is ``renamed_src-v1``; a header reading ``# a_src-v1: a source version`` says otherwise. + """ + from tallyman_core.aliases import rename_alias + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "a_src") + rename_alias(project, "a_src", "renamed_src") + + header = (entry_dir(project, out["hash"]) / "expr.py").read_text().splitlines()[0] + assert "imported as a_src-v1" in header, header + assert not header.startswith("# a_src-v1:"), header + + +def test_a_reset_keeps_the_clone_of_an_imported_source_alive(project: str, tmp_path: Path, monkeypatch): + """``gc_cas`` walks the retention closure; an imported version's clone is in it via the entry's provenance.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.catalog_state import _live_source_digests, capture_tallyman_state + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + out = source_import.update_and_depend(str(src), "orders") + capture_tallyman_state(project) # what a checkpoint does; the retention closure reads its pointer list + + assert out["digest"] in (_live_source_digests(project) or set()) + + +# --------------------------------------------------------------------------- +# D12 — reader options are fixed at import +# --------------------------------------------------------------------------- + + +def test_csv_reader_options_are_recorded_on_the_source_entry(project: str, tmp_path: Path, monkeypatch): + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_csv(_outside(tmp_path) / "orders.csv", [("north", 1), ("south", 2)], sep=";") + + out = source_import.update_and_depend(str(src), "orders", separator=";") + + frame = _snapshot_frame(project, out["hash"]) + assert frame.columns == ["region", "n", ROW_ORDER] + assert frame["region"].to_list() == ["north", "south"] + reader = read_manifest(entry_dir(project, out["hash"])).provenance.reader + assert reader["kind"] == "csv" + assert reader["scan_kwargs"]["separator"] == ";" + + +def test_the_same_csv_under_two_readers_is_two_entries(project: str, tmp_path: Path, monkeypatch): + """Two recipes cannot read one file two ways: import it twice under two aliases, and the hashes differ.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_csv(_outside(tmp_path) / "orders.csv", [("north", 1), ("south", 2)]) + + plain = source_import.update_and_depend(str(src), "orders") + typed = source_import.update_and_depend(str(src), "orders_typed", schema=(("region", "string"), ("n", "float64"))) + + assert plain["hash"] != typed["hash"] + assert str(_snapshot_frame(project, typed["hash"])["n"].dtype) == "Float64" + + +def test_reader_options_are_not_re_derived_at_build_time(project: str, tmp_path: Path, monkeypatch): + """A parquet source records the reader it was imported with; a build reads the snapshot and derives nothing.""" + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + out = source_import.update_and_depend(str(src), "orders") + + assert read_manifest(entry_dir(project, out["hash"])).provenance.reader == {"kind": "parquet"} + + +# --------------------------------------------------------------------------- +# D9 — ingest verifies what it wrote +# --------------------------------------------------------------------------- + + +def test_ensure_cas_path_rejects_a_clone_that_does_not_match_its_name(project: str, tmp_path: Path): + """A file edited mid-copy would otherwise produce a clone whose name lies about its content.""" + from tallyman_xorq import source_identity as si + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + with pytest.raises(si.CloneDigestMismatch): + si.ensure_cas_path(project, src, "0" * 32) + + +def test_recon_cas_path_raises_instead_of_serving_drifted_live_bytes(project: str, tmp_path: Path): + """A version tallyman promised and then lost is a failure, not a downgrade to whatever is on disk now.""" + from tallyman_xorq import source_identity as si + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + digest = si._digest_file(src) + _write_parquet(src, 20) # the live bytes drift and no clone was ever written + + with pytest.raises(si.LostSourceVersion): + si.recon_cas_path(project, src, digest) + + +# --------------------------------------------------------------------------- +# D10 — the MCP surface +# --------------------------------------------------------------------------- + + +def test_catalog_load_parquet_is_gone(project: str): + """D10: the old tool's semantics change too much to keep the name.""" + import tallyman_mcp.server as server + + assert not hasattr(server, "catalog_load_parquet") + + +def test_catalog_import_source_mints_and_records_the_event(project: str, tmp_path: Path, monkeypatch): + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.events import read_events + from tallyman_mcp.server import catalog_import_source + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + + out = catalog_import_source(str(src), "orders") + + assert "error" not in out, out + assert out["alias"] == "orders" + assert out["version"] == 1 + assert out["row_count"] == 10 + assert any(f["name"] == "region" for f in out["schema"]["fields"]) + kinds = [e.get("kind") for e in read_events(project)] + assert "alias_set" in kinds + + +def test_catalog_import_source_takes_a_checkpoint(project: str, tmp_path: Path, monkeypatch): + """An import is a catalog operation (D7), so it lands as one revision like a revise.""" + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.catalog_state import list_revisions + from tallyman_mcp.server import catalog_import_source + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + before = len(list_revisions(project)) + + catalog_import_source(str(src), "orders") + + assert len(list_revisions(project)) == before + 1 + + +def test_catalog_import_source_triggers_auto_recalc(project: str, tmp_path: Path, monkeypatch): + """A source advancing is indistinguishable downstream from an alias being revised.""" + monkeypatch.setenv("TALLYMAN_PROJECT", project) + monkeypatch.setenv("TALLYMAN_AUTO_RECALC", "1") + from tallyman_mcp.server import catalog_create, catalog_import_source + + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + catalog_import_source(str(src), "orders") + assert "error" not in catalog_create("totals", _child_code("orders", project)) + first = get_alias(project, "totals") + + _write_parquet(src, 20) + out = catalog_import_source(str(src), "orders") + + assert out["recalc"]["remap"], out + assert get_alias(project, "totals") != first + + +def test_catalog_import_source_reports_an_error_as_a_dict(project: str, tmp_path: Path, monkeypatch): + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_mcp.server import catalog_import_source + + out = catalog_import_source(str(_outside(tmp_path) / "nope.parquet"), "orders") + + assert "error" in out + assert "nope.parquet" in out["error"] + + +def test_catalog_import_source_reports_a_duplicate_import_as_an_error(project: str, tmp_path: Path, monkeypatch): + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.catalog_state import list_revisions + from tallyman_mcp.server import catalog_import_source + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + catalog_import_source(str(src), "orders") + before = len(list_revisions(project)) + + out = catalog_import_source(str(src), "orders_again") + + assert "orders-v1" in out.get("error", ""), out + assert get_alias(project, "orders_again") is None + assert len(list_revisions(project)) == before + + +def test_catalog_revise_refuses_a_source_alias(project: str, tmp_path: Path, monkeypatch): + """There is no recipe to revise: a source alias advances only by an import.""" + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_mcp.server import catalog_import_source, catalog_revise + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + catalog_import_source(str(src), "orders") + + out = catalog_revise("orders", _child_code("orders", project)) + + assert "error" in out + assert "catalog_import_source" in out["error"] + + +def test_promote_diff_refuses_a_source_alias_as_its_target(project: str, tmp_path: Path, monkeypatch): + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_mcp.server import catalog_create, catalog_import_source, catalog_promote_diff, catalog_revise + + src = _outside(tmp_path) / "orders.parquet" + _write_parquet(src, 10) + catalog_import_source(str(src), "orders") + catalog_create("totals", _child_code("orders", project)) + catalog_revise("totals", _child_code("orders", project) + "expr = expr.mutate(x=1)\n") + + out = catalog_promote_diff("totals", alias="orders") + + assert "error" in out + assert "source alias" in out["error"] + + +def test_catalog_alias_refuses_a_source_entry(project: str, tmp_path: Path, monkeypatch): + """A source version cannot take a catalog name: the alias's kind must match its entry's. + + catalog_alias checked only the kind of the *name*, so ``catalog_alias(, "x")`` made a catalog alias + whose head was an imported file, and ``catalog_revise("x", ...)`` was then allowed on it. The refusal names the + source alias-version that holds the entry and steers to a catalog entry that reads it, which follows the source + when it is imported again. + """ + monkeypatch.setenv("TALLYMAN_PROJECT", project) + from tallyman_core.aliases import alias_kind + from tallyman_mcp.server import catalog_alias, catalog_create, catalog_import_source + + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + imported = catalog_import_source(str(src), "orders") + + out = catalog_alias(imported["hash"], "x") + + assert "error" in out, out + assert "orders-v1" in out["error"] + assert "catalog_create" in out["error"] + assert get_alias(project, "x") is None + assert alias_kind(project, "x") is None + + steer = "from tallyman_xorq.io import tracked_expr_from_alias\nexpr = tracked_expr_from_alias('orders')" + followed = catalog_create("x", steer) + + assert "error" not in followed, followed + assert followed["hash"] != imported["hash"] + assert alias_kind(project, "x") == "catalog" + + +# --------------------------------------------------------------------------- +# D6 — the ordered-copy store is subsumed into the source entry's snapshot +# --------------------------------------------------------------------------- + + +def test_the_ordered_copy_store_is_gone(project: str, tmp_path: Path, monkeypatch): + """There is one shape of file under ``compute_cache/``: a snapshot named by an entry hash. + + ADR-008 D2 gave every source an ordered copy keyed by ``md5(digest ǀ reader signature)``. With the + source itself an entry, the entry hash names the file and the reader options live on the entry that + used them (D12), so the second store and the key that addressed it have nothing left to do. + """ + from tallyman_core.paths import compute_cache_dir + from tallyman_xorq import ordered_copy as oc + from tallyman_xorq import source_import + from tallyman_xorq.materialize import RESULT_CACHE_DIRNAME + + for name in ( + "copy_key", + "ensure_ordered_copy", + "existing_ordered_copy", + "recreate_ordered_copy", + "ORDERED_COPIES_DIRNAME", + "ordered_copies_dir", + "is_ordered_copy_path", + "note_ordered_copy", + "begin_collect", + "end_collect", + "SourceUnavailable", + ): + assert not hasattr(oc, name), f"ordered_copy.{name} is deleted by ADR-011 D6" + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + out = source_import.update_and_depend(str(src), "orders") + from tallyman_xorq.build import build_and_persist + + build_and_persist(project, _child_code("orders", project)) + + assert snapshot_path(project, out["hash"]).is_file() + assert [d.name for d in sorted(compute_cache_dir(project).iterdir())] == [RESULT_CACHE_DIRNAME] + + +# --------------------------------------------------------------------------- +# a second name for a source is an expression over it, not a second import +# --------------------------------------------------------------------------- + + +def test_a_second_name_for_a_source_is_a_catalog_entry_over_it(project: str, tmp_path: Path, monkeypatch): + """What the duplicate-import error tells the user to do instead, and why it needs nothing to force the hash apart. + + A source entry's hash is an md5 of its bytes and reader options; a recipe's hash is xorq's hash of the expression. + An expression that only reads the source is therefore a different entry already, with a followed parent edge, so a + re-import of the source advances it like any other follower. + """ + from tallyman_mcp.server import catalog_create + from tallyman_xorq import source_import + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 10) + source = source_import.update_and_depend(str(src), "orders") + + out = catalog_create( + "orders_eu", "from tallyman_xorq.io import tracked_expr_from_alias\nexpr = tracked_expr_from_alias('orders')\n" + ) + + assert "error" not in out, out + assert out["hash"] != source["hash"] + parents = read_manifest(entry_dir(project, out["hash"])).parents + assert [(p.ref, p.hash, p.follow) for p in parents] == [("orders", source["hash"], True)] + + +# --------------------------------------------------------------------------- +# two projects importing one file share nothing +# --------------------------------------------------------------------------- + + +@pytest.fixture +def two_projects(isolated_home: Path) -> tuple[str, str]: + from tallyman_core import ensure_project, set_active_project + + for name in ("alpha", "beta"): + ensure_project(name) + set_active_project("alpha") + return "alpha", "beta" + + +def test_two_projects_importing_one_file_each_get_their_own_copy(two_projects, tmp_path: Path): + """Same bytes, same entry hash, but each project holds its own entry, snapshot and clone.""" + from tallyman_xorq import source_import + from tallyman_xorq.source_import import source_clone_path + + alpha, beta = two_projects + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + + a = source_import.update_and_depend(str(src), "orders", project=alpha) + b = source_import.update_and_depend(str(src), "orders", project=beta) + + assert a["hash"] == b["hash"] + assert a["created"] and b["created"], "the second project's import is not a no-op on the first's entry" + for proj in (alpha, beta): + manifest = read_manifest(entry_dir(proj, a["hash"])) + assert manifest.project == proj + assert snapshot_path(proj, a["hash"]).is_file() + assert source_clone_path(proj, manifest.provenance).is_file() + assert get_alias(proj, "orders") == a["hash"] + assert snapshot_path(alpha, a["hash"]) != snapshot_path(beta, b["hash"]) + + +def test_the_duplicate_import_check_is_per_project(two_projects, tmp_path: Path): + """One file under different alias names in two projects is two unrelated imports, not a duplicate.""" + from tallyman_xorq import source_import + + alpha, beta = two_projects + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + + source_import.update_and_depend(str(src), "orders", project=alpha) + out = source_import.update_and_depend(str(src), "sales", project=beta) + + assert out["created"] is True + assert history_for(beta, "sales") == [out["hash"]] + assert history_for(beta, "orders") == [] + + +def test_deleting_and_healing_one_projects_snapshot_leaves_the_others_alone(two_projects, tmp_path: Path): + from tallyman_xorq import source_import + from tallyman_xorq.build import build_and_persist + from tallyman_xorq.digest import content_digest + from tallyman_xorq.materialize import ensure_materialized + from tallyman_xorq.result_cache import cached_result_expr + + alpha, beta = two_projects + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + h = source_import.update_and_depend(str(src), "orders", project=alpha)["hash"] + source_import.update_and_depend(str(src), "orders", project=beta) + child_a = build_and_persist(alpha, _child_code("orders", alpha)).content_hash + child_b = build_and_persist(beta, _child_code("orders", beta)).content_hash + beta_snapshot = snapshot_path(beta, h) + beta_digest, beta_mtime = content_digest(beta_snapshot), beta_snapshot.stat().st_mtime_ns + + snapshot_path(alpha, h).unlink() + ensure_materialized(alpha, h) + + assert snapshot_path(alpha, h).is_file() + assert (content_digest(beta_snapshot), beta_snapshot.stat().st_mtime_ns) == (beta_digest, beta_mtime) + assert len(cached_result_expr(alpha, child_a).execute()) == 3 + assert len(cached_result_expr(beta, child_b).execute()) == 3 + + +def test_retiring_one_projects_clones_leaves_the_others(two_projects, tmp_path: Path): + """``.cas`` lives under each project's ``data/``; a sweep in one project never reaches another's.""" + from tallyman_xorq import source_identity as si + from tallyman_xorq import source_import + from tallyman_xorq.source_import import source_clone_path + + alpha, beta = two_projects + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + h = source_import.update_and_depend(str(src), "orders", project=alpha)["hash"] + source_import.update_and_depend(str(src), "orders", project=beta) + beta_clone = source_clone_path(beta, read_manifest(entry_dir(beta, h)).provenance) + + assert si.gc_cas(alpha, set(), bullpen=tmp_path / "bullpen") == 1 + + assert beta_clone.is_file() + + +def test_a_re_import_in_one_project_does_not_stale_the_other(two_projects, tmp_path: Path): + from tallyman_core.aliases import set_alias + from tallyman_xorq import source_import + from tallyman_xorq.build import build_and_persist + from tallyman_xorq.staleness import scan + + alpha, beta = two_projects + src = _write_parquet(_outside(tmp_path) / "orders.parquet", 12) + source_import.update_and_depend(str(src), "orders", project=alpha) + source_import.update_and_depend(str(src), "orders", project=beta) + child_a = build_and_persist(alpha, _child_code("orders", alpha)).content_hash + child_b = build_and_persist(beta, _child_code("orders", beta)).content_hash + for proj, child in ((alpha, child_a), (beta, child_b)): + set_alias(proj, "totals", child) # the scan judges live alias heads (#154) + + _write_parquet(src, 20) + source_import.update_and_depend(str(src), "orders", project=beta) + + assert scan(alpha)[child_a].stale is False + assert scan(beta)[child_b].stale is True diff --git a/tests/test_staleness.py b/tests/test_staleness.py index 840926dd..67a4a499 100644 --- a/tests/test_staleness.py +++ b/tests/test_staleness.py @@ -1,24 +1,22 @@ from __future__ import annotations -from tallyman_cli.fixtures import write_shoe_orders -from tallyman_core import data_dir from tallyman_core.aliases import get_alias from tallyman_mcp.server import catalog_create, catalog_revise from tallyman_xorq.staleness import entry_staleness, scan -def _base_code(project: str) -> str: # read_project_file root over orders.parquet +def _base_code(project: str) -> str: # root over the imported orders source return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order") """ def _base_code_v2(project: str) -> str: # a different graph → a new content hash return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias("orders_src", project={project!r}) expr = t.select("region", "price", "__row_order").mutate(extra=1) """ @@ -31,10 +29,10 @@ def _child_code(alias: str) -> str: # tracked child off an alias """ -def _pinned_child_code(content_hash: str) -> str: # pinned (no lineage) child off a specific hash +def _pinned_child_code(ref: str) -> str: # pinned (no lineage) child off one version of an alias return f""" from tallyman_xorq.io import pinned_expr_from_alias -t = pinned_expr_from_alias({content_hash!r}) +t = pinned_expr_from_alias({ref!r}) expr = t.mutate(doubled=t.price * 2) """ @@ -60,8 +58,7 @@ def _hash(result: dict) -> str: return result["hash"] -def test_followed_alias_advance_marks_child_stale(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_followed_alias_advance_marks_child_stale(project, orders_src, monkeypatch): base_v1 = _hash(catalog_create("base", _base_code(project))) child_hash = _hash(catalog_create("child", _child_code("base"))) @@ -83,12 +80,11 @@ def test_followed_alias_advance_marks_child_stale(project, orders_parquet, monke ) -def test_hash_pinned_parent_is_not_stale_when_alias_advances(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_hash_pinned_parent_is_not_stale_when_alias_advances(project, orders_src, monkeypatch): base_v1 = _hash(catalog_create("base", _base_code(project))) - # A literal-hash argument is recorded follow=False (a pin), so the alias + # A version reference is recorded follow=False (a pin), so the alias # advancing must not make this child stale. - child_hash = _hash(catalog_create("child", _pinned_child_code(base_v1))) + child_hash = _hash(catalog_create("child", _pinned_child_code("base-v1"))) _hash(catalog_revise("base", _base_code_v2(project))) assert get_alias(project, "base") != base_v1 @@ -98,38 +94,7 @@ def test_hash_pinned_parent_is_not_stale_when_alias_advances(project, orders_par assert [r for r in v.reasons if r.axis == "alias"] == [] -def test_source_edit_marks_entry_stale_under_cas(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") - base_hash = _hash(catalog_create("base", _base_code(project))) - assert entry_staleness(project, base_hash).stale is False - - # Rewrite the source in place with different data (new bytes, new mtime). - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=200, seed=1) - - v = entry_staleness(project, base_hash) - assert v.stale is True - source_reasons = [r for r in v.reasons if r.axis == "source"] - assert len(source_reasons) == 1 - assert source_reasons[0].ref == "orders.parquet" - - -def test_off_mode_reports_source_axis_unknown_not_fresh(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "off") - base_hash = _hash(catalog_create("base", _base_code(project))) - - v = entry_staleness(project, base_hash) - assert v.stale is False - assert "source" in v.unknown_axes - - # off mode records no source digests, so it cannot see drift — unknown, not stale. - write_shoe_orders(data_dir(project) / "orders.parquet", n_rows=200, seed=1) - v2 = entry_staleness(project, base_hash) - assert v2.stale is False - assert "source" in v2.unknown_axes - - -def test_scan_distinguishes_direct_from_transitive_staleness(project, orders_parquet, monkeypatch): - monkeypatch.setenv("TALLYMAN_SOURCE_IDENTITY", "cas") +def test_scan_distinguishes_direct_from_transitive_staleness(project, orders_src, monkeypatch): # Opt out of auto-recalc-on-revise: this test needs the revise to advance "a" # WITHOUT cascading, so the scan can observe b directly-stale and c only # transitively-stale. With the cascade on (the default), the revise would @@ -153,3 +118,62 @@ def test_scan_distinguishes_direct_from_transitive_staleness(project, orders_par assert verdicts[b].transitively_stale is False assert verdicts[c].stale is False assert verdicts[c].transitively_stale is True + + +# --------------------------------------------------------------------------- +# ADR-011 D6 — staleness has one axis. An entry is stale when a followed alias +# has moved, and that is the only reason: the source axis, and the digests it +# took of files outside the arena, are gone. +# --------------------------------------------------------------------------- + + +def _import(project: str, path, alias: str) -> dict: + from tallyman_xorq.source_import import update_and_depend + + return update_and_depend(path, alias, project=project) + + +def test_a_scan_has_one_axis_and_digests_no_file(project, tmp_path, monkeypatch): + """Three imported sources and two children: the scan reads aliases and manifests, and nothing else. + + The digest counter is its own positive control — each import takes two digests, one of the file as + given and one of the clone as written (D9) — so a counter that stays at zero across the scan is a + measurement that the scan took none, not an assertion that the code looks like it wouldn't. + ``manifest.sources`` and the ``unknown`` source axis go with it: a verdict never carries a source + reason, and never reports an axis it could not evaluate. + """ + import pandas as pd + + from tallyman_xorq import source_identity as si + from tallyman_xorq import staleness + + monkeypatch.setenv("TALLYMAN_PROJECT", project) + outside = tmp_path / "outside" + outside.mkdir(parents=True, exist_ok=True) + + digested: list[str] = [] + real = si._digest_file + + def counting_digest(path): + digested.append(str(path)) + return real(path) + + monkeypatch.setattr(si, "_digest_file", counting_digest) + + for alias in ("orders", "orders_b", "orders_c"): + frame = pd.DataFrame({"region": ["n", "s", "e"], "price": [1.0, 2.0, 3.0], "who": [alias] * 3}) + frame.to_parquet(outside / f"{alias}.parquet") + _import(project, outside / f"{alias}.parquet", alias) + catalog_create("totals", _agg_child_code("orders")) + catalog_create("flagged", _const_child_code("orders_b")) + assert len(digested) == 6, f"three imports, each digesting the file and then its clone; got {digested}" + + digested.clear() + verdicts = scan(project) + + assert digested == [], f"a staleness scan must digest nothing; it digested {digested}" + assert not hasattr(si, "digest_for"), "the stat-memoized digest went with source_digests.json (D6)" + assert not hasattr(staleness, "_force_source_rehash"), "nothing forces a rehash; nothing hashes" + for content_hash, verdict in verdicts.items(): + assert verdict.unknown_axes == [], (content_hash, verdict.unknown_axes) + assert [r.axis for r in verdict.reasons if r.axis != "alias"] == [], (content_hash, verdict.reasons) diff --git a/tests/test_summary_stats.py b/tests/test_summary_stats.py index 8a3edb3e..7d15ea2a 100644 --- a/tests/test_summary_stats.py +++ b/tests/test_summary_stats.py @@ -101,7 +101,7 @@ def test_write_does_not_create_disabled_dir_on_active_listing(project: str): # --------------------------------------------------------------------------- -def test_ensure_session_includes_project_root_in_load_expr_body(project: str, orders_parquet: Path, monkeypatch): +def test_ensure_session_includes_project_root_in_load_expr_body(project: str, orders_src: str, monkeypatch): """The /load_expr POST must carry the project's absolute path so buckaroo's load_project_stat_klasses (PR #784) can scan its stats/ directory.""" @@ -109,8 +109,8 @@ def test_ensure_session_includes_project_root_in_load_expr_body(project: str, or from tallyman_xorq import build_and_persist code = f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({orders_src!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ res = build_and_persist(project, code) diff --git a/tests/test_tallyman_read_csv.py b/tests/test_tallyman_read_csv.py index 53c2518c..ded9d734 100644 --- a/tests/test_tallyman_read_csv.py +++ b/tests/test_tallyman_read_csv.py @@ -1,10 +1,17 @@ -"""Tests for tallyman_read_csv — CSV ingest with a stable ``__row_order``. - -ADR-004-result-digest-canonical-ordering gave a CSV read an ``original_row_order`` column and a trailing ``order_by`` -so that a snapshot's bytes are deterministic. ADR-008 (plans/ADR-008-row-order-of-reads.md) replaces both: the column -is ``__row_order`` (ADR-008 D7, amending ADR-005 INV-1), the trailing sort is gone so a CSV root is a cheap read -(ADR-008 D7, amending ADR-005 INV-2), and the ordered copy of the CSV lives in the project's -``compute_cache/ordered_sources`` (ADR-007 D13), keyed by the CSV's content (ADR-008 D2, #168). +"""Tests for CSV ingest — the schema DSL and a stable ``__row_order``, at import time. + +ADR-004-result-digest-canonical-ordering gave a CSV read an ``original_row_order`` column and a trailing +``order_by`` so that a snapshot's bytes are deterministic. ADR-008 (plans/ADR-008-row-order-of-reads.md) replaced +both: the column is ``__row_order`` (ADR-008 D7, amending ADR-005 INV-1), and polars numbers the rows in file order +rather than a datafusion scan that may repartition (ADR-008 D2, #168). + +ADR-011 (plans/ADR-011-sources-are-aliases.md) moved *when* all of that happens. ``tallyman_read_csv`` is a build +error in an authored recipe (ADR-011 D2; that refusal is pinned in tests/test_io.py). A CSV enters the catalog +through ``update_and_depend(path, alias, schema=..., **reader_options)``, which parses it with polars under the same +schema DSL, the same 100 -> 10k -> whole-file inference ladder and the same error messages, and writes ONE parquet +snapshot: the source entry's own, at ``compute_cache/result_cache/.parquet`` (ADR-011 D1). That +snapshot is what ``compute_cache/ordered_sources/.parquet`` used to be, so the tests below read it, and +``manifest.result_digest`` is the digest record that ``manifest.ordered_copies`` used to hold. """ from __future__ import annotations @@ -14,128 +21,88 @@ from tallyman_core import data_dir, entry_dir, read_manifest from tallyman_core.paths import compute_cache_dir, tallyman_home -from tallyman_mcp.server import catalog_create -from tallyman_xorq.build import list_entries -from tallyman_xorq.result_cache import ( - baked_snapshot_path, - cache_worthy, - cached_result_expr, - snapshot_file_digest, - verify_result_faithful, -) +from tallyman_xorq.materialize import ensure_materialized, snapshot_path, snapshots_dir +from tallyman_xorq.result_cache import cached_result_expr, snapshot_file_digest, verify_result_faithful +from tallyman_xorq.source_import import update_and_depend @pytest.fixture def sample_csv(project: str) -> Path: - """Small CSV under the project data dir with a known row ordering.""" + """Small CSV under the project data dir with a known row ordering. + + ``data/`` stopped being special with ADR-011 D2 — an import takes any path — but the file has to live + somewhere, and putting it here keeps it inside the per-test isolated home. + """ p = data_dir(project) / "sample.csv" # Deliberately write rows in an order that is NOT alphabetical by name, so # any test that verifies file order can distinguish a correctly-ordered - # copy from an arbitrarily-ordered one. + # snapshot from an arbitrarily-ordered one. p.write_text("id,name,value\n3,charlie,30\n1,alice,10\n2,bob,20\n") return p -def _hash_of(project: str) -> str: - return list_entries(project)[0]["content_hash"] +def _import_sample(project: str, csv_path: Path, alias: str = "sample_csv") -> dict: + """Import *csv_path* under *alias*, with its three data columns pinned.""" + import xorq.vendor.ibis as ibis + schema = ibis.schema({"id": "int64", "name": "string", "value": "int64"}) + return update_and_depend(csv_path, alias, project=project, schema=schema) -def _read_csv_code(csv_path: Path) -> str: - return f""" -import xorq.vendor.ibis as ibis -from tallyman_xorq.io import tallyman_read_csv -schema = ibis.schema({{"id": "int64", "name": "string", "value": "int64"}}) -expr = tallyman_read_csv({str(csv_path)!r}, schema=schema) -""" +def _snapshot_table(project: str, content_hash: str): + """The source entry's snapshot, read straight off disk.""" + import pyarrow.parquet as pq -def _ordered_copies(project: str) -> list[Path]: - """The ordered copies of sources: ADR-007 D13 puts them under the project's compute cache.""" - return sorted((compute_cache_dir(project) / "ordered_sources").glob("*.parquet")) + return pq.read_table(str(snapshot_path(project, content_hash))) -def test_tallyman_read_csv_adds_row_order(project, sample_csv, monkeypatch): - """tallyman_read_csv returns an expression whose schema ends in __row_order: int64 (ADR-008 D7).""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - res = catalog_create("csv_entry", _read_csv_code(sample_csv)) - assert "error" not in res, res - names = [f["name"] for f in res["schema"]["fields"]] - schema_fields = {f["name"]: f["type"] for f in res["schema"]["fields"]} +def test_tallyman_read_csv_adds_row_order(project, sample_csv): + """A CSV source entry's recorded schema ends in __row_order: int64 (ADR-008 D7).""" + out = _import_sample(project, sample_csv) + names = [f["name"] for f in out["schema"]["fields"]] + schema_fields = {f["name"]: f["type"] for f in out["schema"]["fields"]} assert names == ["id", "name", "value", "__row_order"], f"one row-order column, and it is last: {names}" assert schema_fields["__row_order"] == "int64" assert "original_row_order" not in schema_fields -def test_tallyman_read_csv_entry_is_cheap(project, sample_csv, monkeypatch): - """A tallyman_read_csv entry is a plain read of its ordered copy, so it is cheap (ADR-008 D7). - - Before ADR-008 its trailing ``order_by`` made every entry in a CSV lineage worthy for that Sort alone, and each - revision baked a full sorted copy. - """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("csv_entry", _read_csv_code(sample_csv)) - h = _hash_of(project) - assert cache_worthy(project, h) is False, "a tallyman_read_csv root has no Sort, so it is a cheap read" - assert read_manifest(entry_dir(project, h)).cache_worthy is False - - -def test_tallyman_read_csv_bakes_no_snapshot(project, sample_csv, monkeypatch): - """A CSV root writes no snapshot: its rows are fixed by the content-keyed ordered copy (ADR-008 D7).""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - res = catalog_create("csv_entry", _read_csv_code(sample_csv)) - assert "error" not in res, res - h = _hash_of(project) +def test_csv_source_snapshot_is_in_file_order(project, sample_csv): + """The snapshot holds the CSV's rows in file order, numbered 0..N-1 in ``__row_order`` (ADR-008 D2).""" + out = _import_sample(project, sample_csv) - assert baked_snapshot_path(project, h) is None, "a cheap entry has no snapshot" - snapshots = compute_cache_dir(project) / "result_cache" - assert not snapshots.exists() or not list(snapshots.glob("*.parquet")), "nothing was materialized" - - -def test_tallyman_read_csv_ordered_copy_is_in_file_order(project, sample_csv, monkeypatch): - """The ordered copy holds the CSV's rows in file order, numbered 0..N-1 in ``__row_order`` (ADR-008 D2).""" - import pyarrow.parquet as pq - - monkeypatch.setenv("TALLYMAN_PROJECT", project) - catalog_create("csv_entry", _read_csv_code(sample_csv)) - - copies = _ordered_copies(project) - assert len(copies) == 1, f"expected one ordered copy under compute_cache/ordered_sources, found {copies}" - table = pq.read_table(str(copies[0])) + snapshots = sorted(snapshots_dir(project).glob("*.parquet")) + assert snapshots == [snapshot_path(project, out["hash"])], f"one import, one snapshot: {snapshots}" + table = _snapshot_table(project, out["hash"]) assert table.column_names == ["id", "name", "value", "__row_order"] assert table["id"].to_pylist() == [3, 1, 2], "file order, not sorted order" assert table["__row_order"].to_pylist() == [0, 1, 2] -def test_a_recreated_ordered_copy_matches_its_recorded_digest(project, sample_csv, monkeypatch): - """The manifest records the copy's content digest, and a re-created copy reproduces it (ADR-007 D13). +def test_a_recreated_source_snapshot_matches_its_recorded_digest(project, sample_csv): + """The manifest records the snapshot's digest, and a re-created snapshot reproduces it (ADR-011 D1). - Replaces the check that a CSV root's snapshot digest is stable across a heal: a CSV root is cheap now, so the - file whose reproducibility matters is the ordered copy. + The ordered copy this replaces had its own digest record in ``manifest.ordered_copies``. A source version's + snapshot IS that copy now, so the file whose reproducibility matters is the one ``manifest.result_digest`` + already covers, and ``ensure_materialized`` is what makes it again (from the clone of the imported bytes). """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) - res = catalog_create("csv_entry", _read_csv_code(sample_csv)) - assert "error" not in res, res - h = _hash_of(project) - - records = read_manifest(entry_dir(project, h)).ordered_copies - assert records and len(records) == 1, f"the manifest must record the ordered copy: {records}" - ((key, record),) = records.items() - recorded = record["content_digest"] + out = _import_sample(project, sample_csv) + h = out["hash"] + recorded = read_manifest(entry_dir(project, h)).result_digest assert recorded.startswith("arrow-sha256:"), recorded - copy = compute_cache_dir(project) / "ordered_sources" / f"{key}.parquet" - assert copy.exists() and snapshot_file_digest(copy) == recorded - copy.unlink() + snapshot = snapshot_path(project, h) + assert snapshot.exists() and snapshot_file_digest(snapshot) == recorded + snapshot.unlink() cached_result_expr.cache_clear() - cached_result_expr(project, h) # opening the entry makes the copy again from the clone + ensure_materialized(project, h) - assert copy.exists(), "opening the entry must re-create its ordered copy" - assert snapshot_file_digest(copy) == recorded, "the re-created copy must reproduce the recorded digest" + assert snapshot.exists(), "ensure_materialized must write the source snapshot again" + assert snapshot_file_digest(snapshot) == recorded, "the re-created snapshot must reproduce the recorded digest" @pytest.fixture def repartitioned_csv(project: str) -> tuple[Path, int]: - """A CSV large enough that datafusion repartitions the scan across threads. + """A CSV large enough that a parallel scan repartitions it across threads. The marker column holds the source row index (0..N-1) in file order, so a correct ``__row_order`` must line up with it row-for-row. The file is @@ -156,34 +123,26 @@ def repartitioned_csv(project: str) -> tuple[Path, int]: return p, n -def _big_read_csv_code(csv_path: Path) -> str: - return f""" -import xorq.vendor.ibis as ibis -from tallyman_xorq.io import tallyman_read_csv -schema = ibis.schema({{"marker": "int64", "pad": "string"}}) -expr = tallyman_read_csv({str(csv_path)!r}, schema=schema) -""" +def _import_big(project: str, csv_path: Path, alias: str = "big_csv") -> dict: + import xorq.vendor.ibis as ibis + + schema = ibis.schema({"marker": "int64", "pad": "string"}) + return update_and_depend(csv_path, alias, project=project, schema=schema) -def test_tallyman_read_csv_preserves_file_order_under_repartition(project, repartitioned_csv, monkeypatch): - """__row_order must equal true file order even when the scan repartitions. +def test_tallyman_read_csv_preserves_file_order_under_repartition(project, repartitioned_csv): + """__row_order must equal true file order even when a scan of this size repartitions. Regression for the canonical-ordering bug: a bare ``ibis.row_number()`` over a repartitioned datafusion CSV scan numbers rows in nondeterministic arrival - order. With the marker column = source row index, the ordered copy's row at + order. With the marker column = source row index, the snapshot's row at ``__row_order == k`` must carry ``marker == k`` (polars numbers the rows in file order, ADR-008 D2). """ - import pyarrow.parquet as pq - - monkeypatch.setenv("TALLYMAN_PROJECT", project) csv_path, n = repartitioned_csv - res = catalog_create("big_csv", _big_read_csv_code(csv_path)) - assert "error" not in res, res + out = _import_big(project, csv_path) - copies = _ordered_copies(project) - assert len(copies) == 1, f"expected one ordered copy under compute_cache/ordered_sources, found {copies}" - table = pq.read_table(str(copies[0])) + table = _snapshot_table(project, out["hash"]) assert table.column("__row_order").to_pylist() == list(range(n)), "__row_order must be 0..N-1, contiguous" marker = table.column("marker").to_pylist() # The crux: row k of the source file must land at __row_order == k. @@ -194,129 +153,110 @@ def test_tallyman_read_csv_preserves_file_order_under_repartition(project, repar ) -def test_ordered_copy_digest_stable_under_repartition(project, repartitioned_csv, monkeypatch): - """A re-created ordered copy of a repartitioned CSV has the digest recorded when it was first written.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) +def test_source_snapshot_digest_stable_under_repartition(project, repartitioned_csv): + """A re-created snapshot of a repartitioned CSV has the digest recorded when it was first written.""" csv_path, _ = repartitioned_csv - res = catalog_create("big_csv", _big_read_csv_code(csv_path)) - assert "error" not in res, res - h = _hash_of(project) - ((key, record),) = read_manifest(entry_dir(project, h)).ordered_copies.items() - recorded = record["content_digest"] + out = _import_big(project, csv_path) + h = out["hash"] + recorded = read_manifest(entry_dir(project, h)).result_digest - copy = compute_cache_dir(project) / "ordered_sources" / f"{key}.parquet" - assert copy.exists() - copy.unlink() + snapshot = snapshot_path(project, h) + assert snapshot.exists() + snapshot.unlink() cached_result_expr.cache_clear() - cached_result_expr(project, h) # re-created from the clone by the read + ensure_materialized(project, h) # parsed again from the clone, by polars, with the recorded reader options - assert copy.exists() - assert snapshot_file_digest(copy) == recorded, ( - "the ordered copy's digest drifted across two independent ingests of a repartitioned CSV" + assert snapshot.exists() + assert snapshot_file_digest(snapshot) == recorded, ( + "the snapshot's digest drifted across two independent parses of a repartitioned CSV" ) -def test_tallyman_read_csv_reconstructs_after_source_deleted(project, sample_csv, monkeypatch): - """#6: once the ordered copy is written, reading the entry must not touch the CSV. +def test_tallyman_read_csv_reconstructs_after_source_deleted(project, sample_csv): + """#6: once the import has run, reading the entry must not touch the CSV. - The CSV is read exactly once — at ingest. After that, deleting (or moving) the source CSV must not break the - entry: it is a cheap read of the ordered copy, so there is no snapshot to resolve and no digest to verify. + The CSV is read exactly once, at import. After that, deleting (or moving) it must not break the entry: the rows + are in the entry's own snapshot, and the bytes they were parsed from are in the clone under ``data/.cas``. """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) - res = catalog_create("csv_entry", _read_csv_code(sample_csv)) - assert "error" not in res, res - h = _hash_of(project) - assert baked_snapshot_path(project, h) is None, "a CSV root is a cheap entry: nothing is baked" + out = _import_sample(project, sample_csv) + h = out["hash"] - # Delete the source CSV; the ordered copy remains. + # Delete the source CSV; the snapshot and the clone remain. sample_csv.unlink() cached_result_expr.cache_clear() df = cached_result_expr(project, h).execute() assert df["id"].tolist() == [3, 1, 2], "the entry must read without the source CSV" - assert verify_result_faithful(project, h) is None, "a cheap entry records no digest to verify" + assert verify_result_faithful(project, h) is True, "the snapshot still matches the digest recorded at import" -def test_ordered_csv_copy_lives_in_the_project_compute_cache(project, sample_csv, monkeypatch): - """The ordered copy is cache, so it lives under the project's compute_cache (ADR-007 D13). +def test_a_csv_source_snapshot_lives_in_the_project_compute_cache(project, sample_csv): + """The snapshot is cache, so it lives under the project's compute_cache (ADR-007 D13, ADR-011 D1). - It used to live under TALLYMAN_HOME/csv_ordered, outside every project: never collected, not packed, and - outside the project root, so a CSV entry's build was not portable. + The ordered copy it replaces started under TALLYMAN_HOME/csv_ordered, outside every project: never collected, + not packed, and outside the project root, so a CSV entry's build was not portable. ADR-007 D13 moved it into + ``compute_cache/ordered_sources``; ADR-011 D1 folded that second store into the entry's own snapshot. """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) - res = catalog_create("csv_entry", _read_csv_code(sample_csv)) - assert "error" not in res, res + out = _import_sample(project, sample_csv) - assert _ordered_copies(project), "the ordered copy must live under compute_cache/ordered_sources" - assert not list((tallyman_home() / "csv_ordered").glob("*.parquet")), "csv_ordered under TALLYMAN_HOME is retired" + snapshot = snapshot_path(project, out["hash"]) + assert snapshot.is_file(), f"the snapshot must be under the project's compute cache: {snapshot}" + assert snapshot.parent.parent == compute_cache_dir(project) + assert not (compute_cache_dir(project) / "ordered_sources").exists(), "the second source store is retired" + assert not (tallyman_home() / "csv_ordered").exists(), "csv_ordered under TALLYMAN_HOME is retired" -def test_tallyman_read_csv_forwards_reader_kwargs(project, monkeypatch): +def test_tallyman_read_csv_forwards_reader_kwargs(project): """#10: reader options (separator, skip_rows, ...) are forwarded to polars scan_csv. - The rewrite dropped **kwargs; a documented call like tallyman_read_csv(path, schema, - separator=';') must parse the alternate delimiter instead of raising TypeError. + They are named in the import call now and recorded on the entry (ADR-011 D12), so a documented + ``separator=';'`` must parse the alternate delimiter instead of raising TypeError. """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "semi.csv" p.write_text("id;name\n3;charlie\n1;alice\n2;bob\n") - code = f""" -import xorq.vendor.ibis as ibis -from tallyman_xorq.io import tallyman_read_csv -schema = ibis.schema({{"id": "int64", "name": "string"}}) -expr = tallyman_read_csv({str(p)!r}, schema=schema, separator=";") -""" - res = catalog_create("semi", code) - assert "error" not in res, res - fields = {f["name"] for f in res["schema"]["fields"]} - assert {"id", "name", "__row_order"} <= fields, ( - f"separator=';' not forwarded — columns did not split: {fields}" + + out = update_and_depend(p, "semi_src", project=project, schema={"id": "int64", "name": "string"}, separator=";") + + names = [f["name"] for f in out["schema"]["fields"]] + assert names == ["id", "name", "__row_order"], ( + f"separator=';' not forwarded — columns did not split: {names}" ) + assert _snapshot_table(project, out["hash"])["name"].to_pylist() == ["charlie", "alice", "bob"] # --------------------------------------------------------------------------- # # The reserved name is now the exact string '__row_order' (ADR-008 D6). A CSV that already # has a column of that name (a file tallyman exported) has it overwritten, with no -# validation of its values: the ordered copy numbers the rows in file order (ADR-008 D2). +# validation of its values: the import numbers the rows in file order (ADR-008 D2). # 'original_row_order' is not special any more: it is ordinary data. # --------------------------------------------------------------------------- # -def test_existing_row_order_column_is_overwritten(project, monkeypatch): +def test_existing_row_order_column_is_overwritten(project): """A CSV that already carries a '__row_order' column ingests with it overwritten by 0..N-1 in file order, whatever its values were — no error, no validation, and still exactly one row-order column (last).""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "hasorder.csv" # Rows deliberately NOT sorted by name; the incoming __row_order values are not the file sequence. p.write_text("__row_order,name\n7,charlie\n3,alice\n5,bob\n") - code = f""" -from tallyman_xorq.io import tallyman_read_csv -expr = tallyman_read_csv({str(p)!r}) -""" - res = catalog_create("hasorder", code) - assert "error" not in res, res - h = _hash_of(project) - df = cached_result_expr(project, h).execute() + + out = update_and_depend(p, "hasorder", project=project) + + df = cached_result_expr(project, out["hash"]).execute() assert list(df.columns) == ["name", "__row_order"] assert df["__row_order"].tolist() == [0, 1, 2] assert df["name"].tolist() == ["charlie", "alice", "bob"] -def test_existing_row_order_column_with_explicit_schema_accepted(project, monkeypatch): +def test_existing_row_order_column_with_explicit_schema_accepted(project): """A CSV carrying a '__row_order' column plus an explicit schema for its DATA columns must ingest. The reserved column is tallyman's, not the caller's to spec, so the totality check must not demand it be named.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "orderschema.csv" # __row_order present; rows deliberately NOT sorted by name. p.write_text("__row_order,id,name\n7,1,charlie\n3,2,alice\n5,3,bob\n") - code = f""" -from tallyman_xorq.io import tallyman_read_csv -expr = tallyman_read_csv({str(p)!r}, schema={{"id": "int64", "name": "string"}}) -""" - res = catalog_create("orderschema", code) - assert "error" not in res, res - h = _hash_of(project) - df = cached_result_expr(project, h).execute() + + out = update_and_depend(p, "orderschema", project=project, schema={"id": "int64", "name": "string"}) + + df = cached_result_expr(project, out["hash"]).execute() assert df["__row_order"].tolist() == [0, 1, 2] assert df["id"].tolist() == [1, 2, 3] assert df["name"].tolist() == ["charlie", "alice", "bob"] @@ -329,21 +269,17 @@ def test_existing_row_order_column_with_explicit_schema_accepted(project, monkey pytest.param("a,alice\nb,bob\n", ["a", "b"], id="not integers"), ], ) -def test_original_row_order_is_ordinary_data(project, monkeypatch, body, values): +def test_original_row_order_is_ordinary_data(project, body, values): """'original_row_order' is not reserved any more: ADR-008 D6 reserves only the exact name '__row_order'. Before ADR-008 a CSV with such a column raised unless it was the canonical 0..N-1 sequence. """ - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "oro.csv" p.write_text("original_row_order,name\n" + body) - code = f""" -from tallyman_xorq.io import tallyman_read_csv -expr = tallyman_read_csv({str(p)!r}) -""" - res = catalog_create("oro", code) - assert "error" not in res, res - df = cached_result_expr(project, _hash_of(project)).execute() + + out = update_and_depend(p, "oro", project=project) + + df = cached_result_expr(project, out["hash"]).execute() assert df["original_row_order"].tolist() == values, "an ordinary column keeps its values" assert df["__row_order"].tolist() == list(range(len(values))) @@ -355,17 +291,14 @@ def test_original_row_order_is_ordinary_data(project, monkeypatch, body, values) (("a", "int64"), ("__row_order", "int64")), # positional, second column ], ) -def test_schema_output_name_row_order_raises(project, monkeypatch, schema): +def test_schema_output_name_row_order_raises(project, schema): """A schema that maps a DATA column onto the reserved '__row_order' output name collides with tallyman's - row index: assigning to the column is not allowed (ADR-008 D6). Ingest must reject the reserved output + row index: assigning to the column is not allowed (ADR-008 D6). The import must reject the reserved output name with a clear ValueError.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - from tallyman_xorq.io import tallyman_read_csv - p = data_dir(project) / "renameoro.csv" p.write_text("a,b\n1,10\n2,20\n") with pytest.raises(ValueError, match="__row_order"): - tallyman_read_csv(str(p), schema=schema) + update_and_depend(p, "renameoro", project=project, schema=schema) # --------------------------------------------------------------------------- # @@ -380,33 +313,41 @@ def test_schema_output_name_row_order_raises(project, monkeypatch, schema): # 2. schema-error diagnostics must not hide the reserved column, and # 3. positional binding must not silently rebind onto the wrong data column. # --------------------------------------------------------------------------- # -def test_suggested_schema_recovery_is_pasteable_with_reserved_column(project, monkeypatch): +def test_suggested_schema_recovery_is_pasteable_with_reserved_column(project): """#143 recovery contract, reserved-column path: when an explicit schema fails to parse a CSV that carries a '__row_order' column, the suggested schema in the error must be paste-ready. The whole-file suggestion must NOT emit a cell for the reserved column (which the caller cannot spec) — - pasting the suggestion back would otherwise over-count the columns.""" + pasting the suggestion back would otherwise over-count the columns. + + The retry also pins that the failed import left nothing behind: the alias is still unclaimed and no entry was + written, so the second call mints v1 exactly as the first would have. + """ import ast - monkeypatch.setenv("TALLYMAN_PROJECT", project) - from tallyman_xorq.io import tallyman_read_csv + from tallyman_core.aliases import alias_kind + from tallyman_xorq.build import list_entries p = data_dir(project) / "suggest_oro.csv" # __row_order present (as tallyman exports it); the amount column holds a float # that an int64 pin cannot parse, so the explicit-mode suggestion path fires. p.write_text("id,amount,__row_order\n1,12.5,0\n2,3.0,1\n") with pytest.raises(ValueError) as exc: - tallyman_read_csv(str(p), schema=(("id", "int64"), ("amount", "int64"))) + update_and_depend(p, "suggest_oro", project=project, schema=(("id", "int64"), ("amount", "int64"))) msg = str(exc.value) assert "Suggested schema" in msg, msg assert "__row_order" not in msg, f"suggestion leaked the reserved column: {msg}" + assert alias_kind(project, "suggest_oro") is None, "a failed import must not claim the alias" + assert list_entries(project) == [], "a failed import must not leave a half-written entry" + # The suggestion must paste back and parse. suggested = ast.literal_eval(msg.rsplit("schema=", 1)[-1].strip()) - expr = tallyman_read_csv(str(p), schema=suggested) - out = {k: str(v) for k, v in expr.schema().items()} - assert out.get("amount") == "float64", out - assert "__row_order" in out # the reserved column is still there, numbered by tallyman + out = update_and_depend(p, "suggest_oro", project=project, schema=suggested) + assert (out["version"], out["created"]) == (1, True) + types = {name: str(dtype) for name, dtype in cached_result_expr(project, out["hash"]).schema().items()} + assert types.get("amount") == "float64", types + assert "__row_order" in types # the reserved column is still there, numbered by tallyman @pytest.mark.parametrize( @@ -417,36 +358,30 @@ def test_suggested_schema_recovery_is_pasteable_with_reserved_column(project, mo pytest.param({"zzz": "int64", "&rest": "infer"}, id="by-name miss (a guard)"), ], ) -def test_schema_error_diagnostic_names_reserved_column(project, monkeypatch, schema): +def test_schema_error_diagnostic_names_reserved_column(project, schema): """A schema error against a CSV that carries '__row_order' must not hide that column. The diagnostic must not print the reserved-stripped header: a 3-column file (a,b,__row_order) reported as 'has 2 [a, b]' is an off-by-one that the user, staring at a 3-column file, cannot reconcile. The reserved column must appear in the message.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - from tallyman_xorq.io import tallyman_read_csv - p = data_dir(project) / "diag_oro.csv" p.write_text("a,b,__row_order\n1,2,0\n3,4,1\n") with pytest.raises(ValueError) as exc: - tallyman_read_csv(str(p), schema=schema) + update_and_depend(p, "diag_oro", project=project, schema=schema) assert "__row_order" in str(exc.value), ( f"diagnostic hides the reserved column: {exc.value}" ) -def test_positional_schema_rejects_nontrailing_reserved_column(project, monkeypatch): +def test_positional_schema_rejects_nontrailing_reserved_column(project): """Positional cells bind by physical column position. A '__row_order' column that is NOT the last column shifts that mapping — excluding it from the middle silently rebinds later cells onto the wrong data - column (renaming/dropping a column with no error). Ingest must reject a + column (renaming/dropping a column with no error). The import must reject a positional schema in this layout rather than silently corrupt the output.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - from tallyman_xorq.io import tallyman_read_csv - p = data_dir(project) / "oro_middle.csv" # __row_order sits in the MIDDLE, not trailing. p.write_text("sku,__row_order,qty\nA,0,10\nB,1,20\n") with pytest.raises(ValueError) as exc: - tallyman_read_csv(str(p), schema=(("sku", "string"), ("row", "int64"))) + update_and_depend(p, "oro_middle", project=project, schema=(("sku", "string"), ("row", "int64"))) msg = str(exc.value).lower() assert "__row_order" in msg and ("position" in msg or "last" in msg or "by-name" in msg), ( f"expected a positional/last-column guard message; got: {exc.value}" diff --git a/tests/test_tallyman_read_csv_contract.py b/tests/test_tallyman_read_csv_contract.py index cbd76e44..6f8f8e82 100644 --- a/tests/test_tallyman_read_csv_contract.py +++ b/tests/test_tallyman_read_csv_contract.py @@ -1,4 +1,4 @@ -"""Contract tests for tallyman_read_csv — the intelligent-import redesign. +"""Contract tests for tallyman's CSV reader — the intelligent-import redesign. ADR plans/ADR-005-intelligent-csv-import.md. Covers the #137-review cluster: @@ -10,7 +10,11 @@ - #143 — no-schema inference escalates past the default window; an explicit pinned type that can't parse raises with a paste-ready schema suggestion. -These are RED on the pre-fix tree (seen failing on CI), green after the fixes. +ADR-011 (plans/ADR-011-sources-are-aliases.md) moved the reader out of the recipe: the DSL, the ladder and the +error messages are unchanged, but they run when the file is imported +(``update_and_depend(path, alias, schema=..., **reader_options)``), not when a build executes. So a schema error is +raised by the import call rather than returned as a ``catalog_create`` error, and the types a spec produces are read +off the source entry the import minted. """ from __future__ import annotations @@ -19,7 +23,7 @@ import pytest from tallyman_core import data_dir -from tallyman_mcp.server import catalog_create +from tallyman_xorq.source_import import SourceImportError, update_and_depend # --------------------------------------------------------------------------- # @@ -75,94 +79,84 @@ def test_polars_overrides_preserves_timestamp_tz_and_precision(): # --------------------------------------------------------------------------- # # #141 — schema spec contract # --------------------------------------------------------------------------- # -def _types_of(res: dict) -> dict[str, str]: - return {f["name"]: f["type"] for f in res["schema"]["fields"]} +def _types_of(project: str, out: dict) -> dict[str, str]: + """The column types a recipe reading this source alias sees, by name. + The entry's recorded schema is arrow's spelling of the same types (``large_string`` where polars wrote a + string, ``double`` where it wrote a float); this reads the ibis view a recipe gets, which is what a schema + spec is written against. + """ + from tallyman_xorq.result_cache import cached_result_expr -def _build(name: str, csv: Path, schema_literal: str) -> dict: - code = f""" -from tallyman_xorq.io import tallyman_read_csv -expr = tallyman_read_csv({str(csv)!r}, schema={schema_literal}) -""" - return catalog_create(name, code) + return {name: str(dtype) for name, dtype in cached_result_expr(project, out["hash"]).schema().items()} + + +def _import(project: str, alias: str, csv: Path, schema=None, **reader_options) -> dict: + return update_and_depend(csv, alias, project=project, schema=schema, **reader_options) -def test_schema_as_plain_dict_binds_by_name(project, monkeypatch): +def test_schema_as_plain_dict_binds_by_name(project): """A plain dict (not just an ibis schema) binds by header name.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "dict.csv" p.write_text("id,name\n1,alice\n2,bob\n") - res = _build("dict_named", p, '{"id": "int64", "name": "string"}') - assert "error" not in res, res - types = _types_of(res) + out = _import(project, "dict_named", p, {"id": "int64", "name": "string"}) + types = _types_of(project, out) assert types["id"] == "int64" assert types["name"] == "string" -def test_schema_name_not_in_header_raises_listed_suggestion(project, monkeypatch): +def test_schema_name_not_in_header_raises_listed_suggestion(project): """#141: a by-name schema whose name is absent from the header raises a listed, actionable error steering toward the positional tuple form.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) + import xorq.vendor.ibis as ibis + p = data_dir(project) / "yf.csv" p.write_text("Price,Close\n2020-01-01,10.0\n2020-01-02,11.0\n") - code = f""" -import xorq.vendor.ibis as ibis -from tallyman_xorq.io import tallyman_read_csv -schema = ibis.schema({{"Date": "date", "Close": "float64"}}) -expr = tallyman_read_csv({str(p)!r}, schema=schema) -""" - res = catalog_create("yf_named", code) - assert "error" in res - err = res["error"] + with pytest.raises(ValueError) as exc: + _import(project, "yf_named", p, ibis.schema({"Date": "date", "Close": "float64"})) + err = str(exc.value) assert "Price" in err # the actual header is listed assert "&rest" in err # steered toward the positional tuple form (tallyman phrasing) -def test_tuple_schema_renames_by_position(project, monkeypatch): +def test_tuple_schema_renames_by_position(project): """#141: tuple-of-tuples binds by position — the yfinance Price->Date rename.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "yf2.csv" p.write_text("Price,Close\n2020-01-01,10.0\n2020-01-02,11.0\n") - res = _build("yf_pos", p, '(("Date", "date"), ("Close", "float64"))') - assert "error" not in res, res - types = _types_of(res) + out = _import(project, "yf_pos", p, (("Date", "date"), ("Close", "float64"))) + types = _types_of(project, out) assert "Date" in types and "Price" not in types # col 0 renamed positionally - assert types["Date"].startswith("date") # date32[day] in the serialized schema + assert types["Date"].startswith("date") -def test_tuple_schema_rest_infers_tail(project, monkeypatch): +def test_tuple_schema_rest_infers_tail(project): """#141: ('&rest', 'infer') keeps the tail's names and infers their types.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "rest.csv" p.write_text("a,b,c\n2020-01-01,5,xy\n2020-01-02,6,zz\n") - res = _build("rest_tail", p, '(("when", "date"), ("&rest", "infer"))') - assert "error" not in res, res - types = _types_of(res) - assert types["when"].startswith("date") # col 0 renamed + pinned (date32[day]) + out = _import(project, "rest_tail", p, (("when", "date"), ("&rest", "infer"))) + types = _types_of(project, out) + assert types["when"].startswith("date") # col 0 renamed + pinned assert types["b"] == "int64" # tail kept name, inferred int assert types["c"] == "string" # tail kept name, inferred string -def test_dict_schema_rest_infers_others(project, monkeypatch): +def test_dict_schema_rest_infers_others(project): """#141: dict '&rest' pins some columns by name and infers the rest.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "drest.csv" p.write_text("id,amount\n01,5\n02,6\n") - res = _build("drest", p, '{"id": "string", "&rest": "infer"}') - assert "error" not in res, res - types = _types_of(res) + out = _import(project, "drest", p, {"id": "string", "&rest": "infer"}) + types = _types_of(project, out) assert types["id"] == "string" # pinned string keeps leading zeros assert types["amount"] == "int64" # inferred -def test_partial_spec_without_rest_raises(project, monkeypatch): +def test_partial_spec_without_rest_raises(project): """#141: a partial spec with no wildcard is non-total and must raise.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "partial.csv" p.write_text("a,b,c\n1,2,3\n4,5,6\n") - res = _build("partial", p, '(("a", "int64"), ("b", "int64"))') - assert "error" in res - err = res["error"] + with pytest.raises(ValueError) as exc: + _import(project, "partial", p, (("a", "int64"), ("b", "int64"))) + err = str(exc.value) assert "&rest" in err or "total" in err.lower() @@ -183,33 +177,20 @@ def late_poison_csv(project: str) -> Path: return p -def test_no_schema_escalates_past_default_window(project, late_poison_csv, monkeypatch): +def test_no_schema_escalates_past_default_window(project, late_poison_csv): """#143: no-schema inference escalates the window and resolves the messy tail.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - code = f""" -from tallyman_xorq.io import tallyman_read_csv -expr = tallyman_read_csv({str(late_poison_csv)!r}) -""" - res = catalog_create("escal", code) - assert "error" not in res, res - types = _types_of(res) + out = _import(project, "escal", late_poison_csv) + types = _types_of(project, out) assert types["v"] == "string" # whole-file infer fell it back to string -def test_explicit_type_failure_suggests_schema(project, late_poison_csv, monkeypatch): +def test_explicit_type_failure_suggests_schema(project, late_poison_csv): """#143: a pinned int64 that can't parse raises with a paste-ready suggestion.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - code = f""" -import xorq.vendor.ibis as ibis -from tallyman_xorq.io import tallyman_read_csv -schema = ibis.schema({{"k": "int64", "v": "int64"}}) -expr = tallyman_read_csv({str(late_poison_csv)!r}, schema=schema) -""" - res = catalog_create("explicit_fail", code) - assert "error" in res - err = res["error"] - # tallyman emits a paste-ready suggestion (absent pre-fix; "schema=" alone - # would false-match the recipe code echoed in the traceback). + import xorq.vendor.ibis as ibis + + with pytest.raises(ValueError) as exc: + _import(project, "explicit_fail", late_poison_csv, ibis.schema({"k": "int64", "v": "int64"})) + err = str(exc.value) assert "suggested schema" in err.lower() assert "'v'" in err or '"v"' in err # names the failing column assert "string" in err # suggests string for the unparseable column @@ -219,33 +200,28 @@ def test_explicit_type_failure_suggests_schema(project, late_poison_csv, monkeyp # #148-review — reserved scan kwargs must not collide with the internal ones # --------------------------------------------------------------------------- # @pytest.mark.parametrize("bad_kwarg", ["infer_schema_length", "schema_overrides"]) -def test_reserved_scan_kwarg_raises_clear_error(project, monkeypatch, bad_kwarg): +def test_reserved_scan_kwarg_raises_clear_error(project, bad_kwarg): """``infer_schema_length`` and ``schema_overrides`` are managed internally — the former by the escalation ladder, the latter by the ``schema=`` parameter — so - forwarding one as a ``**kwargs`` reader option must raise a clear ValueError, not + forwarding one as a reader option must raise a clear SourceImportError, not the raw polars ``TypeError: got multiple values for keyword argument`` (or, for - ``schema_overrides`` with no schema, silently bypass the schema system).""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) - from tallyman_xorq.io import tallyman_read_csv + ``schema_overrides`` with no schema, silently bypass the schema system). + The import decides the reader (``source_import._reader_for``), so that is where the guard has to be: it must + reject the call before any bytes are copied into the arena. + """ p = data_dir(project) / "kw.csv" p.write_text("a,b\n1,x\n2,y\n") kwargs = {bad_kwarg: 1000 if bad_kwarg == "infer_schema_length" else {"a": "int64"}} - with pytest.raises(ValueError, match="managed internally"): - tallyman_read_csv(str(p), **kwargs) + with pytest.raises(SourceImportError, match="managed internally"): + _import(project, "kw_src", p, **kwargs) -def test_ordinary_reader_kwarg_still_forwarded(project, monkeypatch): +def test_ordinary_reader_kwarg_still_forwarded(project): """The guard must reject only the two reserved keys — a genuine reader option such as ``separator`` still reaches polars.scan_csv and parses correctly.""" - monkeypatch.setenv("TALLYMAN_PROJECT", project) p = data_dir(project) / "semi.csv" p.write_text("a;b\n1;x\n2;y\n") - code = f""" -from tallyman_xorq.io import tallyman_read_csv -expr = tallyman_read_csv({str(p)!r}, separator=";") -""" - res = catalog_create("semicsv", code) - assert "error" not in res, res - types = _types_of(res) + out = _import(project, "semicsv", p, separator=";") + types = _types_of(project, out) assert "a" in types and "b" in types # split into two columns, not one "a;b" diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 629c2e4d..d8bf2d17 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -17,7 +17,6 @@ import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path from typing import Any import httpx @@ -139,7 +138,7 @@ def test_endpoint_read_empty_trace(project: str, fresh_companion_app): assert got["spans"] == [] -def test_ingest_does_not_checkpoint_the_catalog(project: str, orders_parquet, fresh_companion_app): +def test_ingest_does_not_checkpoint_the_catalog(project: str, orders_src: str, fresh_companion_app): """A telemetry POST must be exempt from the dispatch-boundary catalog checkpoint. The middleware checkpoints every non-GET project route by default (so new authored-state routes can't silently skip it), but spans @@ -150,7 +149,7 @@ def test_ingest_does_not_checkpoint_the_catalog(project: str, orders_parquet, fr # Build one entry so the project has a catalog with a real step counter. from tallyman_xorq import build_and_persist - build_and_persist(project, _code_for(project)) + build_and_persist(project, _code_for(project, orders_src)) c = TestClient(fresh_companion_app) before = current_step(project) @@ -188,33 +187,33 @@ def fake_post(url, json=None, timeout=None): return mgr, captured -def test_load_session_includes_telemetry_url_when_companion_known(project, orders_parquet, monkeypatch): +def test_load_session_includes_telemetry_url_when_companion_known(project, orders_src, monkeypatch): from tallyman_xorq import build_and_persist - res = build_and_persist(project, _code_for(project)) + res = build_and_persist(project, _code_for(project, orders_src)) mgr, captured = _running_manager(monkeypatch, companion_base_url="http://127.0.0.1:7860/") mgr.load_session(res.content_hash, project) # Trailing slash on the base url is normalized; the path is per-project. assert captured["json"]["telemetry_url"] == f"http://127.0.0.1:7860/{project}/api/telemetry" -def test_load_session_omits_telemetry_url_when_companion_unknown(project, orders_parquet, monkeypatch): +def test_load_session_omits_telemetry_url_when_companion_unknown(project, orders_src, monkeypatch): from tallyman_xorq import build_and_persist - res = build_and_persist(project, _code_for(project)) + res = build_and_persist(project, _code_for(project, orders_src)) mgr, captured = _running_manager(monkeypatch) # no companion_base_url mgr.load_session(res.content_hash, project) assert "telemetry_url" not in captured["json"] -def test_grid_load_event_records_session_id_for_join(project, orders_parquet, monkeypatch): +def test_grid_load_event_records_session_id_for_join(project, orders_src, monkeypatch): """The buckaroo activity event must carry the session_id so the Log UI can join it to the firstpull.* spans (keyed by that id as their `trace`).""" from tallyman_companion import create_app from tallyman_core.events import read_events from tallyman_xorq import build_and_persist - res = build_and_persist(project, _code_for(project)) + res = build_and_persist(project, _code_for(project, orders_src)) class _StubBuckaroo: bound_port = 8700 @@ -232,10 +231,10 @@ def load_session(self, content_hash, project, column_config_overrides=None): assert evs and evs[0]["session_id"] == "sess-join" -def _code_for(project: str) -> str: +def _code_for(project: str, src: str) -> str: return f""" -from tallyman_xorq.io import read_project_file -t = read_project_file("orders.parquet", project={project!r}) +from tallyman_xorq.io import tracked_expr_from_alias +t = tracked_expr_from_alias({src!r}, project={project!r}) expr = t.group_by("region").aggregate(n=t.count()) """ @@ -267,7 +266,7 @@ def log_message(self, *a): # silence the default stderr access log @pytest.mark.integration -def test_integration_buckaroo_posts_firstpull_spans(project: str, orders_parquet: Path): +def test_integration_buckaroo_posts_firstpull_spans(project: str, orders_src: str): """End-to-end: a real buckaroo 0.15.3 server, told a ``telemetry_url`` on ``/load_expr``, fire-and-forget POSTs ``firstpull.*`` perf spans back in the OTel-shaped record this module reads (``{trace, source, name, t_start_ms, @@ -276,7 +275,7 @@ def test_integration_buckaroo_posts_firstpull_spans(project: str, orders_parquet (summary_stats, ws_first_payload) need a widget and aren't asserted.""" from tallyman_xorq import build_and_persist - res = build_and_persist(project, _code_for(project)) + res = build_and_persist(project, _code_for(project, orders_src)) _CaptureHandler.captured = [] server = ThreadingHTTPServer(("127.0.0.1", 0), _CaptureHandler)