Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
230110c
docs: plan native tallyman_core.catalog (drop xorq catalog package)
paddymul Jun 16, 2026
1f5034b
docs: refine native-catalog-store plan from review pass
paddymul Jun 16, 2026
08efa47
docs: fix connect/notebook/aliases/line-num/zip-cost findings in nati…
paddymul Jun 16, 2026
83dcf72
docs: propagate notebook relocation + fix citations (verification pass)
paddymul Jun 16, 2026
3126126
docs: be honest about what dropping the entry wheel forfeits
paddymul Jun 16, 2026
f24e761
docs: per-catalog-repo reproducibility via a tracked pyproject.toml +…
paddymul Jun 17, 2026
7689a05
docs: cross-link #106 (runtime per-project venv isolation) as deferre…
paddymul Jun 17, 2026
ba2d942
docs: record why expr.py stays durable (chaining needs a shared backend)
paddymul Jun 17, 2026
cb44fb2
chore: gitignore .claude/ and .claude-worktrees/ (multi-GB local tool…
paddymul Jun 17, 2026
c3b38de
docs: pin the missing test-plan coverage in native-catalog-store
paddymul Jun 17, 2026
411f701
test: native catalog store — no xorq subprocess, checkpoint owns the …
paddymul Jun 17, 2026
6c82866
feat: native content-addressed recipe store; checkpoint owns the zip …
paddymul Jun 17, 2026
ec0320d
refactor: decompose catalog.yaml into tracked files (native store, #52)
paddymul Jun 17, 2026
863a450
refactor: Cut B — narrow catalog-free imports + datafusion connect shim
paddymul Jun 17, 2026
356ee1c
chore: sweep stale xorq-catalog docs + dead code after the native cut
paddymul Jun 17, 2026
62fe9d0
test: pin the decomposed tracked tree end to end; mark the plan imple…
paddymul Jun 17, 2026
778c55f
test: native catalog store integrity + warm-evict self-heal (failing-…
paddymul Jun 17, 2026
a0d639f
fix: close three tracked-surface integrity holes in the native store
paddymul Jun 17, 2026
ee0a90a
fix: cached_result_expr re-checks the baked snapshot on every read
paddymul Jun 17, 2026
e029657
feat: rebuild a project's catalog into the native store by re-execing…
paddymul Jun 17, 2026
b51216f
test: read aliases from the native aliases.jsonl in the Tier-B perf h…
paddymul Jun 17, 2026
acaa031
docs: record the bug-hunt hardening pass + close the rebuild-mechanis…
paddymul Jun 17, 2026
40f09aa
feat: rebuild handles catalog.yaml-era corpora, self-chaining revises…
paddymul Jun 17, 2026
b7acc10
test: failing tests for PR #103 review findings (M1/L1, L4, L6)
paddymul Jun 17, 2026
e4e5527
fix: address PR #103 review — consistency guard, atomic writes, cleanup
paddymul Jun 17, 2026
4f7f78b
test: cheap build must validate the row-level projection (failing-first)
paddymul Jun 17, 2026
8e686f8
fix: cheap build validates row-level ops; doc-nit cleanup
paddymul Jun 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,6 @@ catalog/
projects/
# cache lab measurement reports (tests/test_cache_lab.py)
tests/cache_lab_reports/
# Claude Code local tooling state + agent worktrees (multi-GB; never commit)
.claude/
.claude-worktrees/
615 changes: 615 additions & 0 deletions plans/native-catalog-store.md

Large diffs are not rendered by default.

375 changes: 375 additions & 0 deletions scripts/rebuild_native_catalog.py

Large diffs are not rendered by default.

19 changes: 10 additions & 9 deletions src/tallyman_companion/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ def _build_compare_expr(project: str, a_hash: str, b_hash: str, keys: tuple[str,
"""
import tempfile

import xorq.api as xo
from xorq.ibis_yaml.compiler import build_expr

a = cached_result_expr(project, a_hash)
b = cached_result_expr(project, b_hash)
Expand All @@ -211,7 +211,7 @@ def _build_compare_expr(project: str, a_hash: str, b_hash: str, keys: tuple[str,

builds_dir = Path(tempfile.gettempdir()) / "tallyman_diff_builds"
builds_dir.mkdir(parents=True, exist_ok=True)
build_path = Path(xo.build_expr(expr, builds_dir=str(builds_dir)))
build_path = Path(build_expr(expr, builds_dir=str(builds_dir)))

return build_path, overrides

Expand Down Expand Up @@ -569,7 +569,7 @@ def api_entry_detail(project: str, content_hash: str):
for i, h in enumerate(hist_hashes, start=1):
forensic_history.append({"hash": h, "version": i, "is_current": h == content_hash})

prompt_history = read_prompts(entry)
prompt_history = read_prompts(project, content_hash)
chart_spec = get_chart(project, content_hash)

build_artifacts: list[dict] = []
Expand Down Expand Up @@ -1099,12 +1099,13 @@ async def api_delete_result_cache(project: str, content_hash: str):
p.unlink()
except OSError as exc:
raise HTTPException(500, str(exc))
# Drop the read-path memo so the snapshot actually self-heals on next read.
# A warm cached_result_expr holds a deferred_read of the file just unlinked;
# left in place, the next viewer read (api_data) — and ensure_session's
# pre-/load_expr heal — would dereference a missing file instead of
# re-baking. lru_cache has no per-key eviction, and a full clear is cheap
# for an admin delete: entries are content-addressed, so warm exprs simply
# Defense-in-depth, not load-bearing for correctness: since ee0a90a,
# cached_result_expr is a thin non-memoised wrapper that re-checks
# path.exists() and self-heals on every read, so the next viewer read
# (api_data) / ensure_session heal re-bakes the evicted snapshot even with a
# warm memo left in place (proven by
# test_cached_result_expr_self_heals_after_warm_then_evict). We still clear
# it because it's cheap for an admin delete: content-addressed entries just
# re-reconstruct (and the evicted one re-bakes) on the next read.
cached_result_expr.cache_clear()
return {"ok": True, "hash": content_hash}
Expand Down
2 changes: 2 additions & 0 deletions src/tallyman_core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
write_display_klass,
)
from tallyman_core.errors import clear_errors, get_error, list_errors, record_error
from tallyman_core.fsutil import atomic_write_text
from tallyman_core.manifest import Manifest, read_manifest, write_manifest
from tallyman_core.marimo_export import export_notebook_path, notebook_to_marimo
from tallyman_core.notebook import CellNotFound
Expand Down Expand Up @@ -93,6 +94,7 @@
"write_post_processing",
"write_stat",
"alias_for_hash",
"atomic_write_text",
"artifacts_dir",
"catalog_dir",
"clear_errors",
Expand Down
68 changes: 43 additions & 25 deletions src/tallyman_core/aliases.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,13 @@
alias_map: {alias: latest_hash}
alias_history: {alias: [hash_v1, hash_v2, ...]} # oldest first

Both live as tallyman-owned keys in the catalog's ``catalog.yaml`` (read and
written through ``catalog_state``), not as separate files. catalog.yaml is the
versioned state carrier the catalog git repo tracks, so alias state clones and
versions with the catalog and ``reset_to``'s ``git reset`` rolls it back for
free — a sidecar store would have to be reconciled separately, and committing
one *inside* the repo broke xorq's ``assert_consistency`` (#48). xorq keeps its
own ``aliases`` key + alias symlinks in the same file; these tallyman keys sit
alongside, and xorq round-trips them untouched.
Both live in a tracked ``aliases.jsonl`` in the catalog repo — one line per
alias, ``{"alias", "latest", "history": [...]}``. The native store tracks the
file directly, so alias state clones and versions with the catalog and
``reset_to``'s ``git reset`` rolls it back with the rest of the tree (no
separate reconcile). Before the native cut this had to be smuggled into
catalog.yaml keys, because committing a separate file inside the (then xorq)
catalog repo broke its ``assert_consistency`` (#48); that constraint is gone.

Identity is the content hash; the alias is a *name* for a concept that may
evolve. The latest hash is always the current best answer; older hashes
Expand All @@ -21,8 +20,10 @@

from __future__ import annotations

from tallyman_core import xorq_catalog as _xcat
from tallyman_core.paths import ensure_project
import json

from tallyman_core.fsutil import atomic_write_text
from tallyman_core.paths import catalog_dir, ensure_project


class AliasExists(ValueError):
Expand All @@ -33,24 +34,44 @@ class AliasNotFound(KeyError):
pass


def _aliases_file(project: str):
return catalog_dir(project) / "aliases.jsonl"


def _read(project: str) -> tuple[dict[str, str], dict[str, list[str]]]:
"""Read the alias map + history out of catalog.yaml.
"""Read the alias map + history out of the tracked ``aliases.jsonl``.

Lazy import: ``aliases`` is imported very early in ``tallyman_core``'s
package init (before ``catalog_state``'s transitive deps), so resolving
``catalog_state`` at call time keeps that ordering clean.
One line per alias: ``{"alias", "latest", "history": [...]}``. The native
store tracks this file directly, so ``reset_to``'s ``git reset`` rolls alias
state back with the rest of the tree (no separate reconcile).
"""
from tallyman_core import catalog_state as cs

state = cs.read_tallyman_state(project)
return state["alias_map"], state["alias_history"]
p = _aliases_file(project)
alias_map: dict[str, str] = {}
history: dict[str, list[str]] = {}
if p.exists():
for line in p.read_text().splitlines():
if not line.strip():
continue
rec = json.loads(line)
alias_map[rec["alias"]] = rec["latest"]
history[rec["alias"]] = rec.get("history", [])
return alias_map, history


def _write(project: str, aliases: dict[str, str], history: dict[str, list[str]]) -> None:
"""Persist the alias map + history into catalog.yaml (preserving xorq's keys)."""
from tallyman_core import catalog_state as cs

cs.write_tallyman_state(project, alias_map=aliases, alias_history=history)
"""Persist the alias map + history to ``aliases.jsonl`` (sorted for a stable
diff; an empty map removes the file)."""
ensure_project(project)
p = _aliases_file(project)
p.parent.mkdir(parents=True, exist_ok=True)
if not aliases:
p.unlink(missing_ok=True)
return
body = "".join(
json.dumps({"alias": name, "latest": aliases[name], "history": history.get(name, [])}) + "\n"
for name in sorted(aliases)
)
atomic_write_text(p, body)


def load_aliases(project: str) -> dict[str, str]:
Expand Down Expand Up @@ -126,8 +147,6 @@ def set_alias(project: str, name: str, content_hash: str, *, expect_exists: bool

_write(project, aliases, history)

_xcat.add_alias(project, content_hash, alias=name)

return {
"name": name,
"hash": content_hash,
Expand All @@ -152,4 +171,3 @@ def remove_alias(project: str, name: str) -> None:
aliases.pop(name, None)
history.pop(name, None)
_write(project, aliases, history)
_xcat.remove_alias(project, name)
Loading
Loading