Finance reconciliation: reconcile-receipts skill + cron (closes unimatrix27/ideas#25)

finance/migrate.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Apply or roll back finance migrations.
+
+Usage:
+    SUPABASE_DB_URL=... python3 finance/migrate.py up           # apply every up migration in order
+    SUPABASE_DB_URL=... python3 finance/migrate.py down         # roll back every down migration in reverse order
+    SUPABASE_DB_URL=... python3 finance/migrate.py up 001       # apply just 001_*.up.sql
+    SUPABASE_DB_URL=... python3 finance/migrate.py down 002     # roll back just 002_*.down.sql
+
+Every SQL file is idempotent on re-apply (CREATE/DROP IF EXISTS) and wrapped in
+its own BEGIN/COMMIT, so partial states never linger.
+"""
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+import psycopg2
+
+MIGRATIONS_DIR = Path(__file__).parent / "migrations"
+
+
+def _discover(direction: str) -> list[Path]:
+    suffix = ".up.sql" if direction == "up" else ".down.sql"
+    files = sorted(MIGRATIONS_DIR.glob(f"*{suffix}"))
+    if direction == "down":
+        files = list(reversed(files))
+    return files
+
+
+def _filter(files: list[Path], prefix: str | None) -> list[Path]:
+    if prefix is None:
+        return files
+    matched = [f for f in files if f.name.startswith(prefix + "_") or f.stem.startswith(prefix)]
+    if not matched:
+        raise SystemExit(f"no migration matched prefix {prefix!r}")
+    return matched
+
+
+def main() -> int:
+    if len(sys.argv) < 2 or sys.argv[1] not in ("up", "down"):
+        print("usage: migrate.py {up|down} [prefix]", file=sys.stderr)
+        return 2
+
+    direction = sys.argv[1]
+    prefix = sys.argv[2] if len(sys.argv) > 2 else None
+
+    url = os.environ.get("SUPABASE_DB_URL")
+    if not url:
+        print("SUPABASE_DB_URL not set", file=sys.stderr)
+        return 2
+
+    files = _filter(_discover(direction), prefix)
+    if not files:
+        print("no migrations found")
+        return 0
+
+    with psycopg2.connect(url) as conn:
+        for sql_file in files:
+            sql = sql_file.read_text()
+            with conn.cursor() as cur:
+                cur.execute(sql)
+            print(f"Migration {direction!r} applied from {sql_file.name}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

finance/migrations/001_receipt_tables.down.sql

@@ -0,0 +1,9 @@
+-- Rollback for 001_receipt_tables.up.sql.
+-- Drops only what the up migration created. Legacy bank.* tables are untouched.
+
+BEGIN;
+
+DROP TABLE IF EXISTS bank.receipt_matches;
+DROP TABLE IF EXISTS bank.receipt_candidates;
+
+COMMIT;

finance/migrations/001_receipt_tables.up.sql

@@ -0,0 +1,100 @@
+-- Issue unimatrix27/ideas#20 — Finance reconciliation data foundation.
+--
+-- Creates two new tables in the existing bank.* schema. Legacy tables
+-- (transactions, belege_sent, belege_to_send, belege_missing, match_proposals)
+-- are left untouched.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS bank.receipt_candidates (
+    id                    bigserial PRIMARY KEY,
+    source_system         text        NOT NULL,
+    mailbox               text,
+    outlook_message_id    text,
+    internet_message_id   text,
+    received_at           timestamptz,
+    sent_at               timestamptz,
+    from_email            text,
+    subject               text,
+    attachment_name       text,
+    attachment_sha256     text,
+    local_blob_path       text,
+    text_sha256           text,
+    extracted_text        text,
+    extracted_json        jsonb       NOT NULL DEFAULT '{}'::jsonb,
+    parse_status          text        NOT NULL DEFAULT 'pending',
+    parse_error           text,
+    created_at            timestamptz NOT NULL DEFAULT now(),
+    updated_at            timestamptz NOT NULL DEFAULT now(),
+    CONSTRAINT receipt_candidates_source_system_chk
+        CHECK (source_system IN ('graph', 'manual_upload', 'portal', 'datev_sent')),
+    CONSTRAINT receipt_candidates_parse_status_chk
+        CHECK (parse_status IN ('pending', 'ok', 'failed', 'portal_required'))
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS receipt_candidates_attachment_sha256_uniq
+    ON bank.receipt_candidates (attachment_sha256)
+    WHERE attachment_sha256 IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS receipt_candidates_internet_message_id_idx
+    ON bank.receipt_candidates (internet_message_id);
+
+CREATE INDEX IF NOT EXISTS receipt_candidates_outlook_message_id_idx
+    ON bank.receipt_candidates (outlook_message_id);
+
+CREATE INDEX IF NOT EXISTS receipt_candidates_source_parse_idx
+    ON bank.receipt_candidates (source_system, parse_status);
+
+CREATE INDEX IF NOT EXISTS receipt_candidates_extracted_json_gin
+    ON bank.receipt_candidates USING gin (extracted_json);
+
+
+CREATE TABLE IF NOT EXISTS bank.receipt_matches (
+    id                       bigserial PRIMARY KEY,
+    receipt_candidate_id     bigint REFERENCES bank.receipt_candidates(id) ON DELETE RESTRICT,
+    bank_tx_id               bigint NOT NULL REFERENCES bank.transactions(id) ON DELETE RESTRICT,
+    confidence               text,
+    match_type               text   NOT NULL,
+    reason_codes             jsonb  NOT NULL DEFAULT '[]'::jsonb,
+    decision_status          text   NOT NULL,
+    decided_by               text   NOT NULL,
+    legacy_belege_sent_id    bigint,
+    legacy_meta              jsonb,
+    created_at               timestamptz NOT NULL DEFAULT now(),
+    updated_at               timestamptz NOT NULL DEFAULT now(),
+    CONSTRAINT receipt_matches_confidence_chk
+        CHECK (confidence IS NULL OR confidence IN ('very_high', 'high', 'medium', 'low')),
+    CONSTRAINT receipt_matches_match_type_chk
+        CHECK (match_type IN (
+            'exact_invoice_number',
+            'exact_amount_date',
+            'vendor_period',
+            'refund_to_invoice',
+            'manual',
+            'portal_only',
+            'manual_review_legacy'
+        )),
+    CONSTRAINT receipt_matches_decision_status_chk
+        CHECK (decision_status IN (
+            'proposed', 'approved', 'rejected', 'sent', 'ignored', 'manual_needed'
+        )),
+    CONSTRAINT receipt_matches_decided_by_chk
+        CHECK (decided_by IN ('code', 'user', 'llm', 'legacy_outlook_rule')),
+    CONSTRAINT receipt_matches_candidate_tx_uniq
+        UNIQUE (receipt_candidate_id, bank_tx_id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS receipt_matches_legacy_belege_sent_uniq
+    ON bank.receipt_matches (legacy_belege_sent_id)
+    WHERE legacy_belege_sent_id IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS receipt_matches_tx_idx
+    ON bank.receipt_matches (bank_tx_id);
+
+CREATE INDEX IF NOT EXISTS receipt_matches_decision_status_idx
+    ON bank.receipt_matches (decision_status);
+
+CREATE INDEX IF NOT EXISTS receipt_matches_legacy_meta_gin
+    ON bank.receipt_matches USING gin (legacy_meta);
+
+COMMIT;

finance/migrations/002_indexer_state.down.sql

@@ -0,0 +1,8 @@
+-- Rollback for 002_indexer_state.up.sql.
+-- Drops only what the up migration created.
+
+BEGIN;
+
+DROP TABLE IF EXISTS bank.indexer_state;
+
+COMMIT;

finance/migrations/002_indexer_state.up.sql

@@ -0,0 +1,19 @@
+-- Issue unimatrix27/ideas#21 — Finance reconciliation indexer state.
+--
+-- Adds bank.indexer_state, keyed by mailbox, for the receipt indexer's
+-- delta-token cursor + last-run summary. Idempotent on re-apply; the legacy
+-- bank.* tables and receipt_candidates / receipt_matches (#20) are untouched.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS bank.indexer_state (
+    mailbox          text PRIMARY KEY,
+    delta_token      text,
+    last_run_at      timestamptz,
+    last_error       text,
+    last_summary     jsonb,
+    created_at       timestamptz NOT NULL DEFAULT now(),
+    updated_at       timestamptz NOT NULL DEFAULT now()
+);
+
+COMMIT;

finance/migrations/003_receipt_status_view.down.sql

@@ -0,0 +1,4 @@
+-- Roll back unimatrix27/ideas#23 view.
+BEGIN;
+DROP VIEW IF EXISTS bank.receipt_status_v;
+COMMIT;

finance/migrations/003_receipt_status_view.up.sql

@@ -0,0 +1,68 @@
+-- Issue unimatrix27/ideas#23 — Finance reconciliation toolbox.
+--
+-- Adds the additive `bank.receipt_status_v` view used by the
+-- `list_open_transactions` tool to surface the six-bucket classification
+-- agreed in #27:
+--
+--   done              — already forwarded to DATEV (legacy bank.belege_sent
+--                       or a receipt_matches row with decision_status='sent')
+--   available_to_send — an approved match is waiting for send_match()
+--   manual_needed     — a portal-only / manual_review_legacy match needs a human
+--   ambiguous         — at least one 'proposed' row, but no approval yet
+--   missing           — non-ignored tx with no match rows at all
+--   ignored           — transactions.ignored = true (never touched by the agent)
+--
+-- bank.belege_missing (existing view) is NOT modified.
+
+BEGIN;
+
+CREATE OR REPLACE VIEW bank.receipt_status_v AS
+WITH match_rollup AS (
+    SELECT
+        bank_tx_id,
+        COUNT(*) FILTER (WHERE decision_status = 'proposed')      AS proposed_count,
+        COUNT(*) FILTER (WHERE decision_status = 'approved')      AS approved_count,
+        COUNT(*) FILTER (WHERE decision_status = 'sent')          AS sent_count,
+        COUNT(*) FILTER (WHERE decision_status = 'manual_needed') AS manual_needed_count,
+        COUNT(*) FILTER (WHERE decision_status = 'ignored')       AS ignored_count,
+        COUNT(*) FILTER (WHERE decision_status = 'rejected')      AS rejected_count
+    FROM bank.receipt_matches
+    GROUP BY bank_tx_id
+)
+SELECT
+    t.id                              AS bank_tx_id,
+    t.booking_date,
+    t.amount,
+    t.signed_amount,
+    t.currency,
+    t.credit_debit,
+    t.counterparty_name,
+    t.remittance_information,
+    t.ignored,
+    COALESCE(mr.proposed_count, 0)      AS proposed_count,
+    COALESCE(mr.approved_count, 0)      AS approved_count,
+    COALESCE(mr.sent_count, 0)          AS sent_count,
+    COALESCE(mr.manual_needed_count, 0) AS manual_needed_count,
+    COALESCE(mr.ignored_count, 0)       AS match_ignored_count,
+    COALESCE(mr.rejected_count, 0)      AS rejected_count,
+    EXISTS (
+        SELECT 1 FROM bank.belege_sent bs WHERE bs.bank_tx_id = t.id
+    ) AS legacy_belege_sent_exists,
+    CASE
+        WHEN t.ignored = true THEN 'ignored'
+        WHEN COALESCE(mr.sent_count, 0) > 0
+          OR EXISTS (SELECT 1 FROM bank.belege_sent bs WHERE bs.bank_tx_id = t.id)
+            THEN 'done'
+        WHEN COALESCE(mr.manual_needed_count, 0) > 0 THEN 'manual_needed'
+        WHEN COALESCE(mr.approved_count, 0) > 0     THEN 'available_to_send'
+        WHEN COALESCE(mr.proposed_count, 0) > 0     THEN 'ambiguous'
+        ELSE 'missing'
+    END AS status
+FROM bank.transactions t
+LEFT JOIN match_rollup mr ON mr.bank_tx_id = t.id;
+
+COMMENT ON VIEW bank.receipt_status_v IS
+    'Six-bucket reconciliation status per bank.transactions row. '
+    'Additive — bank.belege_missing remains the legacy authoritative view.';
+
+COMMIT;

finance/migrations/004_agent_tables.down.sql

@@ -0,0 +1,5 @@
+-- Roll back unimatrix27/ideas#23 agent tables.
+BEGIN;
+DROP TABLE IF EXISTS bank.agent_anomalies;
+DROP TABLE IF EXISTS bank.agent_reconcile_runs;
+COMMIT;

finance/migrations/004_agent_tables.up.sql

@@ -0,0 +1,71 @@
+-- Issue unimatrix27/ideas#23 — agent persistence tables.
+--
+-- Per #23: "define the tool against the tables' shapes as documented in
+-- #25, and ship a migration 004 that creates them. Otherwise #25 cannot
+-- even start without an extra round-trip." This migration creates the
+-- TABLES only; the agent skill that writes to them ships in #25.
+--
+-- Two tables:
+--
+--   bank.agent_anomalies         — destinations for `flag_anomaly`
+--   bank.agent_reconcile_runs    — destinations for `finalize_run`
+--
+-- Neither table replaces or modifies legacy artefacts (bank.agent_runs
+-- and bank.agent_call_log already exist for the older outlook_auto_rule
+-- pipeline and are left untouched).
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS bank.agent_reconcile_runs (
+    id                bigserial PRIMARY KEY,
+    started_at        timestamptz NOT NULL DEFAULT now(),
+    finalized_at      timestamptz,
+    summary_md        text,
+    proposed_changes  jsonb,
+    tool_call_summary jsonb,
+    invoked_by        text NOT NULL DEFAULT 'llm',
+    notes             text,
+    created_at        timestamptz NOT NULL DEFAULT now(),
+    CONSTRAINT agent_reconcile_runs_invoked_by_chk
+        CHECK (invoked_by IN ('llm', 'user', 'cron'))
+);
+
+CREATE INDEX IF NOT EXISTS agent_reconcile_runs_started_at_idx
+    ON bank.agent_reconcile_runs (started_at DESC);
+
+CREATE INDEX IF NOT EXISTS agent_reconcile_runs_finalized_at_idx
+    ON bank.agent_reconcile_runs (finalized_at DESC);
+
+
+CREATE TABLE IF NOT EXISTS bank.agent_anomalies (
+    id           bigserial PRIMARY KEY,
+    bank_tx_id   bigint REFERENCES bank.transactions(id) ON DELETE RESTRICT,
+    reason       text   NOT NULL,
+    severity     text   NOT NULL,
+    status       text   NOT NULL DEFAULT 'open',
+    raised_by    text   NOT NULL DEFAULT 'llm',
+    run_id       bigint REFERENCES bank.agent_reconcile_runs(id) ON DELETE SET NULL,
+    legacy_meta  jsonb,
+    created_at   timestamptz NOT NULL DEFAULT now(),
+    resolved_at  timestamptz,
+    CONSTRAINT agent_anomalies_severity_chk
+        CHECK (severity IN ('info', 'warn', 'block')),
+    CONSTRAINT agent_anomalies_status_chk
+        CHECK (status IN ('open', 'acknowledged', 'resolved')),
+    CONSTRAINT agent_anomalies_raised_by_chk
+        CHECK (raised_by IN ('llm', 'user', 'system'))
+);
+
+CREATE INDEX IF NOT EXISTS agent_anomalies_bank_tx_id_idx
+    ON bank.agent_anomalies (bank_tx_id);
+
+CREATE INDEX IF NOT EXISTS agent_anomalies_status_idx
+    ON bank.agent_anomalies (status);
+
+CREATE INDEX IF NOT EXISTS agent_anomalies_created_at_idx
+    ON bank.agent_anomalies (created_at DESC);
+
+CREATE INDEX IF NOT EXISTS agent_anomalies_severity_idx
+    ON bank.agent_anomalies (severity);
+
+COMMIT;