From 5f3d9286ae50b147d80f52bc13e624e7678fe0bb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:30:25 +0000 Subject: [PATCH 1/6] test(recovery): require isolated restore catalog acceptance Add the failing contract for caller-owned catalog inspection of a restored PostgreSQL target. Protected main cannot yet prove required package tables, tenant-status indexes, or forced lifecycle RLS. Co-authored-by: Seongho Bae --- tests/test_postgres_restore_acceptance.py | 262 ++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 tests/test_postgres_restore_acceptance.py diff --git a/tests/test_postgres_restore_acceptance.py b/tests/test_postgres_restore_acceptance.py new file mode 100644 index 000000000..4dd9e7c30 --- /dev/null +++ b/tests/test_postgres_restore_acceptance.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Catalog-acceptance regressions for an isolated PostgreSQL restore target.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from pg_llm_batch.postgres_restore_acceptance import ( + PostgresRestoreAcceptanceError, + inspect_postgres_restore_catalog, +) +from pg_llm_batch.postgres_schema_evidence import inspect_postgres_schema + + +REQUIRED_TABLES = ( + "com_config", + "com_secrets", + "llm_queues", + "llm_batches", + "llm_remote_batch_jobs", + "llm_batch_file_payloads", + "llm_batch_files", + "llm_requests", + "llm_jsonl_lines", + "llm_endpoints", + "llm_endpoint_models", +) +REQUIRED_INDEXES = ( + "idx_llm_remote_batch_jobs_tenant_status_observed", + "uq_llm_remote_batch_jobs_tenant_endpoint_id", +) +CHECKPOINT_TABLE = "llm_result_stream_checkpoints" + + +class _CatalogCursor: + """Return one finite catalog snapshot through the caller-owned cursor seam.""" + + def __init__(self, rows: list[tuple[object, ...]]) -> None: + self.rows = rows + self.executed_sql: str | None = None + self.executed_params: tuple[object, ...] | None = None + + def __enter__(self) -> _CatalogCursor: + return self + + def __exit__(self, *_exc: object) -> bool: + return False + + def execute(self, sql: object, params: object = None) -> None: + if type(sql) is not str: + raise AssertionError("catalog SQL must be an exact built-in string") + self.executed_sql = sql + self.executed_params = params if type(params) is tuple else None + + def fetchall(self) -> list[tuple[object, ...]]: + return list(self.rows) + + +class _CatalogConnection: + """Expose one caller-owned connection that never carries a DSN.""" + + def __init__(self, rows: list[tuple[object, ...]]) -> None: + self.cursor_handle = _CatalogCursor(rows) + + def cursor(self) -> _CatalogCursor: + return self.cursor_handle + + +class _HostileName(str): + """Refuse rendering if rejected catalog names leak into diagnostics.""" + + def __str__(self) -> str: + raise AssertionError("must not render hostile catalog names") + + +class _ExecuteFailureCursor(_CatalogCursor): + """Surface a lower-layer catalog failure without becoming operator evidence.""" + + def execute(self, sql: object, params: object = None) -> None: + del sql, params + raise RuntimeError("password=supersecret host=db.internal") + + +class _ExecuteFailureConnection(_CatalogConnection): + """Return a cursor whose execute path leaks deployment text.""" + + def cursor(self) -> _CatalogCursor: + return _ExecuteFailureCursor([]) + + +def _relation( + name: str, + kind: str, + row_security: bool = False, + force_row_security: bool = False, +) -> tuple[str, str, bool, bool]: + return (name, kind, row_security, force_row_security) + + +def _complete_catalog( + *, + lifecycle_rls: bool = True, + include_checkpoint: bool = True, +) -> list[tuple[object, ...]]: + rows: list[tuple[object, ...]] = [ + _relation(name, "r") for name in REQUIRED_TABLES if name != "llm_remote_batch_jobs" + ] + rows.append( + _relation( + "llm_remote_batch_jobs", + "r", + row_security=lifecycle_rls, + force_row_security=lifecycle_rls, + ) + ) + rows.extend(_relation(name, "i") for name in REQUIRED_INDEXES) + if include_checkpoint: + rows.append( + _relation( + CHECKPOINT_TABLE, + "r", + row_security=True, + force_row_security=True, + ) + ) + return rows + + +def test_inspect_restore_catalog_accepts_complete_isolated_target() -> None: + connection = _CatalogConnection(_complete_catalog()) + + evidence = inspect_postgres_restore_catalog(connection) + schema = inspect_postgres_schema() + + assert evidence.required_table_count == len(REQUIRED_TABLES) + assert evidence.required_index_count == len(REQUIRED_INDEXES) + assert evidence.lifecycle_rls_enabled is True + assert evidence.lifecycle_rls_forced is True + assert evidence.checkpoint_store_present is True + assert evidence.checkpoint_store_rls_forced is True + assert evidence.expected_schema_sha256 == schema.sha256 + assert evidence.expected_schema_size_bytes == schema.size_bytes + assert "password" not in evidence.as_dict() + assert "host" not in evidence.as_dict() + + +def test_inspect_restore_catalog_rejects_missing_lifecycle_table() -> None: + rows = [ + row + for row in _complete_catalog() + if row[0] != "llm_remote_batch_jobs" + ] + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog is incomplete", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_unforced_lifecycle_rls() -> None: + connection = _CatalogConnection(_complete_catalog(lifecycle_rls=False)) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog failed tenant isolation checks", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_allows_schema_init_without_checkpoint_store() -> None: + connection = _CatalogConnection(_complete_catalog(include_checkpoint=False)) + + evidence = inspect_postgres_restore_catalog(connection) + + assert evidence.checkpoint_store_present is False + assert evidence.checkpoint_store_rls_forced is False + assert evidence.required_table_count == len(REQUIRED_TABLES) + + +def test_inspect_restore_catalog_rejects_checkpoint_store_without_forced_rls() -> None: + rows = _complete_catalog(include_checkpoint=False) + rows.append(_relation(CHECKPOINT_TABLE, "r", row_security=True, force_row_security=False)) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog failed tenant isolation checks", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_hostile_relation_name() -> None: + rows = _complete_catalog() + rows[0] = (_HostileName("com_config"), "r", False, False) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_hides_lower_layer_diagnostics() -> None: + connection = _ExecuteFailureConnection([]) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog could not be inspected", + ) as raised: + inspect_postgres_restore_catalog(connection) + + assert "supersecret" not in str(raised.value) + assert "db.internal" not in str(raised.value) + assert raised.value.__cause__ is None + + +def test_inspect_restore_catalog_rejects_oversized_catalog() -> None: + rows = _complete_catalog() + [_relation(f"extra_table_{index}", "r") for index in range(8)] + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_uses_parameterized_current_schema_probe() -> None: + connection = _CatalogConnection(_complete_catalog()) + + inspect_postgres_restore_catalog(connection) + + sql = connection.cursor_handle.executed_sql + params = connection.cursor_handle.executed_params + assert sql is not None + assert "current_schema()" in sql + assert "%s" in sql + assert params is not None + assert "llm_remote_batch_jobs" in params[1] + assert CHECKPOINT_TABLE in params[1] + + +def test_inspect_restore_catalog_rejects_non_tuple_params_leakage_path() -> None: + class _BadFetchCursor(_CatalogCursor): + def fetchall(self) -> list[tuple[object, ...]]: + return [("com_config", "r", False, False)] # type: ignore[return-value] + + class _BadFetchConnection(_CatalogConnection): + def cursor(self) -> _CatalogCursor: + return _BadFetchCursor([]) + + # A single required table is incomplete even if the row shape is valid. + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog is incomplete", + ): + inspect_postgres_restore_catalog(_BadFetchConnection([])) From 8f0ef04c233ea784c746df8429a2ac45007cb034 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:32:18 +0000 Subject: [PATCH 2/6] feat(recovery): inspect isolated restore catalog objects Add a caller-owned pg_class probe that accepts a restored target only when required package tables, tenant-status indexes, and forced lifecycle RLS are present. The probe does not execute pg_dump or pg_restore and keeps diagnostics content-free. Co-authored-by: Seongho Bae --- pg_llm_batch/postgres_restore_acceptance.py | 163 ++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 pg_llm_batch/postgres_restore_acceptance.py diff --git a/pg_llm_batch/postgres_restore_acceptance.py b/pg_llm_batch/postgres_restore_acceptance.py new file mode 100644 index 000000000..4153802c2 --- /dev/null +++ b/pg_llm_batch/postgres_restore_acceptance.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Inspect a caller-owned PostgreSQL catalog after an isolated restore.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pg_llm_batch.postgres_schema_evidence import inspect_postgres_schema + + +_REQUIRED_TABLES = ( + "com_config", + "com_secrets", + "llm_queues", + "llm_batches", + "llm_remote_batch_jobs", + "llm_batch_file_payloads", + "llm_batch_files", + "llm_requests", + "llm_jsonl_lines", + "llm_endpoints", + "llm_endpoint_models", +) +_REQUIRED_INDEXES = ( + "idx_llm_remote_batch_jobs_tenant_status_observed", + "uq_llm_remote_batch_jobs_tenant_endpoint_id", +) +_CHECKPOINT_TABLE = "llm_result_stream_checkpoints" +_ALLOWED_NAMES = frozenset(_REQUIRED_TABLES + _REQUIRED_INDEXES + (_CHECKPOINT_TABLE,)) +_RELATION_KINDS = frozenset({"r", "i"}) +_MAX_CATALOG_ROWS = 16 +_CATALOG_SQL = """ +SELECT c.relname, c.relkind, c.relrowsecurity, c.relforcerowsecurity +FROM pg_catalog.pg_class AS c +INNER JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace +WHERE n.nspname = current_schema() + AND c.relkind = ANY(%s) + AND c.relname = ANY(%s) +""".strip() + + +class PostgresRestoreAcceptanceError(ValueError): + """Report a fail-closed isolated restore catalog acceptance violation.""" + + +@dataclass(frozen=True, slots=True) +class PostgresRestoreCatalogEvidence: + """Represent content-free catalog acceptance for one isolated restore target.""" + + required_table_count: int + required_index_count: int + lifecycle_rls_enabled: bool + lifecycle_rls_forced: bool + checkpoint_store_present: bool + checkpoint_store_rls_forced: bool + expected_schema_sha256: str + expected_schema_size_bytes: int + + def as_dict(self) -> dict[str, object]: + """Return the stable machine-readable restore-catalog evidence schema.""" + return { + "required_table_count": self.required_table_count, + "required_index_count": self.required_index_count, + "lifecycle_rls_enabled": self.lifecycle_rls_enabled, + "lifecycle_rls_forced": self.lifecycle_rls_forced, + "checkpoint_store_present": self.checkpoint_store_present, + "checkpoint_store_rls_forced": self.checkpoint_store_rls_forced, + "expected_schema_sha256": self.expected_schema_sha256, + "expected_schema_size_bytes": self.expected_schema_size_bytes, + } + + +def _invalid_catalog() -> None: + """Reject malformed catalog evidence without reflecting row contents.""" + raise PostgresRestoreAcceptanceError( + "PostgreSQL restore catalog evidence is invalid" + ) + + +def _evaluate_catalog_rows( + rows: object, +) -> PostgresRestoreCatalogEvidence: + """Validate one finite catalog snapshot against the packaged restore contract.""" + if type(rows) is not list or len(rows) > _MAX_CATALOG_ROWS: + _invalid_catalog() + tables: dict[str, tuple[bool, bool]] = {} + indexes: dict[str, bool] = {} + for row in rows: + if type(row) is not tuple or len(row) != 4: + _invalid_catalog() + relation_name, relation_kind, row_security, force_row_security = row + if ( + type(relation_name) is not str + or relation_name not in _ALLOWED_NAMES + or type(relation_kind) is not str + or relation_kind not in _RELATION_KINDS + or type(row_security) is not bool + or type(force_row_security) is not bool + ): + _invalid_catalog() + if relation_kind == "r": + if relation_name in tables: + _invalid_catalog() + tables[relation_name] = (row_security, force_row_security) + else: + if relation_name in indexes: + _invalid_catalog() + indexes[relation_name] = True + if any(name not in tables for name in _REQUIRED_TABLES): + raise PostgresRestoreAcceptanceError( + "PostgreSQL restore catalog is incomplete" + ) + if any(name not in indexes for name in _REQUIRED_INDEXES): + raise PostgresRestoreAcceptanceError( + "PostgreSQL restore catalog is incomplete" + ) + lifecycle_security = tables["llm_remote_batch_jobs"] + if not lifecycle_security[0] or not lifecycle_security[1]: + raise PostgresRestoreAcceptanceError( + "PostgreSQL restore catalog failed tenant isolation checks" + ) + checkpoint_present = _CHECKPOINT_TABLE in tables + checkpoint_security = tables.get(_CHECKPOINT_TABLE, (False, False)) + if checkpoint_present and (not checkpoint_security[0] or not checkpoint_security[1]): + raise PostgresRestoreAcceptanceError( + "PostgreSQL restore catalog failed tenant isolation checks" + ) + schema = inspect_postgres_schema() + return PostgresRestoreCatalogEvidence( + required_table_count=len(_REQUIRED_TABLES), + required_index_count=len(_REQUIRED_INDEXES), + lifecycle_rls_enabled=lifecycle_security[0], + lifecycle_rls_forced=lifecycle_security[1], + checkpoint_store_present=checkpoint_present, + checkpoint_store_rls_forced=checkpoint_security[1], + expected_schema_sha256=schema.sha256, + expected_schema_size_bytes=schema.size_bytes, + ) + + +def inspect_postgres_restore_catalog( + connection: object, +) -> PostgresRestoreCatalogEvidence: + """Prove required package catalog objects on a caller-owned restore target. + + The callable inspects ``pg_class`` through the caller-owned connection. It + does not execute ``pg_dump`` or ``pg_restore``, open a package-owned + connection, or claim that a backup artifact is restorable. Missing required + tables or tenant-status indexes fail closed. Lifecycle row-level security + must be enabled and forced. A present checkpoint store must also be forced. + """ + try: + with connection.cursor() as cursor: + cursor.execute( + _CATALOG_SQL, + (("r", "i"), _REQUIRED_TABLES + _REQUIRED_INDEXES + (_CHECKPOINT_TABLE,)), + ) + rows = cursor.fetchall() + except Exception: + raise PostgresRestoreAcceptanceError( + "PostgreSQL restore catalog could not be inspected" + ) from None + return _evaluate_catalog_rows(rows) From e3e0e6f77911a063676a0b58536a0242e7f6ac3a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:32:18 +0000 Subject: [PATCH 3/6] docs(recovery): record restore catalog acceptance contract Add ADR 0018 and operator doctoring so an isolated restore target is checked before production traffic is pointed at it. Co-authored-by: Seongho Bae --- ...018-postgres-restore-catalog-acceptance.md | 77 +++++++++++++++++++ docs/doctoring/postgres-restore-acceptance.md | 66 ++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 docs/adr/0018-postgres-restore-catalog-acceptance.md create mode 100644 docs/doctoring/postgres-restore-acceptance.md diff --git a/docs/adr/0018-postgres-restore-catalog-acceptance.md b/docs/adr/0018-postgres-restore-catalog-acceptance.md new file mode 100644 index 000000000..2d514cdc0 --- /dev/null +++ b/docs/adr/0018-postgres-restore-catalog-acceptance.md @@ -0,0 +1,77 @@ +# ADR 0018: Isolated restore catalog acceptance without executor authority + +- **Status:** Proposed +- **Date:** 2026-08-16 + +## Context + +Protected main can hash a backup artifact and the packaged `schema.sql` resource, +and it can encode a content-free recovery receipt. Those primitives do not prove +that an isolated restore target contains the package-owned tables, tenant-status +indexes, or forced row-level security that a buyer needs before using the +database again. + +Executable `pg_dump` and `pg_restore` remain separate active writers. A catalog +acceptance probe must not open a package-owned connection, must not execute +backup or restore, and must not treat a receipt or hash as restorability. + +PostgreSQL publishes `relrowsecurity` and `relforcerowsecurity` on `pg_class` +as the catalog truth for whether row-level security is enabled and whether it +also applies to the table owner (PostgreSQL Global Development Group, 2026b, +2026c). Contingency planning requires proving that recovered information-system +state is usable, not only that a backup command returned zero (Swanson et al., +2010). + +## Decision + +Add `inspect_postgres_restore_catalog(connection)` as a caller-owned catalog +probe: + +1. The caller supplies an already-authorized connection. The package does not + parse a DSN, inherit libpq defaults, or create a connection. +2. The probe reads `pg_class` in `current_schema()` through one parameterized + query. Relation names and kinds are bound; they are never concatenated into + SQL. +3. Required packaged-schema tables and the tenant-qualified lifecycle unique + index plus tenant-status index must be present. Missing required objects fail + closed. +4. `llm_remote_batch_jobs` must have row-level security enabled and forced. + If `llm_result_stream_checkpoints` is present, it must also be forced. A + schema-init target that has not applied migration 0007 may omit the + checkpoint store. +5. Evidence is content-free: counts, boolean RLS flags, and the packaged schema + SHA-256/size from `inspect_postgres_schema()`. Paths, DSNs, SQL text, and + lower-layer diagnostics never enter the evidence or exception text. +6. Success does not mean a backup is restorable, a live cluster matches every + constraint/function body, PITR works, or a stated RPO/RTO is met. + +## Consequences + +Operators can fail an isolated restore target before they point production +traffic at it, using the same packaged schema identity already hashed on +protected main. Backup and restore executors stay on their own lanes. Canonical +PRD/TRD/CHANGELOG status remains owned by those writers until this probe +integrates. + +## Rollback + +Remove `pg_llm_batch/postgres_restore_acceptance.py`, its tests, this ADR, and +the doctoring record. No schema migration or durable state conversion is +required. + +## References + +PostgreSQL Global Development Group. (2026a). *Backup and restore*. In +*PostgreSQL 18 documentation*. https://www.postgresql.org/docs/18/backup.html + +PostgreSQL Global Development Group. (2026b). *pg_class*. In *PostgreSQL 18 +documentation*. https://www.postgresql.org/docs/18/catalog-pg-class.html + +PostgreSQL Global Development Group. (2026c). *Row security policies*. In +*PostgreSQL 18 documentation*. +https://www.postgresql.org/docs/18/ddl-rowsecurity.html + +Swanson, M., Bowen, P., Phillips, A., Gallup, D., & Lynes, D. (2010). +*Contingency planning guide for federal information systems* (NIST Special +Publication 800-34 Rev. 1). National Institute of Standards and Technology. +https://doi.org/10.6028/NIST.SP.800-34r1 diff --git a/docs/doctoring/postgres-restore-acceptance.md b/docs/doctoring/postgres-restore-acceptance.md new file mode 100644 index 000000000..801d9c5c8 --- /dev/null +++ b/docs/doctoring/postgres-restore-acceptance.md @@ -0,0 +1,66 @@ +# Isolated PostgreSQL restore catalog acceptance + +## What to do next + +After you restore into an isolated libpq target, call +`inspect_postgres_restore_catalog(connection)` on a connection you already +opened to that target. Do not point production traffic at the target until the +probe returns. If it raises, keep the target isolated, do not retry a +transactional restore into the same service, and compare the packaged schema +hash on the evidence with the backup you intended to apply. + +## Claim boundary + +This probe proves that the caller-owned connection can see the required +package-owned tables, the tenant-qualified lifecycle unique index, the +tenant-status index, and forced row-level security on +`llm_remote_batch_jobs`. If `llm_result_stream_checkpoints` is present, it +must also be forced. + +It does **not** execute `pg_dump` or `pg_restore`, open a package-owned +connection, prove every constraint or function body, prove live-cluster parity +with `schema.sql` byte-for-byte, authenticate an operator, or establish +PITR/RPO/RTO/HA/DR, CSAP, or SOC 2 readiness. A recovery receipt or artifact +hash is not acceptance. + +## Operator contract + +1. Restore only into an isolated target whose credentials you already trust. +2. Open the connection yourself. Pass that connection object. The package never + reads a DSN for this probe. +3. On success, record the returned `expected_schema_sha256` and + `expected_schema_size_bytes` next to the backup artifact hash you already + have. Those values come from the packaged `schema.sql` resource, not from + hashing the live cluster. +4. Treat `checkpoint_store_present=false` as a schema-init target that has not + applied migration 0007. Do not treat it as proof that checkpoints were + intentionally empty. +5. On `PostgresRestoreAcceptanceError`, read only the fixed category. The + exception text never contains SQL, DSNs, passwords, or catalog dumps. + +## Verification + +Focused tests prove a complete isolated catalog is accepted, a missing +lifecycle table or tenant-status index is incomplete, unforced lifecycle or +checkpoint RLS fails tenant-isolation checks, hostile name subclasses and +lower-layer diagnostics stay out of exceptions, and the probe binds +`current_schema()` with parameters. Coverage for +`pg_llm_batch.postgres_restore_acceptance` is 100% statement and branch. +Public docstrings are complete. + +## References + +PostgreSQL Global Development Group. (2026a). *Backup and restore*. In +*PostgreSQL 18 documentation*. https://www.postgresql.org/docs/18/backup.html + +PostgreSQL Global Development Group. (2026b). *pg_class*. In *PostgreSQL 18 +documentation*. https://www.postgresql.org/docs/18/catalog-pg-class.html + +PostgreSQL Global Development Group. (2026c). *Row security policies*. In +*PostgreSQL 18 documentation*. +https://www.postgresql.org/docs/18/ddl-rowsecurity.html + +Swanson, M., Bowen, P., Phillips, A., Gallup, D., & Lynes, D. (2010). +*Contingency planning guide for federal information systems* (NIST Special +Publication 800-34 Rev. 1). National Institute of Standards and Technology. +https://doi.org/10.6028/NIST.SP.800-34r1 From 6ac26702d9ada5bbdca86a738cc12a2464dc85b8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:32:21 +0000 Subject: [PATCH 4/6] test(recovery): close restore catalog acceptance branches Cover malformed rows, duplicate relations, missing tenant-status indexes, and partial RLS flags so the probe stays at 100% branch coverage. Co-authored-by: Seongho Bae --- tests/test_postgres_restore_acceptance.py | 179 +++++++++++++++++++++- 1 file changed, 177 insertions(+), 2 deletions(-) diff --git a/tests/test_postgres_restore_acceptance.py b/tests/test_postgres_restore_acceptance.py index 4dd9e7c30..c3e58819e 100644 --- a/tests/test_postgres_restore_acceptance.py +++ b/tests/test_postgres_restore_acceptance.py @@ -3,8 +3,6 @@ from __future__ import annotations -from typing import Any - import pytest from pg_llm_batch.postgres_restore_acceptance import ( @@ -245,6 +243,183 @@ def test_inspect_restore_catalog_uses_parameterized_current_schema_probe() -> No assert CHECKPOINT_TABLE in params[1] +def test_inspect_restore_catalog_rejects_non_list_catalog_rows() -> None: + class _TupleFetchCursor(_CatalogCursor): + def fetchall(self) -> list[tuple[object, ...]]: + return tuple(_complete_catalog()) # type: ignore[return-value] + + class _TupleFetchConnection(_CatalogConnection): + def cursor(self) -> _CatalogCursor: + return _TupleFetchCursor([]) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(_TupleFetchConnection([])) + + +def test_inspect_restore_catalog_rejects_list_row_instead_of_tuple() -> None: + rows = _complete_catalog() + rows[0] = ["com_config", "r", False, False] # type: ignore[assignment] + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_unknown_relation_name() -> None: + rows = _complete_catalog(include_checkpoint=False) + rows.append(_relation("secret_audit_dump", "r")) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_non_string_relation_kind() -> None: + rows = _complete_catalog() + rows[0] = ("com_config", b"r", False, False) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_non_bool_force_flag() -> None: + rows = _complete_catalog() + rows[0] = ("com_config", "r", False, 1) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_malformed_row_shape() -> None: + rows = _complete_catalog() + rows[0] = ("com_config", "r", False) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_unknown_relation_kind() -> None: + rows = _complete_catalog() + rows[0] = ("com_config", "v", False, False) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_non_bool_security_flags() -> None: + rows = _complete_catalog() + rows[0] = ("com_config", "r", 0, False) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_duplicate_table_row() -> None: + rows = _complete_catalog() + rows.append(_relation("com_config", "r")) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_duplicate_index_row() -> None: + rows = _complete_catalog() + rows.append(_relation(REQUIRED_INDEXES[0], "i")) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog evidence is invalid", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_missing_tenant_status_index() -> None: + rows = [ + row + for row in _complete_catalog() + if row[0] != "idx_llm_remote_batch_jobs_tenant_status_observed" + ] + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog is incomplete", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_lifecycle_enabled_without_force() -> None: + rows = _complete_catalog(lifecycle_rls=False) + for index, row in enumerate(rows): + if row[0] == "llm_remote_batch_jobs": + rows[index] = _relation( + "llm_remote_batch_jobs", + "r", + row_security=True, + force_row_security=False, + ) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog failed tenant isolation checks", + ): + inspect_postgres_restore_catalog(connection) + + +def test_inspect_restore_catalog_rejects_checkpoint_without_enabled_rls() -> None: + rows = _complete_catalog(include_checkpoint=False) + rows.append( + _relation( + CHECKPOINT_TABLE, + "r", + row_security=False, + force_row_security=True, + ) + ) + connection = _CatalogConnection(rows) + + with pytest.raises( + PostgresRestoreAcceptanceError, + match="PostgreSQL restore catalog failed tenant isolation checks", + ): + inspect_postgres_restore_catalog(connection) + + def test_inspect_restore_catalog_rejects_non_tuple_params_leakage_path() -> None: class _BadFetchCursor(_CatalogCursor): def fetchall(self) -> list[tuple[object, ...]]: From 4f2d3fc5873d62898afa00682d3b48befc54b45f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 01:16:36 +0900 Subject: [PATCH 5/6] test(recovery): prove restore catalog index semantics --- ...gres_restore_acceptance_index_semantics.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/test_postgres_restore_acceptance_index_semantics.py diff --git a/tests/test_postgres_restore_acceptance_index_semantics.py b/tests/test_postgres_restore_acceptance_index_semantics.py new file mode 100644 index 000000000..f1e9be287 --- /dev/null +++ b/tests/test_postgres_restore_acceptance_index_semantics.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for PostgreSQL restore-catalog query semantics.""" + +from __future__ import annotations + +from pg_llm_batch.postgres_restore_acceptance import inspect_postgres_restore_catalog + + +_REQUIRED_TABLES = ( + "com_config", + "com_secrets", + "llm_queues", + "llm_batches", + "llm_remote_batch_jobs", + "llm_batch_file_payloads", + "llm_batch_files", + "llm_requests", + "llm_jsonl_lines", + "llm_endpoints", + "llm_endpoint_models", +) +_REQUIRED_INDEXES = ( + "idx_llm_remote_batch_jobs_tenant_status_observed", + "uq_llm_remote_batch_jobs_tenant_endpoint_id", +) +_CHECKPOINT_TABLE = "llm_result_stream_checkpoints" + + +class _RecordingCursor: + """Record the exact PostgreSQL query and parameters supplied by the probe.""" + + def __init__(self) -> None: + self.sql: str | None = None + self.params: tuple[object, ...] | None = None + + def __enter__(self) -> _RecordingCursor: + return self + + def __exit__(self, *_exc: object) -> bool: + return False + + def execute(self, sql: object, params: object = None) -> None: + assert type(sql) is str + assert type(params) is tuple + self.sql = sql + self.params = params + + def fetchall(self) -> list[tuple[object, ...]]: + rows: list[tuple[object, ...]] = [ + (name, "r", False, False) + for name in _REQUIRED_TABLES + if name != "llm_remote_batch_jobs" + ] + rows.append(("llm_remote_batch_jobs", "r", True, True)) + rows.extend((name, "i", False, False) for name in _REQUIRED_INDEXES) + rows.append((_CHECKPOINT_TABLE, "r", True, True)) + return rows + + +class _RecordingConnection: + """Expose a caller-owned cursor without constructing a package connection.""" + + def __init__(self) -> None: + self.cursor_handle = _RecordingCursor() + + def cursor(self) -> _RecordingCursor: + return self.cursor_handle + + +def test_any_parameters_use_psycopg_array_values() -> None: + """Bind list values to ``ANY(%s)`` while retaining tuple execute parameters.""" + connection = _RecordingConnection() + + inspect_postgres_restore_catalog(connection) + + params = connection.cursor_handle.params + assert params is not None + assert type(params) is tuple + assert type(params[0]) is list + assert params[0] == ["r", "i"] + assert type(params[1]) is list + assert "llm_remote_batch_jobs" in params[1] + assert _CHECKPOINT_TABLE in params[1] + + +def test_catalog_query_authenticates_index_structure() -> None: + """Require index ownership, keys, uniqueness, validity, and plain-index shape.""" + connection = _RecordingConnection() + + inspect_postgres_restore_catalog(connection) + + sql = connection.cursor_handle.sql + assert sql is not None + required_fragments = ( + "pg_catalog.pg_index", + "indrelid", + "indisunique", + "indisvalid", + "indisready", + "indpred", + "indexprs", + "pg_catalog.pg_get_indexdef", + "llm_remote_batch_jobs", + "tenant_scope", + "endpoint_alias", + "remote_batch_id", + "batch_status", + "last_observed_at", + ) + for fragment in required_fragments: + assert fragment in sql From e74afe8ddd9a8dec27160195dfc218bff36b9b32 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 01:17:26 +0900 Subject: [PATCH 6/6] fix(recovery): authenticate restore index semantics --- pg_llm_batch/postgres_restore_acceptance.py | 44 ++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/pg_llm_batch/postgres_restore_acceptance.py b/pg_llm_batch/postgres_restore_acceptance.py index 4153802c2..98de89355 100644 --- a/pg_llm_batch/postgres_restore_acceptance.py +++ b/pg_llm_batch/postgres_restore_acceptance.py @@ -33,9 +33,48 @@ SELECT c.relname, c.relkind, c.relrowsecurity, c.relforcerowsecurity FROM pg_catalog.pg_class AS c INNER JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace +LEFT JOIN pg_catalog.pg_index AS idx ON idx.indexrelid = c.oid +LEFT JOIN pg_catalog.pg_class AS indexed_table ON indexed_table.oid = idx.indrelid WHERE n.nspname = current_schema() AND c.relkind = ANY(%s) AND c.relname = ANY(%s) + AND ( + c.relkind = 'r' + OR ( + c.relkind = 'i' + AND indexed_table.relname = 'llm_remote_batch_jobs' + AND indexed_table.relnamespace = n.oid + AND idx.indisvalid + AND idx.indisready + AND idx.indpred IS NULL + AND idx.indexprs IS NULL + AND idx.indnkeyatts = 3 + AND idx.indnatts = 3 + AND pg_catalog.pg_get_indexdef(c.oid, 1, TRUE) = 'tenant_scope' + AND ( + ( + c.relname = 'idx_llm_remote_batch_jobs_tenant_status_observed' + AND NOT idx.indisunique + AND pg_catalog.pg_get_indexdef(c.oid, 2, TRUE) = 'batch_status' + AND pg_catalog.pg_get_indexdef(c.oid, 3, TRUE) = 'last_observed_at' + ) + OR ( + c.relname = 'uq_llm_remote_batch_jobs_tenant_endpoint_id' + AND idx.indisunique + AND pg_catalog.pg_get_indexdef(c.oid, 2, TRUE) = 'endpoint_alias' + AND pg_catalog.pg_get_indexdef(c.oid, 3, TRUE) = 'remote_batch_id' + AND EXISTS ( + SELECT 1 + FROM pg_catalog.pg_constraint AS constraint_row + WHERE constraint_row.conindid = c.oid + AND constraint_row.conrelid = indexed_table.oid + AND constraint_row.contype = 'u' + AND NOT constraint_row.condeferrable + ) + ) + ) + ) + ) """.strip() @@ -148,12 +187,15 @@ def inspect_postgres_restore_catalog( connection, or claim that a backup artifact is restorable. Missing required tables or tenant-status indexes fail closed. Lifecycle row-level security must be enabled and forced. A present checkpoint store must also be forced. + Required lifecycle indexes must belong to that lifecycle table and match the + packaged key order, uniqueness, validity, readiness, and plain-index shape. """ + relation_names = list(_REQUIRED_TABLES + _REQUIRED_INDEXES + (_CHECKPOINT_TABLE,)) try: with connection.cursor() as cursor: cursor.execute( _CATALOG_SQL, - (("r", "i"), _REQUIRED_TABLES + _REQUIRED_INDEXES + (_CHECKPOINT_TABLE,)), + (["r", "i"], relation_names), ) rows = cursor.fetchall() except Exception: