Skip to content

fix(durable): enforce lifecycle progress invariants #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
seonghobae merged 33 commits into from
Aug 12, 2026
Merged

fix(durable): enforce lifecycle progress invariants #148

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
4a8ba6f
test(durable): reject impossible lifecycle progress
seonghobae Aug 12, 2026
c0f278d
fix(durable): preserve lifecycle progress invariant
seonghobae Aug 12, 2026
d282b4a
Merge protected main into durable progress invariants
seonghobae Aug 12, 2026
b95dea3
Merge branch 'main' into fix/durable-progress-invariants
opencode-agent[bot] Aug 12, 2026
a9bb139
test(durable): prove request total knownness boundary
seonghobae Aug 12, 2026
ad5626e
test(durable): exercise progress upsert against PostgreSQL
seonghobae Aug 12, 2026
887cc75
fix(durable): preserve request total knownness
seonghobae Aug 12, 2026
8c211d4
fix(schema): persist lifecycle total knownness
seonghobae Aug 12, 2026
3250bc7
fix(schema): mirror lifecycle total knownness
seonghobae Aug 12, 2026
660677c
fix(db): preserve model metadata helper during lifecycle reconciliation
seonghobae Aug 12, 2026
4dea49e
fix(durable): preserve lifecycle persistence compatibility
seonghobae Aug 12, 2026
3bc3264
noop
seonghobae Aug 12, 2026
32b28cb
chore: remove accidental branch placeholder
seonghobae Aug 12, 2026
ca2e02d
noop
seonghobae Aug 12, 2026
3f55f99
chore: remove accidental branch placeholder
seonghobae Aug 12, 2026
7b9f287
merge(main): reconcile durable invariant with current protected head
seonghobae Aug 12, 2026
94512e0
fix(stack): preserve current protected-main tree
seonghobae Aug 12, 2026
4f1c2a1
test(durable): verify internal total-known persistence
seonghobae Aug 12, 2026
ddc783f
merge(main): compose durable progress invariants with current protect…
seonghobae Aug 12, 2026
c4d2528
test(durable): require persisted state on rejected progress
seonghobae Aug 12, 2026
ce4097c
test(durable): expose rejected-upsert and migration evidence gaps
seonghobae Aug 12, 2026
e4c1fea
fix(durable): return persisted state after rejected progress merge
seonghobae Aug 12, 2026
aebf237
fix(schema): fail closed on historical progress corruption
seonghobae Aug 12, 2026
43ccbb9
fix(schema): preserve endpoint definition while hardening migration
seonghobae Aug 12, 2026
803494a
fix(schema): mirror fail-closed progress migration
seonghobae Aug 12, 2026
3140ee3
test(durable): cover rejected progress without persisted row
seonghobae Aug 12, 2026
9116637
Merge branch 'main' into fix/durable-progress-invariants
opencode-agent[bot] Aug 12, 2026
186f7da
fix(durable): tolerate minimal cursor doubles
seonghobae Aug 12, 2026
ac1157c
Merge branch 'main' into fix/durable-progress-invariants
opencode-agent[bot] Aug 12, 2026
40c29e6
test(db): reject caller-selected schema authority
seonghobae Aug 12, 2026
01d9dac
fix(db): remove caller-selected schema execution
seonghobae Aug 12, 2026
2291d4f
test(db): bind schema application to package asset
seonghobae Aug 12, 2026
7667078
merge(main): refresh durable invariants on protected head
seonghobae Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 39 additions & 2 deletions docker/postgres/init/02_schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ CREATE TABLE IF NOT EXISTS llm_remote_batch_jobs (
),
total_requests INTEGER NOT NULL DEFAULT 0
CHECK (total_requests >= 0),
total_requests_known BOOLEAN NOT NULL DEFAULT FALSE,
completed_requests INTEGER NOT NULL DEFAULT 0
CHECK (completed_requests >= 0),
failed_requests INTEGER NOT NULL DEFAULT 0
Expand All @@ -120,6 +121,12 @@ CREATE TABLE IF NOT EXISTS llm_remote_batch_jobs (
updated_at TIMESTAMPTZ NOT NULL,
CONSTRAINT ck_llm_remote_batch_jobs_tenant_scope
CHECK (tenant_scope ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'),
CONSTRAINT ck_llm_remote_batch_jobs_known_progress
CHECK (
NOT total_requests_known OR
completed_requests::BIGINT + failed_requests::BIGINT
<= total_requests::BIGINT
),
CONSTRAINT uq_llm_remote_batch_jobs_tenant_endpoint_id
UNIQUE (tenant_scope, endpoint_alias, remote_batch_id)
);
Expand All @@ -142,6 +149,20 @@ BEGIN
ALTER TABLE llm_remote_batch_jobs
ALTER COLUMN tenant_scope SET NOT NULL;

ALTER TABLE llm_remote_batch_jobs
ADD COLUMN IF NOT EXISTS total_requests_known BOOLEAN;
UPDATE llm_remote_batch_jobs
SET total_requests_known = (
total_requests > 0 AND
completed_requests::BIGINT + failed_requests::BIGINT
<= total_requests::BIGINT
)
WHERE total_requests_known IS NULL;
ALTER TABLE llm_remote_batch_jobs
ALTER COLUMN total_requests_known SET DEFAULT FALSE;
ALTER TABLE llm_remote_batch_jobs
ALTER COLUMN total_requests_known SET NOT NULL;

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
IF EXISTS (
SELECT 1
FROM pg_constraint
Expand All @@ -167,6 +188,23 @@ BEGIN
ALTER TABLE llm_remote_batch_jobs
VALIDATE CONSTRAINT ck_llm_remote_batch_jobs_tenant_scope;

IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_llm_remote_batch_jobs_known_progress'
AND conrelid = 'llm_remote_batch_jobs'::regclass
) THEN
ALTER TABLE llm_remote_batch_jobs
ADD CONSTRAINT ck_llm_remote_batch_jobs_known_progress
CHECK (
NOT total_requests_known OR
completed_requests::BIGINT + failed_requests::BIGINT
<= total_requests::BIGINT
) NOT VALID;
END IF;
ALTER TABLE llm_remote_batch_jobs
VALIDATE CONSTRAINT ck_llm_remote_batch_jobs_known_progress;

Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
IF NOT EXISTS (
SELECT 1
FROM pg_constraint
Expand All @@ -177,7 +215,6 @@ BEGIN
ADD CONSTRAINT uq_llm_remote_batch_jobs_tenant_endpoint_id
UNIQUE (tenant_scope, endpoint_alias, remote_batch_id);
END IF;

ALTER TABLE llm_remote_batch_jobs ENABLE ROW LEVEL SECURITY;
ALTER TABLE llm_remote_batch_jobs FORCE ROW LEVEL SECURITY;
END $$;
Expand Down Expand Up @@ -376,4 +413,4 @@ BEGIN
WHERE table_name = 'com_config'),
'kv config store'::TEXT;
END;
$$ LANGUAGE plpgsql;
$$ LANGUAGE plpgsql;
48 changes: 43 additions & 5 deletions pg_llm_batch/db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,18 @@ def _normalize_remote_batch_snapshot(
)
counts_value = provider_batch.get("request_counts")
request_counts = counts_value if isinstance(counts_value, Mapping) else {}
raw_total_requests = request_counts.get("total")
total_requests_known = (
type(raw_total_requests) is int and raw_total_requests >= 0
)
total_requests = _provider_count(raw_total_requests)
completed_requests = _provider_count(request_counts.get("completed"))
failed_requests = _provider_count(request_counts.get("failed"))
if (
total_requests_known
and completed_requests + failed_requests > total_requests
):
raise ValueError("request_counts progress is inconsistent")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
provider_metadata, metadata_json = _provider_metadata(
provider_batch.get("metadata")
)
Expand All @@ -400,9 +412,10 @@ def _normalize_remote_batch_snapshot(
"batch_status": batch_status,
"output_file_id": output_file_id,
"error_file_id": error_file_id,
"total_requests": _provider_count(request_counts.get("total")),
"completed_requests": _provider_count(request_counts.get("completed")),
"failed_requests": _provider_count(request_counts.get("failed")),
"total_requests": total_requests,
"total_requests_known": total_requests_known,
"completed_requests": completed_requests,
"failed_requests": failed_requests,
"provider_metadata": provider_metadata,
"observed_at": observed,
"terminal_at": terminal_at,
Expand Down Expand Up @@ -444,14 +457,15 @@ def _persist_remote_batch_state(
completed_requests,
failed_requests,
provider_metadata,
total_requests_known,
first_seen_at,
last_observed_at,
terminal_at,
updated_at
)
VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s::jsonb, %s, %s, %s, %s
%s::jsonb, %s, %s, %s, %s, %s
)
ON CONFLICT (tenant_scope, endpoint_alias, remote_batch_id) DO UPDATE
SET observation_order = EXCLUDED.observation_order,
Expand All @@ -476,6 +490,10 @@ def _persist_remote_batch_state(
llm_remote_batch_jobs.total_requests,
EXCLUDED.total_requests
),
total_requests_known = (
llm_remote_batch_jobs.total_requests_known
OR EXCLUDED.total_requests_known
),
completed_requests = GREATEST(
llm_remote_batch_jobs.completed_requests,
EXCLUDED.completed_requests
Expand All @@ -496,6 +514,22 @@ def _persist_remote_batch_state(
),
updated_at = EXCLUDED.updated_at
WHERE EXCLUDED.observation_order > llm_remote_batch_jobs.observation_order
AND (
NOT (
llm_remote_batch_jobs.total_requests_known
OR EXCLUDED.total_requests_known
)
OR GREATEST(
llm_remote_batch_jobs.completed_requests,
EXCLUDED.completed_requests
) + GREATEST(
llm_remote_batch_jobs.failed_requests,
EXCLUDED.failed_requests
) <= GREATEST(
llm_remote_batch_jobs.total_requests,
EXCLUDED.total_requests
)
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
AND (
llm_remote_batch_jobs.batch_status NOT IN (
'completed', 'failed', 'expired', 'cancelled'
Expand All @@ -517,6 +551,7 @@ def _persist_remote_batch_state(
snapshot["completed_requests"],
snapshot["failed_requests"],
metadata_json,
snapshot["total_requests_known"],
observed,
observed,
terminal_at,
Expand Down Expand Up @@ -549,6 +584,7 @@ def persist_remote_batch_state(
observed_at=observed_at,
)
snapshot.pop("tenant_scope", None)
snapshot.pop("total_requests_known", None)
return snapshot


Expand All @@ -562,14 +598,16 @@ def persist_tenant_remote_batch_state(
observed_at: Optional[datetime] = None,
) -> Dict[str, Any]:
"""Persist one lifecycle projection for an explicit trusted tenant scope."""
return _persist_remote_batch_state(
snapshot = _persist_remote_batch_state(
dsn,
tenant_scope,
endpoint_alias,
provider_batch,
observation_order,
observed_at=observed_at,
)
snapshot.pop("total_requests_known", None)
return snapshot


def get_tenant_remote_batch_state(
Expand Down
41 changes: 39 additions & 2 deletions pg_llm_batch/schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ CREATE TABLE IF NOT EXISTS llm_remote_batch_jobs (
),
total_requests INTEGER NOT NULL DEFAULT 0
CHECK (total_requests >= 0),
total_requests_known BOOLEAN NOT NULL DEFAULT FALSE,
completed_requests INTEGER NOT NULL DEFAULT 0
CHECK (completed_requests >= 0),
failed_requests INTEGER NOT NULL DEFAULT 0
Expand All @@ -120,6 +121,12 @@ CREATE TABLE IF NOT EXISTS llm_remote_batch_jobs (
updated_at TIMESTAMPTZ NOT NULL,
CONSTRAINT ck_llm_remote_batch_jobs_tenant_scope
CHECK (tenant_scope ~ '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'),
CONSTRAINT ck_llm_remote_batch_jobs_known_progress
CHECK (
NOT total_requests_known OR
completed_requests::BIGINT + failed_requests::BIGINT
<= total_requests::BIGINT
),
CONSTRAINT uq_llm_remote_batch_jobs_tenant_endpoint_id
UNIQUE (tenant_scope, endpoint_alias, remote_batch_id)
);
Expand All @@ -142,6 +149,20 @@ BEGIN
ALTER TABLE llm_remote_batch_jobs
ALTER COLUMN tenant_scope SET NOT NULL;

ALTER TABLE llm_remote_batch_jobs
ADD COLUMN IF NOT EXISTS total_requests_known BOOLEAN;
UPDATE llm_remote_batch_jobs
SET total_requests_known = (
total_requests > 0 AND
completed_requests::BIGINT + failed_requests::BIGINT
<= total_requests::BIGINT
)
WHERE total_requests_known IS NULL;
ALTER TABLE llm_remote_batch_jobs
ALTER COLUMN total_requests_known SET DEFAULT FALSE;
ALTER TABLE llm_remote_batch_jobs
ALTER COLUMN total_requests_known SET NOT NULL;

IF EXISTS (
SELECT 1
FROM pg_constraint
Expand All @@ -167,6 +188,23 @@ BEGIN
ALTER TABLE llm_remote_batch_jobs
VALIDATE CONSTRAINT ck_llm_remote_batch_jobs_tenant_scope;

IF NOT EXISTS (
SELECT 1
FROM pg_constraint
WHERE conname = 'ck_llm_remote_batch_jobs_known_progress'
AND conrelid = 'llm_remote_batch_jobs'::regclass
) THEN
ALTER TABLE llm_remote_batch_jobs
ADD CONSTRAINT ck_llm_remote_batch_jobs_known_progress
CHECK (
NOT total_requests_known OR
completed_requests::BIGINT + failed_requests::BIGINT
<= total_requests::BIGINT
) NOT VALID;
END IF;
ALTER TABLE llm_remote_batch_jobs
VALIDATE CONSTRAINT ck_llm_remote_batch_jobs_known_progress;

IF NOT EXISTS (
SELECT 1
FROM pg_constraint
Expand All @@ -177,7 +215,6 @@ BEGIN
ADD CONSTRAINT uq_llm_remote_batch_jobs_tenant_endpoint_id
UNIQUE (tenant_scope, endpoint_alias, remote_batch_id);
END IF;

ALTER TABLE llm_remote_batch_jobs ENABLE ROW LEVEL SECURITY;
ALTER TABLE llm_remote_batch_jobs FORCE ROW LEVEL SECURITY;
END $$;
Expand Down Expand Up @@ -376,4 +413,4 @@ BEGIN
WHERE table_name = 'com_config'),
'kv config store'::TEXT;
END;
$$ LANGUAGE plpgsql;
$$ LANGUAGE plpgsql;
Loading
Loading