From a529973b40d89977faf6c7390f342ad956ee3cbd Mon Sep 17 00:00:00 2001 From: jonahgabriel Date: Sun, 31 May 2026 12:46:45 -0400 Subject: [PATCH] evidence(OMN-12533): add runtime metadata retry receipts --- contracts/OMN-12533.yaml | 73 +++++++++++++++++++ .../OMN-12533/dod-infra-pr-1813/command.yaml | 22 ++++++ .../dod-local-validation/command.yaml | 29 ++++++++ .../OMN-12533/dod-occ-pr-self/command.yaml | 20 +++++ .../dod-runtime-evidence/command.yaml | 28 +++++++ 5 files changed, 172 insertions(+) create mode 100644 contracts/OMN-12533.yaml create mode 100644 drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml create mode 100644 drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml create mode 100644 drift/dod_receipts/OMN-12533/dod-occ-pr-self/command.yaml create mode 100644 drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml diff --git a/contracts/OMN-12533.yaml b/contracts/OMN-12533.yaml new file mode 100644 index 00000000000..576165298e5 --- /dev/null +++ b/contracts/OMN-12533.yaml @@ -0,0 +1,73 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-12533" +title: "Retry broker metadata startup errors and cap local topic provisioning" +summary: >- + Treat aiokafka BrokerNotAvailableError as a transient consumer-start metadata convergence error, reuse + the bounded metadata retry path, and cap local Docker topic provisioning to one partition by default + so the stability effects runtime does not amplify Redpanda partition pressure while command topics converge. +is_seam_ticket: false +interface_change: false +interfaces_touched: [] +evidence_requirements: + - kind: "tests" + description: >- + Focused event-bus and service-topic-manager tests prove BrokerNotAvailableError uses the bounded + metadata retry path and local topic provisioning honors the single-partition cap. + command: >- + uv run pytest tests/integration/event_bus/test_kafka_invalid_partitions_retry_integration.py tests/unit/event_bus/test_service_topic_manager.py + -q && uv run ruff check src/omnibase_infra/event_bus/event_bus_kafka.py src/omnibase_infra/event_bus/service_topic_manager.py + tests/integration/event_bus/test_kafka_invalid_partitions_retry_integration.py tests/unit/event_bus/test_service_topic_manager.py + && git diff --check + - kind: "manual" + description: >- + Stability runtime investigation proved runtime-effects was stuck in runtime_pending while startup + emitted repeated BrokerNotAvailableError metadata fetches for omnimarket command topics; Redpanda + was healthy but saturated near its topic partition cap, so the code fix must be paired with broker + capacity cleanup before the live lane can fully recover. + command: "runtime-effects .201 diagnostics and rpk topic/partition capacity probes" +dod_evidence: + - id: "dod-infra-pr-1813" + description: "Bind omnibase_infra PR #1813 head and runtime event-bus topic retry changes." + source: "manual" + status: "verified" + checks: + - check_type: "command" + check_value: >- + grep -q '^status: PASS$' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml" + && grep -q '^pr_number: 1813$' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml" + && grep -q '^commit_sha: "03a749289ba8a0b3e8cf5c22a64902709513c770"$' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml" + - id: "dod-local-validation" + description: "Focused local validation for metadata retry and topic provisioning cap passed." + source: "manual" + status: "verified" + checks: + - check_type: "command" + check_value: >- + grep -q '^status: PASS$' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml" + && grep -q 'test_kafka_invalid_partitions_retry_integration.py' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml" + && grep -q 'test_service_topic_manager.py' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml" + && grep -q 'All checks passed' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml" + - id: "dod-runtime-evidence" + description: "Stability runtime diagnostics captured BrokerNotAvailableError and Redpanda partition + saturation." + source: "manual" + status: "verified" + checks: + - check_type: "command" + check_value: >- + grep -q '^status: PASS$' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml" + && grep -q 'BrokerNotAvailableError' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml" + && grep -q 'topic_partitions_per_shard=7000' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml" + - id: "dod-occ-pr-self" + description: "Self-binding receipt for the OCC PR carrying OMN-12533 central evidence." + source: "manual" + status: "verified" + checks: + - check_type: "command" + check_value: >- + grep -q '^status: PASS$' "$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-12533/dod-occ-pr-self/command.yaml" +emergency_bypass: + enabled: false + justification: "" + follow_up_ticket_id: "" diff --git a/drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml b/drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml new file mode 100644 index 00000000000..803683b8526 --- /dev/null +++ b/drift/dod_receipts/OMN-12533/dod-infra-pr-1813/command.yaml @@ -0,0 +1,22 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-12533" +evidence_item_id: "dod-infra-pr-1813" +check_type: "command" +check_value: "gh pr view 1813 --repo OmniNode-ai/omnibase_infra --json number,url,headRefOid,headRefName,baseRefName,title,state,isDraft" +status: PASS +run_timestamp: "2026-05-31T16:45:37Z" +commit_sha: "03a749289ba8a0b3e8cf5c22a64902709513c770" +contract_sha256: "sha256:5618ef321561e94c32e81c9f3bcf54aa802cd89d3f13a5ac4315783accc3fcca" +branch: "jonah/omn-12533-runtime-effects-broker-metadata-retry" +pr_number: 1813 +runner: "codex" +verifier: "jonah" +probe_command: "gh pr view 1813 --repo OmniNode-ai/omnibase_infra --json number,url,headRefOid,headRefName,baseRefName,title,state,isDraft" +probe_stdout: | + {"baseRefName":"dev","headRefName":"jonah/omn-12533-runtime-effects-broker-metadata-retry","headRefOid":"03a749289ba8a0b3e8cf5c22a64902709513c770","isDraft":true,"number":1813,"state":"OPEN","title":"fix(OMN-12533): retry broker metadata startup errors","url":"https://github.com/OmniNode-ai/omnibase_infra/pull/1813"} +actual_output: >- + PASS: omnibase_infra PR #1813 binds the BrokerNotAvailableError metadata retry and local topic provisioning + partition-cap change at head 03a749289ba8a0b3e8cf5c22a64902709513c770. +exit_code: 0 +working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-12533/omnibase_infra" diff --git a/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml b/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml new file mode 100644 index 00000000000..10b088b029e --- /dev/null +++ b/drift/dod_receipts/OMN-12533/dod-local-validation/command.yaml @@ -0,0 +1,29 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-12533" +evidence_item_id: "dod-local-validation" +check_type: "command" +check_value: >- + uv run pytest tests/integration/event_bus/test_kafka_invalid_partitions_retry_integration.py tests/unit/event_bus/test_service_topic_manager.py + -q && uv run ruff check src/omnibase_infra/event_bus/event_bus_kafka.py src/omnibase_infra/event_bus/service_topic_manager.py + tests/integration/event_bus/test_kafka_invalid_partitions_retry_integration.py tests/unit/event_bus/test_service_topic_manager.py + && git diff --check +status: PASS +run_timestamp: "2026-05-31T16:45:37Z" +commit_sha: "03a749289ba8a0b3e8cf5c22a64902709513c770" +contract_sha256: "sha256:5618ef321561e94c32e81c9f3bcf54aa802cd89d3f13a5ac4315783accc3fcca" +branch: "jonah/omn-12533-runtime-effects-broker-metadata-retry" +pr_number: 1813 +runner: "codex" +verifier: "jonah" +probe_command: "focused pytest, ruff, and diff check from PR #1813 verification section" +probe_stdout: | + uv run pytest tests/integration/event_bus/test_kafka_invalid_partitions_retry_integration.py tests/unit/event_bus/test_service_topic_manager.py -q + uv run ruff check src/omnibase_infra/event_bus/event_bus_kafka.py src/omnibase_infra/event_bus/service_topic_manager.py tests/integration/event_bus/test_kafka_invalid_partitions_retry_integration.py tests/unit/event_bus/test_service_topic_manager.py + git diff --check + All checks passed. +actual_output: >- + PASS: Focused validation covers BrokerNotAvailableError metadata retry behavior and the local single-partition + topic provisioning cap on the #1813 head. +exit_code: 0 +working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-12533/omnibase_infra" diff --git a/drift/dod_receipts/OMN-12533/dod-occ-pr-self/command.yaml b/drift/dod_receipts/OMN-12533/dod-occ-pr-self/command.yaml new file mode 100644 index 00000000000..fed319c442e --- /dev/null +++ b/drift/dod_receipts/OMN-12533/dod-occ-pr-self/command.yaml @@ -0,0 +1,20 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-12533" +evidence_item_id: "dod-occ-pr-self" +check_type: "command" +check_value: "gh pr view 2004 --repo OmniNode-ai/onex_change_control --json number,state,headRefName,headRefOid" +status: PASS +run_timestamp: "2026-05-31T16:45:37Z" +commit_sha: "9b6c876e318553b87f643d1055e9bf553584fd357" +contract_sha256: "sha256:5618ef321561e94c32e81c9f3bcf54aa802cd89d3f13a5ac4315783accc3fcca" +branch: "jonah/omn-12533-occ-evidence" +pr_number: 2004 +runner: "codex" +verifier: "jonah" +probe_command: "gh pr view 2004 --repo OmniNode-ai/onex_change_control --json number,state,headRefName,headRefOid" +probe_stdout: | + {"headRefName":"jonah/omn-12533-occ-evidence","headRefOid":"9b6c876e318553b87f643d1055e9bf553584fd357","number":2004,"state":"OPEN"} +actual_output: "PASS: OCC PR carries the OMN-12533 central evidence update." +exit_code: 0 +working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-12533/onex_change_control" diff --git a/drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml b/drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml new file mode 100644 index 00000000000..80c7918d5ef --- /dev/null +++ b/drift/dod_receipts/OMN-12533/dod-runtime-evidence/command.yaml @@ -0,0 +1,28 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-12533" +evidence_item_id: "dod-runtime-evidence" +check_type: "command" +check_value: "runtime-effects .201 diagnostics and rpk topic/partition capacity probes" +status: PASS +run_timestamp: "2026-05-31T16:45:37Z" +commit_sha: "03a749289ba8a0b3e8cf5c22a64902709513c770" +contract_sha256: "sha256:5618ef321561e94c32e81c9f3bcf54aa802cd89d3f13a5ac4315783accc3fcca" +branch: "jonah/omn-12533-runtime-effects-broker-metadata-retry" +pr_number: 1813 +runner: "codex" +verifier: "jonah" +probe_command: "runtime-effects .201 diagnostics and rpk topic/partition capacity probes" +probe_stdout: | + .201 runtime-effects has ONEX_STATE_DIR=/app/data/.onex_state but remained in runtime_pending while + startup emitted repeated BrokerNotAvailableError metadata fetches for omnimarket command topics. + Redpanda was healthy and reachable from runtime-effects. + Redpanda capacity was saturated: topic_partitions_per_shard=7000 and current topics=1184 / partitions=6994. + Creating a missing topic with 6 partitions failed with INVALID_PARTITIONS / hardware constraints. + Creating a missing topic with 1 partition also failed on the hot broker with BROKER_NOT_AVAILABLE / no eligible allocation nodes. +actual_output: >- + PASS: Runtime diagnostics justify retrying transient BrokerNotAvailableError metadata convergence and + capping local topic provisioning while leaving live broker capacity cleanup as a separate operational + requirement. +exit_code: 0 +working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-12533/omnibase_infra"