diff --git a/contracts/OMN-11513.yaml b/contracts/OMN-11513.yaml index cc1aa3e747e..2017e955ebc 100644 --- a/contracts/OMN-11513.yaml +++ b/contracts/OMN-11513.yaml @@ -51,6 +51,9 @@ evidence_requirements: - kind: "manual" description: "onex_change_control PR #2301 carries the 2026-06-07 post-deploy cloud-tier retest evidence." command: "See drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml" + - kind: "manual" + description: "onex_change_control PR #2355 carries the 2026-06-08 SEA delegation verification evidence." + command: "See drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml" emergency_bypass: enabled: false justification: "" @@ -161,3 +164,11 @@ dod_evidence: - check_type: "command" check_value: "grep -q '^status: PASS$' drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml && grep -q '^pr_number: 2301$' drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml" + - id: "dod-occ-pr-2355" + description: "onex_change_control PR #2355 carries the 2026-06-08 SEA delegation verification evidence." + source: "manual" + status: "verified" + checks: + - check_type: "command" + check_value: "grep -q '^status: PASS$' drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml + && grep -q '^pr_number: 2355$' drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml" diff --git a/drift/dod_receipts/OMN-11513/dod-deploy-assessment-pr-1739/command.yaml b/drift/dod_receipts/OMN-11513/dod-deploy-assessment-pr-1739/command.yaml index a36afe7b5e7..6e7029343a1 100644 --- a/drift/dod_receipts/OMN-11513/dod-deploy-assessment-pr-1739/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-deploy-assessment-pr-1739/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-deploy-assessment-pr-1739" check_type: "command" check_value: "gh pr view 1739 --repo OmniNode-ai/omnibase_infra --json number,state,headRefOid,mergedAt,mergeCommit" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-05-25T15:08:00Z" commit_sha: "a21892cf47c4aa7fe88771d6d5a564a7e575595a" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-cross-repo-test/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-cross-repo-test/command.yaml index 8660e93d83a..cc993f1377e 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-cross-repo-test/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-cross-repo-test/command.yaml @@ -16,6 +16,6 @@ actual_output: "PASS: full OCC test suite no longer fails on missing asyncpg; op gaps are explicit skips." exit_code: 0 pr_number: 1323 -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" branch: "jonah/omn-11513-occ-receipt-evidence" working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-11513/onex_change_control" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-1323/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-1323/command.yaml index cf6598fcbb9..aea0ac801b0 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-1323/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-1323/command.yaml @@ -16,6 +16,6 @@ probe_stdout: | actual_output: "PASS: onex_change_control PR #1323 is open and binds OMN-11513 OCC evidence to head c5e191f7a4ad90d2f597e406802aad3b1eb562f5." exit_code: 0 pr_number: 1323 -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" branch: "jonah/omn-11513-occ-receipt-evidence" working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-11513/onex_change_control" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-1607/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-1607/command.yaml index 80046159c15..c60fe209b4e 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-1607/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-1607/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-occ-pr-1607" check_type: "command" check_value: "gh pr view 1607 --repo OmniNode-ai/onex_change_control --json number,state,headRefOid,headRefName,title" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-05-25T14:48:00Z" commit_sha: "0aa641dc5d1cffd015d1a30dee2aad5cb6cc18d5" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-1608/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-1608/command.yaml index 45bdbf3d524..86360f1490c 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-1608/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-1608/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-occ-pr-1608" check_type: "command" check_value: "gh pr view 1608 --repo OmniNode-ai/onex_change_control --json number,state,headRefOid,headRefName,title" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-05-25T15:08:00Z" commit_sha: "1e0159288e3a3784163fd49f9fcb0ecd6a8d5236" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-1886/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-1886/command.yaml index e08aaff5abb..d8c69c3bca2 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-1886/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-1886/command.yaml @@ -16,6 +16,6 @@ actual_output: "PASS: onex_change_control PR #1886 carries the OMN-11513 Days 1- bundle." exit_code: 0 pr_number: 1886 -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" branch: "clone45/omn-11513-integration-evidence-days-1-7" working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-11513/onex_change_control-pr1886" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-2166/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-2166/command.yaml index 6a3a55cec11..2121053b158 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-2166/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-2166/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-occ-pr-2166" check_type: "command" check_value: "grep -q '^pr_number: 2166$' \"$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-11513/dod-occ-pr-2166/command.yaml\"" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-06-04T17:36:00Z" commit_sha: "6b52cec72b2f43685b1cd5cd78d56a80cdb8296d" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-2176/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-2176/command.yaml index eb29726f68f..c8b90086460 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-2176/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-2176/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-occ-pr-2176" check_type: "command" check_value: "grep -q '^pr_number: 2176$' \"$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-11513/dod-occ-pr-2176/command.yaml\"" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-06-05T02:54:00Z" commit_sha: "aec52162ab3a65c9ba89ff02448e2dafd379e844" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-2203/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-2203/command.yaml index c25ce6ebbc8..f590f641de0 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-2203/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-2203/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-occ-pr-2203" check_type: "command" check_value: "grep -q '^pr_number: 2203$' \"$CONTRACT_REPO_DIR/drift/dod_receipts/OMN-11513/dod-occ-pr-2203/command.yaml\"" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-06-07T13:50:00Z" commit_sha: "1741235a54cbc83466bc2ef220fe6f6a183cc31f" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml index e6b688f04a4..c4cf999fbf2 100644 --- a/drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-occ-pr-2301" check_type: "command" check_value: "grep -q '^pr_number: 2301$' drift/dod_receipts/OMN-11513/dod-occ-pr-2301/command.yaml" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-06-08T15:05:00Z" commit_sha: "ccc5d7e8673cb04f6909e3ab22391599033edea9" diff --git a/drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml b/drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml new file mode 100644 index 00000000000..5aaf5b6f842 --- /dev/null +++ b/drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml @@ -0,0 +1,22 @@ +--- +schema_version: "1.0.0" +ticket_id: "OMN-11513" +evidence_item_id: "dod-occ-pr-2355" +check_type: "command" +check_value: "grep -q '^pr_number: 2355$' drift/dod_receipts/OMN-11513/dod-occ-pr-2355/command.yaml" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" +status: PASS +run_timestamp: "2026-06-08T21:30:00Z" +commit_sha: "134f2162e61336c955596f6118ce9b61deb80719" +branch: "clone45/omn-11513-2026-06-08-verification" +runner: "codex-pr-sweep-20260608" +verifier: "occ-preflight-local" +probe_command: "gh pr view 2355 --repo OmniNode-ai/onex_change_control --json number,state,headRefOid,headRefName,title" +probe_stdout: | + {"headRefName":"clone45/omn-11513-2026-06-08-verification","headRefOid":"134f2162e61336c955596f6118ce9b61deb80719","number":2355,"state":"OPEN","title":"evidence(OMN-11513): 2026-06-08 SEA delegation verification — broker + local-ADK retests"} +actual_output: "PASS: onex_change_control PR #2355 carries the OMN-11513 2026-06-08 SEA delegation verification + evidence, including broker clearance, local-ADK retest evidence, Phase 0 endpoint-resolution correction, + and projection-boundary notes." +exit_code: 0 +working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-11513/onex_change_control-pr2355" +pr_number: 2355 diff --git a/drift/dod_receipts/OMN-11513/dod-omni-home-pr-145/command.yaml b/drift/dod_receipts/OMN-11513/dod-omni-home-pr-145/command.yaml index 921dddca098..af0ad35abd8 100644 --- a/drift/dod_receipts/OMN-11513/dod-omni-home-pr-145/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-omni-home-pr-145/command.yaml @@ -16,6 +16,6 @@ actual_output: "PASS: omni_home PR #145 is open and carries the OMN-11513 plan/e 7f20351afd25fb58c2cd94f7b5be9a166b3b394b." exit_code: 0 pr_number: 145 -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" branch: "jonah/omn-11513-may-monthly-integration-testing-plan-may-1-22" working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-11513/omni_home" diff --git a/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1735/command.yaml b/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1735/command.yaml index 72692fedf14..d6697f509bc 100644 --- a/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1735/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1735/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-omnibase-infra-pr-1735" check_type: "command" check_value: "gh pr view 1735 --repo OmniNode-ai/omnibase_infra --json number,state,headRefOid,headRefName,title" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-05-25T14:45:00Z" commit_sha: "fed67777c6034ee0f88496722ea1d88c46aef31a" diff --git a/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1739/command.yaml b/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1739/command.yaml index a86a62b1972..d5f6a62077c 100644 --- a/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1739/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-omnibase-infra-pr-1739/command.yaml @@ -4,7 +4,7 @@ ticket_id: "OMN-11513" evidence_item_id: "dod-omnibase-infra-pr-1739" check_type: "command" check_value: "gh pr view 1739 --repo OmniNode-ai/omnibase_infra --json number,state,headRefOid,headRefName,title" -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" status: PASS run_timestamp: "2026-05-25T15:02:00Z" commit_sha: "a21892cf47c4aa7fe88771d6d5a564a7e575595a" diff --git a/drift/dod_receipts/OMN-11513/dod-stability-runtime-proof/command.yaml b/drift/dod_receipts/OMN-11513/dod-stability-runtime-proof/command.yaml index 18a3c6f8ec3..25c5c4a9593 100644 --- a/drift/dod_receipts/OMN-11513/dod-stability-runtime-proof/command.yaml +++ b/drift/dod_receipts/OMN-11513/dod-stability-runtime-proof/command.yaml @@ -17,6 +17,6 @@ probe_stdout: | actual_output: "PASS: .201 stability runtime exposes /v1/introspection/manifest and OMN-11513 evidence recorded synthetic delegation correlation 1efcd854-f2cf-49b2-8772-8e2595b58b1b." exit_code: 0 -contract_sha256: "sha256:5f5b090ac9419606cd8f2c5dbf95ed7597253c0aaa99648f5c92d730f52160ec" +contract_sha256: "sha256:5c28a9804569ca5e6274c2491aa8a92d1d7f726f32c10aab499d680e30d0cd57" branch: "jonah/omn-11513-may-monthly-integration-testing-plan-may-1-22" working_dir: "/Users/jonah/Code/omni_home/omni_worktrees/OMN-11513/omni_home" diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/afternoon-retest-unblocked.txt b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/afternoon-retest-unblocked.txt new file mode 100644 index 00000000000..30a6ec90436 --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/afternoon-retest-unblocked.txt @@ -0,0 +1,183 @@ +AFTERNOON RETEST — 2026-06-08 (after Jonah's broker fix) +========================================================= + +Context: Jonah recreated the omnibase-infra-stability-test-redpanda broker and +reported broker 0 now advertised at 100.109.203.94:39092 from his side. He asked +Bret to rerun the broker check and Phase 2 from the WSL/Tailscale seat. + +Pre-retest setup done earlier this afternoon: +- SEA contractor clone pulled June 5 build (80dc069) -> current dev (27f4f03). +- uv sync ran clean (Resolved 242 packages). Resolver canary PASS: 24576. + +Run context for the retest: + branch: dev + SHA: 27f4f03 (fix(OMN-12816): raise SEA generation retry budget #218) + python: 3.12.13 + + +------------------------------------------------------------------------------- +PHASE 1 — BROKER REACHABILITY -> GREEN (advertised host fixed) +------------------------------------------------------------------------------- + +For contrast, this morning (17:14Z, before the fix) the raw-socket metadata parse +returned: node=0 ADVERTISED_HOST=192.168.86.201 port=39092 (the LAN redirect). + +Retest tooling note: ad-hoc probes via confluent_kafka and a raw-socket parse both +hung / timed out on the MetadataRequest, but NOT with this morning's _TRANSPORT +"following the LAN address" signature. librdkafka was now talking to +thrd:100.109.203.94:39092/0 directly and timing out on the request itself. That is +the auto-API-version-negotiation quirk the stability-test overlay explicitly warns +about ("KAFKA_API_VERSION 2.8.0 ... auto fails .201 metadata bootstrap"). The +sanctioned client path resolves it. + +Authoritative check — aiokafka (the real pipeline client): + $ KAFKA_BOOTSTRAP_SERVERS=100.109.203.94:39092 KAFKA_API_VERSION=2.8.0 \ + python -c "" + brokers: [(0, '100.109.203.94', 39092)] + RESULT: bus reachable, metadata OK + +=> Broker 0 now advertises the Tailscale-routable host. Phase 1 GREEN, confirmed + from the WSL/Tailscale seat. Jonah's fix holds from our side. + + +------------------------------------------------------------------------------- +PHASE 3 — DIRECT InferenceClient DELEGATION SMOKE -> PASS (proof-quality) +------------------------------------------------------------------------------- + +(This morning this phase failed at aiokafka bus.start() with InfraTimeoutError — + the LAN-redirect blocker. It now completes the full round-trip.) + + timestamp: 2026-06-08T17:58:12Z SHA: 27f4f03 + + max_context_tokens 24576 + overlay sea-delegation-bootstrap-stability-test + correlation_id 8c0ddf2a-762b-4365-b714-42ac73a0c3e5 + model_used Qwen3.6-27B-MTP-IQ4_XS.gguf + latency_ms 2824 + prompt_tokens 33 + completion_tokens 188 + error_message (empty) + content healthy=true + + Acceptance check: + overlay = stability-test ............ PASS + max_context_tokens = 24576 .......... PASS + model_used = Qwen3.6-27B-MTP-IQ4_XS .. PASS + content indicates healthy=true ...... PASS + error_message empty/None ............ PASS + +=> Bus round-trip + local delegated inference proven end to end. PASS. + + +------------------------------------------------------------------------------- +PHASE 2 — SEA --agent (keyless, per plan) -> broker CLEARED; stops at ADK finding +------------------------------------------------------------------------------- + + timestamp: 2026-06-08T18:00:15Z SHA: 27f4f03 + command: uv run python -m src --agent \ + --task "Generate a compute node that returns whether a status string is healthy." + + [MODE:LIVE] [STAGE:AGENT] Self-extending agent loop + [OVERLAY] sea-delegation-bootstrap-stability-test@1.0.0 lane=stability-test + topic=onex.cmd.omnibase-infra.delegation-inference-request.v1 + [READINESS] requested=sanctioned_live bootstrap=100.109.203.94:39092 + probe=ready (38135ms) transport=bus_backed + NOTE: Cloud key not set — cloud fallback disabled. Routing local-first. + Correlation ID: 1a7f6369-2399-4011-b2c6-d408207e4a89 + Error: selected ADK backend cloud_gemini requires ONEX_TRACK_A_API_KEY or + GOOGLE_API_KEY. Select a local ADK tier in contract.local.yaml or + provide the configured credential. + + Reading: + - Broker is NO LONGER the blocker: probe ready, transport bus_backed, + bootstrap 100.109.203.94, fresh correlation, no direct-mode fallback, + routes local-first. + - Run stops at the agent-loop ADK backend selection (defaults cloud_gemini, + needs a key). This is the SEPARATE finding flagged this morning, now + reproduced cleanly on the far side of the broker fix. It is a config + choice, not a broker/runtime defect; the error is typed and actionable. + - Note: probe readiness was 38135ms (vs ~9s this morning) — freshly recreated + broker, slower initial metadata sync; it did become ready. + + Classification: diagnostic (bus-backed, fresh correlation, no fallback, then + typed ADK-backend stop). NOT discarded (correct host/overlay, no mutation). + + +------------------------------------------------------------------------------- +SUMMARY +------------------------------------------------------------------------------- +Phase 1 broker ........ GREEN (advertises 100.109.203.94 from our seat) +Phase 3 inference ..... PASS (proof; correlation 8c0ddf2a; content healthy=true) +Phase 2 --agent ....... broker cleared; stops at out-of-scope ADK backend (cloud_gemini) + +Jonah's specific blocker (broker LAN-advertise) is confirmed cleared from the +WSL/Tailscale seat. A complete --agent generation proof remains gated on the ADK +reasoning backend: either a local ADK tier (in scope, needs config) or a cloud key +(out of today's scope per the verification plan). + + +------------------------------------------------------------------------------- +APPENDIX — CONTRACTOR DIAGNOSTIC EXPERIMENT (off-plan, Bret-approved): can Phase 2 +run FULLY LOCAL with no cloud key? -> NO (local server 400s the ADK request) +------------------------------------------------------------------------------- +NOT part of Jonah's plan; a self-initiated in-scope experiment. Overlay created and +deleted; clone left clean. + +Setup: created a gitignored contract.local.yaml next to +src/nodes/node_agent_orchestrator/contract.yaml with + config.agent_runtime.adk_model_ref: local_deepseek_reasoning +(the loader supports this overlay seam; contract.py:103). This repoints the agent's +ADK reasoning backend from the default cloud_gemini to the local vLLM tier at :8001. + +What worked (no cloud key needed): + - overlay loaded, deep-merged over the default contract; + - ADK selected the local OpenAI-compatible adapter (_OpenAICompatibleLlm, + Authorization: Bearer EMPTY); + - endpoint rewrite via SEA_LOCAL_INFERENCE_BASE_URL resolved to + http://100.109.203.94:8001/v1/chat/completions (the same proven endpoint); + - got PAST the cloud_gemini key stop entirely. + +What failed: + - httpx.HTTPStatusError: 400 Bad Request from the local endpoint, raised inside + google/adk/models/apigee_llm.py during the agent loop -> + "RuntimeError: ADK runner background thread failed". + +Root cause ISOLATED (captured the exact request + 400 body via httpx instrumentation, +2026-06-08T18:3xZ; full capture in local-adk-400-capture.json): + + 400 response body from the local llama.cpp server (:8001): + "Unable to generate parser for this template. Automatic parser generation + failed: JSON schema conversion failed: Unrecognized schema: {\"type\":\"STRING\"}" + (repeated for each tool parameter) + + The request the ADK adapter sent serializes every tool-function parameter with + GEMINI-DIALECT JSON Schema types — UPPERCASE — e.g. + "parameters":{"properties":{"description":{"type":"STRING"}},"type":"OBJECT"} + Gemini's function-calling schema uses STRING/OBJECT/INTEGER. The local + OpenAI-compatible server (llama.cpp build b9206) requires standard lowercase + JSON Schema ("string"/"object") and rejects the uppercase types when building its + tool-calling grammar -> 400. + + Why earlier probe got 200: my direct probe used a hand-written lowercase tool + ({"type":"object"/"string"}), which the server accepts. The agent's REAL tools come + from ADK FunctionDeclaration objects (Gemini Schema), serialized uppercase. So it + is specifically the Gemini->OpenAI tool-schema dialect that is not converted. + + Where: SEA's _OpenAICompatibleLlm adapter / build_http_chat_completions_model in + src/nodes/node_agent_orchestrator/adk_backend_adapters.py passes the request + through Google ADK's CompletionsHTTPClient (apigee_llm) WITHOUT normalizing the + function-declaration schema types to lowercase for the OpenAI-compatible target. + The Gemini backend works because Gemini accepts uppercase natively. + + Likely fix: when targeting an OpenAI-compatible endpoint, lowercase the JSON Schema + "type" values (and any other Gemini->OpenAI schema normalization) on the tool + function parameters before sending. Candidate site: the _OpenAICompatibleLlm + adapter path. (Could not isolate without authorization to dig; observation only.) + +Conclusion: fully-local SEA --agent generation fails because the local-ADK adapter +ships Gemini-dialect (uppercase) tool schemas to the OpenAI-compatible local server, +which rejects them (400). The local INFERENCE path is proven (Phase 3). The local +ORCHESTRATION path (ADK agent loop on a local model) is blocked on this single schema +conversion. Cloud Gemini remains the only orchestrator backend that completes today, +and that is out of today's scope. Not a demo blocker; a gap in the advertised +local-ADK fallback. diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/local-adk-400-capture.json b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/local-adk-400-capture.json new file mode 100644 index 00000000000..8e51831bba0 --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/local-adk-400-capture.json @@ -0,0 +1,18 @@ +[ + { + "url": "http://100.109.203.94:8001/v1/chat/completions", + "method": "POST", + "req_headers": { + "host": "100.109.203.94:8001", + "accept": "*/*", + "accept-encoding": "gzip, deflate, zstd", + "connection": "keep-alive", + "user-agent": "python-httpx/0.28.1", + "content-type": "application/json", + "content-length": "4260" + }, + "req_body": "{\"model\":\"Qwen3.6-27B-MTP-IQ4_XS.gguf\",\"messages\":[{\"role\":\"system\",\"content\":\"You are a self-extending agent. When asked to perform a task that requires a capability you don't have, you:\\n1. Call scaffold_onex_node to generate a new ONEX compute node for the task\\n2. If the result has returncode != 0, report the error and stop\\n3. Call register_generated_tool with the node_name, contract_yaml, and handler_source from the scaffold result\\n4. If registration fails, report the error and stop\\n5. Call invoke_generated_tool with the registered tool_name and a sample JSON input\\n that exercises the generated capability\\n6. Report: the tool name, what it does, the validation checks it passed, and the invocation result\\n\\nYou only generate COMPUTE nodes (pure transformations, no side effects).\\nMaximum 2 generation attempts before reporting failure.\\nFor invoke_generated_tool, construct a realistic sample input as a JSON string\\nthat matches the generated node's input_model.\\n\\nYou are an agent. Your internal name is \\\"node_self_extending_agent\\\".\"},{\"role\":\"user\",\"content\":\"Task: Generate a compute node that returns whether a status string is healthy.\\nPlease generate, validate, register, and invoke an ONEX compute node for this task. Report the tool name, validation checks passed, and invocation result.\"}],\"stream\":false,\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"scaffold_onex_node\",\"description\":\"Generate an ONEX compute node scaffold from a natural language description.\\n\\n Routes through the DelegationExecutor which uses a local-first escalation\\n ladder built from the model registry contract. Local tiers (Qwen, DeepSeek)\\n are tried before cloud fallback \u2014 no cloud API key is required when a local\\n endpoint is healthy.\\n\\n SEA owns no ladder logic: tier ordering and escalation policy come entirely\\n from load_default_config() / the model registry YAML. This satisfies the\\n OMN-12286 constraint that SEA consumes a routing decision rather than\\n selecting tiers itself.\\n\\n Returns a JSON string matching ModelNodeGenerationResult on success.\\n\\n Raises:\\n SeaGenerationError: when the pipeline cannot produce a contract-valid node.\\n A generation failure is surfaced as a typed exception carrying the\\n correlation id, the structured per-attempt validation errors, and the\\n failing artifact identity \u2014 never a benign ``node_name=\\\"unknown\\\"`` stub.\\n \",\"parameters\":{\"properties\":{\"description\":{\"type\":\"STRING\"}},\"required\":[\"description\"],\"type\":\"OBJECT\"}}},{\"type\":\"function\",\"function\":{\"name\":\"validate_contract\",\"description\":\"Validate an ONEX contract YAML string.\\n\\n Returns a JSON string matching ModelContractValidationResult.\\n Uses ModelValidationResult internally for structured result handling.\\n \",\"parameters\":{\"properties\":{\"yaml_str\":{\"type\":\"STRING\"}},\"required\":[\"yaml_str\"],\"type\":\"OBJECT\"}}},{\"type\":\"function\",\"function\":{\"name\":\"register_generated_tool\",\"description\":\"Register a dynamically generated ONEX node as an invocable tool.\\n\\n Validates the contract + handler, exec()s the handler in a restricted sandbox,\\n and stores it in the in-process registry.\\n\\n ``node_name`` is used as a correlation hint only \u2014 the authoritative node name\\n is read from the contract YAML by ToolRegistry.register().\\n\\n Returns a JSON string with {\\\"registered\\\": true, \\\"tool_name\\\": ..., \\\"handler_hash\\\": ...}\\n or {\\\"registered\\\": false, \\\"error\\\": ...} on failure.\\n \",\"parameters\":{\"properties\":{\"node_name\":{\"type\":\"STRING\"},\"contract_yaml\":{\"type\":\"STRING\"},\"handler_source\":{\"type\":\"STRING\"}},\"required\":[\"node_name\",\"contract_yaml\",\"handler_source\"],\"type\":\"OBJECT\"}}},{\"type\":\"function\",\"function\":{\"name\":\"invoke_generated_tool\",\"description\":\"Invoke a previously registered generated tool by name.\\n\\n Parses input_json, calls the registered handler in a sandboxed environment\\n with a 5-second timeout, and returns the JSON-serialised result.\\n\\n Returns the tool's JSON output string, or {\\\"error\\\": ...} on failure.\\n \",\"parameters\":{\"properties\":{\"tool_name\":{\"type\":\"STRING\"},\"input_json\":{\"type\":\"STRING\"}},\"required\":[\"tool_name\",\"input_json\"],\"type\":\"OBJECT\"}}}]}", + "status": 400, + "resp_body": "{\"error\":{\"code\":400,\"message\":\"Unable to generate parser for this template. Automatic parser generation failed: JSON schema conversion failed:\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\\nUnrecognized schema: {\\\"type\\\":\\\"STRING\\\"}\",\"type\":\"invalid_request_error\"}}" + } +] diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase0-unit-tests.txt b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase0-unit-tests.txt new file mode 100644 index 00000000000..2b9822a62b9 --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase0-unit-tests.txt @@ -0,0 +1,96 @@ +2026-06-08T16:04:00Z +.......................................s......................F......... [ 5%] +........................................................................ [ 10%] +........................................................................ [ 15%] +........................................................................ [ 20%] +........................................................................ [ 25%] +........................................................................ [ 31%] +........................................................................ [ 36%] +........................................................................ [ 41%] +........................................................................ [ 46%] +........................................................................ [ 51%] +........................................................................ [ 57%] +........................................................................ [ 62%] +.....................................F................F................. [ 67%] +........................................................................ [ 72%] +........................................................................ [ 77%] +........................................................................ [ 82%] +........................................................................ [ 88%] +........................................................................ [ 93%] +........................................................................ [ 98%] +..................... [100%] +=================================== FAILURES =================================== +_ test_build_runtime_uses_local_handler_when_contract_selects_local_model_ref __ + + @pytest.mark.unit + def test_build_runtime_uses_local_handler_when_contract_selects_local_model_ref() -> None: + from src.nodes.node_agent_orchestrator import build_agent_runtime + + contract = load_default_agent_orchestrator_contract() + local_contract = contract.model_copy( + update={ + "config": contract.config.model_copy( + update={ + "agent_runtime": contract.config.agent_runtime.model_copy( + update={"adk_model_ref": "local_qwen_coder"} + ) + } + ) + } + ) + + runtime = build_agent_runtime(contract=local_contract) + model = runtime.root_agent.canonical_model + + assert runtime.auth_mode == EnumModelAuthMode.NONE + assert runtime.served_model_id == "Qwen3.6-35B-A3B" + assert runtime.backend_id == "local-coder" + assert model.model == "Qwen3.6-35B-A3B" +> assert model.endpoint_url == "http://localhost:8000/v1/chat/completions" +E AssertionError: assert 'http://100.1...t/completions' == 'http://local...t/completions' +E +E - http://localhost:8000/v1/chat/completions +E + http://100.109.203.94:8000/v1/chat/completions + +tests/unit/test_agent_orchestrator.py:239: AssertionError +____________ test_default_registry_loads_with_served_vllm_model_ids ____________ + + def test_default_registry_loads_with_served_vllm_model_ids() -> None: + registry = load_default_model_registry() + + qwen = registry.entry_for_tier(EnumModelTier.CODING_PRIMARY) + deepseek = registry.entry_for_tier(EnumModelTier.REASONING_FALLBACK) + cloud = registry.entry_for_tier(EnumModelTier.CLOUD_FALLBACK) + + assert qwen.served_model_id == "Qwen3.6-35B-A3B" + assert deepseek.served_model_id == "Qwen3.6-27B-MTP-IQ4_XS.gguf" +> assert qwen.endpoint_url == "http://localhost:8000/v1/chat/completions" +E AssertionError: assert 'http://100.1...t/completions' == 'http://local...t/completions' +E +E - http://localhost:8000/v1/chat/completions +E + http://100.109.203.94:8000/v1/chat/completions + +tests/unit/test_model_registry_contract.py:42: AssertionError +________________ test_load_default_registry_honors_env_override ________________ + +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x762a9aa15190> + + def test_load_default_registry_honors_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + """load_default_model_registry (the path __main__ uses) honors the env seam.""" + monkeypatch.setenv(LOCAL_INFERENCE_BASE_URL_ENV, "http://192.168.86.201:8000") + registry = load_default_model_registry() + qwen = registry.entry_for_tier(EnumModelTier.CODING_PRIMARY) +> assert qwen.endpoint_url == "http://192.168.86.201:8000/v1/chat/completions" +E AssertionError: assert 'http://100.1...t/completions' == 'http://192.1...t/completions' +E +E - http://192.168.86.201:8000/v1/chat/completions +E ? ^^ ^^^^^ ^ +E + http://100.109.203.94:8000/v1/chat/completions +E ? ^^ ^^ ^^^^ + +tests/unit/test_model_registry_contract.py:279: AssertionError +=========================== short test summary info ============================ +FAILED tests/unit/test_agent_orchestrator.py::test_build_runtime_uses_local_handler_when_contract_selects_local_model_ref +FAILED tests/unit/test_model_registry_contract.py::test_default_registry_loads_with_served_vllm_model_ids +FAILED tests/unit/test_model_registry_contract.py::test_load_default_registry_honors_env_override +3 failed, 1385 passed, 1 skipped in 285.51s (0:04:45) diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase2-agent-e2e.txt b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase2-agent-e2e.txt new file mode 100644 index 00000000000..7c512d763ff --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase2-agent-e2e.txt @@ -0,0 +1,11 @@ +2026-06-08T16:12:35Z +[MODE:LIVE] [STAGE:AGENT] Self-extending agent loop +[OVERLAY] sea-delegation-bootstrap-stability-test@1.0.0 lane=stability-test topic=onex.cmd.omnibase-infra.delegation-inference-request.v1 hash=sha256:7ba7c84e3aa3be32... command=--agent -> evidence=.onex_state/hackathon/event_chains/delegation_overlay/preflight-9c3d8343-9719-49b9-bc04-c6446737d0ba.delegation_overlay_evidence.json +[READINESS] requested=sanctioned_live bootstrap=100.109.203.94:39092 probe=ready (9181ms) transport=bus_backed +NOTE: Cloud key not set — cloud fallback disabled. Routing local-first. +╭───────────────────────── ONEX Self-Extending Agent ──────────────────────────╮ +│ Task: Generate a compute node that returns whether a status string is │ +│ healthy. │ +│ Correlation ID: 48626687-f08b-4baa-a657-e83d1344068f │ +╰──────────────────────────────────────────────────────────────────────────────╯ +Error: selected ADK backend cloud_gemini requires ONEX_TRACK_A_API_KEY or GOOGLE_API_KEY. Select a local ADK tier in contract.local.yaml or provide the configured credential. diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase2-local-adk-PASS-postfix.txt b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase2-local-adk-PASS-postfix.txt new file mode 100644 index 00000000000..890f86049ca --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase2-local-adk-PASS-postfix.txt @@ -0,0 +1,54 @@ +=== run context === +2026-06-08T20:18:41Z +56d5a45 +=== Phase 2 (local ADK tier, keyless, post-fix) === +[MODE:LIVE] [STAGE:AGENT] Self-extending agent loop +[OVERLAY] sea-delegation-bootstrap-stability-test@1.0.0 lane=stability-test topic=onex.cmd.omnibase-infra.delegation-inference-request.v1 hash=sha256:7ba7c84e3aa3be32... command=--agent -> evidence=.onex_state/hackathon/event_chains/delegation_overlay/preflight-551c8af7-b07d-4586-8f21-19c898424d24.delegation_overlay_evidence.json +[READINESS] requested=sanctioned_live bootstrap=100.109.203.94:39092 probe=ready (1334ms) transport=bus_backed +NOTE: Cloud key not set — cloud fallback disabled. Routing local-first. +NOTE: Local model not available. Track B will be skipped. +╭───────────────────────── ONEX Self-Extending Agent ──────────────────────────╮ +│ Task: Generate a compute node that returns whether a status string is │ +│ healthy. │ +│ Correlation ID: 0cdc2410-3fa5-4da6-b561-52d4a7075352 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +Deprecated. Please migrate to the async method. + 50 chain(s) on disk, 3 usable example(s) +╭───────────────────────────── Context Injection ──────────────────────────────╮ +│ Injecting 3 prior example(s) │ +│ Task: A compute node that takes a status string as input and returns a │ +│ boolean indicat │ +╰──────────────────────────────────────────────────────────────────────────────╯ + Example # Similarity + 1 0.042 + 2 0.042 + 3 0.021 + 50 chain(s) on disk, 3 usable example(s) + 3 prior example(s) available (not triggered — first-attempt success) + register_generated_tool {'node_name': 'status_health_checker', +'contract_yaml': 'name: status_health_checker\ncontract_version: +1.0.0\nnode_type +╭──────────────────────────────── Registration ────────────────────────────────╮ +│ Node registered as MCP tool │ +│ Name: status_health_checker │ +│ Hash: │ +│ sha256:49620c3dededa776cdda37a12421533fe4b92a261508ac9808223a67e5f55881 │ +╰──────────────────────────────────────────────────────────────────────────────╯ + invoke_generated_tool {'tool_name': 'status_health_checker', 'input_json': +'{"status": "Running"}'} +╭────────────────────────────── Tool Invocation ───────────────────────────────╮ +│ Tool: status_health_checker │ +│ Input: {"status": "Running"} │ +│ Result: {'result': '{"is_healthy": true}'} │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────── Run Summary ─────────────────────────────────╮ +│ Result: PASS │ +│ Cost: $0.0000 │ +│ Latency: 64827 ms │ +│ Attempts: 1 │ +╰──────────────────────────────────────────────────────────────────────────────╯ +correlation_id: 0cdc2410-3fa5-4da6-b561-52d4a7075352 +├── onex.cmd.omnimarket.node-generation-requested.v1 +│ └── source: onex_agent_runner at: 2026-06-08T20:20:45.842037+00:00 +└── onex.evt.omnimarket.node-generation-completed.v1 + └── source: onex_agent_runner at: 2026-06-08T20:20:45.842089+00:00 diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase3-inference-smoke.txt b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase3-inference-smoke.txt new file mode 100644 index 00000000000..f548e01a335 --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase3-inference-smoke.txt @@ -0,0 +1,99 @@ +2026-06-08T16:15:10Z +Timeout connecting to Kafka after 30s +max_context_tokens 24576 +overlay sea-delegation-bootstrap-stability-test +Traceback (most recent call last): + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/delegation_bus_stability.py", line 240, in _run_round_trip + asyncio.get_running_loop() +RuntimeError: no running event loop + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/producer/producer.py", line 358, in start + await self._sender.start() + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/producer/sender.py", line 73, in start + await self._maybe_wait_for_pid() + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/producer/sender.py", line 206, in _maybe_wait_for_pid + success = await self._do_init_pid(node_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/producer/sender.py", line 271, in _do_init_pid + return await handler.do(node_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/producer/sender.py", line 397, in do + resp = await self._sender.client.send(node_id, req, group=self.group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/client.py", line 463, in send + if not (await self.ready(node_id, group=group)): + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/client.py", line 444, in ready + conn = await self._get_conn(node_id, group=group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/client.py", line 418, in _get_conn + self._conns[conn_id] = await create_conn( + ^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/conn.py", line 89, in create_conn + await conn.connect() + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/aiokafka/conn.py", line 216, in connect + transport, _ = await loop.create_connection( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 1107, in create_connection + sock = await self._connect_sock( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 1010, in _connect_sock + await self.sock_connect(sock, address) + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/selector_events.py", line 651, in sock_connect + return await fut + ^^^^^^^^^ +asyncio.exceptions.CancelledError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/omnibase_infra/event_bus/event_bus_kafka.py", line 681, in start + await asyncio.wait_for( + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 519, in wait_for + async with timeouts.timeout(timeout): + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/timeouts.py", line 115, in __aexit__ + raise TimeoutError from exc_val +TimeoutError + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "", line 17, in + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/inference.py", line 299, in infer + response_envelope = self._bus.publish_and_wait( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/delegation_bus_stability.py", line 214, in publish_and_wait + return self._run_round_trip(envelope, correlation_id, timeout_s) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/delegation_bus_stability.py", line 243, in _run_round_trip + return asyncio.run(self._bounded_round_trip(envelope, correlation_id, timeout_s)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 195, in run + return runner.run(main) + ^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/delegation_bus_stability.py", line 275, in _bounded_round_trip + return await asyncio.wait_for(self._round_trip(envelope, correlation_id, timeout_s), timeout=timeout_s) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/clone45/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for + return await fut + ^^^^^^^^^ + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/delegation_bus_stability.py", line 310, in _round_trip + await self._transport.start() + File "/mnt/c/Code/onex-self-extending-agent/src/pipeline/delegation_bus_stability.py", line 367, in start + await bus.start() + File "/mnt/c/Code/onex-self-extending-agent/.venv/lib/python3.12/site-packages/omnibase_infra/event_bus/event_bus_kafka.py", line 747, in start + raise InfraTimeoutError( +omnibase_infra.errors.error_infra.InfraTimeoutError: [ONEX_CORE_092_TIMEOUT_ERROR] Timeout connecting to Kafka after 30s diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase4-projection-readback.txt b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase4-projection-readback.txt new file mode 100644 index 00000000000..1a423287634 --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/phase4-projection-readback.txt @@ -0,0 +1,110 @@ +PHASE 4 — PROJECTION API READBACK + generation_events binding check (2026-06-08) +================================================================================ +Contractor seat (WSL2 over Tailscale), read-only. +Lane note: this evidence uses the stability-test projection API on :13002 +intentionally. OMN-12760's contract/API surface is :3002; :13002 is the +stability-test lane surface used for this contractor verification pass. + +--- Health --- +$ curl -fsS http://100.109.203.94:13002/health +{"status":"ok","postgres":"ok"} + +--- Plan's named topic (Phase 4 step 2) --- +GET /projection/onex.snapshot.projection.hackathon_pipeline_events.v1 +-> 404 unknown_topic (topic not present on this lane) + +--- Projection index: the generation_events projection descriptor --- +GET /projections (excerpt) +{ + "topic": "onex.evt.omnimarket.node-generation-completed.v1", + "table": "generation_events", + "status": "ok", + "columns": ["id","correlation_id","task_description","provider","model_id", + "endpoint_class","attempt_count","total_latency_e2e_ms", + "contract_passed","cost_inference_usd","contract_yaml", + "handler_source","timestamp","created_at"] +} + +--- Binding check: does our Phase 2b proof (correlation 0cdc2410) have a row? --- +GET /projection/onex.evt.omnimarket.node-generation-completed.v1?correlation_id=0cdc2410-3fa5-4da6-b561-52d4a7075352 +{ + "topic": "onex.evt.omnimarket.node-generation-completed.v1", + "projection_version": "1.0.0", + "data_freshness": "degraded", + "latest_event_at": null, + "row_count": 0, + "rows": [] +} + +=> No projection row for our --agent generation. + +--- WHY (root cause, code-confirmed) --- +This is BY DESIGN, not a projection defect: + + src/__main__.py:819-839 — after the agent loop completes, --agent builds the + node-generation-requested/completed envelopes and records them to a LOCAL + event-chain capture file via EventChainCapture.record()/.save(): + capture = EventChainCapture(output_dir=Path(".onex_state/hackathon/event_chains")) + capture.record(REQUESTED_TOPIC, request_envelope) + capture.record(COMPLETED_TOPIC, completion_envelope) + capture.save(correlation_id) + EventChainCapture writes a local JSON file; it does NOT publish to the bus. + + Source snippet captured from the SEA dev worktree: + + capture = EventChainCapture(output_dir=Path(".onex_state/hackathon/event_chains")) + request_envelope = build_envelope( + source_node="onex_agent_runner", + correlation_id=correlation_id, + causation_id=None, + payload={"task_description": task_desc, "session_id": session_id}, + ) + capture.record(REQUESTED_TOPIC, request_envelope) + completion_envelope = build_envelope( + source_node="onex_agent_runner", + correlation_id=correlation_id, + causation_id=request_envelope["envelope_id"], + payload={ + "summary": final_text, + "session_id": session_id, + "contract_passed": _contract_passed, + **latest_scaffold_artifact, + }, + ) + capture.record(COMPLETED_TOPIC, completion_envelope) + chain_path = capture.save(correlation_id) + + grep confirms there is no bus.publish() of the generation-completed topic + anywhere in the --agent path. The only publisher of GENERATION_REQUESTED is the + server-side ingress node (node_kafka_ingress_effect/handlers/handler_kafka_ingress.py). + The generation_events projection is fed by the server-side omnimarket generation + node, reached via the runtime/dashboard/API path — NOT by the contractor CLI. + + Verification command: + rg -n "bus\.publish\(|publish_envelope\(|GENERATION_COMPLETED|node-generation-completed|COMPLETED_TOPIC" src/__main__.py src -g "*.py" + + Relevant stdout: + src/__main__.py:158:COMPLETED_TOPIC = _PIPELINE_TOPICS["generation_completed"] + src/__main__.py:826: capture.record(REQUESTED_TOPIC, request_envelope) + src/__main__.py:838: capture.record(COMPLETED_TOPIC, completion_envelope) + src/pipeline/topics.py:1:GENERATION_COMPLETED_TOPIC = "onex.evt.omnimarket.node-generation-completed.v1" + src/pipeline/consumer.py:394: topic = COMPLETED_TOPIC if result.contract_passed else FAILED_TOPIC + + The command output contains capture.record() and consumer/server-side topic + references, but no bus.publish() or publish_envelope() call for the + node-generation-completed topic in the --agent path. + + Corroboration: local capture file exists for our run — + .onex_state/hackathon/event_chains/0cdc2410-3fa5-4da6-b561-52d4a7075352.json + with both events captured_at 2026-06-08T20:20:45Z (same second; built back-to-back + at end of run, not emitted across the 64s generation). + That capture file is local runtime output and is not committed in this PR; + the committed evidence is the terminal transcript plus the projection readback + above showing row_count 0 for the same correlation. + +--- Conclusion --- +The Phase 2b generation is runtime-observed (terminal output + local event chain), +bus-backed for delegated inference, but NOT projection-backed. A projection row +requires the server-side runtime path, which the contractor seat cannot trigger +(no raw bus publish / runtime trigger per plan rules). Projection confirmation +needs operator-side capture. This is a scope/architecture boundary, not a defect. diff --git a/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/report.md b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/report.md new file mode 100644 index 00000000000..a45f1ce7d99 --- /dev/null +++ b/evidence/OMN-11513/integration-test-passes/2026-06-08-sea-delegation-verification/report.md @@ -0,0 +1,86 @@ +# SEA Delegation Verification — Integration Test Pass Report (2026-06-08) + +**Engagement:** OmniNode Hackathon Integration (umbrella OMN-11513) +**Lane:** stability-test only, from the contractor seat (WSL2 over Tailscale) +**Plan executed:** `2026-06-08-bret-verification-plan.md` (8 phases) +**SEA branch/SHA:** `dev` / `27f4f03` for the afternoon retest (Phases 1–3). Morning preflight (Phase 0) ran on `dev` / `80dc069` before the contractor clone was pulled to current dev. Python 3.12.13. +**New since the previous baseline:** `27f4f03` raises the SEA generation retry budget (#218), plus #215 retiring legacy demo nodes. + +--- + +## Final verdict — partially green / projection limited + +The broker blocker (OMN-12832) is **cleared and confirmed from the contractor seat**. After Jonah's OMN-12834 fix, the contractor-seat `--agent` generation completes **end to end, fully local, with no cloud key** — bus-backed (for delegated inference), generate → register → invoke, fresh correlation, contract-valid node, correct terminal result (`{"is_healthy": true}`, PASS, attempts 1, Cost $0.0000). Local delegated inference is independently proven (Phase 3 PASS), and the projection API is reachable. + +**This is runtime-observed, not projection-backed.** The `--agent` path records its `node-generation-requested/completed` events to a local event-chain file; it does not publish them to the bus, so they never reach the `generation_events` projection (queried for our correlation `0cdc2410`: row_count 0). A projection-backed generation row is produced only by the server-side runtime path (dashboard/API-triggered). The contractor seat cannot exercise that path — both because `--agent` records locally and because triggering the server path would require a raw bus publish or runtime trigger the plan forbids. So projection confirmation needs operator-side capture. This is a scope/architecture boundary, **not a projection defect** (root cause code-confirmed in `__main__.py:819-839`; see the Phase 4 transcript). + +Scope note: the proven generation path uses the supported local-ADK overlay (local reasoning backend); the default cloud-Gemini orchestrator path still requires a key and was not exercised (out of scope today). Prod and cloud/escalation were not verified (out of scope). + +**Day timeline:** +- **AM:** blocked at broker reachability (metadata advertised `192.168.86.201`). Filed OMN-12832. +- **Midday:** Jonah recreated the stability broker. Retest from our seat: Phase 1 green, Phase 3 PASS. Phase 2 broker-cleared but stopped at the ADK backend; filed OMN-12834 for the local-ADK schema-dialect defect. +- **Afternoon:** Jonah merged the OMN-12834 fix (#219, `56d5a45`). Retest: fully-local `--agent` generation completes, PASS. + +--- + +## Evidence table + +| Phase | Command / Surface | Branch/SHA | Correlation ID | Transport | Model / Endpoint | Result | Classification | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 0 Preflight | `pytest tests/unit -q` | dev/`80dc069` (AM) | n/a | n/a | n/a | 1385 pass / 3 fail / 1 skip | endpoint-resolution defect / env override ignored; not classified as no-regression | +| 1 Broker | aiokafka metadata (api 2.8.0) | dev/`27f4f03` | n/a | Tailscale | `100.109.203.94:39092` | advertises `100.109.203.94` (AM: `192.168.86.201`) | **PROOF / cleared** | +| 2 SEA `--agent` (keyless) | `python -m src --agent` | dev/`27f4f03` | `1a7f6369-2399-4011-b2c6-d408207e4a89` | bus_backed | stability-test overlay | bus reached, no direct-mode fallback; typed stop at ADK backend `cloud_gemini` (no key) | **diagnostic** | +| 2b SEA `--agent` (local-ADK overlay, post OMN-12834 fix) | `python -m src --agent` | dev/`56d5a45` | `0cdc2410-3fa5-4da6-b561-52d4a7075352` | bus_backed | local `:8001`, no cloud ($0.00) | generated `status_health_checker`, registered as MCP tool + invoked → `{"is_healthy": true}`; PASS, attempts 1 | **PROOF (runtime-observed; not projection-backed — see verdict)** | +| 3 InferenceClient smoke | inline smoke | dev/`27f4f03` | `8c0ddf2a-762b-4365-b714-42ac73a0c3e5` | bus_backed | `Qwen3.6-27B-MTP-IQ4_XS.gguf` / `:8001` | resolver `24576`; `content healthy=true`; latency 2824ms; empty error | **PROOF** | +| 4 Projection API | `curl :13002` | n/a | `0cdc2410…` (queried) | read-only | projection API | `/health` ok; `generation_events` queried for our correlation → row_count 0 (run is local-capture, not bus-published — see verdict); plan's `hackathon_pipeline_events.v1` → 404 | **reachable**; run not projection-backed (by design) | +| 5 Evidence packet | docs review | n/a | n/a | n/a | n/a | `sea-final-runtime-e2e-20260608/` not in contractor checkout, not on dev/any SEA branch/OCC | **not accessible to contractor** (as the plan anticipated) | +| 6 Dashboard claims | observed surfaces | n/a | n/a | n/a | n/a | classified (table below) | safe / degraded / out-of-scope | +| 7 Progressive | `--agent --progressive` | n/a | n/a | n/a | n/a | not run — gated on a clean Phase 2; Phase 2 stops at ADK backend | **held** | + +--- + +## Findings + +**1. OMN-12832 — broker advertised LAN address (cleared).** AM: broker at `100.109.203.94:39092` accepted TCP over Tailscale but advertised `192.168.86.201` in metadata, unreachable from our seat; every bus round-trip failed after the handshake. PM, after Jonah recreated the broker: the sanctioned aiokafka client (api 2.8.0) reports `brokers: [(0, '100.109.203.94', 39092)]`, and the Phase 3 round-trip — which had failed at `bus.start()` with `InfraTimeoutError` — now completes in 2.8s. Retest confirmation posted to the ticket. + +**2. Phase 0 endpoint override defect (observed, not closed here).** The three unit failures are not a `model_registry.local.yaml` false alarm. `phase0-unit-tests.txt` shows endpoint resolution returning `http://100.109.203.94:8000/v1/chat/completions` where tests expected `localhost` or the `LOCAL_INFERENCE_BASE_URL_ENV` override. The failing `test_load_default_registry_honors_env_override` specifically verifies the env override path and shows it being ignored, so this report does not claim "no code regression" for Phase 0. + +**3. OMN-12834 — local-ADK orchestrator backend sends Gemini-dialect tool schemas (new, medium).** Phase 2 keyless stops with "Select a local ADK tier in `contract.local.yaml`." Following that path (a contractor-created local overlay, since removed): the agent routes to the local `:8001` endpoint with no cloud key, then the local llama.cpp server returns HTTP 400 — `JSON schema conversion failed: Unrecognized schema: {"type":"STRING"}`. The request serializes every tool parameter in Gemini dialect (uppercase `STRING`/`OBJECT`); the OpenAI-compatible server requires lowercase JSON Schema. Confirmed it is the dialect: a hand-written lowercase tool returns 200 from the same endpoint; the agent's ADK-serialized tools return 400. Candidate site: `_OpenAICompatibleLlm` / `build_http_chat_completions_model` in `adk_backend_adapters.py`, which forwards the request without normalizing schema types for the OpenAI target. **Fixed same day:** Jonah merged #219 (`56d5a45`, adds `normalize_openai_tool_schema`). Retest from our seat: the 400 no longer reproduces and the fully-local generation completes (table row 2b; correlation `0cdc2410`). + +**4. Phase 2 generation — runtime-proven fully local (not projection-backed).** After the OMN-12834 fix, the local-ADK orchestration path completes a full generation with no cloud (row 2b). Both the local *inference* path (Phase 3) and the local *orchestration* path (the agent loop) are runtime-proven from the contractor seat. The result is runtime-observed (terminal + local event-chain capture), not projection-backed: `--agent` records its generation events to a local file rather than publishing to the bus, so no `generation_events` projection row is created (root cause code-confirmed; see verdict and the Phase 4 transcript). Producing a projection row needs the server-side runtime path, which is out of the contractor seat's allowed scope. The default cloud-Gemini orchestrator path remains out of scope today (needs a key). + +--- + +## Phase 6 — dashboard claim classification (omnidash dev `6b0a6b6`, file mode) + +| Surface | Authority | Safe claim | +| --- | --- | --- | +| cost-savings-overview | fixture, provisioned, `captured_at` 2026-05-20 | safe with date disclosure | +| delegation-model-output (new this sprint) | fixture, `provisioned=false`, `captured_at` 2026-05-05 | degraded / disclosure only | +| most other delegation + cost widgets | fixture, no freshness metadata | degraded — do not present as live | +| delegation savings / model-routing / quality-gate / token-usage / control-plane | no fixture (OMN-12662 gap) | out-of-scope / no claim | +| cloud escalation, prod | gated / none today | not claimed | + +Net: file mode is fixture-fallback-with-disclosure; the only date-backed cost surface is the savings overview. + +--- + +## Phase 8 — answers to the 2026-06-06 feedback ledger + +- **OMN-12719 (SubagentStop verifier):** no longer blocks the routine; agents stop cleanly. Did not re-patch; reporting observed behavior only. +- **OCC #2203:** merged 2026-06-07. Keyless run marked no-correlation/diagnostic; raw transcript annotated rather than rewritten (CodeRabbit's unsupported-ID finding corrected). +- **OMN-12700 cloud-tier:** not verified today (cloud out of scope). Post-deploy verification on v0.38.3 still showed the two-URL 404; filed OMN-12790 with the topic-binding hypothesis and source location. +- **OMN-12662 / cost widgets:** no cost/savings claims made; surfaces classified degraded or out-of-scope above. +- **Linear inbox digest:** ran (step 0); process hygiene, not evidence. + +--- + +## Not verified today (explicit) + +Prod deployment; cloud/Gemini/OpenRouter escalation; the full self-extension loop (generate → register → expose as MCP → invoke); receipt-honesty gate; URL authority remediation beyond observing sanctioned overlay values. + +--- + +## Evidence + +Transcripts in this folder: morning `phase0-unit-tests.txt`, `phase2-agent-e2e.txt`, `phase3-inference-smoke.txt`; afternoon `afternoon-retest-unblocked.txt`; ADK 400 capture `local-adk-400-capture.json`; post-fix fully-local proof `phase2-local-adk-PASS-postfix.txt`; projection readback + binding check `phase4-projection-readback.txt`.