From 0065ed5b09c33f09fc706c0605bc37be8a867055 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 5 Sep 2026 22:06:23 +0000 Subject: [PATCH] The harness binds to a serving replica; one Spark down no longer refuses every dispatch harness_observe_backend folded the replica roster into one standing and answered with the first non-serving replica's, so with .232 unreachable and .225 advertising the model, dispatch on srv1 refused at provider selection (measured 2026-09-05, first dispatch after #10611). The roster's point is that attempts spread across it, so the reading was all-or-nothing over a set. harness_backend now observes each replica once and keeps both halves: HarnessBackendRoster carries the serving addresses and the declined replicas with their typed standing, so what is down is counted and never absorbed. Selection and the belt preflight admit when any replica serves and refuse naming every replica's cause when none does. Each harness entry point binds its backend URL at process start from the serving set (harness_serving_backend_url; the seed spreads attempts within it) and exits with the roster's refusal before the first POST when nothing serves. harness_wire gains the pure pick over an arbitrary roster, witnessed against the deleted behaviour: a one-element roster is chosen whatever the seed says. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_019iXPtAqzPaNBZbaaM5rxrY --- dag/gunbc/harness/harness_backend.dag | 64 ++++++- dag/gunbc/harness/harness_cli.dag | 159 +++++++++--------- dag/gunbc/harness/harness_wire.dag | 17 +- dag/gunbc/roadmap/roadmap_belt_actuate.dag | 27 +-- .../roadmap/roadmap_dispatch_actuator.dag | 22 +-- dag/test/claim/harness_chat_shape_test.dag | 14 +- 6 files changed, 182 insertions(+), 121 deletions(-) diff --git a/dag/gunbc/harness/harness_backend.dag b/dag/gunbc/harness/harness_backend.dag index f891bb8c7a2..b93e3a49b27 100644 --- a/dag/gunbc/harness/harness_backend.dag +++ b/dag/gunbc/harness/harness_backend.dag @@ -9,6 +9,7 @@ import extdeps.languages.json.parse { } import gunbc.harness.harness_wire { harness_engine_replica_addresses, harness_engine_port, harness_engine_model, + HarnessWireShape, harness_replica_pick, harness_backend_url_for_address, } // THE HARNESS REPORTS ITS OWN STANDING, WHICH IS THE POINT RATHER THAN A CONVENIENCE. @@ -121,9 +122,66 @@ fn harness_standing_is_serving(standing: HarnessBackendStanding) -> Bool { } } -fn harness_observe_backend() -> HarnessBackendStanding { - fold(harness_engine_replica_addresses, init: HarnessBackendServing { model: harness_engine_model }, f: (acc, a) => - if harness_standing_is_serving(standing: acc) { harness_observe_replica(address: a) } else { acc }) +// THE ROSTER IS OBSERVED ONE REPLICA AT A TIME AND THE ANSWER KEEPS BOTH HALVES. The previous fold +// answered for the whole backend with the first non-serving replica's standing, so one Spark down +// refused every dispatch while the other served -- measured 2026-09-05 on srv1, .232 unreachable, +// .225 advertising the model, dispatch refused at selection. That was the all-or-nothing reading of +// a roster whose whole point is that attempts spread across it. Now: serving replicas are the set an +// attempt may bind to; declined replicas are carried with their typed standing, so the count of what +// is down is observable and never absorbed. None serving refuses, naming every replica's cause. +type HarnessReplicaDecline { + address: String + standing: HarnessBackendStanding +} + +type HarnessBackendRoster + = HarnessRosterServing { model: String, serving: List, declined: List } + | HarnessRosterNoneServing { model: String, declined: List } + +type HarnessRosterFold { + serving: List + declined: List +} + +fn harness_observe_roster() -> HarnessBackendRoster { + let folded = fold(harness_engine_replica_addresses, init: HarnessRosterFold { serving: [], declined: [] }, f: (acc, a) => { + let standing = harness_observe_replica(address: a) + if harness_standing_is_serving(standing: standing) { + HarnessRosterFold { serving: concat(acc.serving, [a]), declined: acc.declined } + } else { + HarnessRosterFold { serving: acc.serving, declined: concat(acc.declined, [HarnessReplicaDecline { address: a, standing: standing }]) } + } + }) + if count(folded.serving) == 0 { + HarnessRosterNoneServing { model: harness_engine_model, declined: folded.declined } + } else { + HarnessRosterServing { model: harness_engine_model, serving: folded.serving, declined: folded.declined } + } +} + +fn harness_declines_detail(declined: List) -> String { + join(map(declined, d => join([ + d.address, ": ", harness_standing_label(standing: d.standing), " -- ", harness_standing_detail(standing: d.standing), + ], "")), "; ") +} + +// THE URL AN ATTEMPT POSTS TO, bound at process start from the replicas serving at that moment. The +// seed spreads concurrent attempts across the serving set; a roster with nothing serving is a typed +// refusal the entry point exits on, never a URL that will fail on the first turn. +type HarnessBackendUrl + = BackendUrlBound { url: String, declined_count: Int } + | BackendUrlRefused { detail: String } + +fn harness_serving_backend_url(shape: HarnessWireShape, seed: String) -> HarnessBackendUrl { + match harness_observe_roster() { + HarnessRosterServing { model: _, serving, declined } => + BackendUrlBound { + url: harness_backend_url_for_address(shape: shape, address: harness_replica_pick(seed: seed, replicas: serving)), + declined_count: count(declined), + } + HarnessRosterNoneServing { model: m, declined } => + BackendUrlRefused { detail: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], "") } + } } fn harness_backend_report() -> String { diff --git a/dag/gunbc/harness/harness_cli.dag b/dag/gunbc/harness/harness_cli.dag index 8e351d84375..db9af7fabd2 100644 --- a/dag/gunbc/harness/harness_cli.dag +++ b/dag/gunbc/harness/harness_cli.dag @@ -17,9 +17,8 @@ import gunbc.harness.harness_guidance { harness_worker_guidance_at, harness_guid import gunbc.harness.harness_review_guidance { harness_reviewer_guidance_at, review_guidance_prose } import gunbc.harness.harness_goal_audit_guidance { harness_auditor_guidance_at, audit_guidance_prose } import gunbc.roadmap.roadmap_review_role { review_verdict_shape_text } -import gunbc.harness.harness_wire { - harness_engine_backend_url, harness_engine_model, harness_engine_context_window, -} +import gunbc.harness.harness_wire { harness_engine_model, harness_engine_context_window } +import gunbc.harness.harness_backend { HarnessBackendUrl, BackendUrlBound, BackendUrlRefused, harness_serving_backend_url } // THE FIRST WET ENTRY FOR THE DEDICATED HARNESS. It exists to make DCH-2 provable by execution // rather than by typecheck -- DESIGN section 5's specification-without-execution trap names exactly @@ -165,9 +164,9 @@ fn harness_macbook_environment() -> HarnessToolEnvironment { // different authority. data harness_context_budget: TokenCount = harness_engine_context_window -fn harness_macbook_config() -> HarnessTurnConfig { +fn harness_macbook_config(backend_url: String) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: ""), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -188,9 +187,9 @@ fn harness_macbook_config() -> HarnessTurnConfig { // posted, received a tool_use, executed it, posted the result back, and received a terminal. data harness_probe_prompt: String = "How many files in the dag/gunbc/harness directory of this repository end in .dag? Use run_command to find out, then state the number." -fn harness_probe_outcome() -> HarnessTurnOutcome { +fn harness_probe_outcome(backend_url: String) -> HarnessTurnOutcome { harness_run_turn( - config: harness_macbook_config(), + config: harness_macbook_config(backend_url: backend_url), prompt: harness_probe_prompt, step_budget: harness_default_step_budget, ) @@ -200,8 +199,7 @@ fn harness_probe_outcome() -> HarnessTurnOutcome { // happened rather than whether the task succeeded. Nothing here inspects the assistant's text for // correctness: that is external verification's subject, and a harness that graded its own worker's // answer would be the same conflation DCH-2 forbids between a stop reason and a completed task. -fn harness_probe_cli() -> ProcessExit { - let outcome = harness_probe_outcome() +fn harness_turn_exit(outcome: HarnessTurnOutcome) -> ProcessExit { match outcome { TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => ExitSuccess @@ -215,6 +213,19 @@ fn harness_probe_cli() -> ProcessExit { } } +// THE BACKEND IS BOUND BEFORE THE TURN: a roster with nothing serving exits here with every +// replica's cause, instead of a config whose first POST would refuse. +fn harness_backend_refusal_exit(detail: String) -> ProcessExit { + ExitFailure { code: 1, reason: join(["harness backend: ", detail], "") } +} + +fn harness_probe_cli() -> ProcessExit { + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: "") { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => harness_turn_exit(outcome: harness_probe_outcome(backend_url: url)) + } +} + // THE WORKER ENTRY THE BELT SPAWNS. Its two facts arrive as named arguments rather than through the // environment, because `gunbc run` has had --arg since the scm read landed and the belt's own // annotation saying otherwise ("gunbc run --function cannot pass an argument (no --arg flag)") is @@ -244,9 +255,10 @@ fn harness_worker_config( timeout_program: FilePath, request_scratch_path: FilePath, events_path: FilePath, + backend_url: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: worktree as String), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -286,26 +298,20 @@ fn harness_worker_cli( request_scratch_path: String, events_path: String, ) -> ProcessExit { - let outcome = harness_run_turn( - config: harness_worker_config( - worktree: worktree as FilePath, - timeout_program: timeout_program as FilePath, - request_scratch_path: request_scratch_path as FilePath, - events_path: events_path as FilePath, - ), - prompt: brief, - step_budget: harness_worker_step_budget, - ) - match outcome { - TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => - ExitSuccess - _ => - ExitFailure { - code: 1, - reason: join([ - harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome), - ], ""), - } + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: worktree) { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => + harness_turn_exit(outcome: harness_run_turn( + config: harness_worker_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + backend_url: url, + ), + prompt: brief, + step_budget: harness_worker_step_budget, + )) } } @@ -333,9 +339,10 @@ fn harness_reviewer_config( request_scratch_path: FilePath, events_path: FilePath, verdict_path: String, + backend_url: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: verdict_path), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -361,27 +368,21 @@ fn harness_reviewer_cli( events_path: String, verdict_path: String, ) -> ProcessExit { - let outcome = harness_run_turn( - config: harness_reviewer_config( - worktree: worktree as FilePath, - timeout_program: timeout_program as FilePath, - request_scratch_path: request_scratch_path as FilePath, - events_path: events_path as FilePath, - verdict_path: verdict_path, - ), - prompt: brief, - step_budget: harness_reviewer_step_budget, - ) - match outcome { - TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => - ExitSuccess - _ => - ExitFailure { - code: 1, - reason: join([ - harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome), - ], ""), - } + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: verdict_path) { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => + harness_turn_exit(outcome: harness_run_turn( + config: harness_reviewer_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + verdict_path: verdict_path, + backend_url: url, + ), + prompt: brief, + step_budget: harness_reviewer_step_budget, + )) } } @@ -396,9 +397,10 @@ fn harness_auditor_config( request_scratch_path: FilePath, events_path: FilePath, verdict_path: String, + backend_url: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: verdict_path), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -424,34 +426,33 @@ fn harness_auditor_cli( events_path: String, verdict_path: String, ) -> ProcessExit { - let outcome = harness_run_turn( - config: harness_auditor_config( - worktree: worktree as FilePath, - timeout_program: timeout_program as FilePath, - request_scratch_path: request_scratch_path as FilePath, - events_path: events_path as FilePath, - verdict_path: verdict_path, - ), - prompt: brief, - step_budget: harness_reviewer_step_budget, - ) - match outcome { - TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => - ExitSuccess - _ => - ExitFailure { - code: 1, - reason: join([ - harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome), - ], ""), - } + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: verdict_path) { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => + harness_turn_exit(outcome: harness_run_turn( + config: harness_auditor_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + verdict_path: verdict_path, + backend_url: url, + ), + prompt: brief, + step_budget: harness_reviewer_step_budget, + )) } } fn harness_probe_report() -> String { - let outcome = harness_probe_outcome() - join([ - "harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n", - harness_outcome_detail(outcome: outcome), "\n", - ], "") + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: "") { + BackendUrlRefused { detail } => join(["harness probe refused before the turn: ", detail, "\n"], "") + BackendUrlBound { url, declined_count: _ } => { + let outcome = harness_probe_outcome(backend_url: url) + join([ + "harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n", + harness_outcome_detail(outcome: outcome), "\n", + ], "") + } + } } diff --git a/dag/gunbc/harness/harness_wire.dag b/dag/gunbc/harness/harness_wire.dag index 857ffc9d0e3..763026187ee 100644 --- a/dag/gunbc/harness/harness_wire.dag +++ b/dag/gunbc/harness/harness_wire.dag @@ -359,12 +359,25 @@ fn harness_engine_replica_index(seed: String) -> Int { } fn harness_engine_replica_address(seed: String) -> String { - let want = harness_engine_replica_index(seed: seed) - let picked = fold(harness_engine_replica_addresses, init: ReplicaPick { at: 0, chosen: "" }, f: (acc, a) => + harness_replica_pick(seed: seed, replicas: harness_engine_replica_addresses) +} + +// THE PICK OVER AN ARBITRARY ROSTER, so an attempt binds to a replica chosen from the ones OBSERVED +// SERVING (gunbc.harness.harness_backend harness_serving_backend_url) rather than from the declared +// list. A declared replica that is down is a typed, counted decline there; here it is simply not in +// the roster handed in, so the seed spreads attempts over the machines that can take them. Callers +// hand in a non-empty roster (the serving arm carries at least one address by construction). +fn harness_replica_pick(seed: String, replicas: List) -> String { + let want = fold(chars(s: seed), init: 0, f: (acc, cp) => acc + cp) % list_length(items: replicas) + let picked = fold(replicas, init: ReplicaPick { at: 0, chosen: "" }, f: (acc, a) => ReplicaPick { at: acc.at + 1, chosen: if acc.at == want { a } else { acc.chosen } }) picked.chosen } +fn harness_backend_url_for_address(shape: HarnessWireShape, address: String) -> String { + harness_backend_url_on_port(address: address, port: harness_engine_port, shape: shape) +} + type ReplicaPick { at: Int chosen: String diff --git a/dag/gunbc/roadmap/roadmap_belt_actuate.dag b/dag/gunbc/roadmap/roadmap_belt_actuate.dag index 5d629ab7b17..6be2fea905e 100644 --- a/dag/gunbc/roadmap/roadmap_belt_actuate.dag +++ b/dag/gunbc/roadmap/roadmap_belt_actuate.dag @@ -83,8 +83,8 @@ import extdeps.languages.json.parse { } import gunbc.clock_read { clock_now_probed_at } import gunbc.harness.harness_backend { - HarnessBackendStanding, HarnessBackendServing, HarnessBackendModelAbsent, - HarnessBackendUnreachable, HarnessBackendUnreadable, harness_observe_backend, + HarnessBackendRoster, HarnessRosterServing, HarnessRosterNoneServing, + harness_observe_roster, harness_declines_detail, } import gunbc.roadmap_dispatch_actuator { HostExecArgv, @@ -1394,25 +1394,12 @@ fn belt_provider_preflight_for_instance( detail: "cursor dispatch is not yet wired on this actuator path", } GunbcHarnessProvider => - match harness_observe_backend() { - HarnessBackendServing { model: _ } => ProviderPreflightOk - HarnessBackendModelAbsent { model: m, advertised: a } => + match harness_observe_roster() { + HarnessRosterServing { model: _, serving: _, declined: _ } => ProviderPreflightOk + HarnessRosterNoneServing { model: m, declined } => ProviderPreflightRefused { - step: "harness-backend-model", - detail: join([ - "the harness backend is reachable but does not advertise ", m, - "; it advertises: ", a, - ], ""), - } - HarnessBackendUnreachable { url: u } => - ProviderPreflightRefused { - step: "harness-backend-reachability", - detail: join(["the harness backend did not answer ", u], ""), - } - HarnessBackendUnreadable { cause: c } => - ProviderPreflightRefused { - step: "harness-backend-model-list", - detail: join(["the harness backend answered unreadably: ", c], ""), + step: "harness-backend-roster", + detail: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], ""), } } } diff --git a/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag b/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag index 1428945c3b7..f95e8a85054 100644 --- a/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag +++ b/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag @@ -95,8 +95,8 @@ import extdeps.llm.cli_lifecycle { CursorCliProvider, } import gunbc.harness.harness_backend { - HarnessBackendStanding, HarnessBackendServing, HarnessBackendModelAbsent, - HarnessBackendUnreachable, HarnessBackendUnreadable, harness_observe_backend, + HarnessBackendRoster, HarnessRosterServing, HarnessRosterNoneServing, + harness_observe_roster, harness_declines_detail, } import gunbc.dispatch_selection { ExecutionProfile, @@ -294,26 +294,16 @@ type DispatchActuatorSelection data harness_selection_process_fingerprint: NonEmptyStr = "gunbc" as NonEmptyStr fn dispatch_harness_selection() -> DispatchActuatorSelection { - match harness_observe_backend() { - HarnessBackendServing { model: _ } => + match harness_observe_roster() { + HarnessRosterServing { model: _, serving: _, declined: _ } => DispatchActuatorSelectionOk { provider: GunbcHarnessProvider, effort: ReasoningMedium, process_fingerprint: harness_selection_process_fingerprint, } - HarnessBackendModelAbsent { model: m, advertised: a } => + HarnessRosterNoneServing { model: m, declined } => DispatchActuatorSelectionRefused { - reason: join([ - "harness backend is reachable but does not advertise ", m, "; it advertises: ", a, - ], "") as NonEmptyStr, - } - HarnessBackendUnreachable { url: u } => - DispatchActuatorSelectionRefused { - reason: join(["harness backend did not answer ", u], "") as NonEmptyStr, - } - HarnessBackendUnreadable { cause: c } => - DispatchActuatorSelectionRefused { - reason: join(["harness backend answered unreadably: ", c], "") as NonEmptyStr, + reason: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], "") as NonEmptyStr, } } } diff --git a/dag/test/claim/harness_chat_shape_test.dag b/dag/test/claim/harness_chat_shape_test.dag index 8dc71f7f993..a57744e8ea6 100644 --- a/dag/test/claim/harness_chat_shape_test.dag +++ b/dag/test/claim/harness_chat_shape_test.dag @@ -7,7 +7,7 @@ import gunbc.harness.harness_wire { HarnessWireBlock, WireTextBlock, WireThinkingBlock, WireToolUseBlock, HarnessResponseRead, HarnessResponseParsed, HarnessResponseUnreadable, HarnessWireShape, AnthropicMessagesShape, OpenAiChatCompletionsShape, - harness_decode_chat_completion, harness_backend_path, harness_engine_replica_index, harness_engine_replica_address, harness_engine_replica_addresses, harness_engine_backend_url } + harness_decode_chat_completion, harness_backend_path, harness_engine_replica_index, harness_engine_replica_address, harness_replica_pick, harness_backend_url_for_address, harness_engine_replica_addresses, harness_engine_backend_url } import gunbc.harness.harness_turn { HarnessToolResult, harness_assistant_message_json, harness_tool_result_messages_json, harness_user_text_message_json, @@ -114,3 +114,15 @@ test fn the_replica_pick_is_deterministic_and_covers_the_roster() -> Bool { && harness_engine_replica_address(seed: "a") != harness_engine_replica_address(seed: "b") && string_contains(s: harness_engine_backend_url(shape: OpenAiChatCompletionsShape, seed: "b"), pattern: ":30000/v1/chat/completions") } + +// DISCRIMINATING against the all-or-nothing roster: a pick over the SERVING subset binds to what +// serves regardless of the seed, so a seed that would have chosen the down replica from the declared +// list chooses the live one from a one-element roster; two live replicas still spread by seed. +test fn the_replica_pick_binds_within_the_roster_it_is_handed() -> Bool { + let only = ["10.0.0.2"] + let pair = ["10.0.0.1", "10.0.0.2"] + harness_replica_pick(seed: "a", replicas: only) == "10.0.0.2" + && harness_replica_pick(seed: "b", replicas: only) == "10.0.0.2" + && harness_replica_pick(seed: "a", replicas: pair) != harness_replica_pick(seed: "b", replicas: pair) + && string_contains(s: harness_backend_url_for_address(shape: OpenAiChatCompletionsShape, address: "10.0.0.2"), pattern: "http://10.0.0.2:30000/v1/chat/completions") +}