diff --git a/dag/gunbc/harness/harness_backend.dag b/dag/gunbc/harness/harness_backend.dag index f891bb8c7a2..b93e3a49b27 100644 --- a/dag/gunbc/harness/harness_backend.dag +++ b/dag/gunbc/harness/harness_backend.dag @@ -9,6 +9,7 @@ import extdeps.languages.json.parse { } import gunbc.harness.harness_wire { harness_engine_replica_addresses, harness_engine_port, harness_engine_model, + HarnessWireShape, harness_replica_pick, harness_backend_url_for_address, } // THE HARNESS REPORTS ITS OWN STANDING, WHICH IS THE POINT RATHER THAN A CONVENIENCE. @@ -121,9 +122,66 @@ fn harness_standing_is_serving(standing: HarnessBackendStanding) -> Bool { } } -fn harness_observe_backend() -> HarnessBackendStanding { - fold(harness_engine_replica_addresses, init: HarnessBackendServing { model: harness_engine_model }, f: (acc, a) => - if harness_standing_is_serving(standing: acc) { harness_observe_replica(address: a) } else { acc }) +// THE ROSTER IS OBSERVED ONE REPLICA AT A TIME AND THE ANSWER KEEPS BOTH HALVES. The previous fold +// answered for the whole backend with the first non-serving replica's standing, so one Spark down +// refused every dispatch while the other served -- measured 2026-09-05 on srv1, .232 unreachable, +// .225 advertising the model, dispatch refused at selection. That was the all-or-nothing reading of +// a roster whose whole point is that attempts spread across it. Now: serving replicas are the set an +// attempt may bind to; declined replicas are carried with their typed standing, so the count of what +// is down is observable and never absorbed. None serving refuses, naming every replica's cause. +type HarnessReplicaDecline { + address: String + standing: HarnessBackendStanding +} + +type HarnessBackendRoster + = HarnessRosterServing { model: String, serving: List, declined: List } + | HarnessRosterNoneServing { model: String, declined: List } + +type HarnessRosterFold { + serving: List + declined: List +} + +fn harness_observe_roster() -> HarnessBackendRoster { + let folded = fold(harness_engine_replica_addresses, init: HarnessRosterFold { serving: [], declined: [] }, f: (acc, a) => { + let standing = harness_observe_replica(address: a) + if harness_standing_is_serving(standing: standing) { + HarnessRosterFold { serving: concat(acc.serving, [a]), declined: acc.declined } + } else { + HarnessRosterFold { serving: acc.serving, declined: concat(acc.declined, [HarnessReplicaDecline { address: a, standing: standing }]) } + } + }) + if count(folded.serving) == 0 { + HarnessRosterNoneServing { model: harness_engine_model, declined: folded.declined } + } else { + HarnessRosterServing { model: harness_engine_model, serving: folded.serving, declined: folded.declined } + } +} + +fn harness_declines_detail(declined: List) -> String { + join(map(declined, d => join([ + d.address, ": ", harness_standing_label(standing: d.standing), " -- ", harness_standing_detail(standing: d.standing), + ], "")), "; ") +} + +// THE URL AN ATTEMPT POSTS TO, bound at process start from the replicas serving at that moment. The +// seed spreads concurrent attempts across the serving set; a roster with nothing serving is a typed +// refusal the entry point exits on, never a URL that will fail on the first turn. +type HarnessBackendUrl + = BackendUrlBound { url: String, declined_count: Int } + | BackendUrlRefused { detail: String } + +fn harness_serving_backend_url(shape: HarnessWireShape, seed: String) -> HarnessBackendUrl { + match harness_observe_roster() { + HarnessRosterServing { model: _, serving, declined } => + BackendUrlBound { + url: harness_backend_url_for_address(shape: shape, address: harness_replica_pick(seed: seed, replicas: serving)), + declined_count: count(declined), + } + HarnessRosterNoneServing { model: m, declined } => + BackendUrlRefused { detail: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], "") } + } } fn harness_backend_report() -> String { diff --git a/dag/gunbc/harness/harness_cli.dag b/dag/gunbc/harness/harness_cli.dag index 8e351d84375..db9af7fabd2 100644 --- a/dag/gunbc/harness/harness_cli.dag +++ b/dag/gunbc/harness/harness_cli.dag @@ -17,9 +17,8 @@ import gunbc.harness.harness_guidance { harness_worker_guidance_at, harness_guid import gunbc.harness.harness_review_guidance { harness_reviewer_guidance_at, review_guidance_prose } import gunbc.harness.harness_goal_audit_guidance { harness_auditor_guidance_at, audit_guidance_prose } import gunbc.roadmap.roadmap_review_role { review_verdict_shape_text } -import gunbc.harness.harness_wire { - harness_engine_backend_url, harness_engine_model, harness_engine_context_window, -} +import gunbc.harness.harness_wire { harness_engine_model, harness_engine_context_window } +import gunbc.harness.harness_backend { HarnessBackendUrl, BackendUrlBound, BackendUrlRefused, harness_serving_backend_url } // THE FIRST WET ENTRY FOR THE DEDICATED HARNESS. It exists to make DCH-2 provable by execution // rather than by typecheck -- DESIGN section 5's specification-without-execution trap names exactly @@ -165,9 +164,9 @@ fn harness_macbook_environment() -> HarnessToolEnvironment { // different authority. data harness_context_budget: TokenCount = harness_engine_context_window -fn harness_macbook_config() -> HarnessTurnConfig { +fn harness_macbook_config(backend_url: String) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: ""), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -188,9 +187,9 @@ fn harness_macbook_config() -> HarnessTurnConfig { // posted, received a tool_use, executed it, posted the result back, and received a terminal. data harness_probe_prompt: String = "How many files in the dag/gunbc/harness directory of this repository end in .dag? Use run_command to find out, then state the number." -fn harness_probe_outcome() -> HarnessTurnOutcome { +fn harness_probe_outcome(backend_url: String) -> HarnessTurnOutcome { harness_run_turn( - config: harness_macbook_config(), + config: harness_macbook_config(backend_url: backend_url), prompt: harness_probe_prompt, step_budget: harness_default_step_budget, ) @@ -200,8 +199,7 @@ fn harness_probe_outcome() -> HarnessTurnOutcome { // happened rather than whether the task succeeded. Nothing here inspects the assistant's text for // correctness: that is external verification's subject, and a harness that graded its own worker's // answer would be the same conflation DCH-2 forbids between a stop reason and a completed task. -fn harness_probe_cli() -> ProcessExit { - let outcome = harness_probe_outcome() +fn harness_turn_exit(outcome: HarnessTurnOutcome) -> ProcessExit { match outcome { TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => ExitSuccess @@ -215,6 +213,19 @@ fn harness_probe_cli() -> ProcessExit { } } +// THE BACKEND IS BOUND BEFORE THE TURN: a roster with nothing serving exits here with every +// replica's cause, instead of a config whose first POST would refuse. +fn harness_backend_refusal_exit(detail: String) -> ProcessExit { + ExitFailure { code: 1, reason: join(["harness backend: ", detail], "") } +} + +fn harness_probe_cli() -> ProcessExit { + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: "") { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => harness_turn_exit(outcome: harness_probe_outcome(backend_url: url)) + } +} + // THE WORKER ENTRY THE BELT SPAWNS. Its two facts arrive as named arguments rather than through the // environment, because `gunbc run` has had --arg since the scm read landed and the belt's own // annotation saying otherwise ("gunbc run --function cannot pass an argument (no --arg flag)") is @@ -244,9 +255,10 @@ fn harness_worker_config( timeout_program: FilePath, request_scratch_path: FilePath, events_path: FilePath, + backend_url: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: worktree as String), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -286,26 +298,20 @@ fn harness_worker_cli( request_scratch_path: String, events_path: String, ) -> ProcessExit { - let outcome = harness_run_turn( - config: harness_worker_config( - worktree: worktree as FilePath, - timeout_program: timeout_program as FilePath, - request_scratch_path: request_scratch_path as FilePath, - events_path: events_path as FilePath, - ), - prompt: brief, - step_budget: harness_worker_step_budget, - ) - match outcome { - TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => - ExitSuccess - _ => - ExitFailure { - code: 1, - reason: join([ - harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome), - ], ""), - } + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: worktree) { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => + harness_turn_exit(outcome: harness_run_turn( + config: harness_worker_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + backend_url: url, + ), + prompt: brief, + step_budget: harness_worker_step_budget, + )) } } @@ -333,9 +339,10 @@ fn harness_reviewer_config( request_scratch_path: FilePath, events_path: FilePath, verdict_path: String, + backend_url: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: verdict_path), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -361,27 +368,21 @@ fn harness_reviewer_cli( events_path: String, verdict_path: String, ) -> ProcessExit { - let outcome = harness_run_turn( - config: harness_reviewer_config( - worktree: worktree as FilePath, - timeout_program: timeout_program as FilePath, - request_scratch_path: request_scratch_path as FilePath, - events_path: events_path as FilePath, - verdict_path: verdict_path, - ), - prompt: brief, - step_budget: harness_reviewer_step_budget, - ) - match outcome { - TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => - ExitSuccess - _ => - ExitFailure { - code: 1, - reason: join([ - harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome), - ], ""), - } + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: verdict_path) { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => + harness_turn_exit(outcome: harness_run_turn( + config: harness_reviewer_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + verdict_path: verdict_path, + backend_url: url, + ), + prompt: brief, + step_budget: harness_reviewer_step_budget, + )) } } @@ -396,9 +397,10 @@ fn harness_auditor_config( request_scratch_path: FilePath, events_path: FilePath, verdict_path: String, + backend_url: String, ) -> HarnessTurnConfig { HarnessTurnConfig { - backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: verdict_path), + backend_url: backend_url, model: harness_default_model(), max_tokens: harness_default_max_tokens, reasoning_mode: harness_reasoning_mode, @@ -424,34 +426,33 @@ fn harness_auditor_cli( events_path: String, verdict_path: String, ) -> ProcessExit { - let outcome = harness_run_turn( - config: harness_auditor_config( - worktree: worktree as FilePath, - timeout_program: timeout_program as FilePath, - request_scratch_path: request_scratch_path as FilePath, - events_path: events_path as FilePath, - verdict_path: verdict_path, - ), - prompt: brief, - step_budget: harness_reviewer_step_budget, - ) - match outcome { - TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } => - ExitSuccess - _ => - ExitFailure { - code: 1, - reason: join([ - harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome), - ], ""), - } + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: verdict_path) { + BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail) + BackendUrlBound { url, declined_count: _ } => + harness_turn_exit(outcome: harness_run_turn( + config: harness_auditor_config( + worktree: worktree as FilePath, + timeout_program: timeout_program as FilePath, + request_scratch_path: request_scratch_path as FilePath, + events_path: events_path as FilePath, + verdict_path: verdict_path, + backend_url: url, + ), + prompt: brief, + step_budget: harness_reviewer_step_budget, + )) } } fn harness_probe_report() -> String { - let outcome = harness_probe_outcome() - join([ - "harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n", - harness_outcome_detail(outcome: outcome), "\n", - ], "") + match harness_serving_backend_url(shape: harness_default_wire_shape, seed: "") { + BackendUrlRefused { detail } => join(["harness probe refused before the turn: ", detail, "\n"], "") + BackendUrlBound { url, declined_count: _ } => { + let outcome = harness_probe_outcome(backend_url: url) + join([ + "harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n", + harness_outcome_detail(outcome: outcome), "\n", + ], "") + } + } } diff --git a/dag/gunbc/harness/harness_wire.dag b/dag/gunbc/harness/harness_wire.dag index 857ffc9d0e3..763026187ee 100644 --- a/dag/gunbc/harness/harness_wire.dag +++ b/dag/gunbc/harness/harness_wire.dag @@ -359,12 +359,25 @@ fn harness_engine_replica_index(seed: String) -> Int { } fn harness_engine_replica_address(seed: String) -> String { - let want = harness_engine_replica_index(seed: seed) - let picked = fold(harness_engine_replica_addresses, init: ReplicaPick { at: 0, chosen: "" }, f: (acc, a) => + harness_replica_pick(seed: seed, replicas: harness_engine_replica_addresses) +} + +// THE PICK OVER AN ARBITRARY ROSTER, so an attempt binds to a replica chosen from the ones OBSERVED +// SERVING (gunbc.harness.harness_backend harness_serving_backend_url) rather than from the declared +// list. A declared replica that is down is a typed, counted decline there; here it is simply not in +// the roster handed in, so the seed spreads attempts over the machines that can take them. Callers +// hand in a non-empty roster (the serving arm carries at least one address by construction). +fn harness_replica_pick(seed: String, replicas: List) -> String { + let want = fold(chars(s: seed), init: 0, f: (acc, cp) => acc + cp) % list_length(items: replicas) + let picked = fold(replicas, init: ReplicaPick { at: 0, chosen: "" }, f: (acc, a) => ReplicaPick { at: acc.at + 1, chosen: if acc.at == want { a } else { acc.chosen } }) picked.chosen } +fn harness_backend_url_for_address(shape: HarnessWireShape, address: String) -> String { + harness_backend_url_on_port(address: address, port: harness_engine_port, shape: shape) +} + type ReplicaPick { at: Int chosen: String diff --git a/dag/gunbc/roadmap/roadmap_belt_actuate.dag b/dag/gunbc/roadmap/roadmap_belt_actuate.dag index 5d629ab7b17..6be2fea905e 100644 --- a/dag/gunbc/roadmap/roadmap_belt_actuate.dag +++ b/dag/gunbc/roadmap/roadmap_belt_actuate.dag @@ -83,8 +83,8 @@ import extdeps.languages.json.parse { } import gunbc.clock_read { clock_now_probed_at } import gunbc.harness.harness_backend { - HarnessBackendStanding, HarnessBackendServing, HarnessBackendModelAbsent, - HarnessBackendUnreachable, HarnessBackendUnreadable, harness_observe_backend, + HarnessBackendRoster, HarnessRosterServing, HarnessRosterNoneServing, + harness_observe_roster, harness_declines_detail, } import gunbc.roadmap_dispatch_actuator { HostExecArgv, @@ -1394,25 +1394,12 @@ fn belt_provider_preflight_for_instance( detail: "cursor dispatch is not yet wired on this actuator path", } GunbcHarnessProvider => - match harness_observe_backend() { - HarnessBackendServing { model: _ } => ProviderPreflightOk - HarnessBackendModelAbsent { model: m, advertised: a } => + match harness_observe_roster() { + HarnessRosterServing { model: _, serving: _, declined: _ } => ProviderPreflightOk + HarnessRosterNoneServing { model: m, declined } => ProviderPreflightRefused { - step: "harness-backend-model", - detail: join([ - "the harness backend is reachable but does not advertise ", m, - "; it advertises: ", a, - ], ""), - } - HarnessBackendUnreachable { url: u } => - ProviderPreflightRefused { - step: "harness-backend-reachability", - detail: join(["the harness backend did not answer ", u], ""), - } - HarnessBackendUnreadable { cause: c } => - ProviderPreflightRefused { - step: "harness-backend-model-list", - detail: join(["the harness backend answered unreadably: ", c], ""), + step: "harness-backend-roster", + detail: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], ""), } } } diff --git a/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag b/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag index 1428945c3b7..f95e8a85054 100644 --- a/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag +++ b/dag/gunbc/roadmap/roadmap_dispatch_actuator.dag @@ -95,8 +95,8 @@ import extdeps.llm.cli_lifecycle { CursorCliProvider, } import gunbc.harness.harness_backend { - HarnessBackendStanding, HarnessBackendServing, HarnessBackendModelAbsent, - HarnessBackendUnreachable, HarnessBackendUnreadable, harness_observe_backend, + HarnessBackendRoster, HarnessRosterServing, HarnessRosterNoneServing, + harness_observe_roster, harness_declines_detail, } import gunbc.dispatch_selection { ExecutionProfile, @@ -294,26 +294,16 @@ type DispatchActuatorSelection data harness_selection_process_fingerprint: NonEmptyStr = "gunbc" as NonEmptyStr fn dispatch_harness_selection() -> DispatchActuatorSelection { - match harness_observe_backend() { - HarnessBackendServing { model: _ } => + match harness_observe_roster() { + HarnessRosterServing { model: _, serving: _, declined: _ } => DispatchActuatorSelectionOk { provider: GunbcHarnessProvider, effort: ReasoningMedium, process_fingerprint: harness_selection_process_fingerprint, } - HarnessBackendModelAbsent { model: m, advertised: a } => + HarnessRosterNoneServing { model: m, declined } => DispatchActuatorSelectionRefused { - reason: join([ - "harness backend is reachable but does not advertise ", m, "; it advertises: ", a, - ], "") as NonEmptyStr, - } - HarnessBackendUnreachable { url: u } => - DispatchActuatorSelectionRefused { - reason: join(["harness backend did not answer ", u], "") as NonEmptyStr, - } - HarnessBackendUnreadable { cause: c } => - DispatchActuatorSelectionRefused { - reason: join(["harness backend answered unreadably: ", c], "") as NonEmptyStr, + reason: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], "") as NonEmptyStr, } } } diff --git a/dag/test/claim/harness_chat_shape_test.dag b/dag/test/claim/harness_chat_shape_test.dag index 8dc71f7f993..a57744e8ea6 100644 --- a/dag/test/claim/harness_chat_shape_test.dag +++ b/dag/test/claim/harness_chat_shape_test.dag @@ -7,7 +7,7 @@ import gunbc.harness.harness_wire { HarnessWireBlock, WireTextBlock, WireThinkingBlock, WireToolUseBlock, HarnessResponseRead, HarnessResponseParsed, HarnessResponseUnreadable, HarnessWireShape, AnthropicMessagesShape, OpenAiChatCompletionsShape, - harness_decode_chat_completion, harness_backend_path, harness_engine_replica_index, harness_engine_replica_address, harness_engine_replica_addresses, harness_engine_backend_url } + harness_decode_chat_completion, harness_backend_path, harness_engine_replica_index, harness_engine_replica_address, harness_replica_pick, harness_backend_url_for_address, harness_engine_replica_addresses, harness_engine_backend_url } import gunbc.harness.harness_turn { HarnessToolResult, harness_assistant_message_json, harness_tool_result_messages_json, harness_user_text_message_json, @@ -114,3 +114,15 @@ test fn the_replica_pick_is_deterministic_and_covers_the_roster() -> Bool { && harness_engine_replica_address(seed: "a") != harness_engine_replica_address(seed: "b") && string_contains(s: harness_engine_backend_url(shape: OpenAiChatCompletionsShape, seed: "b"), pattern: ":30000/v1/chat/completions") } + +// DISCRIMINATING against the all-or-nothing roster: a pick over the SERVING subset binds to what +// serves regardless of the seed, so a seed that would have chosen the down replica from the declared +// list chooses the live one from a one-element roster; two live replicas still spread by seed. +test fn the_replica_pick_binds_within_the_roster_it_is_handed() -> Bool { + let only = ["10.0.0.2"] + let pair = ["10.0.0.1", "10.0.0.2"] + harness_replica_pick(seed: "a", replicas: only) == "10.0.0.2" + && harness_replica_pick(seed: "b", replicas: only) == "10.0.0.2" + && harness_replica_pick(seed: "a", replicas: pair) != harness_replica_pick(seed: "b", replicas: pair) + && string_contains(s: harness_backend_url_for_address(shape: OpenAiChatCompletionsShape, address: "10.0.0.2"), pattern: "http://10.0.0.2:30000/v1/chat/completions") +}