Skip to content

The harness binds to a serving replica; one Spark down no longer refuses every dispatch #10617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
briansrls merged 1 commit into from
Sep 5, 2026
Merged

The harness binds to a serving replica; one Spark down no longer refuses every dispatch #10617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 61 additions & 3 deletions dag/gunbc/harness/harness_backend.dag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import extdeps.languages.json.parse {
}
import gunbc.harness.harness_wire {
harness_engine_replica_addresses, harness_engine_port, harness_engine_model,
HarnessWireShape, harness_replica_pick, harness_backend_url_for_address,
}

// THE HARNESS REPORTS ITS OWN STANDING, WHICH IS THE POINT RATHER THAN A CONVENIENCE.
Expand Down Expand Up @@ -121,9 +122,66 @@ fn harness_standing_is_serving(standing: HarnessBackendStanding) -> Bool {
}
}

fn harness_observe_backend() -> HarnessBackendStanding {
fold(harness_engine_replica_addresses, init: HarnessBackendServing { model: harness_engine_model }, f: (acc, a) =>
if harness_standing_is_serving(standing: acc) { harness_observe_replica(address: a) } else { acc })
// THE ROSTER IS OBSERVED ONE REPLICA AT A TIME AND THE ANSWER KEEPS BOTH HALVES. The previous fold
// answered for the whole backend with the first non-serving replica's standing, so one Spark down
// refused every dispatch while the other served -- measured 2026-09-05 on srv1, .232 unreachable,
// .225 advertising the model, dispatch refused at selection. That was the all-or-nothing reading of
// a roster whose whole point is that attempts spread across it. Now: serving replicas are the set an
// attempt may bind to; declined replicas are carried with their typed standing, so the count of what
// is down is observable and never absorbed. None serving refuses, naming every replica's cause.
type HarnessReplicaDecline {
address: String
standing: HarnessBackendStanding
}

type HarnessBackendRoster
= HarnessRosterServing { model: String, serving: List<String>, declined: List<HarnessReplicaDecline> }
| HarnessRosterNoneServing { model: String, declined: List<HarnessReplicaDecline> }

type HarnessRosterFold {
serving: List<String>
declined: List<HarnessReplicaDecline>
}

fn harness_observe_roster() -> HarnessBackendRoster {
let folded = fold(harness_engine_replica_addresses, init: HarnessRosterFold { serving: [], declined: [] }, f: (acc, a) => {
let standing = harness_observe_replica(address: a)
Comment on lines +147 to +148

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 5, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Bound each replica probe before walking the roster

When one declared replica accepts a connection but never answers /v1/models while another replica is healthy, every roster observation stalls at this fold: harness_observe_replica calls http.Client.Get, whose transport is only curl -fsS with no connection or overall timeout (dag/extdeps/http/client.dag:52-58). Because the probe, worker, reviewer, and auditor entry points now all perform this full-roster observation before selecting a URL, they never reach the healthy replica or the turn's request deadline, so one wedged Spark can still block all dispatches instead of becoming a declined member; use a bounded GET for each replica.

Useful? React with 👍 / 👎.

if harness_standing_is_serving(standing: standing) {
HarnessRosterFold { serving: concat(acc.serving, [a]), declined: acc.declined }
} else {
HarnessRosterFold { serving: acc.serving, declined: concat(acc.declined, [HarnessReplicaDecline { address: a, standing: standing }]) }
}
})
if count(folded.serving) == 0 {
HarnessRosterNoneServing { model: harness_engine_model, declined: folded.declined }
} else {
HarnessRosterServing { model: harness_engine_model, serving: folded.serving, declined: folded.declined }
}
}

fn harness_declines_detail(declined: List<HarnessReplicaDecline>) -> String {
join(map(declined, d => join([
d.address, ": ", harness_standing_label(standing: d.standing), " -- ", harness_standing_detail(standing: d.standing),
], "")), "; ")
}

// THE URL AN ATTEMPT POSTS TO, bound at process start from the replicas serving at that moment. The
// seed spreads concurrent attempts across the serving set; a roster with nothing serving is a typed
// refusal the entry point exits on, never a URL that will fail on the first turn.
type HarnessBackendUrl
= BackendUrlBound { url: String, declined_count: Int }
| BackendUrlRefused { detail: String }

fn harness_serving_backend_url(shape: HarnessWireShape, seed: String) -> HarnessBackendUrl {
match harness_observe_roster() {
HarnessRosterServing { model: _, serving, declined } =>
BackendUrlBound {
url: harness_backend_url_for_address(shape: shape, address: harness_replica_pick(seed: seed, replicas: serving)),
declined_count: count(declined),
}
HarnessRosterNoneServing { model: m, declined } =>
BackendUrlRefused { detail: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], "") }
}
}

fn harness_backend_report() -> String {
Expand Down
159 changes: 80 additions & 79 deletions dag/gunbc/harness/harness_cli.dag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ import gunbc.harness.harness_guidance { harness_worker_guidance_at, harness_guid
import gunbc.harness.harness_review_guidance { harness_reviewer_guidance_at, review_guidance_prose }
import gunbc.harness.harness_goal_audit_guidance { harness_auditor_guidance_at, audit_guidance_prose }
import gunbc.roadmap.roadmap_review_role { review_verdict_shape_text }
import gunbc.harness.harness_wire {
harness_engine_backend_url, harness_engine_model, harness_engine_context_window,
}
import gunbc.harness.harness_wire { harness_engine_model, harness_engine_context_window }
import gunbc.harness.harness_backend { HarnessBackendUrl, BackendUrlBound, BackendUrlRefused, harness_serving_backend_url }

// THE FIRST WET ENTRY FOR THE DEDICATED HARNESS. It exists to make DCH-2 provable by execution
// rather than by typecheck -- DESIGN section 5's specification-without-execution trap names exactly
Expand Down Expand Up @@ -165,9 +164,9 @@ fn harness_macbook_environment() -> HarnessToolEnvironment {
// different authority.
data harness_context_budget: TokenCount = harness_engine_context_window

fn harness_macbook_config() -> HarnessTurnConfig {
fn harness_macbook_config(backend_url: String) -> HarnessTurnConfig {
HarnessTurnConfig {
backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: ""),
backend_url: backend_url,
model: harness_default_model(),
max_tokens: harness_default_max_tokens,
reasoning_mode: harness_reasoning_mode,
Expand All @@ -188,9 +187,9 @@ fn harness_macbook_config() -> HarnessTurnConfig {
// posted, received a tool_use, executed it, posted the result back, and received a terminal.
data harness_probe_prompt: String = "How many files in the dag/gunbc/harness directory of this repository end in .dag? Use run_command to find out, then state the number."

fn harness_probe_outcome() -> HarnessTurnOutcome {
fn harness_probe_outcome(backend_url: String) -> HarnessTurnOutcome {
harness_run_turn(
config: harness_macbook_config(),
config: harness_macbook_config(backend_url: backend_url),
prompt: harness_probe_prompt,
step_budget: harness_default_step_budget,
)
Expand All @@ -200,8 +199,7 @@ fn harness_probe_outcome() -> HarnessTurnOutcome {
// happened rather than whether the task succeeded. Nothing here inspects the assistant's text for
// correctness: that is external verification's subject, and a harness that graded its own worker's
// answer would be the same conflation DCH-2 forbids between a stop reason and a completed task.
fn harness_probe_cli() -> ProcessExit {
let outcome = harness_probe_outcome()
fn harness_turn_exit(outcome: HarnessTurnOutcome) -> ProcessExit {
match outcome {
TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } =>
ExitSuccess
Expand All @@ -215,6 +213,19 @@ fn harness_probe_cli() -> ProcessExit {
}
}

// THE BACKEND IS BOUND BEFORE THE TURN: a roster with nothing serving exits here with every
// replica's cause, instead of a config whose first POST would refuse.
fn harness_backend_refusal_exit(detail: String) -> ProcessExit {
ExitFailure { code: 1, reason: join(["harness backend: ", detail], "") }
}

fn harness_probe_cli() -> ProcessExit {
match harness_serving_backend_url(shape: harness_default_wire_shape, seed: "") {
BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail)
BackendUrlBound { url, declined_count: _ } => harness_turn_exit(outcome: harness_probe_outcome(backend_url: url))
}
}

// THE WORKER ENTRY THE BELT SPAWNS. Its two facts arrive as named arguments rather than through the
// environment, because `gunbc run` has had --arg since the scm read landed and the belt's own
// annotation saying otherwise ("gunbc run --function cannot pass an argument (no --arg flag)") is
Expand Down Expand Up @@ -244,9 +255,10 @@ fn harness_worker_config(
timeout_program: FilePath,
request_scratch_path: FilePath,
events_path: FilePath,
backend_url: String,
) -> HarnessTurnConfig {
HarnessTurnConfig {
backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: worktree as String),
backend_url: backend_url,
model: harness_default_model(),
max_tokens: harness_default_max_tokens,
reasoning_mode: harness_reasoning_mode,
Expand Down Expand Up @@ -286,26 +298,20 @@ fn harness_worker_cli(
request_scratch_path: String,
events_path: String,
) -> ProcessExit {
let outcome = harness_run_turn(
config: harness_worker_config(
worktree: worktree as FilePath,
timeout_program: timeout_program as FilePath,
request_scratch_path: request_scratch_path as FilePath,
events_path: events_path as FilePath,
),
prompt: brief,
step_budget: harness_worker_step_budget,
)
match outcome {
TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } =>
ExitSuccess
_ =>
ExitFailure {
code: 1,
reason: join([
harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome),
], ""),
}
match harness_serving_backend_url(shape: harness_default_wire_shape, seed: worktree) {
BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail)
BackendUrlBound { url, declined_count: _ } =>
harness_turn_exit(outcome: harness_run_turn(
config: harness_worker_config(
worktree: worktree as FilePath,
timeout_program: timeout_program as FilePath,
request_scratch_path: request_scratch_path as FilePath,
events_path: events_path as FilePath,
backend_url: url,
),
prompt: brief,
step_budget: harness_worker_step_budget,
))
}
}

Expand Down Expand Up @@ -333,9 +339,10 @@ fn harness_reviewer_config(
request_scratch_path: FilePath,
events_path: FilePath,
verdict_path: String,
backend_url: String,
) -> HarnessTurnConfig {
HarnessTurnConfig {
backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: verdict_path),
backend_url: backend_url,
model: harness_default_model(),
max_tokens: harness_default_max_tokens,
reasoning_mode: harness_reasoning_mode,
Expand All @@ -361,27 +368,21 @@ fn harness_reviewer_cli(
events_path: String,
verdict_path: String,
) -> ProcessExit {
let outcome = harness_run_turn(
config: harness_reviewer_config(
worktree: worktree as FilePath,
timeout_program: timeout_program as FilePath,
request_scratch_path: request_scratch_path as FilePath,
events_path: events_path as FilePath,
verdict_path: verdict_path,
),
prompt: brief,
step_budget: harness_reviewer_step_budget,
)
match outcome {
TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } =>
ExitSuccess
_ =>
ExitFailure {
code: 1,
reason: join([
harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome),
], ""),
}
match harness_serving_backend_url(shape: harness_default_wire_shape, seed: verdict_path) {
BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail)
BackendUrlBound { url, declined_count: _ } =>
harness_turn_exit(outcome: harness_run_turn(
config: harness_reviewer_config(
worktree: worktree as FilePath,
timeout_program: timeout_program as FilePath,
request_scratch_path: request_scratch_path as FilePath,
events_path: events_path as FilePath,
verdict_path: verdict_path,
backend_url: url,
),
prompt: brief,
step_budget: harness_reviewer_step_budget,
))
}
}

Expand All @@ -396,9 +397,10 @@ fn harness_auditor_config(
request_scratch_path: FilePath,
events_path: FilePath,
verdict_path: String,
backend_url: String,
) -> HarnessTurnConfig {
HarnessTurnConfig {
backend_url: harness_engine_backend_url(shape: harness_default_wire_shape, seed: verdict_path),
backend_url: backend_url,
model: harness_default_model(),
max_tokens: harness_default_max_tokens,
reasoning_mode: harness_reasoning_mode,
Expand All @@ -424,34 +426,33 @@ fn harness_auditor_cli(
events_path: String,
verdict_path: String,
) -> ProcessExit {
let outcome = harness_run_turn(
config: harness_auditor_config(
worktree: worktree as FilePath,
timeout_program: timeout_program as FilePath,
request_scratch_path: request_scratch_path as FilePath,
events_path: events_path as FilePath,
verdict_path: verdict_path,
),
prompt: brief,
step_budget: harness_reviewer_step_budget,
)
match outcome {
TurnProviderStopped { wire_stop_reason: _, assistant_text: _, steps_taken: _, input_tokens: _, output_tokens: _ } =>
ExitSuccess
_ =>
ExitFailure {
code: 1,
reason: join([
harness_outcome_label(outcome: outcome), ": ", harness_outcome_detail(outcome: outcome),
], ""),
}
match harness_serving_backend_url(shape: harness_default_wire_shape, seed: verdict_path) {
BackendUrlRefused { detail } => harness_backend_refusal_exit(detail: detail)
BackendUrlBound { url, declined_count: _ } =>
harness_turn_exit(outcome: harness_run_turn(
config: harness_auditor_config(
worktree: worktree as FilePath,
timeout_program: timeout_program as FilePath,
request_scratch_path: request_scratch_path as FilePath,
events_path: events_path as FilePath,
verdict_path: verdict_path,
backend_url: url,
),
prompt: brief,
step_budget: harness_reviewer_step_budget,
))
}
}

fn harness_probe_report() -> String {
let outcome = harness_probe_outcome()
join([
"harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n",
harness_outcome_detail(outcome: outcome), "\n",
], "")
match harness_serving_backend_url(shape: harness_default_wire_shape, seed: "") {
BackendUrlRefused { detail } => join(["harness probe refused before the turn: ", detail, "\n"], "")
BackendUrlBound { url, declined_count: _ } => {
let outcome = harness_probe_outcome(backend_url: url)
join([
"harness probe outcome: ", harness_outcome_label(outcome: outcome), "\n",
harness_outcome_detail(outcome: outcome), "\n",
], "")
}
}
}
17 changes: 15 additions & 2 deletions dag/gunbc/harness/harness_wire.dag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,12 +359,25 @@ fn harness_engine_replica_index(seed: String) -> Int {
}

fn harness_engine_replica_address(seed: String) -> String {
let want = harness_engine_replica_index(seed: seed)
let picked = fold(harness_engine_replica_addresses, init: ReplicaPick { at: 0, chosen: "" }, f: (acc, a) =>
harness_replica_pick(seed: seed, replicas: harness_engine_replica_addresses)
}

// THE PICK OVER AN ARBITRARY ROSTER, so an attempt binds to a replica chosen from the ones OBSERVED
// SERVING (gunbc.harness.harness_backend harness_serving_backend_url) rather than from the declared
// list. A declared replica that is down is a typed, counted decline there; here it is simply not in
// the roster handed in, so the seed spreads attempts over the machines that can take them. Callers
// hand in a non-empty roster (the serving arm carries at least one address by construction).
fn harness_replica_pick(seed: String, replicas: List<String>) -> String {
let want = fold(chars(s: seed), init: 0, f: (acc, cp) => acc + cp) % list_length(items: replicas)
let picked = fold(replicas, init: ReplicaPick { at: 0, chosen: "" }, f: (acc, a) =>
ReplicaPick { at: acc.at + 1, chosen: if acc.at == want { a } else { acc.chosen } })
picked.chosen
}

fn harness_backend_url_for_address(shape: HarnessWireShape, address: String) -> String {
harness_backend_url_on_port(address: address, port: harness_engine_port, shape: shape)
}

type ReplicaPick {
at: Int
chosen: String
Expand Down
27 changes: 7 additions & 20 deletions dag/gunbc/roadmap/roadmap_belt_actuate.dag
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ import extdeps.languages.json.parse {
}
import gunbc.clock_read { clock_now_probed_at }
import gunbc.harness.harness_backend {
HarnessBackendStanding, HarnessBackendServing, HarnessBackendModelAbsent,
HarnessBackendUnreachable, HarnessBackendUnreadable, harness_observe_backend,
HarnessBackendRoster, HarnessRosterServing, HarnessRosterNoneServing,
harness_observe_roster, harness_declines_detail,
}
import gunbc.roadmap_dispatch_actuator {
HostExecArgv,
Expand Down Expand Up @@ -1394,25 +1394,12 @@ fn belt_provider_preflight_for_instance(
detail: "cursor dispatch is not yet wired on this actuator path",
}
GunbcHarnessProvider =>
match harness_observe_backend() {
HarnessBackendServing { model: _ } => ProviderPreflightOk
HarnessBackendModelAbsent { model: m, advertised: a } =>
match harness_observe_roster() {
HarnessRosterServing { model: _, serving: _, declined: _ } => ProviderPreflightOk
HarnessRosterNoneServing { model: m, declined } =>
ProviderPreflightRefused {
step: "harness-backend-model",
detail: join([
"the harness backend is reachable but does not advertise ", m,
"; it advertises: ", a,
], ""),
}
HarnessBackendUnreachable { url: u } =>
ProviderPreflightRefused {
step: "harness-backend-reachability",
detail: join(["the harness backend did not answer ", u], ""),
}
HarnessBackendUnreadable { cause: c } =>
ProviderPreflightRefused {
step: "harness-backend-model-list",
detail: join(["the harness backend answered unreadably: ", c], ""),
step: "harness-backend-roster",
detail: join(["no harness replica serves ", m, ": ", harness_declines_detail(declined: declined)], ""),
}
}
}
Expand Down
Loading
Loading