From bb1717f499e1dcb75da53bb39724d9adbcc184a8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 7 Sep 2026 00:22:06 +0000 Subject: [PATCH] A request the helper can never decode leaves the pending set, instead of stopping the line on the same bytes forever Measured on srv1 (2026-09-06): two publication requests written under a superseded request schema stopped the helper on every tick for a day. The refusal was correct -- the document does not decode, and decoding is a pure function of its bytes, so no run in that population could have ended differently. The defect was that the refusal RETAINED the request. Every other terminal outcome consumes its request: the answer is written, then the request is deleted, so the next run sees a spool holding only open questions. The unreadable arm was the exception, so one bad document re-stopped the line every tick at a full run's cost, with every request queued behind it waiting, and no analysis could ever restart it. STOPPING THE LINE AND CONSUMING THE REQUEST ARE TWO DECISIONS. The stop is what makes the deficit loud; the retirement is what makes it stop ONCE per bad document rather than once per tick. A new undecodable request stops the line again, which is the intended signal. The judgment is pure and the retirement is the effect. publication_request_admission decides from the entry name and the request bytes alone; publication_helper_retire_request performs the move and returns the caller's refusal unchanged on success, so retiring never renames why a request was refused. An earlier revision of this repair fused them, which put a filesystem move on the decision path and turned two existing witnesses into route gaps -- the available repair there was to mock the move, which would have passed them against a fabricated success. A THIRD DOOR WAS OPEN and is closed here too: HelperSubjectOutsideCapability is equally terminal -- the capability is this repository, a constant -- and it also retained its request. The refusals stay three rather than collapsing into one `terminally refused`. They share a remedy and nothing else: a document that does not parse, one that names a repository this principal may not act on, and one whose name lies about its own subject are three different findings for whoever reads the stopped line, and the second is the anti-forgery arm. HelperRequestUnreadable is split from HelperRequestRefused for the same reason: a failed READ is transient and its request stays pending, a failed DECODE is terminal and its request leaves. The collapse is how the terminal class inherited the transient class's remedy. The quarantine is not an answer and cannot be addressed as one -- an undecodable request has no decoded subject, so no answer address exists for it, and writing one at the caller's entry name is the forgery the request/answer permission split makes unwritable. The deploy creates the third directory owned by the publication principal. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_019iXPtAqzPaNBZbaaM5rxrY --- dag/gunbc/host/host_layout.dag | 20 +++ dag/gunbc/live_deploy/emit.dag | 7 + .../roadmap/roadmap_publication_helper.dag | 148 +++++++++++++++--- ...oadmap_publication_helper_witness_test.dag | 120 +++++++++++--- 4 files changed, 255 insertions(+), 40 deletions(-) diff --git a/dag/gunbc/host/host_layout.dag b/dag/gunbc/host/host_layout.dag index 9c28d619d61..ef6652d2edc 100644 --- a/dag/gunbc/host/host_layout.dag +++ b/dag/gunbc/host/host_layout.dag @@ -330,6 +330,26 @@ fn publication_answer_dir(instance_root: String) -> String { join([publication_spool_root(instance_root: instance_root), "/answers"], "") } +// A THIRD DIRECTORY BECAUSE A REQUEST THE HELPER CANNOT DECODE IS NEITHER PENDING NOR ANSWERED. +// +// Decoding is a pure function of the request's bytes, so a document that refuses once refuses on +// every later run: it is a TERMINAL judgment about that document, not a transient one. Left in +// `requests/` it is a poison pill -- the helper stops the line on it, correctly, and then stops the +// line on the SAME document forever, at one full run's cost per tick, while every request queued +// behind it waits. That is the absorbing arm read from the refusing side: the deficit's frequency is +// unbounded rather than counted, and no analysis can ever restart the line. +// +// The quarantine is not an answer and must not be reachable as one. An undecodable request has no +// decoded subject, so the helper cannot derive an answer address for it (see the helper's own +// header: the answer address comes from the subject, never from the caller), and writing one at the +// caller's entry name is exactly the forgery the request/answer permission split makes unwritable. +// So the bytes move sideways into a directory only the publication principal holds, where they stay +// readable for the analysis the stopped line demands. The requester is not told anything, which is +// correct: its question was never understood, and its own gate re-asks while the question is open. +fn publication_refused_dir(instance_root: String) -> String { + join([publication_spool_root(instance_root: instance_root), "/requests-refused"], "") +} + // srv1 observation (2026-08-13, parent eager-koi-458): directory already briansrls-owned; // EACCES was a root-owned STALE FILE from the pre-b1 unit user, not the directory. // Publication authority is atomic rename in gunbc.fleet_converge_cli — NOT live_deploy diff --git a/dag/gunbc/live_deploy/emit.dag b/dag/gunbc/live_deploy/emit.dag index cebeccab53d..f9025cdf0fb 100644 --- a/dag/gunbc/live_deploy/emit.dag +++ b/dag/gunbc/live_deploy/emit.dag @@ -90,6 +90,7 @@ import gunbc.host_layout { publication_spool_root, publication_request_dir, publication_answer_dir, + publication_refused_dir, } import gunbc.host_axis_caps { deploy_memory_cap_apply_script_for_host, @@ -650,6 +651,12 @@ fn apply_publication_spool_steps(spec: DeploymentSpec) -> List { group: spec.names.publication_principal, path: publication_answer_dir(instance_root: spec.service.instance_root as String), )), + deploy_raw(command: install_directory_owned_command( + mode: publication_spool_answer_mode, + owner: spec.names.publication_principal, + group: spec.names.publication_principal, + path: publication_refused_dir(instance_root: spec.service.instance_root as String), + )), ] } diff --git a/dag/gunbc/roadmap/roadmap_publication_helper.dag b/dag/gunbc/roadmap/roadmap_publication_helper.dag index 6a03e0e8cbe..420c913fc75 100644 --- a/dag/gunbc/roadmap/roadmap_publication_helper.dag +++ b/dag/gunbc/roadmap/roadmap_publication_helper.dag @@ -24,7 +24,7 @@ import extdeps.languages.json.parse { json_object_unique_member, JsonMemberFound, } -import gunbc.host_layout { publication_request_dir, publication_answer_dir } +import gunbc.host_layout { publication_request_dir, publication_answer_dir, publication_refused_dir } import gunbc.roadmap.roadmap_publish_actuate { publish_attempt, PublishActuated, PushRefused, PullRequestRefused } import gunbc.repository { gunbc_repository } import gunbc.roadmap_publish { @@ -477,10 +477,22 @@ fn publication_helper_credential() -> PublicationCredentialSource { // EVERY REQUEST GETS AN ANSWER OR A NAMED REFUSAL, and no arm is silent. A request that does not // decode is not skipped: skipping it would leave the requester waiting forever on a document only // the helper can see, which is a deadlock whose observable form is an ordinary Pending lamp. +// +// UNREADABLE AND REFUSED ARE TWO CLASSES WITH OPPOSITE REMEDIES, and an earlier revision carried +// only the first name for both. `HelperRequestUnreadable` is a failed READ -- a permission, a race, +// a listing that named an entry that then went away -- and it is transient: the same document may +// read cleanly on the next run, so the request stays pending and is retried. `HelperRequestRefused` +// is a failed DECODE, which is a pure function of bytes that will not change, so the judgment is +// terminal and the document leaves the pending set. Collapsing them meant the terminal class +// inherited the transient class's remedy -- retry -- which is precisely how one bad document came +// to stop the line on every tick forever. Both still stop the line; they differ in what happens to +// the request, and that is the distinction the two names carry. type PublicationHelperOutcome = HelperAnswered { key: String, outcome_key: String } | HelperRequestNotConsumed { key: String, detail: String } + | HelperRequestNotQuarantined { entry: String, detail: String } | HelperRequestUnreadable { entry: String, detail: String } + | HelperRequestRefused { entry: String, detail: String } | HelperAnswerWriteFailed { key: String, detail: String } | HelperSubjectOutsideCapability { entry: String, detail: String } | HelperSpoolRefused { detail: String } @@ -489,7 +501,9 @@ fn publication_helper_outcome_key(o: PublicationHelperOutcome) -> String { match o { HelperAnswered { key: _, outcome_key: _ } => "answered" HelperRequestNotConsumed { key: _, detail: _ } => "request-not-consumed" + HelperRequestNotQuarantined { entry: _, detail: _ } => "request-not-quarantined" HelperRequestUnreadable { entry: _, detail: _ } => "request-unreadable" + HelperRequestRefused { entry: _, detail: _ } => "request-refused" HelperAnswerWriteFailed { key: _, detail: _ } => "answer-write-failed" HelperSubjectOutsideCapability { entry: _, detail: _ } => "subject-outside-capability" HelperSpoolRefused { detail: _ } => "spool-refused" @@ -527,6 +541,10 @@ fn publication_helper_outcome_disposition(o: PublicationHelperOutcome) -> Public OutcomeStopsTheLine { detail: join([key, ": ", detail], "") } HelperRequestUnreadable { entry, detail } => OutcomeStopsTheLine { detail: join([entry, ": ", detail], "") } + HelperRequestRefused { entry, detail } => + OutcomeStopsTheLine { detail: join([entry, ": ", detail], "") } + HelperRequestNotQuarantined { entry, detail } => + OutcomeStopsTheLine { detail: join([entry, ": ", detail], "") } HelperAnswerWriteFailed { key, detail } => OutcomeStopsTheLine { detail: join([key, ": ", detail], "") } HelperSubjectOutsideCapability { entry, detail } => @@ -643,32 +661,122 @@ fn publication_helper_publish_if_unpublished( } } -fn publication_helper_answer_request( - instance_root: String, - credential: PublicationCredentialSource, - entry: String, - request_text: String, -) -> PublicationHelperOutcome { +// STOPPING THE LINE AND CONSUMING THE REQUEST ARE TWO DECISIONS, AND THE EARLIER REVISION MADE ONE. +// +// Every other terminal outcome consumes its request -- the answer is written, then the request is +// deleted -- so the next run sees a spool holding only open questions. An UNREADABLE request was the +// exception: the line stopped and the document stayed, so the same bytes were re-read, re-refused +// and re-charged a full run every tick, forever, with every request behind it waiting. The refusal +// was right; the retention was the defect. Measured on srv1 (2026-09-06): two requests written under +// a superseded request schema stopped the helper on every tick for a day at ~2 CPU-minutes each, +// and no run in that population could have ended differently, because decoding is a pure function +// of the bytes. +// +// So the document moves out of the pending set and the line still stops. The stop is what makes the +// deficit loud and analyzable; the move is what makes it stop ONCE per bad document rather than once +// per tick. A NEW undecodable request stops the line again, which is the intended signal -- what is +// removed is only the repetition, never the refusal. +// +// A MOVE THAT FAILS IS ITS OWN OUTCOME, on the precedent the delete-failure arm above sets. Silence +// there would put the helper back in exactly the loop this function exists to end, with nothing +// recording that it had tried to leave it. +// WHETHER A REQUEST IS TERMINALLY REFUSED IS A PURE JUDGMENT, AND RETIRING IT IS AN EFFECT. +// +// Separating them is not tidiness. Every one of these refusals is decidable from the entry name and +// the request bytes alone, so the judgment can be witnessed hermetically -- and an earlier revision +// of this repair fused them, which put a filesystem move on the decision path and turned two +// existing witnesses into route gaps: the hermetic route has no arm for the move, so the claims +// stopped before reaching the subject they were written to assert. The available repair was to mock +// the move, which would have passed them against a fabricated success. The judgment moved instead. +type RequestAdmission + = RequestAdmitted { subject: RoadmapPublicationSubject, worktree: String } + | RequestUndecodableDocument { reason: String } + | RequestOutsideCapability { detail: String } + | RequestMisaddressed { detail: String } + +// THE THREE REFUSALS STAY THREE, rather than collapsing into one `terminally refused`. They share a +// remedy -- leave the pending set -- and share nothing else: a document that does not parse, a +// document that names a repository this principal may not act on, and a document whose name lies +// about its own subject are three different things for whoever reads the stopped line, and the +// second is the anti-forgery arm. One name for all three would be the state-space conflation this +// module already avoids in its outcome vocabulary. +fn publication_request_admission(entry: String, request_text: String) -> RequestAdmission { match publication_request_decode(request: request_text) { - RequestUndecodable { reason } => - HelperRequestUnreadable { entry: entry, detail: reason } + RequestUndecodable { reason } => RequestUndecodableDocument { reason: reason } RequestDecoded { subject, worktree } => match publication_helper_capability_admits(subject: subject) { - SubjectOutsideCapability { detail } => - HelperSubjectOutsideCapability { entry: entry, detail: detail } - SubjectWithinCapability => { - let key = publication_spool_key(subject: subject) - if publication_spool_entry_name(key: key) != entry { - HelperRequestUnreadable { - entry: entry, + SubjectOutsideCapability { detail } => RequestOutsideCapability { detail: detail } + SubjectWithinCapability => + if publication_spool_entry_name(key: publication_spool_key(subject: subject)) != entry { + RequestMisaddressed { detail: join([ "the request's own subject addresses ", - publication_spool_entry_name(key: key), + publication_spool_entry_name(key: publication_spool_key(subject: subject)), ", so the document at this entry is not about the judgment its name claims", ], ""), } } else { - let receipt = publication_receipt_of( + RequestAdmitted { subject: subject, worktree: worktree } + } + } + } +} + +// A MOVE THAT FAILS IS ITS OWN OUTCOME, on the precedent the delete-failure arm below sets. Silence +// there would put the helper back in exactly the loop this function exists to end, with nothing +// recording that it had tried to leave it. The refusal the caller already decided is returned +// unchanged on success, so retiring a request never renames why it was refused. +fn publication_helper_retire_request( + instance_root: String, + entry: String, + refused: PublicationHelperOutcome, +) -> PublicationHelperOutcome { + let moved = shell.Move.File( + source: join([publication_request_dir(instance_root: instance_root), "/", entry], "") as FilePath, + destination: join([publication_refused_dir(instance_root: instance_root), "/", entry], "") as FilePath, + ) + if moved.success { + refused + } else { + HelperRequestNotQuarantined { + entry: entry, + detail: join([ + "the request was terminally refused and could not be moved out of the pending set, so it will stop the line again on the next run with the same bytes: ", + publication_request_dir(instance_root: instance_root), "/", entry, + " -> ", publication_refused_dir(instance_root: instance_root), "/", entry, + ], ""), + } + } +} + +fn publication_helper_answer_request( + instance_root: String, + credential: PublicationCredentialSource, + entry: String, + request_text: String, +) -> PublicationHelperOutcome { + match publication_request_admission(entry: entry, request_text: request_text) { + RequestUndecodableDocument { reason } => + publication_helper_retire_request( + instance_root: instance_root, + entry: entry, + refused: HelperRequestRefused { entry: entry, detail: reason }, + ) + RequestOutsideCapability { detail } => + publication_helper_retire_request( + instance_root: instance_root, + entry: entry, + refused: HelperSubjectOutsideCapability { entry: entry, detail: detail }, + ) + RequestMisaddressed { detail } => + publication_helper_retire_request( + instance_root: instance_root, + entry: entry, + refused: HelperRequestRefused { entry: entry, detail: detail }, + ) + RequestAdmitted { subject, worktree } => { + let key = publication_spool_key(subject: subject) + let receipt = publication_receipt_of( subject: subject, outcome: publication_helper_publish_if_unpublished( credential: credential, @@ -712,9 +820,7 @@ fn publication_helper_answer_request( } } } - } - } - } + } } } diff --git a/dag/test/claim/roadmap_publication_helper_witness_test.dag b/dag/test/claim/roadmap_publication_helper_witness_test.dag index b903a147192..e84405f8655 100644 --- a/dag/test/claim/roadmap_publication_helper_witness_test.dag +++ b/dag/test/claim/roadmap_publication_helper_witness_test.dag @@ -18,12 +18,17 @@ import gunbc.roadmap_publication_helper { publication_answer_path_for, PublicationHelperOutcome, HelperAnswered, HelperRequestNotConsumed, HelperRequestUnreadable, + HelperRequestRefused, HelperRequestNotQuarantined, HelperAnswerWriteFailed, HelperSpoolRefused, HelperSubjectOutsideCapability, PublicationHelperDisposition, OutcomeRecorded, OutcomeStopsTheLine, publication_helper_outcome_disposition, + publication_helper_outcome_key, PublicationHelperRunStanding, HelperRunCompleted, HelperRunStopped, publication_helper_run_standing, publication_helper_answer_request, + publication_request_admission, + RequestAdmission, RequestAdmitted, RequestUndecodableDocument, + RequestOutsideCapability, RequestMisaddressed, publication_helper_credential_env_var, } import gunbc.roadmap_publish_observe { @@ -32,6 +37,7 @@ import gunbc.roadmap_publish_observe { import gunbc.host_layout { publication_request_dir, publication_answer_dir, + publication_refused_dir, } import gunbc.live_deploy.emit { live_deploy_protocol, @@ -224,35 +230,29 @@ test fn a_document_that_is_not_json_is_refused_rather_than_read_as_a_request() - // consulted, and the check runs ahead of the branch that would consult it. test fn a_request_whose_subject_addresses_another_entry_is_refused_before_any_observation() -> Bool { - match publication_helper_answer_request( - instance_root: "/opt/gunbc", - credential: CredentialSourceAbsent { reason: "no credential in this witness" }, + match publication_request_admission( entry: "some-other-node.a1.2222222222222222222222222222222222222222.json", request_text: publication_request_json(subject: ordinary_subject(), worktree: "/tmp/wt"), ) { - HelperRequestUnreadable { entry: e, detail: _ } => - e == "some-other-node.a1.2222222222222222222222222222222222222222.json" - HelperAnswered { key: _, outcome_key: _ } => false - HelperRequestNotConsumed { key: _, detail: _ } => false - HelperAnswerWriteFailed { key: _, detail: _ } => false - HelperSpoolRefused { detail: _ } => false - HelperSubjectOutsideCapability { entry: _, detail: _ } => false + RequestMisaddressed { detail } => + string_contains(s: detail, pattern: publication_spool_entry_name( + key: publication_spool_key(subject: ordinary_subject()), + )) + RequestAdmitted { subject: _, worktree: _ } => false + RequestUndecodableDocument { reason: _ } => false + RequestOutsideCapability { detail: _ } => false } } test fn a_request_document_that_does_not_decode_is_refused_rather_than_skipped() -> Bool { - match publication_helper_answer_request( - instance_root: "/opt/gunbc", - credential: CredentialSourceAbsent { reason: "no credential in this witness" }, + match publication_request_admission( entry: "n.a1.1111111111111111111111111111111111111111.json", request_text: "{}", ) { - HelperRequestUnreadable { entry: _, detail: _ } => true - HelperAnswered { key: _, outcome_key: _ } => false - HelperRequestNotConsumed { key: _, detail: _ } => false - HelperAnswerWriteFailed { key: _, detail: _ } => false - HelperSpoolRefused { detail: _ } => false - HelperSubjectOutsideCapability { entry: _, detail: _ } => false + RequestUndecodableDocument { reason: _ } => true + RequestAdmitted { subject: _, worktree: _ } => false + RequestOutsideCapability { detail: _ } => false + RequestMisaddressed { detail: _ } => false } } @@ -267,6 +267,86 @@ test fn a_request_document_that_does_not_decode_is_refused_rather_than_skipped() // subject is re-observed on every run from then on with nothing recording why -- a cost that grows // silently, which is exactly what a recorded classification would hide. +// THE THREE TERMINAL REFUSALS STAY THREE. They share one remedy -- leave the pending set -- and a +// single `terminally refused` arm would have been the cheaper model. It would also have erased the +// anti-forgery distinction: a document naming another repository and a document whose name lies +// about its own subject are different findings for whoever reads the stopped line. This is the +// control that a later collapse reds. +test fn the_three_terminal_refusals_are_distinguished_rather_than_collapsed() -> Bool { + let base = ordinary_subject() + let foreign = RoadmapPublicationSubject { + node_id: base.node_id, attempt_key: base.attempt_key, + repository: other_repository(), branch: base.branch, + expected_base: base.expected_base, expected_head: base.expected_head, + } + let undecodable = match publication_request_admission( + entry: publication_spool_entry_name(key: publication_spool_key(subject: base)), + request_text: "{}", + ) { + RequestUndecodableDocument { reason: _ } => true + RequestAdmitted { subject: _, worktree: _ } => false + RequestOutsideCapability { detail: _ } => false + RequestMisaddressed { detail: _ } => false + } + let outside = match publication_request_admission( + entry: publication_spool_entry_name(key: publication_spool_key(subject: foreign)), + request_text: publication_request_json(subject: foreign, worktree: "/tmp/wt"), + ) { + RequestOutsideCapability { detail: _ } => true + RequestAdmitted { subject: _, worktree: _ } => false + RequestUndecodableDocument { reason: _ } => false + RequestMisaddressed { detail: _ } => false + } + let admitted = match publication_request_admission( + entry: publication_spool_entry_name(key: publication_spool_key(subject: base)), + request_text: publication_request_json(subject: base, worktree: "/tmp/wt"), + ) { + RequestAdmitted { subject: _, worktree: _ } => true + RequestUndecodableDocument { reason: _ } => false + RequestOutsideCapability { detail: _ } => false + RequestMisaddressed { detail: _ } => false + } + undecodable && outside && admitted +} + +// ── A terminally refused request leaves the pending set; a transiently unreadable one does not ── +// +// THIS IS THE CLASS THE EARLIER REVISION HAD NO NAME FOR, and the cost of not having one was +// measured rather than imagined: two requests written under a superseded schema stopped the srv1 +// helper on every tick for a day, each tick paying a full run, because the refusal was correct and +// the retention was not. Decoding is a pure function of the request bytes, so no run in that +// population could have ended differently. +// +// The discriminating pair is the whole claim. A failed READ is transient -- the same path may read +// next run -- so its request stays pending and nothing is moved. A failed DECODE is terminal, so its +// request is moved out of the pending set. If the two shared one outcome, one of these two arms +// would have to be wrong, and the one that was wrong is the one that ran in production. + +test fn a_terminally_refused_request_is_named_apart_from_a_transiently_unreadable_one() -> Bool { + publication_helper_outcome_key(o: HelperRequestRefused { entry: "e", detail: "d" }) + != publication_helper_outcome_key(o: HelperRequestUnreadable { entry: "e", detail: "d" }) +} + +// A QUARANTINE THAT FAILS IS NOT A SILENCE. The helper that could not move the document is back in +// the loop this repair exists to end, and the only thing that makes that visible is its own name -- +// on the precedent HelperRequestNotConsumed sets for the delete that fails after a written answer. +test fn a_quarantine_that_fails_is_its_own_outcome_and_stops_the_line() -> Bool { + publication_helper_outcome_key(o: HelperRequestNotQuarantined { entry: "e", detail: "d" }) + != publication_helper_outcome_key(o: HelperRequestRefused { entry: "e", detail: "d" }) + && disposition_stops(o: HelperRequestNotQuarantined { entry: "e", detail: "d" }) +} + +// THE QUARANTINE IS NOT AN ANSWER, and it must not be addressable as one. An undecodable request has +// no decoded subject, so no answer address exists for it; the bytes go to a third directory the +// publication principal holds. Asserting the three are pairwise distinct is what keeps a later +// author from resolving the quarantine into the answer directory, where the belt would read it. +test fn the_quarantine_is_a_third_directory_and_not_the_answer_directory() -> Bool { + let requests = publication_request_dir(instance_root: "/opt/gunbc") + let answers = publication_answer_dir(instance_root: "/opt/gunbc") + let refused = publication_refused_dir(instance_root: "/opt/gunbc") + refused != requests && refused != answers && requests != answers +} + fn disposition_stops(o: PublicationHelperOutcome) -> Bool { match publication_helper_outcome_disposition(o: o) { OutcomeStopsTheLine { detail: _ } => true @@ -278,6 +358,8 @@ test fn only_the_helpers_own_failures_stop_the_line() -> Bool { !disposition_stops(o: HelperAnswered { key: "k", outcome_key: "branch-not-published" }) && disposition_stops(o: HelperRequestNotConsumed { key: "k", detail: "d" }) && disposition_stops(o: HelperRequestUnreadable { entry: "e", detail: "d" }) + && disposition_stops(o: HelperRequestRefused { entry: "e", detail: "d" }) + && disposition_stops(o: HelperRequestNotQuarantined { entry: "e", detail: "d" }) && disposition_stops(o: HelperAnswerWriteFailed { key: "k", detail: "d" }) && disposition_stops(o: HelperSpoolRefused { detail: "d" }) }