Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/fleet-converge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,8 @@ jobs:
'jq' '--raw-output' '--exit-status' '. as $answer | [inputs] | if length != 0 then error("the cache-API answer is not exactly one JSON document") else $answer | if type == "object" and (.total_count | type) == "number" and (.actions_caches | type) == "array" and .total_count == (.actions_caches | length) and .total_count <= 100 and all(.actions_caches[]; type == "object" and (.key | type) == "string" and (.ref | type) == "string") then [.actions_caches[] | select(.key == $ENV.RELEASE_BINS_KEY and .ref == "refs/heads/main")] | if length == 0 then "absent" elif length == 1 then "present\nid=\(.[0].id) ref=\(.[0].ref) version=\(.[0].version) created=\(.[0].created_at)" else error("duplicate exact entries") end else error("cache-API answer is not the admitted shape") end end' '.release-bins-lookup-main.json' > "$RUNNER_TEMP"'/lookup-main.decoded' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1)
DECODED_STANDING=$('head' '-n' '1' "$RUNNER_TEMP"'/lookup-main.decoded')
if '[' "$DECODED_STANDING" '=' 'present' ']'; then STANDING='present'; ENTRY=$('sed' '-n' '2p' "$RUNNER_TEMP"'/lookup-main.decoded'); else '[' "$DECODED_STANDING" '=' 'absent' ']' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1); fi
'curl' '-sSf' '-H' 'Authorization: Bearer '"$GH_TOKEN" '-H' 'Accept: application/vnd.github+json' "$GITHUB_API_URL"'/repos/'"$GITHUB_REPOSITORY"'/actions/caches?per_page=100&key='"$RELEASE_BINS_KEY"'&ref='"$GITHUB_REF" '-o' "$ROOT"'/.release-bins-lookup-dispatch.json' || ('echo' '::error title=release-bins LookupTransportFailed::the cache API lookup FAILED (unreachable or refused); this is not absence, so neither reuse nor rebuild is decided'; exit 1)
'jq' '--raw-output' '--exit-status' '. as $answer | [inputs] | if length != 0 then error("the cache-API answer is not exactly one JSON document") else $answer | if type == "object" and (.total_count | type) == "number" and (.actions_caches | type) == "array" and .total_count == (.actions_caches | length) and .total_count <= 100 and all(.actions_caches[]; type == "object" and (.key | type) == "string" and (.ref | type) == "string") then [.actions_caches[] | select(.key == $ENV.RELEASE_BINS_KEY and .ref == $ENV.GITHUB_REF)] | if length == 0 then "absent" elif length == 1 then "present\nid=\(.[0].id) ref=\(.[0].ref) version=\(.[0].version) created=\(.[0].created_at)" else error("duplicate exact entries") end else error("cache-API answer is not the admitted shape") end end' '.release-bins-lookup-dispatch.json' > "$RUNNER_TEMP"'/lookup-dispatch.decoded' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1)
'curl' '-sSf' '-H' 'Authorization: Bearer '"$GH_TOKEN" '-H' 'Accept: application/vnd.github+json' "$GITHUB_API_URL"'/repos/'"$GITHUB_REPOSITORY"'/actions/caches?per_page=100&key='"$RELEASE_BINS_KEY"'&ref='"$RELEASE_BINS_DISPATCH_SCOPE_REF" '-o' "$ROOT"'/.release-bins-lookup-dispatch.json' || ('echo' '::error title=release-bins LookupTransportFailed::the cache API lookup FAILED (unreachable or refused); this is not absence, so neither reuse nor rebuild is decided'; exit 1)
'jq' '--raw-output' '--exit-status' '. as $answer | [inputs] | if length != 0 then error("the cache-API answer is not exactly one JSON document") else $answer | if type == "object" and (.total_count | type) == "number" and (.actions_caches | type) == "array" and .total_count == (.actions_caches | length) and .total_count <= 100 and all(.actions_caches[]; type == "object" and (.key | type) == "string" and (.ref | type) == "string") then [.actions_caches[] | select(.key == $ENV.RELEASE_BINS_KEY and .ref == $ENV.RELEASE_BINS_DISPATCH_SCOPE_REF)] | if length == 0 then "absent" elif length == 1 then "present\nid=\(.[0].id) ref=\(.[0].ref) version=\(.[0].version) created=\(.[0].created_at)" else error("duplicate exact entries") end else error("cache-API answer is not the admitted shape") end end' '.release-bins-lookup-dispatch.json' > "$RUNNER_TEMP"'/lookup-dispatch.decoded' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1)
DECODED_STANDING=$('head' '-n' '1' "$RUNNER_TEMP"'/lookup-dispatch.decoded')
if '[' "$DECODED_STANDING" '=' 'present' ']'; then STANDING='present'; ENTRY=$('sed' '-n' '2p' "$RUNNER_TEMP"'/lookup-dispatch.decoded'); else '[' "$DECODED_STANDING" '=' 'absent' ']' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1); fi
'echo' 'standing='"$STANDING" | 'tee' '-a' "$GITHUB_OUTPUT" > '/dev/null'
Expand All @@ -229,6 +229,7 @@ jobs:
env:
RELEASE_BINS_KEY: ${{ steps.release_bins_key.outputs.key }}
GH_TOKEN: ${{ github.token }}
RELEASE_BINS_DISPATCH_SCOPE_REF: ${{ startsWith(github.ref, 'refs/tags/') && format('refs/heads/{0}', github.ref) || github.ref }}
timeout-minutes: 5
- name: Restore release-bins into isolated staging (transport only; publishes on a verified build)
id: release_bins_cache
Expand Down
32 changes: 31 additions & 1 deletion dag/gunbc/fleet/fleet_release_bins_key.dag
Original file line number Diff line number Diff line change
Expand Up @@ -875,10 +875,40 @@ fn release_bins_main_lookup_ref() -> ReleaseBinsLookupRef {
}
}

// THE DISPATCH'S OWN SCOPE, IN THE SPELLING THE CACHE API RECORDS IT. The admitted second ref is
// "the ref this dispatch runs on", but the cache API does not report that scope as GITHUB_REF for
// every ref kind: a workflow_dispatch on a TAG saves its entry under a scope the API lists as
// "refs/heads/" + GITHUB_REF. Observed, not documented: the key release-bins-25f14b25... saved by
// fleet-converge plan run 36652280985 on tag deploy/door-8e0c1fad16 is listed with
// ref=refs/heads/refs/tags/deploy/door-8e0c1fad16, so both apply runs on that tag (36653382960,
// 36653490246) looked up ref==GITHUB_REF, read absent, and refused
// StoreTransportDisagreeAbsentHit over their own dispatch's entry. A branch dispatch is recorded
// verbatim (refs/heads/main entries answer the main lookup), so only a tag is respelled.
//
// This is ONE spelling of the SAME ref, derived from the dispatch's own ref by one expression, not a
// second admitted scope: an entry under any other tag or branch still does not match, and the
// disagreement check is untouched. Its evidence is
// test.claim.long.fleet_release_bins_key_witness release_bins_tag_dispatch_entry_is_present and
// release_bins_RED_other_tag_entry_is_absence. If the API's spelling changes, the lookup reads
// absent again and the run refuses loudly, never widens.
data release_bins_dispatch_scope_ref_env: String = "RELEASE_BINS_DISPATCH_SCOPE_REF"

fn release_bins_dispatch_scope_ref_expression() -> String {
join(["$", "{{ startsWith(github.ref, 'refs/tags/') && format('refs/heads/{0}', github.ref) || github.ref }}"], "")
}

fn release_bins_dispatch_lookup_ref() -> ReleaseBinsLookupRef {
ReleaseBinsLookupRef {
tag: "dispatch",
url_ref: var(n: release_bins_dispatch_scope_ref_env),
jq_ref_expression: join(["$ENV.", release_bins_dispatch_scope_ref_env], ""),
}
}

fn release_bins_lookup_refs() -> List<ReleaseBinsLookupRef> {
[
release_bins_main_lookup_ref(),
ReleaseBinsLookupRef { tag: "dispatch", url_ref: var(n: "GITHUB_REF"), jq_ref_expression: "$ENV.GITHUB_REF" },
release_bins_dispatch_lookup_ref(),
]
}

Expand Down
3 changes: 2 additions & 1 deletion dag/gunbc/fleet/fleet_workflow_steps.dag
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ import v2.workflow.ci_workflow_run_emit {
}
import v2.workflow.ci_release_build_emit { ci_release_build_script }
import gunbc.fleet_release_bins_key {
release_bins_key_script, release_bins_lookup_script, release_bins_outcome_script,
release_bins_key_script, release_bins_lookup_script, release_bins_dispatch_scope_ref_env, release_bins_dispatch_scope_ref_expression, release_bins_outcome_script,
release_bins_pack_manifest_script, release_bins_consumer_key_check_script,
release_bins_key_expression, release_bins_consumer_key_expression,
release_bins_lookup_standing_expression, release_bins_lookup_entry_expression,
Expand Down Expand Up @@ -347,6 +347,7 @@ fn ci_release_bins_lookup_step() -> Step {
value: [
kv(key: "RELEASE_BINS_KEY", value: yaml_string(s: release_bins_key_expression())),
kv(key: "GH_TOKEN", value: yaml_string(s: join(["$", "{{ github.token }}"], ""))),
kv(key: release_bins_dispatch_scope_ref_env, value: yaml_string(s: release_bins_dispatch_scope_ref_expression())),
]
},
working_directory: none,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
module gunbc.recurring_failure_mode.a_tag_dispatch_cannot_find_its_own_cache_entry

import std.types { NonEmptyStr }
import gunbc.recurring_failure_mode { RecurringFailureMode }

data a_tag_dispatch_cannot_find_its_own_cache_entry: RecurringFailureMode = RecurringFailureMode {
identity: "a_tag_dispatch_cannot_find_its_own_cache_entry" as NonEmptyStr,

receipts: [
"INVALID STATE: the release-bins lookup (gunbc.fleet_release_bins_key release_bins_lookup_refs) admits an entry under refs/heads/main or under 'the ref this dispatch runs on', and matched the latter as .ref == GITHUB_REF. For a workflow_dispatch on a TAG the cache API lists that scope as refs/heads/<GITHUB_REF>, so the lookup reads ABSENT for the dispatch's own entry while actions/cache restores it as an exact hit.",

"HARM, 2026-09-30: the #12482 door roll-forward on tag deploy/door-8e0c1fad16. Plan run 36652280985 built and saved release-bins-25f14b25... (API ref=refs/heads/refs/tags/deploy/door-8e0c1fad16); apply runs 36653382960 and 36653490246 on the same tag both refused StoreTransportDisagreeAbsentHit in the build job, so no tag-pinned deploy at a sha main had not already cached could proceed. The apply on tag deploy/12482-178fb76c71 (run 36581756988) passed only because that sha's bins were already admitted under refs/heads/main.",

"EARLIEST WRONG LINK (DESIGN section 6b): not the disagreement check -- it refused correctly on the two observations it was handed -- but the model's assumption that the cache API spells a dispatch's scope as GITHUB_REF for every ref kind. REPAIR: the dispatch lookup reads RELEASE_BINS_DISPATCH_SCOPE_REF, derived in the step from github.ref by one expression (a tag is respelled refs/heads/<ref>; a branch is verbatim). One spelling of the same ref, not a second scope; the check is unchanged.",

"EVIDENCE: test.claim.long.fleet_release_bins_key_witness release_bins_tag_dispatch_entry_is_present (the real decoder over an answer listing the dispatch's entry under the tag's API spelling reads present; under the old GITHUB_REF match it reads absent) and release_bins_RED_other_tag_entry_is_absence (another tag's entry still reads absent). RUNG FOUND AT: mitigatable (loud refusal, no wrong bits run). RUNG NOW: mitigatable -- the witness sits in the long lane. The spelling is an OBSERVATION of GitHub's cache API, not a documented contract: if it changes, the lookup reads absent again and refuses, never widens. NEXT TRIGGER: the cache API's scope spelling modeled as a cited extdeps fact (extdeps GitHub Actions cache) with its own observation receipt, so the derivation reads an upstream authority rather than a workflow-local expression.",
],

evidence: [],
}
36 changes: 36 additions & 0 deletions dag/test/claim/long/fleet_release_bins_key_witness_test.dag
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import gunbc.fleet_release_bins_key {
release_bins_emitted, release_bins_transport,
release_bins_key_statements, release_bins_pack_statements, release_bins_outcome_statements,
release_bins_consumer_statements, release_bins_download_dir, release_bins_main_lookup_ref, lookup_decode_statements,
release_bins_dispatch_lookup_ref, release_bins_dispatch_scope_ref_env,
release_bins_lookup_file, release_bins_staging_dir, release_bins_manifest_name,
release_bins_preimage_name, source_closure_statements, release_bins_environment_statements,
}
Expand Down Expand Up @@ -387,6 +388,41 @@ test fn release_bins_RED_lookup_two_documents_refuses() -> Bool {
refused_as(script: script_RED_lookup_two_documents_refuses, refusal: "LookupAnswerUnusable")
}

// ── A TAG DISPATCH'S OWN ENTRY IS PRESENT; ANOTHER TAG'S IS NOT ─────────────────────────────────
// The cache API lists a tag dispatch's entry under refs/heads/<GITHUB_REF> (fleet-converge runs
// 36653382960 and 36653490246 refused StoreTransportDisagreeAbsentHit over exactly such an entry).
// The dispatch lookup reads RELEASE_BINS_DISPATCH_SCOPE_REF, which the step derives from github.ref;
// here it is supplied as a tag's API spelling, and the real decoder runs over a real answer shape.
// The RED is the same answer under a DIFFERENT tag's scope: still absence, so respelling the
// dispatch's own ref admits no other scope.
data dispatch_tag_scope_ref: String = "refs/heads/refs/tags/deploy/door-fixture"

fn dispatch_answer(entry_ref: String) -> Outcome<String> {
e(statements: [bash_build_with_redir_to_file(
inner: cmd(ws: [fx(t: "printf"), fx(t: "%s"), words(ws: [
fx(t: "{\"total_count\":1,\"actions_caches\":[{\"id\":7,\"key\":\""), var(n: "RELEASE_BINS_KEY"),
fx(t: join(["\",\"ref\":\"", entry_ref, "\",\"version\":\"v\",\"created_at\":\"t\"}]}"], "")),
])]),
path: fx(t: release_bins_lookup_file(r: release_bins_dispatch_lookup_ref())),
)])
}

data part_decode_dispatch: Outcome<String> = e(statements: concat(
concat([cmdl(ls: ["export", "STANDING=absent", "ENTRY=none", join([release_bins_dispatch_scope_ref_env, "=", dispatch_tag_scope_ref], "")])], lookup_decode_statements(r: release_bins_dispatch_lookup_ref())),
[cmd(ws: [fx(t: "echo"), words(ws: [fx(t: "decoded standing="), var(n: "STANDING")])])],
))

data script_tag_dispatch_entry_is_present: TransportScript? = release_bins_transport(parts: [part_fixture_env, part_supplied_key, dispatch_answer(entry_ref: dispatch_tag_scope_ref), part_decode_dispatch])
data script_RED_other_tag_entry_is_absence: TransportScript? = release_bins_transport(parts: [part_fixture_env, part_supplied_key, dispatch_answer(entry_ref: "refs/heads/refs/tags/deploy/another"), part_decode_dispatch])

test fn release_bins_tag_dispatch_entry_is_present() -> Bool {
succeeded_with(script: script_tag_dispatch_entry_is_present, marker: "decoded standing=present")
}

test fn release_bins_RED_other_tag_entry_is_absence() -> Bool {
succeeded_with(script: script_RED_other_tag_entry_is_absence, marker: "decoded standing=absent")
}

// ── P2-4 and review 5332937814: the effective build environment is keyed losslessly or refused ──
data env_none: Outcome<String> = part_no_mutation
data env_opt_level_0: Outcome<String> = exports(assignments: ["CARGO_PROFILE_RELEASE_OPT_LEVEL=0"])
Expand Down
6 changes: 5 additions & 1 deletion src/v2/workflow/floor_route_gap.dag
Original file line number Diff line number Diff line change
Expand Up @@ -1834,6 +1834,10 @@ fn floor_route_gap_expectation_chunk_27() -> List<FloorRouteGapExpectation> {
tail: Cons {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_lookup_valid_empty_answer_is_absence", operation: "Run", ground: NoMockResponse {} },
tail: Cons {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_tag_dispatch_entry_is_present", operation: "Run", ground: NoMockResponse {} },
tail: Cons {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_other_tag_entry_is_absence", operation: "Run", ground: NoMockResponse {} },
tail: Cons {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_count_without_rows_refuses", operation: "Run", ground: NoMockResponse {} },
tail: Cons {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_truncated_answer_refuses", operation: "Run", ground: NoMockResponse {} },
Expand Down Expand Up @@ -1861,7 +1865,7 @@ fn floor_route_gap_expectation_chunk_27() -> List<FloorRouteGapExpectation> {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_value_stream_refuses", operation: "Run", ground: NoMockResponse {} },
tail: Cons {
head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_two_documents_refuses", operation: "Run", ground: NoMockResponse {} },
tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } } } }
tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } } } } } }
}

// THE RUNNER BROWSER TOOLCHAIN'S REAL-EXECUTION CONTROLS (gunbc#12500) run on the local-repo wet lane
Expand Down
12 changes: 12 additions & 0 deletions src/v2/workflow/local_repo_wet_terminal.dag
Original file line number Diff line number Diff line change
Expand Up @@ -2006,6 +2006,18 @@ fn local_repo_wet_schedule() -> List<WetScheduledClaim> {
function: "release_bins_lookup_valid_empty_answer_is_absence",
expectation: ExpectedToHold {}
},
WetScheduledClaim {
identity: WitnessIdentity { module_path: "test.claim.long.fleet_release_bins_key_witness", function: "release_bins_tag_dispatch_entry_is_present" },
entry: "dag/test/claim/long/fleet_release_bins_key_witness_test.dag",
function: "release_bins_tag_dispatch_entry_is_present",
expectation: ExpectedToHold {}
},
WetScheduledClaim {
identity: WitnessIdentity { module_path: "test.claim.long.fleet_release_bins_key_witness", function: "release_bins_RED_other_tag_entry_is_absence" },
entry: "dag/test/claim/long/fleet_release_bins_key_witness_test.dag",
function: "release_bins_RED_other_tag_entry_is_absence",
expectation: ExpectedToHold {}
},
WetScheduledClaim {
identity: WitnessIdentity { module_path: "test.claim.long.fleet_release_bins_key_witness", function: "release_bins_RED_lookup_count_without_rows_refuses" },
entry: "dag/test/claim/long/fleet_release_bins_key_witness_test.dag",
Expand Down
Loading