diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index c9cb8263891..af11ccd5303 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -219,8 +219,8 @@ jobs: 'jq' '--raw-output' '--exit-status' '. as $answer | [inputs] | if length != 0 then error("the cache-API answer is not exactly one JSON document") else $answer | if type == "object" and (.total_count | type) == "number" and (.actions_caches | type) == "array" and .total_count == (.actions_caches | length) and .total_count <= 100 and all(.actions_caches[]; type == "object" and (.key | type) == "string" and (.ref | type) == "string") then [.actions_caches[] | select(.key == $ENV.RELEASE_BINS_KEY and .ref == "refs/heads/main")] | if length == 0 then "absent" elif length == 1 then "present\nid=\(.[0].id) ref=\(.[0].ref) version=\(.[0].version) created=\(.[0].created_at)" else error("duplicate exact entries") end else error("cache-API answer is not the admitted shape") end end' '.release-bins-lookup-main.json' > "$RUNNER_TEMP"'/lookup-main.decoded' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1) DECODED_STANDING=$('head' '-n' '1' "$RUNNER_TEMP"'/lookup-main.decoded') if '[' "$DECODED_STANDING" '=' 'present' ']'; then STANDING='present'; ENTRY=$('sed' '-n' '2p' "$RUNNER_TEMP"'/lookup-main.decoded'); else '[' "$DECODED_STANDING" '=' 'absent' ']' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1); fi - 'curl' '-sSf' '-H' 'Authorization: Bearer '"$GH_TOKEN" '-H' 'Accept: application/vnd.github+json' "$GITHUB_API_URL"'/repos/'"$GITHUB_REPOSITORY"'/actions/caches?per_page=100&key='"$RELEASE_BINS_KEY"'&ref='"$GITHUB_REF" '-o' "$ROOT"'/.release-bins-lookup-dispatch.json' || ('echo' '::error title=release-bins LookupTransportFailed::the cache API lookup FAILED (unreachable or refused); this is not absence, so neither reuse nor rebuild is decided'; exit 1) - 'jq' '--raw-output' '--exit-status' '. as $answer | [inputs] | if length != 0 then error("the cache-API answer is not exactly one JSON document") else $answer | if type == "object" and (.total_count | type) == "number" and (.actions_caches | type) == "array" and .total_count == (.actions_caches | length) and .total_count <= 100 and all(.actions_caches[]; type == "object" and (.key | type) == "string" and (.ref | type) == "string") then [.actions_caches[] | select(.key == $ENV.RELEASE_BINS_KEY and .ref == $ENV.GITHUB_REF)] | if length == 0 then "absent" elif length == 1 then "present\nid=\(.[0].id) ref=\(.[0].ref) version=\(.[0].version) created=\(.[0].created_at)" else error("duplicate exact entries") end else error("cache-API answer is not the admitted shape") end end' '.release-bins-lookup-dispatch.json' > "$RUNNER_TEMP"'/lookup-dispatch.decoded' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1) + 'curl' '-sSf' '-H' 'Authorization: Bearer '"$GH_TOKEN" '-H' 'Accept: application/vnd.github+json' "$GITHUB_API_URL"'/repos/'"$GITHUB_REPOSITORY"'/actions/caches?per_page=100&key='"$RELEASE_BINS_KEY"'&ref='"$RELEASE_BINS_DISPATCH_SCOPE_REF" '-o' "$ROOT"'/.release-bins-lookup-dispatch.json' || ('echo' '::error title=release-bins LookupTransportFailed::the cache API lookup FAILED (unreachable or refused); this is not absence, so neither reuse nor rebuild is decided'; exit 1) + 'jq' '--raw-output' '--exit-status' '. as $answer | [inputs] | if length != 0 then error("the cache-API answer is not exactly one JSON document") else $answer | if type == "object" and (.total_count | type) == "number" and (.actions_caches | type) == "array" and .total_count == (.actions_caches | length) and .total_count <= 100 and all(.actions_caches[]; type == "object" and (.key | type) == "string" and (.ref | type) == "string") then [.actions_caches[] | select(.key == $ENV.RELEASE_BINS_KEY and .ref == $ENV.RELEASE_BINS_DISPATCH_SCOPE_REF)] | if length == 0 then "absent" elif length == 1 then "present\nid=\(.[0].id) ref=\(.[0].ref) version=\(.[0].version) created=\(.[0].created_at)" else error("duplicate exact entries") end else error("cache-API answer is not the admitted shape") end end' '.release-bins-lookup-dispatch.json' > "$RUNNER_TEMP"'/lookup-dispatch.decoded' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1) DECODED_STANDING=$('head' '-n' '1' "$RUNNER_TEMP"'/lookup-dispatch.decoded') if '[' "$DECODED_STANDING" '=' 'present' ']'; then STANDING='present'; ENTRY=$('sed' '-n' '2p' "$RUNNER_TEMP"'/lookup-dispatch.decoded'); else '[' "$DECODED_STANDING" '=' 'absent' ']' || ('echo' '::error title=release-bins LookupAnswerUnusable::the cache API answer was unreadable or incomplete; it is not treated as absence'; exit 1); fi 'echo' 'standing='"$STANDING" | 'tee' '-a' "$GITHUB_OUTPUT" > '/dev/null' @@ -229,6 +229,7 @@ jobs: env: RELEASE_BINS_KEY: ${{ steps.release_bins_key.outputs.key }} GH_TOKEN: ${{ github.token }} + RELEASE_BINS_DISPATCH_SCOPE_REF: ${{ startsWith(github.ref, 'refs/tags/') && format('refs/heads/{0}', github.ref) || github.ref }} timeout-minutes: 5 - name: Restore release-bins into isolated staging (transport only; publishes on a verified build) id: release_bins_cache diff --git a/dag/gunbc/fleet/fleet_release_bins_key.dag b/dag/gunbc/fleet/fleet_release_bins_key.dag index 2d27ab12408..24234471196 100644 --- a/dag/gunbc/fleet/fleet_release_bins_key.dag +++ b/dag/gunbc/fleet/fleet_release_bins_key.dag @@ -875,10 +875,40 @@ fn release_bins_main_lookup_ref() -> ReleaseBinsLookupRef { } } +// THE DISPATCH'S OWN SCOPE, IN THE SPELLING THE CACHE API RECORDS IT. The admitted second ref is +// "the ref this dispatch runs on", but the cache API does not report that scope as GITHUB_REF for +// every ref kind: a workflow_dispatch on a TAG saves its entry under a scope the API lists as +// "refs/heads/" + GITHUB_REF. Observed, not documented: the key release-bins-25f14b25... saved by +// fleet-converge plan run 36652280985 on tag deploy/door-8e0c1fad16 is listed with +// ref=refs/heads/refs/tags/deploy/door-8e0c1fad16, so both apply runs on that tag (36653382960, +// 36653490246) looked up ref==GITHUB_REF, read absent, and refused +// StoreTransportDisagreeAbsentHit over their own dispatch's entry. A branch dispatch is recorded +// verbatim (refs/heads/main entries answer the main lookup), so only a tag is respelled. +// +// This is ONE spelling of the SAME ref, derived from the dispatch's own ref by one expression, not a +// second admitted scope: an entry under any other tag or branch still does not match, and the +// disagreement check is untouched. Its evidence is +// test.claim.long.fleet_release_bins_key_witness release_bins_tag_dispatch_entry_is_present and +// release_bins_RED_other_tag_entry_is_absence. If the API's spelling changes, the lookup reads +// absent again and the run refuses loudly, never widens. +data release_bins_dispatch_scope_ref_env: String = "RELEASE_BINS_DISPATCH_SCOPE_REF" + +fn release_bins_dispatch_scope_ref_expression() -> String { + join(["$", "{{ startsWith(github.ref, 'refs/tags/') && format('refs/heads/{0}', github.ref) || github.ref }}"], "") +} + +fn release_bins_dispatch_lookup_ref() -> ReleaseBinsLookupRef { + ReleaseBinsLookupRef { + tag: "dispatch", + url_ref: var(n: release_bins_dispatch_scope_ref_env), + jq_ref_expression: join(["$ENV.", release_bins_dispatch_scope_ref_env], ""), + } +} + fn release_bins_lookup_refs() -> List { [ release_bins_main_lookup_ref(), - ReleaseBinsLookupRef { tag: "dispatch", url_ref: var(n: "GITHUB_REF"), jq_ref_expression: "$ENV.GITHUB_REF" }, + release_bins_dispatch_lookup_ref(), ] } diff --git a/dag/gunbc/fleet/fleet_workflow_steps.dag b/dag/gunbc/fleet/fleet_workflow_steps.dag index c6fe0946ccf..797f82a3d66 100644 --- a/dag/gunbc/fleet/fleet_workflow_steps.dag +++ b/dag/gunbc/fleet/fleet_workflow_steps.dag @@ -41,7 +41,7 @@ import v2.workflow.ci_workflow_run_emit { } import v2.workflow.ci_release_build_emit { ci_release_build_script } import gunbc.fleet_release_bins_key { - release_bins_key_script, release_bins_lookup_script, release_bins_outcome_script, + release_bins_key_script, release_bins_lookup_script, release_bins_dispatch_scope_ref_env, release_bins_dispatch_scope_ref_expression, release_bins_outcome_script, release_bins_pack_manifest_script, release_bins_consumer_key_check_script, release_bins_key_expression, release_bins_consumer_key_expression, release_bins_lookup_standing_expression, release_bins_lookup_entry_expression, @@ -347,6 +347,7 @@ fn ci_release_bins_lookup_step() -> Step { value: [ kv(key: "RELEASE_BINS_KEY", value: yaml_string(s: release_bins_key_expression())), kv(key: "GH_TOKEN", value: yaml_string(s: join(["$", "{{ github.token }}"], ""))), + kv(key: release_bins_dispatch_scope_ref_env, value: yaml_string(s: release_bins_dispatch_scope_ref_expression())), ] }, working_directory: none, diff --git a/dag/gunbc/recurring_failure_mode/a_tag_dispatch_cannot_find_its_own_cache_entry.dag b/dag/gunbc/recurring_failure_mode/a_tag_dispatch_cannot_find_its_own_cache_entry.dag new file mode 100644 index 00000000000..f0f61c4661e --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_tag_dispatch_cannot_find_its_own_cache_entry.dag @@ -0,0 +1,20 @@ +module gunbc.recurring_failure_mode.a_tag_dispatch_cannot_find_its_own_cache_entry + +import std.types { NonEmptyStr } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_tag_dispatch_cannot_find_its_own_cache_entry: RecurringFailureMode = RecurringFailureMode { + identity: "a_tag_dispatch_cannot_find_its_own_cache_entry" as NonEmptyStr, + + receipts: [ + "INVALID STATE: the release-bins lookup (gunbc.fleet_release_bins_key release_bins_lookup_refs) admits an entry under refs/heads/main or under 'the ref this dispatch runs on', and matched the latter as .ref == GITHUB_REF. For a workflow_dispatch on a TAG the cache API lists that scope as refs/heads/, so the lookup reads ABSENT for the dispatch's own entry while actions/cache restores it as an exact hit.", + + "HARM, 2026-09-30: the #12482 door roll-forward on tag deploy/door-8e0c1fad16. Plan run 36652280985 built and saved release-bins-25f14b25... (API ref=refs/heads/refs/tags/deploy/door-8e0c1fad16); apply runs 36653382960 and 36653490246 on the same tag both refused StoreTransportDisagreeAbsentHit in the build job, so no tag-pinned deploy at a sha main had not already cached could proceed. The apply on tag deploy/12482-178fb76c71 (run 36581756988) passed only because that sha's bins were already admitted under refs/heads/main.", + + "EARLIEST WRONG LINK (DESIGN section 6b): not the disagreement check -- it refused correctly on the two observations it was handed -- but the model's assumption that the cache API spells a dispatch's scope as GITHUB_REF for every ref kind. REPAIR: the dispatch lookup reads RELEASE_BINS_DISPATCH_SCOPE_REF, derived in the step from github.ref by one expression (a tag is respelled refs/heads/; a branch is verbatim). One spelling of the same ref, not a second scope; the check is unchanged.", + + "EVIDENCE: test.claim.long.fleet_release_bins_key_witness release_bins_tag_dispatch_entry_is_present (the real decoder over an answer listing the dispatch's entry under the tag's API spelling reads present; under the old GITHUB_REF match it reads absent) and release_bins_RED_other_tag_entry_is_absence (another tag's entry still reads absent). RUNG FOUND AT: mitigatable (loud refusal, no wrong bits run). RUNG NOW: mitigatable -- the witness sits in the long lane. The spelling is an OBSERVATION of GitHub's cache API, not a documented contract: if it changes, the lookup reads absent again and refuses, never widens. NEXT TRIGGER: the cache API's scope spelling modeled as a cited extdeps fact (extdeps GitHub Actions cache) with its own observation receipt, so the derivation reads an upstream authority rather than a workflow-local expression.", + ], + + evidence: [], +} diff --git a/dag/test/claim/long/fleet_release_bins_key_witness_test.dag b/dag/test/claim/long/fleet_release_bins_key_witness_test.dag index c5ca69d78ae..70fc4b43831 100644 --- a/dag/test/claim/long/fleet_release_bins_key_witness_test.dag +++ b/dag/test/claim/long/fleet_release_bins_key_witness_test.dag @@ -13,6 +13,7 @@ import gunbc.fleet_release_bins_key { release_bins_emitted, release_bins_transport, release_bins_key_statements, release_bins_pack_statements, release_bins_outcome_statements, release_bins_consumer_statements, release_bins_download_dir, release_bins_main_lookup_ref, lookup_decode_statements, + release_bins_dispatch_lookup_ref, release_bins_dispatch_scope_ref_env, release_bins_lookup_file, release_bins_staging_dir, release_bins_manifest_name, release_bins_preimage_name, source_closure_statements, release_bins_environment_statements, } @@ -387,6 +388,41 @@ test fn release_bins_RED_lookup_two_documents_refuses() -> Bool { refused_as(script: script_RED_lookup_two_documents_refuses, refusal: "LookupAnswerUnusable") } +// ── A TAG DISPATCH'S OWN ENTRY IS PRESENT; ANOTHER TAG'S IS NOT ───────────────────────────────── +// The cache API lists a tag dispatch's entry under refs/heads/ (fleet-converge runs +// 36653382960 and 36653490246 refused StoreTransportDisagreeAbsentHit over exactly such an entry). +// The dispatch lookup reads RELEASE_BINS_DISPATCH_SCOPE_REF, which the step derives from github.ref; +// here it is supplied as a tag's API spelling, and the real decoder runs over a real answer shape. +// The RED is the same answer under a DIFFERENT tag's scope: still absence, so respelling the +// dispatch's own ref admits no other scope. +data dispatch_tag_scope_ref: String = "refs/heads/refs/tags/deploy/door-fixture" + +fn dispatch_answer(entry_ref: String) -> Outcome { + e(statements: [bash_build_with_redir_to_file( + inner: cmd(ws: [fx(t: "printf"), fx(t: "%s"), words(ws: [ + fx(t: "{\"total_count\":1,\"actions_caches\":[{\"id\":7,\"key\":\""), var(n: "RELEASE_BINS_KEY"), + fx(t: join(["\",\"ref\":\"", entry_ref, "\",\"version\":\"v\",\"created_at\":\"t\"}]}"], "")), + ])]), + path: fx(t: release_bins_lookup_file(r: release_bins_dispatch_lookup_ref())), + )]) +} + +data part_decode_dispatch: Outcome = e(statements: concat( + concat([cmdl(ls: ["export", "STANDING=absent", "ENTRY=none", join([release_bins_dispatch_scope_ref_env, "=", dispatch_tag_scope_ref], "")])], lookup_decode_statements(r: release_bins_dispatch_lookup_ref())), + [cmd(ws: [fx(t: "echo"), words(ws: [fx(t: "decoded standing="), var(n: "STANDING")])])], +)) + +data script_tag_dispatch_entry_is_present: TransportScript? = release_bins_transport(parts: [part_fixture_env, part_supplied_key, dispatch_answer(entry_ref: dispatch_tag_scope_ref), part_decode_dispatch]) +data script_RED_other_tag_entry_is_absence: TransportScript? = release_bins_transport(parts: [part_fixture_env, part_supplied_key, dispatch_answer(entry_ref: "refs/heads/refs/tags/deploy/another"), part_decode_dispatch]) + +test fn release_bins_tag_dispatch_entry_is_present() -> Bool { + succeeded_with(script: script_tag_dispatch_entry_is_present, marker: "decoded standing=present") +} + +test fn release_bins_RED_other_tag_entry_is_absence() -> Bool { + succeeded_with(script: script_RED_other_tag_entry_is_absence, marker: "decoded standing=absent") +} + // ── P2-4 and review 5332937814: the effective build environment is keyed losslessly or refused ── data env_none: Outcome = part_no_mutation data env_opt_level_0: Outcome = exports(assignments: ["CARGO_PROFILE_RELEASE_OPT_LEVEL=0"]) diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 8ed8a3c8e1f..e0cdc1c9d97 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1834,6 +1834,10 @@ fn floor_route_gap_expectation_chunk_27() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_lookup_valid_empty_answer_is_absence", operation: "Run", ground: NoMockResponse {} }, tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_tag_dispatch_entry_is_present", operation: "Run", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_other_tag_entry_is_absence", operation: "Run", ground: NoMockResponse {} }, + tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_count_without_rows_refuses", operation: "Run", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_truncated_answer_refuses", operation: "Run", ground: NoMockResponse {} }, @@ -1861,7 +1865,7 @@ fn floor_route_gap_expectation_chunk_27() -> List { head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_value_stream_refuses", operation: "Run", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.long.fleet_release_bins_key_witness.release_bins_RED_lookup_two_documents_refuses", operation: "Run", ground: NoMockResponse {} }, - tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } } } } + tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } } } } } } } // THE RUNNER BROWSER TOOLCHAIN'S REAL-EXECUTION CONTROLS (gunbc#12500) run on the local-repo wet lane diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index edbcc6eb603..df072334310 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -2006,6 +2006,18 @@ fn local_repo_wet_schedule() -> List { function: "release_bins_lookup_valid_empty_answer_is_absence", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.long.fleet_release_bins_key_witness", function: "release_bins_tag_dispatch_entry_is_present" }, + entry: "dag/test/claim/long/fleet_release_bins_key_witness_test.dag", + function: "release_bins_tag_dispatch_entry_is_present", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.long.fleet_release_bins_key_witness", function: "release_bins_RED_other_tag_entry_is_absence" }, + entry: "dag/test/claim/long/fleet_release_bins_key_witness_test.dag", + function: "release_bins_RED_other_tag_entry_is_absence", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.long.fleet_release_bins_key_witness", function: "release_bins_RED_lookup_count_without_rows_refuses" }, entry: "dag/test/claim/long/fleet_release_bins_key_witness_test.dag",