diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index bb830799529..bc2eed1eb5c 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -11,9 +11,9 @@ on: options: [srv1, srv2, srv3, srv4] type: choice mode: - description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown + description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -45,6 +45,14 @@ on: description: Workflow run id of the dashboard deploy — required for rlm_launch_deployment_receipt required: false type: string + runner_ids: + description: "org_runner_roster_observe only: comma-separated captured runner registration ids to observe individually (present / absent / unobserved) in the gunbai-ci organization" + required: false + type: string + roster_page_size: + description: "org_runner_roster_observe only: page size for the first roster request (1..100, default 100); later pages follow the authority's own continuation links" + required: false + type: string transaction_nonce: description: Operator-minted unique nonce for one RLM transaction; echoed into run-name so each dispatched run is API-selectable by its displayTitle, closing the run-id correlation gap (review 5062738052 B4). Not consumed by any step required: false @@ -344,6 +352,59 @@ jobs: retention-days: 30 if: github.event.inputs.mode == 'org_actions_observe' timeout-minutes: 10 + - name: "Org runner roster: paginated read of GitHub self-hosted runner registrations" + id: org_runner_roster_observe + run: | + set -euo pipefail + umask 077 + KEY_FILE="$RUNNER_TEMP/org-admin-app-key" + HDR_FILE="$RUNNER_TEMP/org-admin-auth-header" + JWT_FILE="$RUNNER_TEMP/org-admin-app-jwt" + TOKEN_HEADERS="$RUNNER_TEMP/org-admin-token-headers" + TOKEN_BODY="$RUNNER_TEMP/org-admin-token-body" + cleanup() { rm -f "$KEY_FILE" "$HDR_FILE" "$JWT_FILE" "$TOKEN_HEADERS" "$TOKEN_BODY" "$RUNNER_TEMP/org-admin-app-key-sm-body"; unset GUNBC_ORG_ADMIN_TOKEN GH_TOKEN; } + trap cleanup EXIT + printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE" + SM_BODY="$RUNNER_TEMP/org-admin-app-key-sm-body" + SM_CODE="$(curl -sS -o "$SM_BODY" -w '%{http_code}' -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/ci-github-app-private-key/versions/latest:access")" + rm -f "$HDR_FILE" + if [ "$SM_CODE" != 200 ]; then rm -f "$SM_BODY"; echo "OrgAdminAppKeyUnreadable: Secret Manager answered HTTP $SM_CODE for projects/gunbai-secrets/secrets/ci-github-app-private-key/versions/latest to the fleet-cloud-convergence principal. 403 means the version exists and is not readable OR does not exist -- REMEDY: run org_admin_app_key_access_converge_with_supplied_token (gunbc.fleet.org_actions_converge) with a control-plane token to bind roles/secretmanager.secretAccessor, and confirm the version exists. Minting NO token." >&2; exit 1; fi + python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$SM_BODY" > "$KEY_FILE" + rm -f "$SM_BODY" + chmod 600 "$KEY_FILE" + NOW=$(date +%s) + b64url() { base64 -w0 | tr '+/' '-_' | tr -d '='; } + JWT_HEADER=$(printf '%s' '{"alg":"RS256","typ":"JWT"}' | b64url) + JWT_PAYLOAD=$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' "$((NOW-60))" "$((NOW+540))" "2653532" | b64url) + JWT_SIG=$(printf '%s.%s' "$JWT_HEADER" "$JWT_PAYLOAD" | openssl dgst -sha256 -sign "$KEY_FILE" | b64url) + rm -f "$KEY_FILE" + printf 'Authorization: Bearer %s.%s.%s\n' "$JWT_HEADER" "$JWT_PAYLOAD" "$JWT_SIG" > "$JWT_FILE" + curl -sS -X POST -H @"$JWT_FILE" -H 'Accept: application/vnd.github+json' -D "$TOKEN_HEADERS" -o "$TOKEN_BODY" "https://api.github.com/app/installations/104134109/access_tokens" + rm -f "$JWT_FILE" + if ! head -n1 "$TOKEN_HEADERS" | grep -q ' 201 '; then echo "OrgAdminInstallationTokenRefused: GitHub answered $(head -n1 "$TOKEN_HEADERS" | tr -d '\r') for the gunbai-ci installation; a revoked key, a removed installation, or a missing Self-hosted-runners permission each look like this -- inspect the App's installation on the organization" >&2; exit 1; fi + GUNBC_ORG_ADMIN_TOKEN="$(python3 -c 'import sys,json; print(json.load(open(sys.argv[1]))["token"])' "$TOKEN_BODY")" + rm -f "$TOKEN_HEADERS" "$TOKEN_BODY" + export GUNBC_ORG_ADMIN_TOKEN + export GH_TOKEN="$GUNBC_ORG_ADMIN_TOKEN" + echo "org-admin: installation token minted in-run for app gunbai-ci installation 104134109 (one-hour lifetime; key wiped; token held in this step's environment only)" + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/fleet/organization_runner_roster_read.dag --function organization_runner_roster_observe_wet + env: + WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} + GUNBC_RUNNER_ID_PROBES: ${{ github.event.inputs.runner_ids }} + GUNBC_RUNNER_ROSTER_PAGE_SIZE: ${{ github.event.inputs.roster_page_size }} + if: github.event.inputs.mode == 'org_runner_roster_observe' + timeout-minutes: 5 + - name: Upload org runner roster receipt + id: org_runner_roster_receipt_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: org-runner-roster + path: target/org-runner-roster-receipt.txt + if-no-files-found: error + retention-days: 30 + if: always() && github.event.inputs.mode == 'org_runner_roster_observe' + timeout-minutes: 10 - name: "GitHub App control plane: registration + webhook config observation" id: app_control_plane_observe run: | diff --git a/dag/extdeps/github/ci_runner.dag b/dag/extdeps/github/ci_runner.dag index 2829a9692f7..8b9deb1b0ad 100644 --- a/dag/extdeps/github/ci_runner.dag +++ b/dag/extdeps/github/ci_runner.dag @@ -1,6 +1,6 @@ module extdeps.github.ci_runner -import std.types { String, NonEmptyStr, Bool, List } +import std.types { String, NonEmptyStr, Bool, Int, List } import std.nat { Nat } import std.disposition { Disposition, Terminal } import extdeps.external_authority { ExternalAuthority } @@ -20,6 +20,64 @@ data organization_self_hosted_runners_list_citation: ExternalAuthority = Externa } } +data organization_self_hosted_runner_get_citation: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "docs.github.com/en/rest/actions/self-hosted-runners#get-a-self-hosted-runner-for-an-organization" + } +} + +// THE CLI TRANSPORT, one of N realizations and not a fact about the dependency -- the same gh +// family as extdeps.github.org_actions github.CliOrgRunnerGroups, authenticated by the GH_TOKEN +// environment the caller's step holds, never by an argv field. Both operations are readonly. +// +// --include IS LOAD-BEARING. The list endpoint is paginated by an RFC 8288 Link header +// (extdeps.github.pagination), which only the response head carries, and the per-ID read must tell +// 404 (the registration is gone) from every other failure (nothing is known). gh prints the +// status line and headers before the body on success AND on an HTTP error, so the head is the +// discriminator in both operations; extdeps.github.cli_api_include owns reading it. +// +// ListOrgRunnersPage takes the WHOLE request target, not a page number: the first request is +// composed by the caller from org_self_hosted_runners_first_page_target, and every later one is the +// rel="next" target the authority itself returned. Composing page N client-side would be the +// offset protocol, which the cited pagination page does not promise to be the one it continues by. +service github.CliOrgSelfHostedRunners { + operation ListOrgRunnersPage { + input { + target: NonEmptyStr + } + output { + success: Bool from "exit_success" + stdout: String from "stdout" + stderr: String from "stderr" + } + readonly + transport shell { + argv: ["gh", "api", "--include", "{target}"] + } + } + + operation GetOrgRunner { + input { + org: NonEmptyStr + runner_id: Int + } + output { + success: Bool from "exit_success" + stdout: String from "stdout" + stderr: String from "stderr" + } + readonly + transport shell { + argv: ["gh", "api", "--include", "orgs/{org}/actions/runners/{runner_id}"] + } + } +} + +fn org_self_hosted_runners_first_page_target(org: NonEmptyStr, per_page: Int) -> NonEmptyStr { + join(["orgs/", org as String, "/actions/runners?per_page=", to_string(per_page)], "") as NonEmptyStr +} + type RunnerLifecycleSignal = ShutdownReceived | OperationCanceled @@ -35,18 +93,21 @@ type GitHubRunnerRegistrationStatus = GitHubRunnerOnline | GitHubRunnerOffline -// THE ORG RUNNERS ENDPOINT RETURNS id, name, status, busy, ephemeral, version, AND labels AS -// SEPARATE FIELDS. status IS online OR offline; busy IS its own boolean. version IS the runner -// software version string (the incident corpus pinned Actions Runner v2.334.0). None of these are -// folded into a single synthetic status enum. +// THE ORG RUNNERS ENDPOINT RETURNS id, name, os, status, busy AND labels AS REQUIRED FIELDS, and +// ephemeral and version as OPTIONAL ones (version is `string | null`), per the "Self hosted runners" +// schema of organization_self_hosted_runners_list_citation, read 2026-09-19. status IS online OR +// offline; busy IS its own boolean. None of these are folded into a single synthetic status enum. +// The two optional fields are Optional here because the authority does not promise them: a +// decoder that required them would refuse a conforming response, and one that defaulted them +// would fabricate a value the response never carried (DESIGN section 5). type GitHubSelfHostedRunnerRegistration { id: Nat name: NonEmptyStr status: GitHubRunnerRegistrationStatus busy: Bool - ephemeral: Bool - version: NonEmptyStr + ephemeral: Bool? + version: NonEmptyStr? os: NonEmptyStr labels: List } diff --git a/dag/extdeps/github/cli_api_include.dag b/dag/extdeps/github/cli_api_include.dag new file mode 100644 index 00000000000..e5f8b33845a --- /dev/null +++ b/dag/extdeps/github/cli_api_include.dag @@ -0,0 +1,106 @@ +module extdeps.github.cli_api_include + +import std.types { String, NonEmptyStr, Int, List } +import std.algebra { trim } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import extdeps.github.pagination { GitHubLinkHeader, LinkHeaderPresent, LinkHeaderAbsent } + +// WHAT `gh api --include` PRINTS, as the gh CLI documents it: the HTTP status line and response +// headers, a blank line, then the body -- on stdout, and on an HTTP error as well as on success +// (the process exits non-zero on an error status, but still prints the head). Observed by +// execution 2026-09-19 against api.github.com for a 200 with a Link header and for a 403. This +// module owns reading that framing and nothing about any endpoint: which status means what for a +// given resource is the resource module's fact. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "cli.github.com/manual/gh_api" + } +} + +type GhApiIncludedResponse { + status: Int + link: GitHubLinkHeader + body: String +} + +// AN UNREADABLE HEAD IS ITS OWN ARM. Stdout with no status line is not "no Link header" and not +// "status unknown, assume 200": gh failed before any response existed (no network, no auth, a +// refused request), so nothing about the resource was observed and the caller must say so. +type GhApiIncludedParse + = GhApiIncludedParsed { response: GhApiIncludedResponse } + | GhApiIncludedUnreadable { reason: NonEmptyStr } + +fn strip_carriage_return(line: String) -> String { + if ends_with(s: line, suffix: "\r") { + substring(s: line, start: 0, end: string_length(s: line) - 1) + } else { + line + } +} + +fn http_status_from_line(line: String) -> Int? { + if !starts_with(s: line, prefix: "HTTP/") { + none + } else { + match get(xs: filter(split(s: line, delimiter: " "), f => f != ""), index: 1) { + Absent => none + Present { value: code } => parse_int(s: code) + } + } +} + +// The head ends at the first empty line. A header line named "Link" (gh canonicalises header names +// through Go's net/http, so the spelling is exact) is carried whole to extdeps.github.pagination, +// which owns what its value means; two Link lines are refused rather than one silently chosen. +type HeadScan { + in_head: Bool + links: List + body_lines: List +} + +fn scan_after_status_line(lines: List) -> HeadScan { + fold(enumerate(lines), init: HeadScan { in_head: true, links: [], body_lines: [] }, f: fn(acc, p) { + let line = strip_carriage_return(line: p.second) + if p.first == 0 { + acc + } else if acc.in_head { + if line == "" { + HeadScan { in_head: false, links: acc.links, body_lines: acc.body_lines } + } else if starts_with(s: line, prefix: "Link:") { + HeadScan { + in_head: true, + links: list_push(acc.links, trim(s: substring(s: line, start: 5, end: string_length(s: line)))), + body_lines: acc.body_lines + } + } else { + acc + } + } else { + HeadScan { in_head: false, links: acc.links, body_lines: list_push(acc.body_lines, p.second) } + } + }) +} + +fn parse_gh_api_included(stdout: String) -> GhApiIncludedParse { + let lines = split(s: stdout, delimiter: "\n") + match get(xs: lines, index: 0) { + Absent => GhApiIncludedUnreadable { reason: "gh api --include printed nothing, so no HTTP response was observed" } + Present { value: first } => match http_status_from_line(line: strip_carriage_return(line: first)) { + Absent => GhApiIncludedUnreadable { reason: "gh api --include output does not begin with an HTTP status line, so no HTTP response was observed" } + Present { value: status } => { + let scan = scan_after_status_line(lines: lines) + if scan.in_head { + GhApiIncludedUnreadable { reason: "gh api --include output has no blank line ending the response head" } + } else { + match scan.links { + Empty {} => GhApiIncludedParsed { response: GhApiIncludedResponse { status: status, link: LinkHeaderAbsent, body: join(scan.body_lines, "\n") } } + Cons { head: only, tail: Empty {} } => GhApiIncludedParsed { response: GhApiIncludedResponse { status: status, link: LinkHeaderPresent { field_value: only }, body: join(scan.body_lines, "\n") } } + Cons { head: _, tail: Cons { head: _, tail: _ } } => GhApiIncludedUnreadable { reason: "the response head carries more than one Link header line" } + } + } + } + } + } +} diff --git a/dag/extdeps/github/organizations.dag b/dag/extdeps/github/organizations.dag new file mode 100644 index 00000000000..b9c9fba3931 --- /dev/null +++ b/dag/extdeps/github/organizations.dag @@ -0,0 +1,56 @@ +module extdeps.github.organizations + +import std.types { String, NonEmptyStr, Bool, Int } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import extdeps.languages.json.emit { JsonNumber } +import extdeps.languages.json.parse { parse_json_document, JsonDocumentParsed, JsonDocumentUnreadable, json_object_unique_member, JsonMemberFound } +import extdeps.github.cli_api_include { GhApiIncludedParsed, GhApiIncludedUnreadable, parse_gh_api_included } + +// GET /orgs/{org}: the organization object, whose integer `id` is how GitHub's own id-addressed +// URLs (`/organizations/{id}/...`) name the same organization the login names. +data get_organization_citation: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "docs.github.com/en/rest/orgs/orgs#get-an-organization" + } +} + +// THE CLI TRANSPORT, one of N realizations, in the same gh family as extdeps.github.ci_runner +// github.CliOrgSelfHostedRunners; authenticated by the caller's GH_TOKEN environment. Readonly. +service github.CliOrganizations { + operation GetOrganization { + input { + org: NonEmptyStr + } + output { + success: Bool from "exit_success" + stdout: String from "stdout" + stderr: String from "stderr" + } + readonly + transport shell { + argv: ["gh", "api", "--include", "orgs/{org}"] + } + } +} + +// The id, when the response is a 200 carrying one; otherwise nothing is known and nothing is +// guessed -- a caller that needed the id refuses whatever depended on it. +fn organization_id_from_included(stdout: String) -> Int? { + match parse_gh_api_included(stdout: stdout) { + GhApiIncludedUnreadable { reason: _ } => none + GhApiIncludedParsed { response } => + if response.status != 200 { + none + } else { + match parse_json_document(s: response.body) { + JsonDocumentUnreadable { gap: _ } => none + JsonDocumentParsed { value: doc } => match json_object_unique_member(v: doc, key: "id") { + JsonMemberFound { value: JsonNumber { lexeme } } => parse_int(s: lexeme as String) + _ => none + } + } + } + } +} diff --git a/dag/gunbc/apply.dag b/dag/gunbc/apply.dag index a987e5c2ea0..3f4385fbb73 100644 --- a/dag/gunbc/apply.dag +++ b/dag/gunbc/apply.dag @@ -10,7 +10,7 @@ import gunbc.runner_capacity_plan { parse_positive_width, runner_capacity_plan, } -import gunbc.fleet_runner_connectivity { organization_runner_roster_evidence } +import gunbc.fleet.organization_runner_roster_read { organization_runner_roster_live_read } import gunbc.runner_capacity_realize { runner_capacity_apply_wet_exit } // TOP-LEVEL gunbc.apply dispatcher. gunbc.live_deploy.apply remains the srv1 dashboard child @@ -88,7 +88,7 @@ fn apply_runner_capacity_plan( plan: runner_capacity_plan( srv3_width: w3, srv4_width: w4, - roster: organization_runner_roster_evidence(), + roster: organization_runner_roster_live_read(), ) ) } diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 4d5676f37a6..24d09fd9d8d 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -757,6 +757,11 @@ data gunbc_ci_org_actions_converge_target: GunbcRunStepTarget = GunbcRunStepTarg function: "org_actions_converge_wet", } +data gunbc_ci_org_runner_roster_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/fleet/organization_runner_roster_read.dag", + function: "organization_runner_roster_observe_wet", +} + // THE R2 WRITE MINT AND ITS PREFLIGHT ARE TWO TARGETS, AND THE ORDER IS THE SAFETY ARGUMENT. // gunbc.cloudflare.r2_token_mint_run issues cloudflare.AccountTokens.Create, which is // non-idempotent and whose value is returned exactly once: a run that mints and then cannot reach @@ -1824,6 +1829,19 @@ fn gunbc_ci_org_actions_converge_invoke() -> String { ) } +// SAME PRELUDE AS THE ORG-ACTIONS STEP: the roster read needs the same org-scoped installation token, +// minted in-run and held only in this step's environment. +fn gunbc_ci_org_runner_roster_observe_invoke() -> String { + gunbc_run_step_script_with_prelude( + prelude: gunbc_ci_org_admin_app_token_prelude(), + source_roots: witness_layer_roots, + entry: gunbc_ci_org_runner_roster_observe_target.entry, + function: gunbc_ci_org_runner_roster_observe_target.function, + claim_run: false, + receipt_rel: none + ) +} + // THE MINT RUNS AS THE RUN'S OWN FEDERATED IDENTITY, AND THERE IS NO PRELUDE HERE ON PURPOSE. // The org-actions and App control-plane steps carry shell preludes because they mint credentials // the typed effects could not reach; this pair needs none. gunbc.auth.access_token_source diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 35da4110b44..327a37bd024 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -16,6 +16,7 @@ import gunbc.action_use_admission { } import gunbc.fleet.app_control_plane_converge { app_control_plane_receipt_path } import gunbc.fleet.org_actions_converge { org_actions_validation_receipt_path } +import gunbc.fleet.organization_runner_roster_read { organization_runner_roster_receipt_path, runner_id_probes_env_name, roster_page_size_env_name } import gunbc.cloudflare.r2_permission_group_observe { r2_mint_preflight_receipt_path } import gunbc.cloudflare.r2_token_mint_run { r2_mint_receipt_glob_in, r2_mint_receipt_scope, R2MintReceiptAttemptScoped, R2OriginObjectWrite, @@ -74,6 +75,7 @@ import gunbc.ci_spec { gunbc_ci_app_control_plane_converge_invoke, gunbc_ci_r2_mint_preflight_invoke, gunbc_ci_r2_object_write_mint_invoke, + gunbc_ci_org_runner_roster_observe_invoke, gunbc_ci_runner_guest_image_observe_invoke, gunbc_ci_runner_guest_image_converge_invoke, gunbc_ci_runner_microvm_host_converge_invoke, @@ -187,6 +189,7 @@ type FleetConvergeWorkflowMode | RunnerPasswordSessionToolConverge | R2MintPreflight | R2ObjectWriteMint + | OrgRunnerRosterObserve fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String { match mode { @@ -212,6 +215,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String RunnerPasswordSessionToolConverge => "runner_password_session_tool_converge" R2MintPreflight => "r2_mint_preflight" R2ObjectWriteMint => "r2_object_write_mint" + OrgRunnerRosterObserve => "org_runner_roster_observe" } } @@ -236,7 +240,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // @@ -282,6 +286,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc RunnerPasswordSessionToolConverge => none R2MintPreflight => none R2ObjectWriteMint => none + OrgRunnerRosterObserve => none } } @@ -638,6 +643,7 @@ data fleet_converge_allocation_store_plan_step_if: String = fleet_converge_mode_ data fleet_converge_any_plan_step_if: String = fleet_converge_any_mode_step_if(modes: [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan]) data fleet_converge_apply_step_if: String = fleet_converge_mode_step_if(mode: FleetApply) data fleet_converge_org_actions_step_if: String = fleet_converge_mode_step_if(mode: OrgActionsObserve) +data fleet_converge_org_runner_roster_step_if: String = fleet_converge_mode_step_if(mode: OrgRunnerRosterObserve) data fleet_converge_app_control_plane_step_if: String = fleet_converge_mode_step_if(mode: AppControlPlaneObserve) @@ -1064,6 +1070,46 @@ fn fleet_converge_org_actions_step() -> Step { } } +// THE ORG RUNNER ROSTER READ RIDES THE SAME IN-RUN TOKEN AS org_actions_observe, and is a separate +// mode because it answers a different question: which registrations the organization holds, read +// to exhaustion, rather than whether the runner-group settings match policy. Its receipt uploads on +// every outcome, because a Partial or an Unavailable roster is exactly the evidence worth keeping. +fn fleet_converge_org_runner_roster_step() -> Step { + RunStep { + name: Present { value: "Org runner roster: paginated read of GitHub self-hosted runner registrations" }, + id: Present { value: "org_runner_roster_observe" }, + run: gunbc_ci_org_runner_roster_observe_invoke(), + shell: none, + env: Present { value: [ + kv(key: "WIF_ACCESS_TOKEN", value: yaml_string(s: "${{ steps.wif_auth.outputs.access_token }}")), + kv(key: runner_id_probes_env_name as String, value: yaml_string(s: "${{ github.event.inputs.runner_ids }}")), + kv(key: roster_page_size_env_name as String, value: yaml_string(s: "${{ github.event.inputs.roster_page_size }}")), + ] }, + working_directory: none, + if_condition: Present { value: fleet_converge_org_runner_roster_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +fn fleet_converge_org_runner_roster_receipt_upload_step() -> Step { + UsesStep { + name: Present { value: "Upload org runner roster receipt" }, + id: Present { value: "org_runner_roster_receipt_upload" }, + uses: upload_artifact_action, + with: Present { value: [ + kv(key: "name", value: yaml_string(s: "org-runner-roster")), + kv(key: "path", value: yaml_string(s: organization_runner_roster_receipt_path)), + kv(key: "if-no-files-found", value: yaml_string(s: "error")), + kv(key: "retention-days", value: yaml_int(n: 30)), + ] }, + env: none, + if_condition: Present { value: join(["always() && ", fleet_converge_org_runner_roster_step_if], "") }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } + } +} + fn fleet_converge_app_control_plane_step() -> Step { RunStep { name: Present { value: "GitHub App control plane: registration + webhook config observation" }, @@ -1533,6 +1579,8 @@ fn fleet_converge_job() -> Job { fleet_converge_apply_receipt_upload_step(), fleet_converge_org_actions_step(), fleet_converge_org_actions_receipt_upload_step(), + fleet_converge_org_runner_roster_step(), + fleet_converge_org_runner_roster_receipt_upload_step(), fleet_converge_app_control_plane_step(), fleet_converge_app_control_plane_receipt_upload_step(), fleet_converge_r2_mint_preflight_step(), @@ -1698,7 +1746,7 @@ data fleet_converge_workflow: Workflow = Workflow { }, DispatchInput { name: "mode", - description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown" }, + description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown" }, required: true, default: none, type: InputChoice { options: fleet_converge_mode_options }, @@ -1752,6 +1800,20 @@ data fleet_converge_workflow: Workflow = Workflow { default: none, type: InputString, }, + DispatchInput { + name: "runner_ids", + description: Present { value: "org_runner_roster_observe only: comma-separated captured runner registration ids to observe individually (present / absent / unobserved) in the gunbai-ci organization" }, + required: false, + default: none, + type: InputString, + }, + DispatchInput { + name: "roster_page_size", + description: Present { value: "org_runner_roster_observe only: page size for the first roster request (1..100, default 100); later pages follow the authority's own continuation links" }, + required: false, + default: none, + type: InputString, + }, DispatchInput { name: "transaction_nonce", description: Present { value: "Operator-minted unique nonce for one RLM transaction; echoed into run-name so each dispatched run is API-selectable by its displayTitle, closing the run-id correlation gap (review 5062738052 B4). Not consumed by any step" }, diff --git a/dag/gunbc/fleet/fleet_runner_connectivity.dag b/dag/gunbc/fleet/fleet_runner_connectivity.dag index 5ce0264ac0f..88bfeff49e2 100644 --- a/dag/gunbc/fleet/fleet_runner_connectivity.dag +++ b/dag/gunbc/fleet/fleet_runner_connectivity.dag @@ -2,9 +2,7 @@ module gunbc.fleet_runner_connectivity import std.types { String, NonEmptyStr, Bool, Int, List } import std.nat { Nat, nat_range_inclusive } -import std.disposition { Disposition, Terminal, Scaffold, SingleAuthority } -import std.decl_ref { DeclarationRef, WholeDeclaration } -import std.realization_reconcile { ShowEffectiveRead, ReadAbsent } +import std.disposition { Disposition, Terminal } import product.placement_supply { HostIdentity } import extdeps.github.ci_runner { GitHubSelfHostedRunnerRegistration, @@ -237,8 +235,8 @@ fn incident_registration( name: name, status: status, busy: busy, - ephemeral: true, - version: incident_runner_version, + ephemeral: Present { value: true }, + version: Present { value: incident_runner_version }, os: "Linux" as NonEmptyStr, labels: incident_runner_labels(host: host), } @@ -295,33 +293,23 @@ data incident_2026_08_06_managed_registrations: List OrganizationRunnerRosterEvidence { +fn incident_2026_08_06_managed_roster_fixture() -> OrganizationRunnerRosterEvidence { RosterObserved { observed_on: "2026-08-06" as NonEmptyStr, registrations: incident_2026_08_06_managed_registrations, } } -fn organization_runner_roster_evidence() -> OrganizationRunnerRosterEvidence { +// THE 2026-08-06 INCIDENT ROSTER, KEPT ONLY AS A DATED HISTORICAL FIXTURE FOR WITNESSES. Production +// planning reads gunbc.fleet.organization_runner_roster_read organization_runner_roster_live_read; +// nothing outside a witness may plan from this population. +fn incident_2026_08_06_roster_fixture() -> OrganizationRunnerRosterEvidence { RosterObserved { observed_on: "2026-08-06" as NonEmptyStr, registrations: incident_2026_08_06_registrations, } } -data organization_runner_roster_producer_scaffold: Disposition = Scaffold { - dissolves_to: SingleAuthority, - bind: DeclarationRef { - module_path: "gunbc.fleet_runner_connectivity", - decl_name: "organization_runner_roster_show_effective_read", - field: WholeDeclaration, - } -} - -data organization_runner_roster_show_effective_read: ShowEffectiveRead = ReadAbsent { - reason: "no live org-level GET /orgs/gunb-ai/actions/runners has executed through the gunbai-ci GitHub App yet; the incident fixture in organization_runner_roster_evidence is the pinned acceptance corpus until this read binds ReadObserved" -} - fn registration_matches_name( reg: GitHubSelfHostedRunnerRegistration, name: NonEmptyStr, @@ -563,7 +551,7 @@ fn runner_slot_connectivity_verdict_for_incarnation( } fn runner_slot_connectivity_verdict(slot: RunnerSlotIdentity) -> RunnerSlotConnectivityVerdict { - let roster = organization_runner_roster_evidence() + let roster = incident_2026_08_06_roster_fixture() let incarnations = local_runner_slot_incarnations() match fold( incarnations, @@ -718,7 +706,7 @@ fn fleet_runner_capacity_width_snapshot_from_roster( } fn fleet_runner_capacity_width_snapshot() -> FleetRunnerCapacityWidthSnapshot { - fleet_runner_capacity_width_snapshot_from_roster(roster: organization_runner_roster_evidence()) + fleet_runner_capacity_width_snapshot_from_roster(roster: incident_2026_08_06_roster_fixture()) } fn connectivity_verdicts_for_active_incarnations( diff --git a/dag/gunbc/fleet/organization_runner_roster_read.dag b/dag/gunbc/fleet/organization_runner_roster_read.dag new file mode 100644 index 00000000000..abd1d872b94 --- /dev/null +++ b/dag/gunbc/fleet/organization_runner_roster_read.dag @@ -0,0 +1,954 @@ +module gunbc.fleet.organization_runner_roster_read + +import std.types { String, NonEmptyStr, Bool, Int, List, Map } +import std.algebra { trim } +import std.nat { Nat } +import std.checked_arithmetic { checked_int_magnitude, CheckedNatReady, CheckedNatOverflow } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.page_fold { + PageSize, PageSizeAdmission, PageSizeAdmitted, PageSizeAboveAuthorityMaximum, admit_page_size, + PageBound, PageCursor, OffsetPage, OpaqueToken, LinkNext, + PageContinuation, PageObservation, + PageFoldState, PageFoldDemand, DemandPage, DemandNone, + PageFoldOutcome, PagesClosed, PagesRefused, + page_fold_begin, page_fold_demand, page_fold_observe, page_fold_finish, page_fold_refusal_key +} +import extdeps.uri { uri_wire } +import extdeps.languages.json.emit { JsonValue, JsonString, JsonArray } +import extdeps.languages.json.parse { + parse_json_document, + JsonDocumentParsed, + JsonDocumentUnreadable, + json_document_gap_text, + json_object_unique_member, + JsonMemberFound, + JsonMemberAbsent, + JsonMemberDuplicated, + JsonMemberNotAnObject +} +import extdeps.shell.credentials { env_credential } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.github.github { max_per_page } +import extdeps.github.organizations { organization_id_from_included } +import extdeps.github.pagination { + GitHubLinkHeader, LinkHeaderPresent, LinkHeaderAbsent, github_max_page_size, + GitHubContinuationRead, ContinuationRead, LinkHeaderUnreadable, + github_continuation_from_link_header, github_listing_subject, github_first_page_cursor +} +import extdeps.github.cli_api_include { GhApiIncludedParse, GhApiIncludedParsed, GhApiIncludedUnreadable, parse_gh_api_included } +import extdeps.github.ci_runner { + GitHubSelfHostedRunnerRegistration, + GitHubRunnerRegistrationStatus, GitHubRunnerOnline, GitHubRunnerOffline, + org_self_hosted_runners_first_page_target +} +import gunbc.clock_read { clock_now_probed_at_or_unknown } +import gunbc.runner_slot_allocation { runner_label_sets_equal } +import gunbc.runner_host_deploy { github_app_runner_org, gunbc_ci_github_app } +import gunbc.fleet.org_actions_converge { + org_admin_token_env_name, + json_member_string, + json_member_int, + json_member_bool +} +import gunbc.fleet_runner_connectivity { + OrganizationRunnerRosterEvidence, + RosterObserved, + RosterObservedPartial, + RosterReadUnavailable, + organization_runner_roster_from_page +} + +// THE LIVE ORGANIZATION RUNNER ROSTER. This is the producer the planner's roster parameter was +// waiting for: GET /orgs/{org}/actions/runners, read through the gh CLI under the gunbai-ci App +// installation token that gunbc.ci_spec gunbc_ci_org_admin_app_token_prelude mints in-run into +// GUNBC_ORG_ADMIN_TOKEN and GH_TOKEN -- the same credential route and custody +// gunbc.fleet.org_actions_converge already reads runner groups through. Nothing here holds a key, +// mints a token, or places one on an argv. +// +// WITHOUT THAT CREDENTIAL THE ANSWER IS RosterReadUnavailable, NEVER THE PINNED CORPUS. The +// 2026-08-06 incident corpus survives only as the dated witness fixture +// gunbc.fleet_runner_connectivity incident_2026_08_06_roster_fixture; a production caller that +// cannot read the organization is told so, and the planner refuses on that arm rather than planning +// from a month-old population. + +// ONE PAGE AS THIS READ KEEPS IT. total_count is carried per page, not once, because the population +// can change between requests; pages that disagree on it did not observe one population. +// The request target and the Link field the authority answered with are kept beside the rows, so a +// receipt can show which request produced which ids and how the listing continued -- the HTTP +// boundary facts, not only the conclusion drawn from them. +type OrganizationRunnerRosterPage { + target: String + link_field: String + registrations: List + total_count: Int +} + +type OrganizationRunnerRosterPageRead + = RosterPageDelivered { page: OrganizationRunnerRosterPage, continuation: PageContinuation } + | RosterPageFailed { cause: NonEmptyStr } + +type RunnerRegistrationDecode + = RunnerRegistrationDecoded { registration: GitHubSelfHostedRunnerRegistration } + | RunnerRegistrationDecodeRefused { reason: NonEmptyStr } + +fn decode_runner_status(raw: NonEmptyStr) -> GitHubRunnerRegistrationStatus? { + if (raw as String) == "online" { + Present { value: GitHubRunnerOnline } + } else if (raw as String) == "offline" { + Present { value: GitHubRunnerOffline } + } else { + none + } +} + +// Labels are objects with a name; a label without one refuses the runner rather than dropping it, +// because a label set with a member missing is a different label set. +fn decode_runner_labels(v: JsonValue) -> List? { + match json_object_unique_member(v: v, key: "labels") { + JsonMemberFound { value: JsonArray { elements } } => { + let names = map(elements, l => json_member_string(v: l, key: "name")) + if any(names, n => match n { Absent => true, Present { value: _ } => false }) { + none + } else { + Present { value: flat_map(names, n => match n { Present { value } => [value], Absent => [] }) } + } + } + _ => none + } +} + +// REQUIRED FIELDS REFUSE WHEN ABSENT; OPTIONAL ONES STAY ABSENT. The split is the upstream schema's +// (extdeps.github.ci_runner GitHubSelfHostedRunnerRegistration), not a choice made here. +fn decode_runner_registration(v: JsonValue) -> RunnerRegistrationDecode { + match json_member_int(v: v, key: "id") { + Absent => RunnerRegistrationDecodeRefused { reason: "a runner object has no integer id" } + Present { value: id } => + if id < 0 { + RunnerRegistrationDecodeRefused { reason: "a runner object has a negative id" } + } else { + match json_member_string(v: v, key: "name") { + Absent => RunnerRegistrationDecodeRefused { reason: concat("runner has no name: id ", to_string(id)) as NonEmptyStr } + Present { value: name } => match json_member_string(v: v, key: "status") { + Absent => RunnerRegistrationDecodeRefused { reason: concat("runner has no status: id ", to_string(id)) as NonEmptyStr } + Present { value: raw_status } => match decode_runner_status(raw: raw_status) { + Absent => RunnerRegistrationDecodeRefused { reason: concat("runner status is neither online nor offline: ", raw_status as String) as NonEmptyStr } + Present { value: status } => match json_member_bool(v: v, key: "busy") { + Absent => RunnerRegistrationDecodeRefused { reason: concat("runner has no busy flag: id ", to_string(id)) as NonEmptyStr } + Present { value: busy } => match json_member_string(v: v, key: "os") { + Absent => RunnerRegistrationDecodeRefused { reason: concat("runner has no os: id ", to_string(id)) as NonEmptyStr } + Present { value: os } => match decode_runner_labels(v: v) { + Absent => RunnerRegistrationDecodeRefused { reason: concat("runner labels are absent or unnamed: id ", to_string(id)) as NonEmptyStr } + Present { value: labels } => match checked_int_magnitude(a: id) { + CheckedNatOverflow { cause: _ } => RunnerRegistrationDecodeRefused { reason: concat("runner id is not representable as a natural number: ", to_string(id)) as NonEmptyStr } + CheckedNatReady { value: nat_id } => RunnerRegistrationDecoded { registration: GitHubSelfHostedRunnerRegistration { + id: nat_id, + name: name, + status: status, + busy: busy, + ephemeral: json_member_bool(v: v, key: "ephemeral"), + version: json_member_string(v: v, key: "version"), + os: os, + labels: labels + } } + } + } + } + } + } + } + } + } + } +} + +type RosterPageDecode + = RosterPageDecoded { registrations: List, total_count: Int } + | RosterPageDecodeRefused { reason: NonEmptyStr } + +fn first_registration_refusal(ds: List) -> NonEmptyStr? { + fold(ds, init: none, f: fn(found, d) { match found { + Present { value } => Present { value: value } + Absent => match d { + RunnerRegistrationDecodeRefused { reason } => Present { value: reason } + RunnerRegistrationDecoded { registration: _ } => none + } + } }) +} + +fn decode_roster_page_json(body: String) -> RosterPageDecode { + match parse_json_document(s: body) { + JsonDocumentUnreadable { gap } => RosterPageDecodeRefused { + reason: concat("org runners response is ", json_document_gap_text(gap: gap)) as NonEmptyStr + } + JsonDocumentParsed { value: doc } => match json_member_int(v: doc, key: "total_count") { + Absent => RosterPageDecodeRefused { reason: "org runners response has no integer total_count" } + Present { value: total } => match json_object_unique_member(v: doc, key: "runners") { + JsonMemberFound { value: JsonArray { elements } } => { + let decoded = map(elements, e => decode_runner_registration(v: e)) + match first_registration_refusal(ds: decoded) { + Present { value: reason } => RosterPageDecodeRefused { reason: reason } + Absent => RosterPageDecoded { + registrations: flat_map(decoded, d => match d { + RunnerRegistrationDecoded { registration } => [registration] + RunnerRegistrationDecodeRefused { reason: _ } => [] + }), + total_count: total + } + } + } + JsonMemberFound { value: _ } => RosterPageDecodeRefused { reason: "runners is not a JSON array" } + JsonMemberAbsent => RosterPageDecodeRefused { reason: "runners is absent" } + JsonMemberDuplicated { count: _ } => RosterPageDecodeRefused { reason: "runners is duplicated" } + JsonMemberNotAnObject => RosterPageDecodeRefused { reason: "org runners response is not a JSON object" } + } + } + } +} + +// ONE INCLUDED RESPONSE TO ONE PAGE READ. This is the pure half of the fetch, so the decision a +// response produces -- status, continuation, body -- is exercisable over a supplied response +// without a network (DESIGN section 3: a witness discriminates at one interface). +fn link_field_text(link: GitHubLinkHeader) -> String { + match link { + LinkHeaderPresent { field_value: v } => v + LinkHeaderAbsent => "" + } +} + +fn roster_page_read_from_included(target: String, stdout: String, stderr: String) -> OrganizationRunnerRosterPageRead { + match parse_gh_api_included(stdout: stdout) { + GhApiIncludedUnreadable { reason } => + RosterPageFailed { cause: join([reason as String, "; gh stderr: ", stderr], "") as NonEmptyStr } + GhApiIncludedParsed { response } => + if response.status != 200 { + RosterPageFailed { cause: join(["org runners page answered HTTP ", to_string(response.status), "; gh stderr: ", stderr], "") as NonEmptyStr } + } else { + match github_continuation_from_link_header(header: response.link) { + LinkHeaderUnreadable { parse: _ } => RosterPageFailed { cause: "org runners page carries a Link header that does not parse, so whether another page exists is unknown" } + ContinuationRead { continuation } => match decode_roster_page_json(body: response.body) { + RosterPageDecodeRefused { reason } => RosterPageFailed { cause: reason } + RosterPageDecoded { registrations, total_count } => RosterPageDelivered { + page: OrganizationRunnerRosterPage { + target: target, + link_field: link_field_text(link: response.link), + registrations: registrations, + total_count: total_count + }, + continuation: continuation + } + } + } + } + } +} + +data github_api_origin_prefix: String = "https://api.github.com/" + +// THE SUBJECT A LISTING IS ABOUT: one organization, named by its login and -- where it was read -- +// by its numeric id. GitHub's own rel="next" targets address an organization by id +// (`/organizations//actions/runners`), so the id is what lets a continuation be checked +// against the organization the walk started on rather than merely against the API host. +type OrganizationRunnerScope { + login: NonEmptyStr + id: Int? +} + +// THE LISTING REQUEST: the subject, and the page size its first request asks for. Page size is a +// request parameter admitted against the authority's maximum (std.page_fold admit_page_size); every +// later request is the authority's own continuation target, so the size is never re-composed. +type OrganizationRunnerListing { + scope: OrganizationRunnerScope + page_size: PageSize +} + +// WHICH REQUEST A CURSOR NAMES, AND WHETHER IT IS STILL THIS LISTING. The first request is composed +// from the scope; every later one is the rel="next" target the authority returned, used VERBATIM -- +// but only after its path is shown to be this organization's runners listing. A target on another +// host, another organization, a repository's runners, or any other resource refuses before it is +// fetched: its rows would otherwise be accumulated into this organization's roster, where totals +// and distinct-id counts could still agree. The numeric-id alias is admitted only when the scope +// carries that same id; with no corroborated id it refuses, it is not guessed. +type RosterRequestTarget + = RosterRequestAt { target: NonEmptyStr } + | RosterRequestRefused { cause: NonEmptyStr } + +fn url_path_before_query(rest: String) -> String { + match get(xs: split(s: rest, delimiter: "?"), index: 0) { + Present { value: p } => p + Absent => rest + } +} + +fn continuation_path_admitted(scope: OrganizationRunnerScope, path: String) -> Bool { + if path == join(["orgs/", scope.login as String, "/actions/runners"], "") { + true + } else { + match scope.id { + Absent => false + Present { value: id } => path == join(["organizations/", to_string(id), "/actions/runners"], "") + } + } +} + +fn roster_request_target(listing: OrganizationRunnerListing, cursor: PageCursor) -> RosterRequestTarget { + let scope = listing.scope + match cursor { + OffsetPage { page: n } => + if n == 1 { + RosterRequestAt { target: org_self_hosted_runners_first_page_target(org: scope.login, per_page: listing.page_size.count) } + } else { + RosterRequestRefused { cause: "an offset cursor past page one reached the org runners read, which continues only by Link targets" } + } + OpaqueToken { token: _ } => RosterRequestRefused { cause: "an opaque-token cursor reached the org runners read, which continues only by Link targets" } + LinkNext { target: u } => { + let wire = uri_wire(uri: u) as String + if !starts_with(s: wire, prefix: github_api_origin_prefix) { + RosterRequestRefused { cause: concat("the next-page link leaves api.github.com: ", wire) as NonEmptyStr } + } else { + let path = url_path_before_query(rest: substring(s: wire, start: string_length(s: github_api_origin_prefix), end: string_length(s: wire))) + if continuation_path_admitted(scope: scope, path: path) { + RosterRequestAt { target: wire as NonEmptyStr } + } else { + RosterRequestRefused { cause: join(["the next-page link is not this organization's runners listing (", scope.login as String, "): ", wire], "") as NonEmptyStr } + } + } + } + } +} + +// WHAT ONE REQUEST RETURNED, as the transport hands it back. The live walk fills it from gh; a +// witness fills it from a supplied response. Everything after it -- admission, framing, decode, +// the fold -- is the same code in both. +type RosterTransportResponse { + stdout: String + stderr: String +} + +fn fetch_roster_page_via( + listing: OrganizationRunnerListing, + cursor: PageCursor, + respond: fn(NonEmptyStr) -> RosterTransportResponse, +) -> OrganizationRunnerRosterPageRead { + match roster_request_target(listing: listing, cursor: cursor) { + RosterRequestRefused { cause } => RosterPageFailed { cause: cause } + RosterRequestAt { target } => { + let response = respond(target) + roster_page_read_from_included(target: target as String, stdout: response.stdout, stderr: response.stderr) + } + } +} + +fn gh_roster_transport(target: NonEmptyStr) -> RosterTransportResponse { + let response = github.CliOrgSelfHostedRunners.ListOrgRunnersPage(target: target) + RosterTransportResponse { stdout: response.stdout, stderr: response.stderr } +} + +// THE WALK KEEPS WHAT std.page_fold DELIBERATELY DOES NOT, AND SAYS WHY. +// +// std.page_fold's outcome carries items only when the listing closed, so a caller cannot use a +// truncated listing as if it were whole. This walk still delegates every continuation decision to +// that fold -- advance, the cycle check, the page budget, closure -- and keeps its OWN copy of the +// pages read so far, because the carrier it feeds has a typed partial arm that the fold's outcome +// does not: RosterObservedPartial names its observed_count beside the authority's total_count, and +// gunbc.fleet_runner_connectivity already decides per question whether a prefix may answer. A later +// page failing therefore keeps the pages already read and reports Partial, instead of discarding a +// population that was observed and reporting none. This is a stated divergence from the fold's +// outcome shape, not a second pager: the fold state is the only thing that decides whether another +// request is owed. +type OrganizationRunnerRosterWalk { + state: PageFoldState + pages: List + failure: NonEmptyStr? +} + +fn roster_walk_begin() -> OrganizationRunnerRosterWalk { + OrganizationRunnerRosterWalk { + state: page_fold_begin( + subject: github_listing_subject(operation: "ListOrgSelfHostedRunners"), + bound: organization_runner_roster_page_bound, + first: github_first_page_cursor + ), + pages: [], + failure: none + } +} + +// TERMINATION IS THE FOLD'S. Every recursive call follows a page_fold_observe, which ends the fold, +// refuses, or advances to a cursor not already demanded and counts the page against max_pages; a +// failed or refused fetch ends the walk. +fn roster_walk_pump( + listing: OrganizationRunnerListing, + walk: OrganizationRunnerRosterWalk, + respond: fn(NonEmptyStr) -> RosterTransportResponse, +) -> OrganizationRunnerRosterWalk { + match walk.failure { + Present { value: _ } => walk + Absent => match page_fold_demand(state: walk.state) { + DemandNone => walk + DemandPage { cursor: c } => match fetch_roster_page_via(listing: listing, cursor: c, respond: respond) { + RosterPageFailed { cause } => + OrganizationRunnerRosterWalk { state: walk.state, pages: walk.pages, failure: Present { value: cause } } + RosterPageDelivered { page, continuation } => + roster_walk_pump(listing: listing, walk: OrganizationRunnerRosterWalk { + state: page_fold_observe(state: walk.state, page: PageObservation { items: page.registrations, continuation: continuation }), + pages: list_push(walk.pages, page), + failure: none + }, respond: respond) + } + } + } +} + +// A POLICY BUDGET, NOT A MEASUREMENT: 60 pages of extdeps.github.github max_per_page rows. Whether +// it is adequate is read from the instrument rather than asserted here -- the observe mode's receipt +// (organization_runner_roster_observe_wet) records pages_read and the authority's total_count on +// every run -- and an inadequate bound is loud: PageBudgetExhausted, reported as Partial. +data organization_runner_roster_page_bound: PageBound = PageBound { max_pages: 60 } + +// WHAT THE WALK ESTABLISHED, WITH THE REASON A PARTIAL WAS PARTIAL. The carrier has no reason field +// on its partial arm, so the report keeps it -- and the ids whose readings disagreed -- for the +// receipt the observe mode writes. +type OrganizationRunnerRosterReport { + evidence: OrganizationRunnerRosterEvidence + pages: List + pages_read: Int + disputed_ids: List + note: String +} + +// TWO READINGS OF ONE ID AGREE WHEN EVERY FIELD AGREES, compared as typed values: optional presence +// is compared as presence, not through a sentinel spelling, and labels through the established +// label-equivalence authority (gunbc.runner_slot_allocation runner_label_sets_equal), which compares +// label SETS rather than a joined rendering -- so ["blue,green"] and ["blue", "green"] differ. +fn optional_bool_agrees(a: Bool?, b: Bool?) -> Bool { + match a { + Absent => match b { Absent => true, Present { value: _ } => false } + Present { value: x } => match b { Absent => false, Present { value: y } => x == y } + } +} + +fn optional_text_agrees(a: NonEmptyStr?, b: NonEmptyStr?) -> Bool { + match a { + Absent => match b { Absent => true, Present { value: _ } => false } + Present { value: x } => match b { Absent => false, Present { value: y } => (x as String) == (y as String) } + } +} + +fn registration_status_agrees(a: GitHubRunnerRegistrationStatus, b: GitHubRunnerRegistrationStatus) -> Bool { + match a { + GitHubRunnerOnline => match b { GitHubRunnerOnline => true, GitHubRunnerOffline => false } + GitHubRunnerOffline => match b { GitHubRunnerOnline => false, GitHubRunnerOffline => true } + } +} + +fn registration_readings_agree(a: GitHubSelfHostedRunnerRegistration, b: GitHubSelfHostedRunnerRegistration) -> Bool { + to_string(a.id) == to_string(b.id) + && (a.name as String) == (b.name as String) + && registration_status_agrees(a: a.status, b: b.status) + && a.busy == b.busy + && optional_bool_agrees(a: a.ephemeral, b: b.ephemeral) + && optional_text_agrees(a: a.version, b: b.version) + && (a.os as String) == (b.os as String) + && runner_label_sets_equal(a: map(a.labels, l => l as String), b: map(b.labels, l => l as String)) +} + +// REPEATED IDS ARE RECONCILED, NEVER RESOLVED BY POSITION. A paginated listing is not an atomic +// snapshot, so one id can be read twice. Readings that agree collapse to one; readings that DISAGREE +// make that id disputed: it is withheld from the roster's members (no first-wins or last-wins +// payload is ever handed to a consumer as that registration's standing) and the roster is Partial. +// Ids whose readings are not in dispute are kept. +type RegistrationReconciliation { + kept: List + disputed_ids: List +} + +type ReconcileScan { + first_reading: Map + disputed: Map + first_seen_reversed: List + disputed_reversed: List +} + +fn reconcile_registrations(pages: List) -> RegistrationReconciliation { + let scan = fold(flat_map(pages, p => p.registrations), init: ReconcileScan { + first_reading: empty_map(), + disputed: empty_map(), + first_seen_reversed: [], + disputed_reversed: [] + }, f: fn(acc, r) { + let key = to_string(r.id) + match map_get(acc.first_reading, key) { + Absent => ReconcileScan { + first_reading: map_insert(acc.first_reading, key, r), + disputed: acc.disputed, + first_seen_reversed: concat([r], acc.first_seen_reversed), + disputed_reversed: acc.disputed_reversed + } + Present { value: seen } => + if registration_readings_agree(a: seen, b: r) { + acc + } else { + match map_get(acc.disputed, key) { + Present { value: _ } => acc + Absent => ReconcileScan { + first_reading: acc.first_reading, + disputed: map_insert(acc.disputed, key, true), + first_seen_reversed: acc.first_seen_reversed, + disputed_reversed: concat([key], acc.disputed_reversed) + } + } + } + } + }) + RegistrationReconciliation { + kept: filter(reverse(scan.first_seen_reversed), r => match map_get(scan.disputed, to_string(r.id)) { + Present { value: _ } => false + Absent => true + }), + disputed_ids: reverse(scan.disputed_reversed) + } +} + +fn partial_roster( + observed_on: NonEmptyStr, + registrations: List, + authority_total: Int, +) -> OrganizationRunnerRosterEvidence { + RosterObservedPartial { + observed_on: observed_on, + registrations: registrations, + observed_count: length(xs: registrations), + authority_total: authority_total, + } +} + +fn partial_report( + observed_on: NonEmptyStr, + registrations: List, + authority_total: Int, + pages: List, + disputed_ids: List, + note: String, +) -> OrganizationRunnerRosterReport { + OrganizationRunnerRosterReport { + evidence: partial_roster(observed_on: observed_on, registrations: registrations, authority_total: authority_total), + pages: pages, + pages_read: length(xs: pages), + disputed_ids: disputed_ids, + note: note + } +} + +// OBSERVED ONLY WHEN FOUR INDEPENDENT FACTS AGREE: the authority closed the listing (no rel="next"), +// every page reported the same total_count, no id was read with disagreeing payloads, and the number +// of DISTINCT runner ids equals the total. Any one failing is Partial over the undisputed ids that +// were seen. Distinctness matters because an offset listing that shifts under a concurrent +// registration repeats a row; counting rows would call that repetition coverage. +fn classify_roster_walk(walk: OrganizationRunnerRosterWalk, observed_on: NonEmptyStr) -> OrganizationRunnerRosterReport { + let pages_read = length(xs: walk.pages) + match walk.pages { + Empty {} => OrganizationRunnerRosterReport { + evidence: RosterReadUnavailable { reason: match walk.failure { + Present { value: cause } => concat("the first org runners page was not read: ", cause as String) + Absent => "the org runners read delivered no page" + } }, + pages: [], + pages_read: 0, + disputed_ids: [], + note: "no page read" + } + Cons { head: first, tail: _ } => { + let reconciled = reconcile_registrations(pages: walk.pages) + let regs = reconciled.kept + let disputed = reconciled.disputed_ids + let total = first.total_count + match walk.failure { + Present { value: cause } => partial_report(observed_on: observed_on, registrations: regs, authority_total: total, pages: walk.pages, disputed_ids: disputed, note: concat("a later page failed after the earlier pages were kept: ", cause as String)) + Absent => match page_fold_finish(state: walk.state) { + PagesRefused { refusal } => partial_report(observed_on: observed_on, registrations: regs, authority_total: total, pages: walk.pages, disputed_ids: disputed, note: concat("the page fold refused before the listing closed: ", page_fold_refusal_key(r: refusal) as String)) + PagesClosed { subject: _, pages_read: _, items: _ } => + if !all(walk.pages, p => p.total_count == total) { + partial_report(observed_on: observed_on, registrations: regs, authority_total: total, pages: walk.pages, disputed_ids: disputed, note: "pages disagreed on total_count, so the listing did not observe one population") + } else if length(xs: disputed) > 0 { + partial_report(observed_on: observed_on, registrations: regs, authority_total: total, pages: walk.pages, disputed_ids: disputed, note: concat("runner ids were read with disagreeing payloads and are withheld: ", join(disputed, ","))) + } else { + OrganizationRunnerRosterReport { + evidence: organization_runner_roster_from_page(observed_on: observed_on, registrations: regs, authority_total: total), + pages: walk.pages, + pages_read: pages_read, + disputed_ids: [], + note: "listing closed; distinct runner ids compared against total_count" + } + } + } + } + } + } +} + +// THE ADMITTED CREDENTIAL ROUTE, AND THE ONE BINDING IT REQUIRES. gh authenticates from GH_TOKEN +// before any stored configuration, and gunbc.ci_spec gunbc_ci_org_admin_app_token_prelude exports +// the one minted installation token as BOTH GUNBC_ORG_ADMIN_TOKEN and GH_TOKEN. The route is admitted +// only when both bindings are present AND AGREE: two present but different values would let gh +// authenticate with a credential other than the one the prelude minted while the route claimed the +// prelude's. Equality does not authenticate an arbitrary environment -- the trusted-prelude +// assumption stays explicit -- it refuses a contradictory binding. No value is ever rendered into a +// reason, a receipt or an argv. +type OrgRunnerReadRoute = GhTokenFromOrgAdminPrelude {} + +type OrgRunnerRouteAdmission + = OrgRunnerRouteAdmitted { route: OrgRunnerReadRoute } + | OrgRunnerRouteRefused { reason: NonEmptyStr } + +data gh_token_env_name: NonEmptyStr = "GH_TOKEN" + +fn org_runner_route_from_bindings(admin_token: String?, gh_token: String?) -> OrgRunnerRouteAdmission { + match admin_token { + Absent => OrgRunnerRouteRefused { reason: "GUNBC_ORG_ADMIN_TOKEN is absent: the org runner read runs only under the installation token gunbc.ci_spec gunbc_ci_org_admin_app_token_prelude mints in-run; no pinned population is substituted" } + Present { value: admin } => match gh_token { + Absent => OrgRunnerRouteRefused { reason: "GH_TOKEN is absent, so gh would not authenticate with the prelude's installation token" } + Present { value: gh } => + if admin == "" || gh == "" { + OrgRunnerRouteRefused { reason: "a required credential binding is empty" } + } else if admin == gh { + OrgRunnerRouteAdmitted { route: GhTokenFromOrgAdminPrelude {} } + } else { + OrgRunnerRouteRefused { reason: "GH_TOKEN and GUNBC_ORG_ADMIN_TOKEN are bound to different values, so gh would not authenticate with the credential the prelude minted" } + } + } + } +} + +fn organization_runner_read_route() -> OrgRunnerRouteAdmission { + org_runner_route_from_bindings( + admin_token: shell.Env.Get(name: org_admin_token_env_name).value, + gh_token: shell.Env.Get(name: gh_token_env_name).value + ) +} + + +// WHICH ORGANIZATION IS READ IS NOT AUTHORED HERE. The token is minted for the gunbai-ci App's +// installation, so the organization it can answer for is that App row's owner login -- the same +// derivation gunbc.runner_host_deploy uses for every host's registrations. +fn organization_runner_roster_org() -> NonEmptyStr { + github_app_runner_org(config: gunbc_ci_github_app) +} + +// The numeric id is READ from the authority (GET /orgs/{org}), never authored, and only so that +// GitHub's own id-addressed continuation links can be admitted as this organization's. +fn organization_runner_scope(org: NonEmptyStr) -> OrganizationRunnerScope { + let response = github.CliOrganizations.GetOrganization(org: org) + OrganizationRunnerScope { login: org, id: organization_id_from_included(stdout: response.stdout) } +} + +fn organization_runner_roster_report(org: NonEmptyStr, page_size: PageSize) -> OrganizationRunnerRosterReport { + match organization_runner_read_route() { + OrgRunnerRouteRefused { reason } => OrganizationRunnerRosterReport { + evidence: RosterReadUnavailable { reason: reason as String }, + pages: [], + pages_read: 0, + disputed_ids: [], + note: "credential route refused" + } + OrgRunnerRouteAdmitted { route: _ } => { + let walk = roster_walk_pump( + listing: OrganizationRunnerListing { scope: organization_runner_scope(org: org), page_size: page_size }, + walk: roster_walk_begin(), + respond: fn(t) { gh_roster_transport(target: t) } + ) + classify_roster_walk(walk: walk, observed_on: clock_now_probed_at_or_unknown()) + } + } +} + +// THE PRODUCTION ROSTER. gunbc.apply and gunbc.runner_capacity_realize plan from this. +fn organization_runner_roster_live_read() -> OrganizationRunnerRosterEvidence { + organization_runner_roster_report(org: organization_runner_roster_org(), page_size: github_max_page_size).evidence +} + +// PER-ID PRESENCE, for a registration id captured elsewhere (a JIT mint, a local .runner file). +// A registration id means nothing outside its organization -- GitHub addresses the object by org AND +// id -- so the organization is a parameter and every arm carries it. +// +// Three answers and not two. Absent is the authority saying the registration does not exist IN AN +// ORGANIZATION THIS CREDENTIAL IS SHOWN ABLE TO SEE. A 404 alone does not establish that: GitHub also +// answers 404 for a scope the token cannot address. So a 404 reads as Absent only when the +// visibility evidence names the SAME organization, established under the admitted route by a +// runners listing that answered 200; visibility for any other organization, or none, is Unobserved. +// Access and completeness are separate facts: one first-page probe establishes access, and says +// nothing about whether the population was read in full. +// +// This is a READ. GitHub documents the listing and the single-runner GET under the same +// self-hosted-runners READ permission; neither establishes whether this credential may DELETE a +// registration, which is never probed by attempting one. +type OrganizationRunnerIdObservation + = RunnerIdPresent { org: NonEmptyStr, registration: GitHubSelfHostedRunnerRegistration } + | RunnerIdAbsent { org: NonEmptyStr, runner_id: Int } + | RunnerIdUnobserved { org: NonEmptyStr, runner_id: Int, reason: NonEmptyStr } + +type OrganizationRunnerVisibility + = RunnersVisible { org: NonEmptyStr, route: OrgRunnerReadRoute } + | RunnersVisibilityUnestablished { org: NonEmptyStr, reason: NonEmptyStr } + +fn absence_established_for(visibility: OrganizationRunnerVisibility, org: NonEmptyStr) -> NonEmptyStr? { + match visibility { + RunnersVisible { org: seen, route: _ } => + if (seen as String) == (org as String) { + none + } else { + Present { value: join(["HTTP 404, but the visibility evidence is for organization ", seen as String, ", not ", org as String], "") as NonEmptyStr } + } + RunnersVisibilityUnestablished { org: _, reason } => + Present { value: concat("HTTP 404, but this credential was not shown able to list the organization's runners: ", reason as String) as NonEmptyStr } + } +} + +fn runner_id_observation_from_included( + org: NonEmptyStr, + visibility: OrganizationRunnerVisibility, + runner_id: Int, + stdout: String, + stderr: String, +) -> OrganizationRunnerIdObservation { + match parse_gh_api_included(stdout: stdout) { + GhApiIncludedUnreadable { reason } => + RunnerIdUnobserved { org: org, runner_id: runner_id, reason: join([reason as String, "; gh stderr: ", stderr], "") as NonEmptyStr } + GhApiIncludedParsed { response } => + if response.status == 404 { + match absence_established_for(visibility: visibility, org: org) { + Absent => RunnerIdAbsent { org: org, runner_id: runner_id } + Present { value: why } => RunnerIdUnobserved { org: org, runner_id: runner_id, reason: why } + } + } else if response.status != 200 { + RunnerIdUnobserved { org: org, runner_id: runner_id, reason: join(["runner read answered HTTP ", to_string(response.status), "; gh stderr: ", stderr], "") as NonEmptyStr } + } else { + match parse_json_document(s: response.body) { + JsonDocumentUnreadable { gap } => + RunnerIdUnobserved { org: org, runner_id: runner_id, reason: concat("runner response is ", json_document_gap_text(gap: gap)) as NonEmptyStr } + JsonDocumentParsed { value: doc } => match decode_runner_registration(v: doc) { + RunnerRegistrationDecodeRefused { reason } => RunnerIdUnobserved { org: org, runner_id: runner_id, reason: reason } + RunnerRegistrationDecoded { registration } => + if to_string(registration.id) == to_string(runner_id) { + RunnerIdPresent { org: org, registration: registration } + } else { + RunnerIdUnobserved { org: org, runner_id: runner_id, reason: "the runner read returned a different id than the one requested" } + } + } + } + } + } +} + +// VISIBILITY IS TAKEN FROM THE ROSTER READ THE CALLER ALREADY MADE, NOT FROM A SECOND LISTING. +// Whether this credential can list the organization's runners is one fact; the roster walk's first +// page establishes it, so the per-ID join consumes that reading rather than paying for, and possibly +// disagreeing with, a second request. A roster that delivered no page establishes nothing. +fn runner_visibility_from_report(org: NonEmptyStr, route: OrgRunnerReadRoute, report: OrganizationRunnerRosterReport) -> OrganizationRunnerVisibility { + if report.pages_read > 0 { + RunnersVisible { org: org, route: route } + } else { + RunnersVisibilityUnestablished { + org: org, + reason: concat("the roster read delivered no page: ", report.note) as NonEmptyStr + } + } +} + +// ONE GET PER ID, joined against visibility the caller established for the same organization. +fn organization_runner_id_observations( + org: NonEmptyStr, + visibility: OrganizationRunnerVisibility, + runner_ids: List, +) -> List { + match organization_runner_read_route() { + OrgRunnerRouteRefused { reason } => map(runner_ids, id => RunnerIdUnobserved { org: org, runner_id: id, reason: reason }) + OrgRunnerRouteAdmitted { route: _ } => map(runner_ids, id => { + let response = github.CliOrgSelfHostedRunners.GetOrgRunner(org: org, runner_id: id) + runner_id_observation_from_included(org: org, visibility: visibility, runner_id: id, stdout: response.stdout, stderr: response.stderr) + }) + } +} + +// THE FLEET-CONVERGE OBSERVE ENTRY (mode org_runner_roster_observe). It reads the roster under the +// in-run token, observes any captured registration ids the dispatcher names in +// GUNBC_RUNNER_ID_PROBES (comma-separated), writes a receipt of what was established, and exits +// non-zero unless the roster was Observed in full -- a Partial is evidence worth keeping and is +// still not a green read. +data organization_runner_roster_receipt_path: String = "target/org-runner-roster-receipt.txt" +data runner_id_probes_env_name: NonEmptyStr = "GUNBC_RUNNER_ID_PROBES" + +fn roster_evidence_arm(e: OrganizationRunnerRosterEvidence) -> String { + match e { + RosterObserved { observed_on: _, registrations: regs } => concat("observed count=", to_string(length(xs: regs))) + RosterObservedPartial { observed_on: _, registrations: _, observed_count: seen, authority_total: total } => + join(["partial observed_count=", to_string(seen), " authority_total=", to_string(total)], "") + RosterReadUnavailable { reason: r } => concat("unavailable reason=", r) + } +} + +// ONE LINE PER REGISTRATION THE READ KEPT, so a reader of the receipt can answer "is this id in the +// organization" from the observation itself rather than from a count. +fn roster_registration_lines(e: OrganizationRunnerRosterEvidence) -> String { + let regs = match e { + RosterObserved { observed_on: _, registrations: rs } => rs + RosterObservedPartial { observed_on: _, registrations: rs, observed_count: _, authority_total: _ } => rs + RosterReadUnavailable { reason: _ } => [] + } + join(map(regs, r => join([ + "registration id=", to_string(r.id), + " name=", r.name as String, + " status=", match r.status { GitHubRunnerOnline => "online", GitHubRunnerOffline => "offline" }, + " busy=", if r.busy { "true" } else { "false" }, + "\n" + ], "")), "") +} + +// A MALFORMED PROBE ID REFUSES THE STEP. Dropping an unparsable token would give a receipt with +// one probe line fewer than the dispatcher asked for and no trace of the loss. +type RunnerIdProbeParse + = RunnerIdProbesParsed { ids: List } + | RunnerIdProbesUnparsable { tokens: List } + +fn parse_runner_id_probes(raw: String) -> RunnerIdProbeParse { + let tokens = filter(map(split(s: raw, delimiter: ","), t => trim(s: t)), t => t != "") + let bad = filter(tokens, t => match parse_int(s: t) { Present { value: n } => n < 0, Absent => true }) + if length(xs: bad) > 0 { + RunnerIdProbesUnparsable { tokens: bad } + } else { + RunnerIdProbesParsed { ids: flat_map(tokens, t => match parse_int(s: t) { Present { value: n } => [n], Absent => [] }) } + } +} + +fn runner_id_probe_request() -> RunnerIdProbeParse { + match shell.Env.Get(name: runner_id_probes_env_name).value { + Absent => RunnerIdProbesParsed { ids: [] } + Present { value: raw } => parse_runner_id_probes(raw: raw) + } +} + +fn runner_id_observation_line(o: OrganizationRunnerIdObservation) -> String { + match o { + RunnerIdPresent { org: g, registration: r } => join(["probe id=", to_string(r.id), " org=", g as String, " present name=", r.name as String, "\n"], "") + RunnerIdAbsent { org: g, runner_id: id } => join(["probe id=", to_string(id), " org=", g as String, " absent\n"], "") + RunnerIdUnobserved { org: g, runner_id: id, reason: why } => join(["probe id=", to_string(id), " org=", g as String, " unobserved reason=", why as String, "\n"], "") + } +} + +// THE COMMAND'S TERMINAL. Green only when the roster is complete AND every requested probe reached a +// concrete answer: present and absent are both observations; unobserved is not. A complete listing +// does not excuse a requested subject that was never observed. +type RosterObserveTerminal + = RosterObserveComplete + | RosterObserveRosterPartial + | RosterObserveRosterUnavailable { reason: String } + | RosterObserveProbesUnobserved { runner_ids: List } + +fn unobserved_probe_ids(probes: List) -> List { + flat_map(probes, o => match o { + RunnerIdUnobserved { org: _, runner_id: id, reason: _ } => [id] + RunnerIdPresent { org: _, registration: _ } => [] + RunnerIdAbsent { org: _, runner_id: _ } => [] + }) +} + +fn roster_observe_terminal(evidence: OrganizationRunnerRosterEvidence, probes: List) -> RosterObserveTerminal { + match evidence { + RosterReadUnavailable { reason: r } => RosterObserveRosterUnavailable { reason: r } + RosterObservedPartial { observed_on: _, registrations: _, observed_count: _, authority_total: _ } => RosterObserveRosterPartial + RosterObserved { observed_on: _, registrations: _ } => { + let missing = unobserved_probe_ids(probes: probes) + if length(xs: missing) > 0 { RosterObserveProbesUnobserved { runner_ids: missing } } else { RosterObserveComplete } + } + } +} + +// THE PAGE SIZE THIS RUN ASKS FOR: the authority's maximum unless the dispatcher names a smaller one, +// which is how a real continuation is exercised against a population that fits one full page. +data roster_page_size_env_name: NonEmptyStr = "GUNBC_RUNNER_ROSTER_PAGE_SIZE" + +type RosterPageSizeRequest + = RosterPageSizeRequested { size: PageSize } + | RosterPageSizeRefused { reason: NonEmptyStr } + +fn roster_page_size_from(raw: String?) -> RosterPageSizeRequest { + match raw { + Absent => RosterPageSizeRequested { size: github_max_page_size } + Present { value: text } => + if trim(s: text) == "" { + RosterPageSizeRequested { size: github_max_page_size } + } else { + match parse_int(s: trim(s: text)) { + Absent => RosterPageSizeRefused { reason: concat("roster_page_size is not an integer: ", text) as NonEmptyStr } + Present { value: n } => + if n < 1 { + RosterPageSizeRefused { reason: concat("roster_page_size must be positive: ", text) as NonEmptyStr } + } else { + match admit_page_size(requested: PageSize { count: n }, maximum: github_max_page_size) { + PageSizeAdmitted { size } => RosterPageSizeRequested { size: size } + PageSizeAboveAuthorityMaximum { requested: _, maximum: _ } => + RosterPageSizeRefused { reason: concat("roster_page_size exceeds the authority's maximum page size: ", text) as NonEmptyStr } + } + } + } + } + } +} + +fn roster_page_lines(pages: List) -> String { + join(map(pages, p => join([ + "page target=", p.target, + " total_count=", to_string(p.total_count), + " link=", if p.link_field == "" { "(none)" } else { p.link_field }, + " ids=", join(map(p.registrations, r => to_string(r.id)), ","), + "\n" + ], "")), "") +} + +fn organization_runner_roster_observe_wet() -> ProcessExit { + match runner_id_probe_request() { + RunnerIdProbesUnparsable { tokens } => + exit_failure(reason: concat("OrgRunnerIdProbesUnparsable: runner_ids carries tokens that are not non-negative integer ids: ", join(tokens, ","))) + RunnerIdProbesParsed { ids } => match roster_page_size_from(raw: shell.Env.Get(name: roster_page_size_env_name).value) { + RosterPageSizeRefused { reason } => exit_failure(reason: concat("OrgRunnerRosterPageSizeRefused: ", reason as String)) + RosterPageSizeRequested { size } => organization_runner_roster_observe_with_probes(runner_ids: ids, page_size: size) + } + } +} + +fn organization_runner_roster_observe_with_probes(runner_ids: List, page_size: PageSize) -> ProcessExit { + let org = organization_runner_roster_org() + let route = organization_runner_read_route() + let report = organization_runner_roster_report(org: org, page_size: page_size) + let probes = match route { + OrgRunnerRouteRefused { reason } => organization_runner_id_observations(org: org, visibility: RunnersVisibilityUnestablished { org: org, reason: reason }, runner_ids: runner_ids) + OrgRunnerRouteAdmitted { route: r } => organization_runner_id_observations(org: org, visibility: runner_visibility_from_report(org: org, route: r, report: report), runner_ids: runner_ids) + } + let receipt = join([ + "endpoint=GET-/orgs/\{org\}/actions/runners organization=", org as String, + " page_size=", to_string(page_size.count), + " pages_read=", to_string(report.pages_read), + " roster=", roster_evidence_arm(e: report.evidence), + " disputed_ids=", join(report.disputed_ids, ","), + " note=", report.note, "\n" + ], "") + let wrote = Filesystem.Write( + path: organization_runner_roster_receipt_path, + content: join([ + receipt, + roster_page_lines(pages: report.pages), + roster_registration_lines(e: report.evidence), + join(map(probes, o => runner_id_observation_line(o: o)), "") + ], "") + ) + if !wrote.success { + exit_failure(reason: concat("OrgRunnerRosterReceiptWriteRefused: ", wrote.error)) + } else { + match roster_observe_terminal(evidence: report.evidence, probes: probes) { + RosterObserveComplete => ExitSuccess + RosterObserveRosterPartial => exit_failure(reason: concat("OrgRunnerRosterPartial: ", receipt)) + RosterObserveRosterUnavailable { reason: r } => exit_failure(reason: concat("OrgRunnerRosterUnavailable: ", r)) + RosterObserveProbesUnobserved { runner_ids: ids } => + exit_failure(reason: concat("OrgRunnerIdProbesUnobserved: requested runner ids with no concrete answer: ", join(map(ids, i => to_string(i)), ","))) + } + } +} diff --git a/dag/gunbc/runner/runner_capacity_plan.dag b/dag/gunbc/runner/runner_capacity_plan.dag index 37838d83f7a..5381680fb56 100644 --- a/dag/gunbc/runner/runner_capacity_plan.dag +++ b/dag/gunbc/runner/runner_capacity_plan.dag @@ -50,10 +50,10 @@ import gunbc.runner_slot_allocation { // plan can never add one. Additions are the desired members not represented by the OBSERVED GitHub // population, which is now DECODED FROM THE ROSTER by observed_github_slot_identities rather than // synthesized as a dense local prefix — the previous form measured local systemd incarnations and -// handed them to a check about GitHub registrations, two populations behind one name. The remaining -// rung is one layer further out and is a different fact: organization_runner_roster_evidence is a -// declared Scaffold returning the pinned 2026-08-06 incident corpus, so the decode is real and -// total while the observation it decodes is still a constant. Recorded in +// handed them to a check about GitHub registrations, two populations behind one name. The observation +// it decodes is the live organization read, gunbc.fleet.organization_runner_roster_read +// organization_runner_roster_live_read, at both production call sites; a caller without the +// org-admin credential receives RosterReadUnavailable and the plan refuses. Recorded in // slot_enumeration_narrow_class_note. The admission ceiling DERIVES from // gunbc.runner_slot_allocation rather than restating it, so a width no host is admitted to carry is // refused instead of admitted and left unfulfilled. The plan is not a circular controller that @@ -118,13 +118,11 @@ data slot_enumeration_narrow_class_note: String = "ERROR CLASS: A PLANNER'S ENUM // not staleness but CATEGORY: local systemd incarnations were standing in for GitHub registrations, // two populations behind one name, undetectable because both are lists of slot identities. // -// THIS DECODES AN OBSERVATION THAT IS NOT YET AN OBSERVATION, AND SAYING SO HERE IS THE OBLIGATION. -// The roster reaching this function comes from organization_runner_roster_evidence, a declared -// Scaffold returning the pinned 2026-08-06 incident corpus. So the planner does NOT observe GitHub -// today; it decodes a constant, correctly. That is worth landing -- a correct decoder over a fixture -// becomes a correct decoder over reality on the day the producer lands, and the category error is -// closed either way -- but a reader who takes "the planner observes GitHub" from this code will be -// wrong until that Scaffold dissolves. The gap count went from two to one, not from one to zero. +// THIS DECODES A LIVE OBSERVATION WHERE THE CALLER HOLDS THE CREDENTIAL, AND NOTHING WHERE IT DOES +// NOT. The roster reaching this function in production is gunbc.fleet.organization_runner_roster_read +// organization_runner_roster_live_read: the paginated org read under the in-run gunbai-ci installation +// token, or RosterReadUnavailable without it. The 2026-08-06 incident corpus survives only as the +// witness fixture gunbc.fleet_runner_connectivity incident_2026_08_06_roster_fixture. // // AND THE NAME FORK BELOW IS A PRECONDITION FOR THAT DAY, NOT AN ADJACENT TIDY-UP. Against a fixture // whose names were authored to match, three name builders agreeing is trivially true. Against a live diff --git a/dag/gunbc/runner/runner_capacity_realize.dag b/dag/gunbc/runner/runner_capacity_realize.dag index 8dfe5a03297..e08e4f6623d 100644 --- a/dag/gunbc/runner/runner_capacity_realize.dag +++ b/dag/gunbc/runner/runner_capacity_realize.dag @@ -17,7 +17,7 @@ import gunbc.runner_capacity_plan { runner_capacity_add_count, addition_actions, } -import gunbc.fleet_runner_connectivity { organization_runner_roster_evidence } +import gunbc.fleet.organization_runner_roster_read { organization_runner_roster_live_read } import gunbc.runner_lifecycle { runner_width_disk_preflight, DiskHeadroomSufficient, @@ -264,7 +264,7 @@ fn runner_capacity_apply_wet_exit(srv3_width: Int, srv4_width: Int) -> ProcessEx let plan = runner_capacity_plan( srv3_width: srv3_width, srv4_width: srv4_width, - roster: organization_runner_roster_evidence(), + roster: organization_runner_roster_live_read(), ) let outcome = runner_capacity_apply_plan(plan: plan) if runner_capacity_apply_any_host_refused(outcome: outcome) { diff --git a/dag/test/claim/fleet/fleet_runner_connectivity_witness_test.dag b/dag/test/claim/fleet/fleet_runner_connectivity_witness_test.dag index 5419f3f9da8..8863f8ca114 100644 --- a/dag/test/claim/fleet/fleet_runner_connectivity_witness_test.dag +++ b/dag/test/claim/fleet/fleet_runner_connectivity_witness_test.dag @@ -8,7 +8,7 @@ import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3, operator_host_srv4, } import gunbc.fleet_runner_connectivity { - organization_runner_roster_evidence, + incident_2026_08_06_roster_fixture, runner_slot_connectivity_verdict, runner_slot_connectivity_verdict_for_incarnation, connectivity_verdicts_for_active_incarnations, @@ -139,13 +139,13 @@ test fn materialized_and_unit_active_widths_are_twenty_not_desired() -> Bool { } test fn detached_count_matches_offline_registrations_on_active_slots() -> Bool { - let roster = organization_runner_roster_evidence() + let roster = incident_2026_08_06_roster_fixture() let verdicts = connectivity_verdicts_for_active_incarnations(roster: roster) count_matching(verdicts: verdicts, matched: v => connectivity_verdict_is_detached(v: v)) == 15 } test fn healthy_count_is_one_and_orphaned_count_is_four() -> Bool { - let roster = organization_runner_roster_evidence() + let roster = incident_2026_08_06_roster_fixture() let verdicts = connectivity_verdicts_for_active_incarnations(roster: roster) count_matching(verdicts: verdicts, matched: v => connectivity_verdict_is_healthy(v: v)) == 1 && count_matching(verdicts: verdicts, matched: v => connectivity_verdict_is_orphaned(v: v)) == 4 @@ -276,8 +276,8 @@ fn online_srv4_01_under(labels: List) -> RunnerSlotConnectivityVerd name: "srv4-01" as NonEmptyStr, status: GitHubRunnerOnline, busy: false, - ephemeral: true, - version: "2.334.0" as NonEmptyStr, + ephemeral: Present { value: true }, + version: Present { value: "2.334.0" as NonEmptyStr }, os: "Linux" as NonEmptyStr, labels: labels, }, diff --git a/dag/test/claim/fleet/organization_runner_roster_read_witness_test.dag b/dag/test/claim/fleet/organization_runner_roster_read_witness_test.dag new file mode 100644 index 00000000000..d97de7db8a2 --- /dev/null +++ b/dag/test/claim/fleet/organization_runner_roster_read_witness_test.dag @@ -0,0 +1,441 @@ +module test.claim.organization_runner_roster_read_witness + +import std.logic { Bool } +import std.types { String, NonEmptyStr, Int, List } +import std.page_fold { ContinuationClosed, ContinuationNext, LinkNext } +import extdeps.uri { uri_from_wire } +import extdeps.github.pagination { github_first_page_cursor, github_max_page_size } +import std.page_fold { PageSize } +import extdeps.github.cli_api_include { GhApiIncludedParsed, GhApiIncludedUnreadable, parse_gh_api_included } +import extdeps.github.ci_runner { GitHubSelfHostedRunnerRegistration } +import gunbc.fleet_runner_connectivity { RosterObserved, RosterObservedPartial, RosterReadUnavailable, OrganizationRunnerRosterEvidence } +import gunbc.fleet.organization_runner_roster_read { + OrganizationRunnerScope, + OrganizationRunnerListing, + org_runner_route_from_bindings, + OrgRunnerRouteAdmitted, + OrgRunnerRouteRefused, + roster_observe_terminal, + RosterObserveComplete, + RosterObserveRosterPartial, + RosterObserveRosterUnavailable, + RosterObserveProbesUnobserved, + roster_page_size_from, + RosterPageSizeRequested, + RosterPageSizeRefused, + RosterTransportResponse, + RosterRequestAt, + RosterRequestRefused, + GhTokenFromOrgAdminPrelude, + roster_request_target, + roster_walk_begin, + roster_walk_pump, + RosterPageDelivered, + RosterPageFailed, + RunnerIdPresent, + RunnerIdAbsent, + RunnerIdUnobserved, + RunnersVisible, + RunnersVisibilityUnestablished, + roster_page_read_from_included, + classify_roster_walk, + parse_runner_id_probes, + RunnerIdProbesParsed, + RunnerIdProbesUnparsable, + runner_visibility_from_report, + OrganizationRunnerRosterReport, + runner_id_observation_from_included +} +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// EVERY INPUT BELOW IS A SUPPLIED `gh api --include` STDOUT, shaped as gh printed it on 2026-09-19 +// (status line, headers, blank line, body). These claims discriminate at the response boundary; +// the network route is exercised by the fleet-converge mode org_runner_roster_observe. + +data runner_one_json: String = "\{\"id\":11,\"name\":\"srv3-01\",\"os\":\"Linux\",\"status\":\"online\",\"busy\":false,\"ephemeral\":true,\"labels\":[\{\"name\":\"self-hosted\"}]}" +data runner_two_json: String = "\{\"id\":12,\"name\":\"srv3-02\",\"os\":\"Linux\",\"status\":\"offline\",\"busy\":false,\"labels\":[\{\"name\":\"self-hosted\"}]}" + +fn page_body(total: Int, runners: List) -> String { + join(["\{\"total_count\":", to_string(total), ",\"runners\":[", join(runners, ","), "]}"], "") +} + +fn included(status_line: String, link: String, body: String) -> String { + let head = if link == "" { [status_line, "Content-Type: application/json"] } else { [status_line, concat("Link: ", link), "Content-Type: application/json"] } + join(concat(head, ["", body]), "\n") +} + +data page_two_url: String = "https://api.github.com/organizations/1/actions/runners?per_page=100&page=2" +data other_org_url: String = "https://api.github.com/orgs/other-org/actions/runners?per_page=100&page=2" +data repo_runners_url: String = "https://api.github.com/repos/gunb-ai/gunbc/actions/runners?per_page=100&page=2" + +fn next_link_to(url: String) -> String { + join(["<", url, ">; rel=\"next\""], "") +} + +data next_link: String = next_link_to(url: page_two_url) + +test fn a_page_with_a_next_link_continues() -> Bool { + match roster_page_read_from_included(target: first_target, stdout: included(status_line: "HTTP/2.0 200 OK", link: next_link, body: page_body(total: 2, runners: [runner_one_json])), stderr: "") { + RosterPageDelivered { page, continuation: ContinuationNext { cursor: LinkNext { target: _ } } } => page.total_count == 2 && length(xs: page.registrations) == 1 + _ => false + } +} + +test fn a_page_without_a_link_header_closes_the_listing() -> Bool { + match roster_page_read_from_included(target: first_target, stdout: included(status_line: "HTTP/2.0 200 OK", link: "", body: page_body(total: 1, runners: [runner_one_json])), stderr: "") { + RosterPageDelivered { page: _, continuation: ContinuationClosed } => true + _ => false + } +} + +test fn an_optional_field_absent_upstream_stays_absent() -> Bool { + match roster_page_read_from_included(target: first_target, stdout: included(status_line: "HTTP/2.0 200 OK", link: "", body: page_body(total: 1, runners: [runner_two_json])), stderr: "") { + RosterPageDelivered { page, continuation: _ } => match page.registrations { + Cons { head: r, tail: _ } => match r.ephemeral { Absent => true, Present { value: _ } => false } + Empty {} => false + } + _ => false + } +} + +test fn a_forbidden_page_is_a_failure_not_an_empty_page() -> Bool { + match roster_page_read_from_included(target: first_target, stdout: included(status_line: "HTTP/2.0 403 Forbidden", link: "", body: "\{\"message\":\"Resource not accessible by integration\"}"), stderr: "gh: Resource not accessible by integration (HTTP 403)") { + RosterPageFailed { cause: _ } => true + RosterPageDelivered { page: _, continuation: _ } => false + } +} + +test fn output_without_a_status_line_observed_no_response() -> Bool { + match parse_gh_api_included(stdout: "error connecting to api.github.com") { + GhApiIncludedUnreadable { reason: _ } => true + GhApiIncludedParsed { response: _ } => false + } +} + +// OR-1: A CONTINUATION IS ADMITTED ONLY AS THIS ORGANIZATION'S RUNNERS LISTING. + +fn listing_of(scope: OrganizationRunnerScope) -> OrganizationRunnerListing { + OrganizationRunnerListing { scope: scope, page_size: github_max_page_size } +} + +data scope_with_id: OrganizationRunnerScope = OrganizationRunnerScope { login: "gunb-ai" as NonEmptyStr, id: Present { value: 1 } } +data scope_without_id: OrganizationRunnerScope = OrganizationRunnerScope { login: "gunb-ai" as NonEmptyStr, id: none } + +fn admitted(scope: OrganizationRunnerScope, url: String) -> Bool { + match uri_from_wire(url: url) { + Absent => false + Present { value: u } => match roster_request_target(listing: listing_of(scope: scope), cursor: LinkNext { target: u }) { + RosterRequestAt { target: _ } => true + RosterRequestRefused { cause: _ } => false + } + } +} + +test fn the_id_alias_of_this_organization_is_admitted() -> Bool { + admitted(scope: scope_with_id, url: page_two_url) + && admitted(scope: scope_with_id, url: "https://api.github.com/orgs/gunb-ai/actions/runners?per_page=100&page=2") +} + +test fn an_uncorroborated_id_alias_is_refused() -> Bool { + !admitted(scope: scope_without_id, url: page_two_url) + && !admitted(scope: OrganizationRunnerScope { login: "gunb-ai" as NonEmptyStr, id: Present { value: 2 } }, url: page_two_url) +} + +test fn another_organization_or_resource_is_refused() -> Bool { + !admitted(scope: scope_with_id, url: other_org_url) && !admitted(scope: scope_with_id, url: repo_runners_url) +} + +// THE WALK, DRIVEN THROUGH THE REAL PUMP: admission, framing, decode and std.page_fold, over +// responses supplied per request target. +fn respond_from(pages: List) -> fn(NonEmptyStr) -> RosterTransportResponse { + fn(t) { + fold(pages, init: RosterTransportResponse { stdout: "no scripted response", stderr: "" }, f: fn(acc, p) { + if (t as String) == p.target { RosterTransportResponse { stdout: p.stdout, stderr: "" } } else { acc } + }) + } +} + +type ScriptedPage { + target: String + stdout: String +} + +data first_target: String = "orgs/gunb-ai/actions/runners?per_page=100" + +fn ok(link: String, total: Int, runners: List) -> String { + included(status_line: "HTTP/2.0 200 OK", link: link, body: page_body(total: total, runners: runners)) +} + +fn walk_evidence(scope: OrganizationRunnerScope, pages: List) -> OrganizationRunnerRosterReport { + classify_roster_walk( + walk: roster_walk_pump(listing: listing_of(scope: scope), walk: roster_walk_begin(), respond: respond_from(pages: pages)), + observed_on: "2026-09-19" as NonEmptyStr + ) +} + +data runner_one_offline_json: String = "\{\"id\":11,\"name\":\"srv3-01\",\"os\":\"Linux\",\"status\":\"offline\",\"busy\":false,\"ephemeral\":true,\"labels\":[\{\"name\":\"self-hosted\"}]}" + +test fn a_two_page_listing_that_closes_is_observed() -> Bool { + match walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: next_link, total: 2, runners: [runner_one_json]) }, + ScriptedPage { target: page_two_url, stdout: ok(link: "", total: 2, runners: [runner_two_json]) }, + ]).evidence { + RosterObserved { observed_on: _, registrations: regs } => length(xs: regs) == 2 + _ => false + } +} + +// OR-1 AT THE WALK: another organization's page cannot complete this roster, although its rows would +// make totals and distinct ids agree. +test fn a_continuation_into_another_organization_contributes_no_members() -> Bool { + match walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: next_link_to(url: other_org_url), total: 2, runners: [runner_one_json]) }, + ScriptedPage { target: other_org_url, stdout: ok(link: "", total: 2, runners: [runner_two_json]) }, + ]).evidence { + RosterObservedPartial { observed_on: _, registrations: regs, observed_count: seen, authority_total: total } => length(xs: regs) == 1 && seen == 1 && total == 2 + _ => false + } +} + +// OR-3: one id read twice across a real two-page progression with DISAGREEING payloads. Distinct ids +// (2) still meet the total (2); the disagreement must make the id disputed, not first-wins. +test fn a_disagreeing_repeat_across_pages_is_disputed_not_first_wins() -> Bool { + let report = walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: next_link, total: 2, runners: [runner_one_offline_json]) }, + ScriptedPage { target: page_two_url, stdout: ok(link: "", total: 2, runners: [runner_one_json, runner_two_json]) }, + ]) + let disputed = report.disputed_ids == ["11"] + disputed && match report.evidence { + RosterObservedPartial { observed_on: _, registrations: regs, observed_count: _, authority_total: _ } => + !any(regs, r => to_string(r.id) == "11") && length(xs: regs) == 1 + _ => false + } +} + +test fn an_agreeing_repeat_across_pages_deduplicates() -> Bool { + match walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: next_link, total: 2, runners: [runner_one_json]) }, + ScriptedPage { target: page_two_url, stdout: ok(link: "", total: 2, runners: [runner_one_json, runner_two_json]) }, + ]).evidence { + RosterObserved { observed_on: _, registrations: regs } => length(xs: regs) == 2 + _ => false + } +} + +// A REPEATED ROW IS NOT COVERAGE: two rows, one distinct id, against a total of two. +test fn a_repeated_runner_id_does_not_count_toward_completeness() -> Bool { + match walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: "", total: 2, runners: [runner_one_json, runner_one_json]) }, + ]).evidence { + RosterObservedPartial { observed_on: _, registrations: _, observed_count: seen, authority_total: total } => seen == 1 && total == 2 + _ => false + } +} + +// A LATER PAGE FAILING KEEPS THE EARLIER MEMBERS AS PARTIAL -- never a population-wide absence. +test fn a_failed_later_page_keeps_earlier_members_as_partial() -> Bool { + match walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: next_link, total: 2, runners: [runner_one_json]) }, + ScriptedPage { target: page_two_url, stdout: included(status_line: "HTTP/2.0 502 Bad Gateway", link: "", body: "\{}") }, + ]).evidence { + RosterObservedPartial { observed_on: _, registrations: regs, observed_count: seen, authority_total: total } => length(xs: regs) == 1 && seen == 1 && total == 2 + _ => false + } +} + +test fn a_failed_first_page_is_unavailable() -> Bool { + match walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: included(status_line: "HTTP/2.0 403 Forbidden", link: "", body: "\{}") }, + ]).evidence { + RosterReadUnavailable { reason: _ } => true + _ => false + } +} + +// PER-ID: 404 IS ABSENCE ONLY WHERE THE SAME ORGANIZATION WAS SHOWN LISTABLE UNDER THE ADMITTED ROUTE. + +data not_found: String = "HTTP/2.0 404 Not Found\nContent-Type: application/json\n\n\{\"message\":\"Not Found\"}" +data gunb_ai: NonEmptyStr = "gunb-ai" + +test fn a_404_under_an_established_listing_is_absent() -> Bool { + match runner_id_observation_from_included(org: gunb_ai, visibility: RunnersVisible { org: gunb_ai, route: GhTokenFromOrgAdminPrelude {} }, runner_id: 11, stdout: not_found, stderr: "gh: Not Found (HTTP 404)") { + RunnerIdAbsent { org: o, runner_id: id } => (o as String) == "gunb-ai" && id == 11 + _ => false + } +} + +test fn a_404_without_an_established_listing_is_unobserved() -> Bool { + match runner_id_observation_from_included(org: gunb_ai, visibility: RunnersVisibilityUnestablished { org: gunb_ai, reason: "org runners page answered HTTP 403" as NonEmptyStr }, runner_id: 11, stdout: not_found, stderr: "gh: Not Found (HTTP 404)") { + RunnerIdUnobserved { org: _, runner_id: id, reason: _ } => id == 11 + _ => false + } +} + +// OR-2: visibility established for ANOTHER organization cannot make this one's 404 an absence. +test fn visibility_for_another_organization_cannot_establish_absence() -> Bool { + match runner_id_observation_from_included(org: gunb_ai, visibility: RunnersVisible { org: "other-org" as NonEmptyStr, route: GhTokenFromOrgAdminPrelude {} }, runner_id: 11, stdout: not_found, stderr: "gh: Not Found (HTTP 404)") { + RunnerIdUnobserved { org: o, runner_id: _, reason: _ } => (o as String) == "gunb-ai" + _ => false + } +} + +test fn a_200_for_the_requested_id_is_present_in_its_org() -> Bool { + match runner_id_observation_from_included(org: gunb_ai, visibility: RunnersVisible { org: gunb_ai, route: GhTokenFromOrgAdminPrelude {} }, runner_id: 11, stdout: included(status_line: "HTTP/2.0 200 OK", link: "", body: runner_one_json), stderr: "") { + RunnerIdPresent { org: o, registration: r } => (o as String) == "gunb-ai" && (r.name as String) == "srv3-01" + _ => false + } +} + +test fn a_200_for_a_different_id_is_unobserved() -> Bool { + match runner_id_observation_from_included(org: gunb_ai, visibility: RunnersVisible { org: gunb_ai, route: GhTokenFromOrgAdminPrelude {} }, runner_id: 12, stdout: included(status_line: "HTTP/2.0 200 OK", link: "", body: runner_one_json), stderr: "") { + RunnerIdUnobserved { org: _, runner_id: _, reason: _ } => true + _ => false + } +} + +// A MALFORMED PROBE ID REFUSES; IT IS NEVER DROPPED. +test fn a_malformed_runner_id_token_refuses_rather_than_vanishing() -> Bool { + match parse_runner_id_probes(raw: "11, 1x2") { + RunnerIdProbesUnparsable { tokens } => tokens == ["1x2"] + RunnerIdProbesParsed { ids: _ } => false + } +} + +test fn well_formed_runner_id_tokens_parse_in_order() -> Bool { + match parse_runner_id_probes(raw: " 251463,91529 ") { + RunnerIdProbesParsed { ids } => ids == [251463, 91529] + RunnerIdProbesUnparsable { tokens: _ } => false + } +} + +// VISIBILITY COMES FROM THE ROSTER READ: a delivered page establishes it for that org; none does not. +test fn visibility_follows_the_roster_reads_first_page() -> Bool { + let read = walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: "", total: 1, runners: [runner_one_json]) }, + ]) + let failed = walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: included(status_line: "HTTP/2.0 403 Forbidden", link: "", body: "\{}") }, + ]) + let seen = match runner_visibility_from_report(org: gunb_ai, route: GhTokenFromOrgAdminPrelude {}, report: read) { + RunnersVisible { org: o, route: _ } => (o as String) == "gunb-ai" + RunnersVisibilityUnestablished { org: _, reason: _ } => false + } + let unseen = match runner_visibility_from_report(org: gunb_ai, route: GhTokenFromOrgAdminPrelude {}, report: failed) { + RunnersVisible { org: _, route: _ } => false + RunnersVisibilityUnestablished { org: _, reason: _ } => true + } + seen && unseen +} + +// OR-2: THE ROUTE REQUIRES ONE CREDENTIAL BOUND TWICE, NOT TWO PRESENT VALUES. +test fn the_same_credential_bound_twice_admits_the_route() -> Bool { + match org_runner_route_from_bindings(admin_token: Present { value: "token-a" }, gh_token: Present { value: "token-a" }) { + OrgRunnerRouteAdmitted { route: _ } => true + OrgRunnerRouteRefused { reason: _ } => false + } +} + +test fn mismatching_credential_bindings_refuse_the_route() -> Bool { + match org_runner_route_from_bindings(admin_token: Present { value: "token-a" }, gh_token: Present { value: "token-b" }) { + OrgRunnerRouteRefused { reason: r } => !string_contains(s: r as String, pattern: "token-a") && !string_contains(s: r as String, pattern: "token-b") + OrgRunnerRouteAdmitted { route: _ } => false + } +} + +test fn a_missing_binding_refuses_the_route() -> Bool { + let no_gh = match org_runner_route_from_bindings(admin_token: Present { value: "token-a" }, gh_token: none) { + OrgRunnerRouteRefused { reason: _ } => true + OrgRunnerRouteAdmitted { route: _ } => false + } + let no_admin = match org_runner_route_from_bindings(admin_token: none, gh_token: Present { value: "token-a" }) { + OrgRunnerRouteRefused { reason: _ } => true + OrgRunnerRouteAdmitted { route: _ } => false + } + no_gh && no_admin +} + +// OR-3: READINGS THAT A JOINED RENDERING WOULD CONFUSE ARE COMPARED AS TYPED VALUES. +data runner_one_label_joined_json: String = "\{\"id\":11,\"name\":\"srv3-01\",\"os\":\"Linux\",\"status\":\"online\",\"busy\":false,\"ephemeral\":true,\"labels\":[\{\"name\":\"blue,green\"}]}" +data runner_one_labels_split_json: String = "\{\"id\":11,\"name\":\"srv3-01\",\"os\":\"Linux\",\"status\":\"online\",\"busy\":false,\"ephemeral\":true,\"labels\":[\{\"name\":\"blue\"},\{\"name\":\"green\"}]}" +data runner_one_version_absent_json: String = "\{\"id\":11,\"name\":\"srv3-01\",\"os\":\"Linux\",\"status\":\"online\",\"busy\":false,\"ephemeral\":true,\"labels\":[\{\"name\":\"self-hosted\"}]}" +data runner_one_version_sentinel_json: String = "\{\"id\":11,\"name\":\"srv3-01\",\"os\":\"Linux\",\"status\":\"online\",\"busy\":false,\"ephemeral\":true,\"version\":\"version-unreported\",\"labels\":[\{\"name\":\"self-hosted\"}]}" + +fn disputed_across_two_pages(first: String, second: String) -> Bool { + let report = walk_evidence(scope: scope_with_id, pages: [ + ScriptedPage { target: first_target, stdout: ok(link: next_link, total: 2, runners: [first]) }, + ScriptedPage { target: page_two_url, stdout: ok(link: "", total: 2, runners: [second, runner_two_json]) }, + ]) + report.disputed_ids == ["11"] && match report.evidence { + RosterObservedPartial { observed_on: _, registrations: regs, observed_count: _, authority_total: _ } => + !any(regs, r => to_string(r.id) == "11") && any(regs, r => to_string(r.id) == "12") + _ => false + } +} + +test fn a_joined_label_and_split_labels_are_a_dispute() -> Bool { + disputed_across_two_pages(first: runner_one_label_joined_json, second: runner_one_labels_split_json) +} + +test fn an_absent_version_and_a_sentinel_spelled_version_are_a_dispute() -> Bool { + disputed_across_two_pages(first: runner_one_version_absent_json, second: runner_one_version_sentinel_json) +} + +// OR-4: THE COMMAND IS GREEN ONLY WHEN EVERY REQUESTED PROBE HAS A CONCRETE ANSWER. +data complete_roster: OrganizationRunnerRosterEvidence = RosterObserved { observed_on: "2026-09-19" as NonEmptyStr, registrations: [] } + +test fn no_probes_requested_on_a_complete_roster_is_complete() -> Bool { + match roster_observe_terminal(evidence: complete_roster, probes: []) { + RosterObserveComplete => true + _ => false + } +} + +test fn present_and_absent_probes_are_both_observations() -> Bool { + match roster_observe_terminal(evidence: complete_roster, probes: [RunnerIdAbsent { org: gunb_ai, runner_id: 91529 }]) { + RosterObserveComplete => true + _ => false + } +} + +test fn an_unobserved_probe_fails_a_complete_roster() -> Bool { + match roster_observe_terminal(evidence: complete_roster, probes: [ + RunnerIdAbsent { org: gunb_ai, runner_id: 91529 }, + RunnerIdUnobserved { org: gunb_ai, runner_id: 91513, reason: "runner read answered HTTP 502" as NonEmptyStr }, + ]) { + RosterObserveProbesUnobserved { runner_ids: ids } => ids == [91513] + _ => false + } +} + +test fn a_negative_runner_id_refuses() -> Bool { + match parse_runner_id_probes(raw: "251463,-4") { + RunnerIdProbesUnparsable { tokens } => tokens == ["-4"] + RunnerIdProbesParsed { ids: _ } => false + } +} + +test fn no_probe_input_requests_no_probes() -> Bool { + match parse_runner_id_probes(raw: " ") { + RunnerIdProbesParsed { ids } => length(xs: ids) == 0 + RunnerIdProbesUnparsable { tokens: _ } => false + } +} + +// PAGE SIZE: a smaller supported size is admitted; one above the authority's maximum refuses. +test fn a_smaller_page_size_is_admitted_and_an_oversized_one_refuses() -> Bool { + let fifty = match roster_page_size_from(raw: Present { value: "50" }) { + RosterPageSizeRequested { size } => size.count == 50 + RosterPageSizeRefused { reason: _ } => false + } + let too_big = match roster_page_size_from(raw: Present { value: "101" }) { + RosterPageSizeRefused { reason: _ } => true + RosterPageSizeRequested { size: _ } => false + } + let default_max = match roster_page_size_from(raw: none) { + RosterPageSizeRequested { size } => size.count == 100 + RosterPageSizeRefused { reason: _ } => false + } + fifty && too_big && default_max +} diff --git a/dag/test/claim/runner/runner_capacity_plan_witness_test.dag b/dag/test/claim/runner/runner_capacity_plan_witness_test.dag index ac18eb1f397..d75463cd503 100644 --- a/dag/test/claim/runner/runner_capacity_plan_witness_test.dag +++ b/dag/test/claim/runner/runner_capacity_plan_witness_test.dag @@ -27,8 +27,8 @@ import gunbc.runner_capacity_plan { PlanRefused, } import gunbc.fleet_runner_connectivity { - organization_runner_roster_evidence, - organization_runner_roster_managed_hosts_incident_evidence, + incident_2026_08_06_roster_fixture, + incident_2026_08_06_managed_roster_fixture, RosterReadUnavailable, RosterObserved, OrganizationRunnerRosterEvidence, @@ -87,7 +87,7 @@ test fn a_width_above_the_committed_ceiling_is_refused_not_silently_unfulfilled( let plan = runner_capacity_plan( srv3_width: gunbc_runner_slots_per_host(host: operator_host_srv3) + 1, srv4_width: gunbc_runner_slots_per_host(host: operator_host_srv4) + 1, - roster: organization_runner_roster_managed_hosts_incident_evidence() + roster: incident_2026_08_06_managed_roster_fixture() ) match plan { PlanReady { hosts: _ } => @@ -133,7 +133,7 @@ fn added_indices(actions: List) -> List { // SAME ROSTER MOVE AS THE ROW ABOVE: the managed-pair slice of the 2026-08-06 corpus, which carries // the holes this row is about ({01,02,04,05} on both hosts) and nothing about srv2. test fn at_the_committed_width_additions_follow_purpose_not_count() -> Bool { - let roster = organization_runner_roster_managed_hosts_incident_evidence() + let roster = incident_2026_08_06_managed_roster_fixture() let plan = runner_capacity_plan(srv3_width: 6, srv4_width: 6, roster: roster) match plan { PlanReady { hosts: _ } => @@ -149,10 +149,10 @@ test fn at_the_committed_width_additions_follow_purpose_not_count() -> Bool { } } -// THE ONE CLAIM THAT STILL RUNS THE REAL PRODUCER, AND WHAT IT ESTABLISHES IS A REFUSAL. This is -// the pairing obligation for the supplied slices above (DESIGN 3): organization_runner_roster_evidence -// is what gunbc.apply and gunbc.runner_capacity_realize hand the planner, and against the fleet as -// committed today the planner REFUSES it, naming srv2-01. srv2's commitment is unestablished since +// THE WHOLE 2026-08-06 CORPUS, AND WHAT IT ESTABLISHES IS A REFUSAL. incident_2026_08_06_roster_fixture +// was what gunbc.apply and gunbc.runner_capacity_realize handed the planner until they moved to the +// live read (gunbc.fleet.organization_runner_roster_read); against the fleet as committed today the +// planner REFUSES this corpus, naming srv2-01. srv2's commitment is unestablished since // the 26/25 GiB transition, so its three 2026-08-06 registrations resolve to no committed slot, and // a registration the fleet cannot account for is refused rather than dropped // (a_registration_naming_no_committed_slot_refuses_instead_of_being_dropped is the srv9-01 form of @@ -164,7 +164,7 @@ test fn at_the_committed_width_additions_follow_purpose_not_count() -> Bool { // resolves again and the row is re-derived by the change that re-establishes srv2, not by editing // the 2026-08-06 rows to fit the fleet of the day. test fn the_whole_org_corpus_is_refused_while_srv2_holds_unaccounted_registrations() -> Bool { - match runner_capacity_plan(srv3_width: 6, srv4_width: 6, roster: organization_runner_roster_evidence()) { + match runner_capacity_plan(srv3_width: 6, srv4_width: 6, roster: incident_2026_08_06_roster_fixture()) { PlanReady { hosts: _ } => false PlanRefused { reason: r } => string_contains(s: r, pattern: "resolves to no committed fleet slot") @@ -389,12 +389,12 @@ test fn every_positive_width_is_spellable_at_the_apply_boundary() -> Bool { // THE NUMBERS BELOW ARE ABOUT A FIXTURE AND NOT ABOUT THE FLEET, AND AN EARLIER REVISION OF THIS // BLOCK SAID OTHERWISE. Measured by execution against what the production path currently supplies: // srv1 synthesized 5 / decoded 5, srv2 5 / 3, srv3 5 / 4, srv4 5 / 4, twenty against sixteen. What -// it supplies is organization_runner_roster_evidence, a declared Scaffold returning the pinned -// 2026-08-06 incident corpus, so this decodes a sixteen-day-old constant. The earlier revision also +// it supplied was the pinned 2026-08-06 incident corpus, now the dated witness fixture +// incident_2026_08_06_roster_fixture, so this decodes a constant. The earlier revision also // called the incident note's fifteen-offline-of-sixteen line an INDEPENDENT corroboration; it is the // prose describing the same fixture, so the two agreements are one source agreeing with itself. // That makes this a consistency check on the decoder, which is worth having, and it is not evidence -// about today's fleet -- nothing on the production path has read GitHub since the pin. +// about today's fleet -- the production path now reads gunbc.fleet.organization_runner_roster_read instead. // // WHAT IS TRUE AGAINST ANY ROSTER, AND IS THE HALF THAT MATTERS: the category error is a fact about // the wiring rather than the data, and the direction of the error is fixed -- addition_actions @@ -407,8 +407,8 @@ fn one_registration(name: String, id: Nat) -> GitHubSelfHostedRunnerRegistration name: name as NonEmptyStr, status: GitHubRunnerOnline, busy: false, - ephemeral: true, - version: "2.334.0" as NonEmptyStr, + ephemeral: Present { value: true }, + version: Present { value: "2.334.0" as NonEmptyStr }, os: "Linux" as NonEmptyStr, labels: [], } diff --git a/dag/test/claim/runner/runner_capacity_realize_witness_test.dag b/dag/test/claim/runner/runner_capacity_realize_witness_test.dag index de0284a2f6e..b1b0f96bd1e 100644 --- a/dag/test/claim/runner/runner_capacity_realize_witness_test.dag +++ b/dag/test/claim/runner/runner_capacity_realize_witness_test.dag @@ -9,7 +9,7 @@ import gunbc.runner_capacity_plan { PlanRefused, } import gunbc.fleet_runner_connectivity { - organization_runner_roster_managed_hosts_incident_evidence, + incident_2026_08_06_managed_roster_fixture, organization_runner_roster_materialized_slots_online_evidence, } import gunbc.runner_capacity_realize { @@ -62,7 +62,7 @@ test fn apply_seam_plans_additions_at_the_committed_width_not_count_refusal() -> let plan = runner_capacity_plan( srv3_width: 6, srv4_width: 6, - roster: organization_runner_roster_managed_hosts_incident_evidence(), + roster: incident_2026_08_06_managed_roster_fixture(), ) match plan { PlanReady { hosts: _ } => @@ -130,7 +130,7 @@ test fn apply_seam_preserves_sibling_host_receipt_when_one_refuses() -> Bool { plan: runner_capacity_plan( srv3_width: 6, srv4_width: 6, - roster: organization_runner_roster_managed_hosts_incident_evidence(), + roster: incident_2026_08_06_managed_roster_fixture(), ), ) match outcome {