Fleet converge gaps: ephemeral registrations retire, teardown grant enrolls the slot grammar, diverged is not unrelated

.github/workflows/fleet-converge.yml

@@ -293,11 +293,42 @@ jobs:
       - name: Org Actions credential validation + read-only settings diff
         id: org_actions_observe
         run: |
+          set -euo pipefail
+          umask 077
+          KEY_FILE="$RUNNER_TEMP/org-admin-app-key"
+          HDR_FILE="$RUNNER_TEMP/org-admin-auth-header"
+          JWT_FILE="$RUNNER_TEMP/org-admin-app-jwt"
+          TOKEN_HEADERS="$RUNNER_TEMP/org-admin-token-headers"
+          TOKEN_BODY="$RUNNER_TEMP/org-admin-token-body"
+          cleanup() { rm -f "$KEY_FILE" "$HDR_FILE" "$JWT_FILE" "$TOKEN_HEADERS" "$TOKEN_BODY" "$RUNNER_TEMP/org-admin-app-key-sm-body"; unset GUNBC_ORG_ADMIN_TOKEN GH_TOKEN; }
+          trap cleanup EXIT
+          printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE"
+          SM_BODY="$RUNNER_TEMP/org-admin-app-key-sm-body"
+          SM_CODE="$(curl -sS -o "$SM_BODY" -w '%{http_code}' -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/ci-github-app-private-key/versions/latest:access")"
+          rm -f "$HDR_FILE"
+          if [ "$SM_CODE" != 200 ]; then rm -f "$SM_BODY"; echo "OrgAdminAppKeyUnreadable: Secret Manager answered HTTP $SM_CODE for projects/gunbai-secrets/secrets/ci-github-app-private-key/versions/latest to the fleet-cloud-convergence principal. 403 means the version exists and is not readable OR does not exist -- REMEDY: run org_admin_app_key_access_converge_with_supplied_token (gunbc.fleet.org_actions_converge) with a control-plane token to bind roles/secretmanager.secretAccessor, and confirm the version exists. Minting NO token." >&2; exit 1; fi
+          python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$SM_BODY" > "$KEY_FILE"
+          rm -f "$SM_BODY"
+          chmod 600 "$KEY_FILE"
+          NOW=$(date +%s)
+          b64url() { base64 -w0 | tr '+/' '-_' | tr -d '='; }
+          JWT_HEADER=$(printf '%s' '{"alg":"RS256","typ":"JWT"}' | b64url)
+          JWT_PAYLOAD=$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' "$((NOW-60))" "$((NOW+540))" "2653532" | b64url)
+          JWT_SIG=$(printf '%s.%s' "$JWT_HEADER" "$JWT_PAYLOAD" | openssl dgst -sha256 -sign "$KEY_FILE" | b64url)
+          rm -f "$KEY_FILE"
+          printf 'Authorization: Bearer %s.%s.%s\n' "$JWT_HEADER" "$JWT_PAYLOAD" "$JWT_SIG" > "$JWT_FILE"
+          curl -sS -X POST -H @"$JWT_FILE" -H 'Accept: application/vnd.github+json' -D "$TOKEN_HEADERS" -o "$TOKEN_BODY" "https://api.github.com/app/installations/104134109/access_tokens"
+          rm -f "$JWT_FILE"
+          if ! head -n1 "$TOKEN_HEADERS" | grep -q ' 201 '; then echo "OrgAdminInstallationTokenRefused: GitHub answered $(head -n1 "$TOKEN_HEADERS" | tr -d '\r') for the gunbai-ci installation; a revoked key, a removed installation, or a missing Self-hosted-runners permission each look like this -- inspect the App's installation on the organization" >&2; exit 1; fi
+          GUNBC_ORG_ADMIN_TOKEN="$(python3 -c 'import sys,json; print(json.load(open(sys.argv[1]))["token"])' "$TOKEN_BODY")"
+          rm -f "$TOKEN_HEADERS" "$TOKEN_BODY"
+          export GUNBC_ORG_ADMIN_TOKEN
+          export GH_TOKEN="$GUNBC_ORG_ADMIN_TOKEN"
+          echo "org-admin: installation token minted in-run for app gunbai-ci installation 104134109 (one-hour lifetime; key wiped; token held in this step's environment only)"
           ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
           "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/fleet/org_actions_converge.dag --function org_actions_converge_wet
         env:
-          GUNBC_ORG_ADMIN_TOKEN: ${{ secrets.GUNBC_ORG_ADMIN_TOKEN }}
-          GH_TOKEN: ${{ secrets.GUNBC_ORG_ADMIN_TOKEN }}
+          WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }}
         if: github.event.inputs.mode == 'org_actions_observe'
         timeout-minutes: 5
       - name: Upload org Actions credential validation receipt

dag/extdeps/github/actions_runner.dag

@@ -18,6 +18,11 @@ import extdeps.toolchain.types {
   Wasm32,
 }
 import std.types { NonEmptyStr, String, List }
+import extdeps.languages.json.parse {
+  parse_json_document, JsonDocumentParsed, JsonDocumentUnreadable, json_document_gap_text,
+  json_object_unique_member, JsonMemberFound, JsonMemberAbsent, JsonMemberDuplicated, JsonMemberNotAnObject,
+}
+import extdeps.languages.json.emit { JsonString }
 import std.disposition { Disposition, Scaffold, SingleAuthority }
 import std.decl_ref { DeclarationRef, WholeDeclaration }
 
@@ -134,6 +139,48 @@ data actions_runner_registration_file_name: String = ".runner"
 
 data actions_runner_cache_dir: String = "/opt/actions-runner-cache"
 
+// THE REGISTRATION FILE SAYS WHETHER THE REGISTRATION CAN OUTLIVE ONE JOB. `.runner` is the JSON
+// `config.sh` writes; its `Ephemeral` member is the string "True" when the runner was configured
+// with `--ephemeral`. Upstream semantics (docs.github.com, "Autoscaling with self-hosted runners",
+// Using ephemeral runners): an ephemeral runner is automatically unregistered from GitHub after it
+// completes one job, and is not re-registered by restarting the process. So an ephemeral
+// registration file beside a unit that is neither active nor enabled records a registration GitHub
+// has already dropped; nothing remains to `config.sh remove`, and no credential is needed to know
+// it. A persistent registration (`Ephemeral` absent or "False") stays registered org-side until
+// removed, which needs an org-scoped removal token this repository does not hold, so a persistent
+// registration is NOT decidable from the file alone.
+type ActionsRunnerRegistration
+  = ActionsRunnerRegistrationPersistent
+  | ActionsRunnerRegistrationEphemeral
+
+type ActionsRunnerRegistrationRead
+  = ActionsRunnerRegistrationDecoded { registration: ActionsRunnerRegistration }
+  | ActionsRunnerRegistrationUnreadable { reason: String }
+
+data actions_runner_registration_ephemeral_member: String = "Ephemeral"
+
+fn actions_runner_registration_decode(raw: String) -> ActionsRunnerRegistrationRead {
+  match parse_json_document(s: raw) {
+    JsonDocumentUnreadable { gap } =>
+      ActionsRunnerRegistrationUnreadable { reason: concat(".runner is ", json_document_gap_text(gap: gap)) }
+    JsonDocumentParsed { value: doc } =>
+      match json_object_unique_member(v: doc, key: actions_runner_registration_ephemeral_member) {
+        JsonMemberFound { value: JsonString { value: flag } } =>
+          if flag == "True" {
+            ActionsRunnerRegistrationDecoded { registration: ActionsRunnerRegistrationEphemeral }
+          } else if flag == "False" {
+            ActionsRunnerRegistrationDecoded { registration: ActionsRunnerRegistrationPersistent }
+          } else {
+            ActionsRunnerRegistrationUnreadable { reason: concat(".runner Ephemeral is outside True/False: ", flag) }
+          }
+        JsonMemberFound { value: _ } => ActionsRunnerRegistrationUnreadable { reason: ".runner Ephemeral is not a string" }
+        JsonMemberAbsent => ActionsRunnerRegistrationDecoded { registration: ActionsRunnerRegistrationPersistent }
+        JsonMemberDuplicated { count: _ } => ActionsRunnerRegistrationUnreadable { reason: ".runner Ephemeral is duplicated" }
+        JsonMemberNotAnObject => ActionsRunnerRegistrationUnreadable { reason: ".runner is not a JSON object" }
+      }
+  }
+}
+
 data actions_runner_slot_extract_script_scaffold: Disposition = Scaffold {
   dissolves_to: SingleAuthority,
   bind: DeclarationRef {

dag/extdeps/github/org_admin_auth.dag

@@ -1,6 +1,6 @@
 module extdeps.github.org_admin_auth
 
-import std.types { NonEmptyStr, Timestamp }
+import std.types { NonEmptyStr, Timestamp, String, Int }
 import std.decl_ref { DeclarationRef, WholeDeclaration }
 import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef }
 import extdeps.uri { Uri, Https }
@@ -23,6 +23,21 @@ data installation_token_citation: ExternalAuthority = ExternalAuthority {
   uri: Uri { scheme: Https, locator: "docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-an-installation-access-token-for-a-github-app" }
 }
 
+// The installation access token endpoint and the App JWT bounds are facts of GitHub's API
+// (installation_token_citation): the JWT is signed RS256 with the App's private key, its `iat` may
+// be backdated up to 60 s for clock skew, its `exp` may be at most 10 minutes after issue, and the
+// POST returns a token valid for one hour. Modeled here so the workflow step that mints one names
+// the endpoint through the citation rather than spelling a URL.
+data github_app_installation_access_token_url_prefix: String = "https://api.github.com/app/installations/"
+
+fn github_app_installation_access_token_url(installation_id: Int) -> String {
+  join([github_app_installation_access_token_url_prefix, to_string(installation_id), "/access_tokens"], "")
+}
+
+data github_app_jwt_backdate_seconds: Int = 60
+
+data github_app_jwt_lifetime_seconds: Int = 540
+
 data oauth_flow_citation: ExternalAuthority = ExternalAuthority {
   uri: Uri { scheme: Https, locator: "docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps" }
 }

dag/gunbc/ci/ci_spec.dag

@@ -1,6 +1,12 @@
 module gunbc.ci_spec
 import gunbc.ledger_row_coherence { heal_repair_declaration_artifact_path }
 import extdeps.cloud.gcp.secret_ref { secret_ref_access_url, secret_ref_version_resource }
+import gunbc.auth.github_apps { gunbai_ci_declared, gunbai_ci_app_pem_secret, gunbai_ci_declared_installation_id }
+import extdeps.github.org_admin_auth {
+  github_app_installation_access_token_url,
+  github_app_jwt_backdate_seconds,
+  github_app_jwt_lifetime_seconds,
+}
 import gunbc.spark.credential_workflow { spark_administrator_password_secret_ref }
 import gunbc.spark.grant_privileged_operation { spark_grant_credential_path_env }
 import gunbc.spark.bootstrap_provision { spark_bootstrap_cloud_principal_member }
@@ -1362,8 +1368,116 @@ fn gunbc_ci_fleet_converge_apply_invoke() -> String {
   )
 }
 
+// THE ORG-ADMIN CREDENTIAL IS MINTED IN-RUN FROM THE App KEY THE FLEET ALREADY HOLDS; NO HUMAN
+// TOKEN EXISTS. The interim in docs/plans/org-admin-credential-acquisition.md asked the operator to
+// mint a fine-grained PAT and paste it into an Actions secret. That step was never taken, and it
+// was never necessary: the gunbai-ci GitHub App (gunbc.auth.github_apps gunbai_ci_declared) is
+// installed on the organization, its private key sits in Secret Manager behind the same WIF read
+// the fleet key uses, and the runner installer already mints registration tokens with it. So this
+// prelude follows the fleet-key lifecycle exactly -- WIF token by env, the SecretRef's own access
+// URL, a 0600 file under RUNNER_TEMP, a trap armed before the key touches disk -- then signs a
+// ten-minute RS256 JWT with openssl, exchanges it for a one-hour installation token
+// (extdeps.github.org_admin_auth installation_token_citation), wipes the key, and exports the token
+// into THIS step's environment only. Nothing reaches GITHUB_ENV; the trap unsets it on exit.
+//
+// THE HTTP STATUS IS CLASSIFIED, NOT COLLAPSED (the lesson gunbc_ci_fleet_key_agent_prelude's
+// administrator sibling records): the mint POST writes its headers and body to files, and anything
+// but 201 refuses with the status line, so a revoked key, a removed installation and a missing
+// permission each surface as what they are rather than as a JSON decode traceback.
+fn gunbc_ci_org_admin_app_token_prelude() -> List<PipelineStep> {
+  [
+    Do { run: ci_retry_body_run(command: "set -euo pipefail") },
+    Do { run: ci_retry_body_run(command: "umask 077") },
+    Do { run: ci_retry_body_run(command: "KEY_FILE=\"$RUNNER_TEMP/org-admin-app-key\"") },
+    Do { run: ci_retry_body_run(command: "HDR_FILE=\"$RUNNER_TEMP/org-admin-auth-header\"") },
+    Do { run: ci_retry_body_run(command: "JWT_FILE=\"$RUNNER_TEMP/org-admin-app-jwt\"") },
+    Do { run: ci_retry_body_run(command: "TOKEN_HEADERS=\"$RUNNER_TEMP/org-admin-token-headers\"") },
+    Do { run: ci_retry_body_run(command: "TOKEN_BODY=\"$RUNNER_TEMP/org-admin-token-body\"") },
+    Do {
+      run: ci_retry_body_run(
+        command: "cleanup() { rm -f \"$KEY_FILE\" \"$HDR_FILE\" \"$JWT_FILE\" \"$TOKEN_HEADERS\" \"$TOKEN_BODY\" \"$RUNNER_TEMP/org-admin-app-key-sm-body\"; unset GUNBC_ORG_ADMIN_TOKEN GH_TOKEN; }"
+      )
+    },
+    Do { run: ci_retry_body_run(command: "trap cleanup EXIT") },
+    Do { run: ci_retry_body_run(command: gunbc_ci_auth_header_file_write_command()) },
+    Do { run: ci_retry_body_run(command: "SM_BODY=\"$RUNNER_TEMP/org-admin-app-key-sm-body\"") },
+    Do {
+      run: ci_retry_body_run(
+        command: join([
+          "SM_CODE=\"$(curl -sS -o \"$SM_BODY\" -w '%\{http_code}' -H @\"$HDR_FILE\" \"",
+          secret_ref_access_url(ref: gunbai_ci_app_pem_secret),
+          "\")\""
+        ], "")
+      )
+    },
+    Do { run: ci_retry_body_run(command: "rm -f \"$HDR_FILE\"") },
+    Do {
+      run: ci_retry_body_run(
+        command: join([
+          "if [ \"$SM_CODE\" != 200 ]; then rm -f \"$SM_BODY\"; echo \"OrgAdminAppKeyUnreadable: Secret Manager answered HTTP $SM_CODE for ",
+          secret_ref_version_resource(ref: gunbai_ci_app_pem_secret),
+          " to the fleet-cloud-convergence principal. 403 means the version exists and is not readable OR does not exist -- REMEDY: run org_admin_app_key_access_converge_with_supplied_token (gunbc.fleet.org_actions_converge) with a control-plane token to bind roles/secretmanager.secretAccessor, and confirm the version exists. Minting NO token.\" >&2; exit 1; fi"
+        ], "")
+      )
+    },
+    Do { run: ci_retry_body_run(command: "python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))[\"payload\"][\"data\"]))' \"$SM_BODY\" > \"$KEY_FILE\"") },
+    Do { run: ci_retry_body_run(command: "rm -f \"$SM_BODY\"") },
+    Do { run: ci_retry_body_run(command: "chmod 600 \"$KEY_FILE\"") },
+    Do { run: ci_retry_body_run(command: "NOW=$(date +%s)") },
+    Do { run: ci_retry_body_run(command: "b64url() { base64 -w0 | tr '+/' '-_' | tr -d '='; }") },
+    Do { run: ci_retry_body_run(command: "JWT_HEADER=$(printf '%s' '\{\"alg\":\"RS256\",\"typ\":\"JWT\"}' | b64url)") },
+    Do {
+      run: ci_retry_body_run(
+        command: join([
+          "JWT_PAYLOAD=$(printf '\{\"iat\":%d,\"exp\":%d,\"iss\":\"%s\"}' \"$((NOW-",
+          to_string(github_app_jwt_backdate_seconds),
+          "))\" \"$((NOW+",
+          to_string(github_app_jwt_lifetime_seconds),
+          "))\" \"",
+          to_string(gunbai_ci_declared.app_id.value),
+          "\" | b64url)"
+        ], "")
+      )
+    },
+    Do { run: ci_retry_body_run(command: "JWT_SIG=$(printf '%s.%s' \"$JWT_HEADER\" \"$JWT_PAYLOAD\" | openssl dgst -sha256 -sign \"$KEY_FILE\" | b64url)") },
+    Do { run: ci_retry_body_run(command: "rm -f \"$KEY_FILE\"") },
+    Do { run: ci_retry_body_run(command: "printf 'Authorization: Bearer %s.%s.%s\\n' \"$JWT_HEADER\" \"$JWT_PAYLOAD\" \"$JWT_SIG\" > \"$JWT_FILE\"") },
+    Do {
+      run: ci_retry_body_run(
+        command: join([
+          "curl -sS -X POST -H @\"$JWT_FILE\" -H 'Accept: application/vnd.github+json' -D \"$TOKEN_HEADERS\" -o \"$TOKEN_BODY\" \"",
+          github_app_installation_access_token_url(installation_id: gunbai_ci_declared_installation_id.value),
+          "\""
+        ], "")
+      )
+    },
+    Do { run: ci_retry_body_run(command: "rm -f \"$JWT_FILE\"") },
+    Do {
+      run: ci_retry_body_run(
+        command: "if ! head -n1 \"$TOKEN_HEADERS\" | grep -q ' 201 '; then echo \"OrgAdminInstallationTokenRefused: GitHub answered $(head -n1 \"$TOKEN_HEADERS\" | tr -d '\\r') for the gunbai-ci installation; a revoked key, a removed installation, or a missing Self-hosted-runners permission each look like this -- inspect the App's installation on the organization\" >&2; exit 1; fi"
+      )
+    },
+    Do { run: ci_retry_body_run(command: "GUNBC_ORG_ADMIN_TOKEN=\"$(python3 -c 'import sys,json; print(json.load(open(sys.argv[1]))[\"token\"])' \"$TOKEN_BODY\")\"") },
+    Do { run: ci_retry_body_run(command: "rm -f \"$TOKEN_HEADERS\" \"$TOKEN_BODY\"") },
+    Do { run: ci_retry_body_run(command: "export GUNBC_ORG_ADMIN_TOKEN") },
+    Do { run: ci_retry_body_run(command: "export GH_TOKEN=\"$GUNBC_ORG_ADMIN_TOKEN\"") },
+    Do {
+      run: ci_retry_body_run(
+        command: join([
+          "echo \"org-admin: installation token minted in-run for app ",
+          gunbai_ci_declared.slug as String,
+          " installation ",
+          to_string(gunbai_ci_declared_installation_id.value),
+          " (one-hour lifetime; key wiped; token held in this step's environment only)\""
+        ], "")
+      )
+    }
+  ]
+}
+
 fn gunbc_ci_org_actions_converge_invoke() -> String {
-  gunbc_run_step_script(
+  gunbc_run_step_script_with_prelude(
+    prelude: gunbc_ci_org_admin_app_token_prelude(),
     source_roots: witness_layer_roots,
     entry: gunbc_ci_org_actions_converge_target.entry,
     function: gunbc_ci_org_actions_converge_target.function,

dag/gunbc/executor_privileged_operation.dag

@@ -173,6 +173,18 @@ fn executor_privileged_operations_sudoers_body(
 // longer generated file and removes the traversal entirely.
 data executor_managed_directory_mode: NonEmptyStr = "755" as NonEmptyStr
 
+fn runner_slot_install_operations(
+  slot_dirs: List<NonEmptyStr>,
+  slot_owner: NonEmptyStr,
+) -> List<ExecutorPrivilegedOperation> {
+  map(slot_dirs, d => EnsureOwnedDirectory {
+    path: d,
+    owner: slot_owner,
+    group: slot_owner,
+    mode: executor_managed_directory_mode,
+  })
+}
+
 fn runner_slot_privileged_operations(
   slot_dirs: List<NonEmptyStr>,
   slot_owner: NonEmptyStr,