diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 4cbca5a11be..886ab03fddd 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -1543,7 +1543,7 @@ jobs: compression-level: 0 if: always() && github.event.inputs.mode == 'workspace_checkpoint_restore' timeout-minutes: 10 - - name: Materialize approval MAC keys on srv1 (SM pinned -> sha256 prefix -> /etc/gunbc-roadmap 0640) + - name: Materialize approval MAC keys on the approval broker host (SM pinned -> sha256 prefix -> /etc/gunbc-roadmap 0640) id: approval_keyring_converge run: |- # ๐ŸŸก dissolve-on: gunbc_ci_approval_keyring_converge_invoke - orch-emitted foreign-executor (GitHub Actions run:) credential runner: WIF access token by env, two PINNED secret versions (capability and submission MAC keys) fetched over curl, 0600 files under RUNNER_TEMP with a trap armed BEFORE the credentials touch disk. The pipeline steps are modeled (gunbc_ci_approval_keyring_converge_prelude) but the runner transport itself remains hand-shell; DISSOLVES WHEN the runner is authored as v2.extdeps.languages.bash_build nodes (typed, grammar-quoted words; command substitution; assignment; pipe; if/test; redirect; || true) emitted by v2.workflow.bash_emit bash_emit_stmts -- a route AVAILABLE today (#12422 retired the sibling pack runner on it), so this migration is unauthored, not blocked on a missing capability; set -euo pipefail, umask, export and trap are ordinary commands on that route, and the trap body is itself built from nodes and passed as one grammar-quoted word, so no new orchestration intent is a prerequisite @@ -1572,7 +1572,7 @@ jobs: FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} if: github.event.inputs.mode == 'approval_keyring_converge' timeout-minutes: 5 - - name: Issue a device enrolment code on srv1 and deliver it to the operator channel (receipt only in this log) + - name: Issue a device enrolment code on the approval broker host and deliver it to the operator channel (receipt only in this log) id: approval_device_enrolment_code_issue run: |- ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) @@ -3052,7 +3052,7 @@ jobs: env: RELEASE_BINS_KEY: ${{ needs.build.outputs.release_bins_key }} timeout-minutes: 5 - - name: Install the approval broker dark on srv1 (additive unit + slice; does not restart gunbc-roadmap.service) + - name: Install the approval broker dark on the approval broker host (additive unit + slice; does not restart gunbc-roadmap.service) id: approval_broker_dark_install run: |- ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) diff --git a/dag/extdeps/shell.dag b/dag/extdeps/shell.dag index 3ee4a20ba2c..a480adae742 100644 --- a/dag/extdeps/shell.dag +++ b/dag/extdeps/shell.dag @@ -528,6 +528,22 @@ service shell.Stat { nonzero => String "stat could not read the path" } } + + operation GroupOf { + requires none + input { path: FilePath } + output { + success: Bool from "exit_success" + stdout: String from "stdout" + stderr: String from "stderr" + } + readonly + transport shell { argv: ["stat", "-c", "%G", "--", "{path}"] } + exit { + 0 => Unit "the path exists and its owning group is on stdout" + nonzero => String "stat could not read the path" + } + } } service shell.Move { diff --git a/dag/gunbc/approve_ios_project.dag b/dag/gunbc/approve_ios_project.dag index 027da3d7ea3..5a29b8bdb13 100644 --- a/dag/gunbc/approve_ios_project.dag +++ b/dag/gunbc/approve_ios_project.dag @@ -14,7 +14,8 @@ import extdeps.apple.build_settings { MarketingVersion, CurrentProjectVersion } import gunbc.auth.approval_app_attest_config { approval_app_bundle_id, approval_app_bundle_version } -import gunbc.auth.approval_broker_endpoint { approval_broker_public_host_from, approval_broker_public_port } +import gunbc.auth.approval_broker_endpoint { approval_broker_public_port, ApprovalBrokerAddressResolved, ApprovalBrokerAddressRefused } +import gunbc.auth.approval_broker_placement { approval_broker_public_host_from } import gunbc.fleet_intent_network { fleet_intent_network } import extdeps.apple.xcode { xcode_26_6 } import extdeps.xcodegen.project_spec { @@ -45,7 +46,7 @@ import extdeps.languages.yaml.emit { emit_yaml, EmittedYaml, YamlEmitRefused, ya // server_host ApproveServerHost -- read by ServerConfig at launch. // apns_environment ApproveApnsEnvironment -- read by ServerConfig at launch. // apns_environment is xcconfig-substituted from Config/Team.xcconfig. server_host is NOT: it is the -// broker's public host (gunbc.auth.approval_broker_endpoint approval_broker_public_host_from), the +// broker's public host (gunbc.auth.approval_broker_placement approval_broker_public_host_from), the // same composition the approval links are issued against, written into the spec as a value. It was // an operator-typed APPROVE_SERVER_HOST, a second authority for where the broker answers. type ApproveAppInfo { @@ -194,9 +195,12 @@ fn approve_ios_project_yml_from(domain: NonEmptyStr?, port: Port?) -> ApproveIos "gunbc.approve_ios_project: approval_broker_public_port is ", to_string(p), " and the app's ServerConfig carries no port; project it (an ApproveServerPort Info.plist key read by gunbc.approve_ios_swift_wire ServerConfig) before publishing the broker off the default TLS listener" ], "") } - Absent => match emit_yaml(v: project_spec_yaml(p: approve_ios_project_spec(server_host: approval_broker_public_host_from(domain: domain)))) { - EmittedYaml { text: t } => ApproveIosProjectGenerated { content: concat(approve_ios_project_header, t) } - YamlEmitRefused { path: p, reason: r } => ApproveIosProjectRefused { reason: yaml_emit_refusal_text(module_path: "gunbc.approve_ios_project", path: p, reason: r) } + Absent => match approval_broker_public_host_from(domain: domain) { + ApprovalBrokerAddressRefused { detail } => ApproveIosProjectRefused { reason: join(["gunbc.approve_ios_project: ", detail as String], "") } + ApprovalBrokerAddressResolved { value: host } => match emit_yaml(v: project_spec_yaml(p: approve_ios_project_spec(server_host: host))) { + EmittedYaml { text: t } => ApproveIosProjectGenerated { content: concat(approve_ios_project_header, t) } + YamlEmitRefused { path: p, reason: r } => ApproveIosProjectRefused { reason: yaml_emit_refusal_text(module_path: "gunbc.approve_ios_project", path: p, reason: r) } + } } } } diff --git a/dag/gunbc/auth/approval_broker_endpoint.dag b/dag/gunbc/auth/approval_broker_endpoint.dag index 53a1202570e..2834ced244d 100644 --- a/dag/gunbc/auth/approval_broker_endpoint.dag +++ b/dag/gunbc/auth/approval_broker_endpoint.dag @@ -7,7 +7,8 @@ import extdeps.tailscale.serve { import extdeps.external_authority { ExternalAuthority, CitedFigureStanding, CitedToAuthority, TranscribedUncited } import gunbc.fleet_tailnet { fleet_tailnet_domain, tailnet_base_url_from, tailnet_host_from } import gunbc.serve_liveness { serve_liveness_path } -import gunbc.srv1_dashboard_bind { srv1_dashboard_host_identity } +import extdeps.shell +import v2.std.collection { Present, Absent } import gunbc.auth.approval_device_wire { approval_device_request_path_prefix, approval_device_enrollment_path_prefix } // โ”€โ”€ WHERE THE APPROVAL BROKER ANSWERS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -95,12 +96,6 @@ data approval_broker_listen_port: Port = 8085 // door is the only public path" true rather than merely intended. data approval_broker_listen_host: NonEmptyStr = "127.0.0.1" -// THE HOST THE BROKER RUNS ON, derived from the instance that owns its unit rather than spelled: a -// run that files and polls over the loopback approval client (the GCP IAM converge job) has to run -// there. -fn approval_broker_host_identity() -> NonEmptyStr { - srv1_dashboard_host_identity -} fn approval_broker_loopback_base_url(port: Port) -> NonEmptyStr { join(["http://", approval_broker_listen_host as String, ":", to_string(port)], "") as NonEmptyStr @@ -235,15 +230,43 @@ data approval_broker_front_door_path_preservation: CitedFigureStanding = Transcr // (gunbc.approve_ios_project) -- read one port rather than each assuming 443. data approval_broker_public_port: Port? = none -fn approval_broker_base_url_from(domain: NonEmptyStr?) -> NonEmptyStr { - tailnet_base_url_from(host: approval_broker_host_identity(), domain: domain, port: approval_broker_public_port) -} +// THE BROKER'S PUBLIC ADDRESS ON A GIVEN HOST. Which host is a placement fact, not this module's: +// the prod role decides WHERE the broker is installed (gunbc.auth.approval_broker_placement, over +// gunbc.live_deploy.spec RoleSingletonHolding), and a broker-side reader receives the host it was +// installed on rather than resolving the role -- so the broker's closure never reaches the instance +// roster (#13132). +type ApprovalBrokerAddress + = ApprovalBrokerAddressResolved { value: NonEmptyStr } + | ApprovalBrokerAddressRefused { detail: NonEmptyStr } -fn approval_broker_base_url() -> NonEmptyStr { - approval_broker_base_url_from(domain: fleet_tailnet_domain) +fn approval_broker_base_url_on(host: NonEmptyStr, domain: NonEmptyStr?) -> NonEmptyStr { + tailnet_base_url_from(host: host, domain: domain, port: approval_broker_public_port) } // THE BARE HOST OF THAT SAME ADDRESS, for a client that composes scheme, host and path itself. -fn approval_broker_public_host_from(domain: NonEmptyStr?) -> NonEmptyStr { - tailnet_host_from(host: approval_broker_host_identity(), domain: domain) +fn approval_broker_public_host_on(host: NonEmptyStr, domain: NonEmptyStr?) -> NonEmptyStr { + tailnet_host_from(host: host, domain: domain) +} + +// THE SERVING BROKER'S OWN ADDRESS IS GIVEN, NOT DISCOVERED. Placement already knows which host the +// broker is installed on (gunbc.auth.approval_broker_placement, from the prod-role holder token the +// spec carries), so the emitter binds the base URL into the process's environment: the broker unit's +// Environment= line (gunbc.live_deploy.emit) and the enrolment verb's env prefix +// (gunbc.auth.approval_device_enrolment_code_issue). The broker reads the value it was given -- no +// host-effect read, so its closure never reaches the instance roster (#13132) and there is one +// authority for where it runs. An unset binding, or one that is not an https URL (the emitter's +// refusal marker), refuses; it is never replaced by a host. +data approval_broker_base_url_env_name: NonEmptyStr = "GUNBC_APPROVAL_BROKER_BASE_URL" + +fn approval_broker_serving_base_url_of(bound: String?) -> ApprovalBrokerAddress { + match bound { + Present { value: v } => + if starts_with(s: v, prefix: "https://") { ApprovalBrokerAddressResolved { value: v as NonEmptyStr } } + else { ApprovalBrokerAddressRefused { detail: join(["approval broker address unresolved: ", approval_broker_base_url_env_name as String, " is bound to '", v, "', not an https URL"], "") as NonEmptyStr } } + Absent => ApprovalBrokerAddressRefused { detail: join(["approval broker address unresolved: ", approval_broker_base_url_env_name as String, " is not bound in this process"], "") as NonEmptyStr } + } +} + +fn approval_broker_serving_base_url() -> ApprovalBrokerAddress { + approval_broker_serving_base_url_of(bound: shell.Env.Get(name: approval_broker_base_url_env_name).value) } diff --git a/dag/gunbc/auth/approval_broker_placement.dag b/dag/gunbc/auth/approval_broker_placement.dag new file mode 100644 index 00000000000..a21f9edbefa --- /dev/null +++ b/dag/gunbc/auth/approval_broker_placement.dag @@ -0,0 +1,70 @@ +module gunbc.auth.approval_broker_placement + +import gunbc.live_deploy.desired { prod_role_actuated_realization } +import std.types { NonEmptyStr } +import gunbc.roadmap_dashboard_instance { + ProdRoleHolder, + ProdRoleRealization, + ProdRoleRealizedBy, + ProdRoleRealizationRefused, + ProdRoleRealizationRefusal, + prod_role_realization, + prod_role_realization_refusal_detail, +} +import gunbc.auth.approval_broker_endpoint { + ApprovalBrokerAddress, + ApprovalBrokerAddressResolved, + ApprovalBrokerAddressRefused, + approval_broker_public_host_on, + approval_broker_base_url_on, +} +import gunbc.fleet_tailnet { fleet_tailnet_domain } + +// WHERE THE APPROVAL BROKER IS INSTALLED, for readers OUTSIDE the broker process: the GCP IAM +// converge job that must run beside it (gunbc.fleet_converge_workflow) and the approvals app's build +// config (gunbc.approve_ios_project). It is the host of the prod-role holder -- the same token +// (gunbc.roadmap_dashboard_instance ProdRoleHolder) from which gunbc.live_deploy.spec builds the +// broker's unit, so the job and the app name the host the unit is actually on. Under a refused +// realization there is no such host and each reader refuses through its own arm. +// +// This module is deliberately NOT imported by any broker-side module (gunbc.auth.approval_broker_endpoint, +// approval_decision_store, approval_request_submission, approval_device_enrolment_code): the broker +// receives its own host at serve time (approval_broker_serving_base_url) and never reaches the +// instance roster, which keeps the native broker's closure as #13132 narrowed it. +type ApprovalBrokerHost + = ApprovalBrokerHostResolved { host: NonEmptyStr } + | ApprovalBrokerHostRefused { cause: ProdRoleRealizationRefusal } + +fn approval_broker_host_under(realization: ProdRoleRealization) -> ApprovalBrokerHost { + match realization { + ProdRoleRealizedBy { holder } => ApprovalBrokerHostResolved { host: holder.instance.host_identity } + ProdRoleRealizationRefused { cause } => ApprovalBrokerHostRefused { cause: cause } + } +} + +fn approval_broker_public_host_under(realization: ProdRoleRealization, domain: NonEmptyStr?) -> ApprovalBrokerAddress { + match approval_broker_host_under(realization: realization) { + ApprovalBrokerHostResolved { host } => ApprovalBrokerAddressResolved { value: approval_broker_public_host_on(host: host, domain: domain) } + ApprovalBrokerHostRefused { cause } => + ApprovalBrokerAddressRefused { detail: ("approval broker address unresolved: " + prod_role_realization_refusal_detail(cause: cause)) as NonEmptyStr } + } +} + +// THE BASE URL THE BROKER IS GIVEN, from the holder token itself: the emitter binds it into the unit +// and the enrolment verb's environment (gunbc.auth.approval_broker_endpoint +// approval_broker_base_url_env_name), so the serving broker never resolves its own host. +fn approval_broker_base_url_of_holder(holder: ProdRoleHolder) -> NonEmptyStr { + approval_broker_base_url_on(host: holder.instance.host_identity, domain: fleet_tailnet_domain) +} + +fn approval_broker_base_url_under(realization: ProdRoleRealization) -> ApprovalBrokerAddress { + match realization { + ProdRoleRealizedBy { holder } => ApprovalBrokerAddressResolved { value: approval_broker_base_url_of_holder(holder: holder) } + ProdRoleRealizationRefused { cause } => + ApprovalBrokerAddressRefused { detail: ("approval broker address unresolved: " + prod_role_realization_refusal_detail(cause: cause)) as NonEmptyStr } + } +} + +fn approval_broker_public_host_from(domain: NonEmptyStr?) -> ApprovalBrokerAddress { + approval_broker_public_host_under(realization: prod_role_actuated_realization(), domain: domain) +} diff --git a/dag/gunbc/auth/approval_broker_serve.dag b/dag/gunbc/auth/approval_broker_serve.dag index 8aee18b602b..d93c2629a8a 100644 --- a/dag/gunbc/auth/approval_broker_serve.dag +++ b/dag/gunbc/auth/approval_broker_serve.dag @@ -15,7 +15,7 @@ import gunbc.clock_read { clock_now_probed_at } import gunbc.auth.approval_request_submission { admit_parsed_submission, SubmissionVerified, SubmissionNotAdmitted, file_and_notify, submission_outcome_http_body, read_submission_keyring, read_publisher_token, escalation_status, parse_submission_envelope, SubmissionEnvelopeParsed, SubmissionEnvelopeUnreadable, - SubmissionFiledAndNotified, SubmissionFiledNotNotified, SubmissionFiledPublisherTokenUnavailable, SubmissionFiledLinksNotIssued, + SubmissionFiledAndNotified, SubmissionFiledNotNotified, SubmissionFiledPublisherTokenUnavailable, SubmissionFiledLinksNotIssued, SubmissionFiledBrokerAddressRefused, SubmissionAlreadyFiled, SubmissionStoreRefused, SubmissionRefused, SubmissionFiledCapabilityKeyUnavailable, SubmissionUnreadable, SubmissionKeyUnavailable, SubmissionNotEnrolled, SubmissionAuthorityRefused, } @@ -51,6 +51,7 @@ import gunbc.auth.approval_broker_endpoint { approval_device_request_path_template, approval_device_enrollment_path_template, approval_broker_listen_host, + approval_broker_serving_base_url, } import gunbc.auth.approval_device_wire { approval_device_enrol_path, approval_device_redeem_path, approval_device_pending_path, approval_device_push_path } import gunbc.auth.approval_device_routes { @@ -414,6 +415,7 @@ fn approval_submit_admitted_response(request: ServeHttpRequest 502 SubmissionFiledPublisherTokenUnavailable { escalation_id: _, detail: _ } => 503 SubmissionFiledLinksNotIssued { escalation_id: _, key_id: _ } => 503 + SubmissionFiledBrokerAddressRefused { escalation_id: _, detail: _ } => 503 SubmissionAlreadyFiled { escalation_id: _ } => 409 SubmissionStoreRefused { detail: _ } => 503 SubmissionRefused { refusal: v } => match v { diff --git a/dag/gunbc/auth/approval_decision_store.dag b/dag/gunbc/auth/approval_decision_store.dag index 8038f37b3ae..bbfd614477d 100644 --- a/dag/gunbc/auth/approval_decision_store.dag +++ b/dag/gunbc/auth/approval_decision_store.dag @@ -20,7 +20,7 @@ import extdeps.filesystem.filesystem_io { Filesystem, FilesystemExactRead, Files import extdeps.tailscale.identity { TailnetIdentity } import extdeps.network.address { IpV4, ip_address_is_loopback } import extdeps.network.ipv4 { parse_ipv4_address, Ipv4Parsed, Ipv4ParseFailed } -import gunbc.srv1_dashboard_bind { srv1_dashboard_listen_host } +import gunbc.auth.approval_broker_endpoint { approval_broker_listen_host } import extdeps.crypto.mac { MacKey, MacKeyId, MacSuite, HmacSha256, MacVerification, mac_verify, mac_key_material_hex_length } import extdeps.languages.json.emit { JsonValue, JsonString, JsonBool, serialize_json, json_object, json_kv, json_string, json_bool } import extdeps.languages.json.parse { JsonDocumentParsed, JsonDocumentUnreadable, JsonMemberFound, JsonMemberAbsent, JsonMemberDuplicated, JsonMemberNotAnObject, parse_json_document, json_object_unique_member } @@ -161,7 +161,7 @@ type IdentityTrustStanding // THE TRUST STANDING IS DERIVED FROM THE ROW THE UNIT IS EMITTED FROM, never asserted beside it // (review 67264), AND A LOOPBACK BIND ATTRIBUTES NOTHING (reviews 67306, 67348): -// gunbc.srv1_dashboard_bind srv1_dashboard_listen_host (srv1_live_dashboard_instance.listen_host) is the one fact +// gunbc.auth.approval_broker_endpoint approval_broker_listen_host (the broker unit's --host) is the one fact // live_deploy turns into the unit's --host, so the standing reads that field. Any bind other than // loopback keeps the gate closed: the header is self-assertable by whoever reaches the port. A // loopback bind rules out off-host callers only; an on-host process under another POSIX user can @@ -177,21 +177,26 @@ fn approval_identity_trust_for(listen_host: NonEmptyStr) -> IdentityTrustStandin Ipv4Parsed { address: a } => if ip_address_is_loopback(addr: IpV4 { v4: a }) { IdentityHeaderUnattributed { - residual: join(["gunbc-roadmap.service binds --host ", listen_host as String, " (srv1_live_dashboard_instance.listen_host) behind tailscale serve https://srv1.tailecbe08.ts.net/, so off-host callers cannot reach it; an on-host process under another POSIX user still can, so the header is not attributed and the record names the capability's bearer"], "") as NonEmptyStr, + residual: join(["gunbc-approval-broker.service binds --host ", listen_host as String, " (gunbc.auth.approval_broker_endpoint approval_broker_listen_host) behind its tailscale serve front door, so off-host callers cannot reach it; an on-host process under another POSIX user still can, so the header is not attributed and the record names the capability's bearer"], "") as NonEmptyStr, } } else { IdentityHeaderTrustNotEstablished { - blocker: join(["gunbc-roadmap.service binds --host ", listen_host as String, " (srv1_live_dashboard_instance.listen_host), so a Tailscale-User-Login header can be set by anyone who reaches the port; the gate opens when the row binds loopback"], "") as NonEmptyStr, + blocker: join(["gunbc-approval-broker.service binds --host ", listen_host as String, " (gunbc.auth.approval_broker_endpoint approval_broker_listen_host), so a Tailscale-User-Login header can be set by anyone who reaches the port; the gate opens when the row binds loopback"], "") as NonEmptyStr, } } Ipv4ParseFailed { error: _ } => IdentityHeaderTrustNotEstablished { - blocker: join(["srv1_live_dashboard_instance.listen_host '", listen_host as String, "' is not an IPv4 address, so the bind cannot be classified as loopback"], "") as NonEmptyStr, + blocker: join(["approval_broker_listen_host '", listen_host as String, "' is not an IPv4 address, so the bind cannot be classified as loopback"], "") as NonEmptyStr, } } } -data approval_identity_trust: IdentityTrustStanding = approval_identity_trust_for(listen_host: srv1_dashboard_listen_host) +// THE TRUST OF THE PROCESS THAT SERVES APPROVALS, derived from THE ONE ROW its unit's --host is +// emitted from: gunbc.auth.approval_broker_endpoint approval_broker_listen_host, which +// gunbc.live_deploy.spec approval_broker_service_config binds (reviews 67264/67306). Editing that bind +// to 0.0.0.0 closes this gate in the same commit, and the broker never reaches the instance roster +// to compute it (#13132). Which host the broker runs on is placement, not trust. +data approval_identity_trust: IdentityTrustStanding = approval_identity_trust_for(listen_host: approval_broker_listen_host) // โ”€โ”€ the store โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ diff --git a/dag/gunbc/auth/approval_device_enrolment_code.dag b/dag/gunbc/auth/approval_device_enrolment_code.dag index 04cfda3c2b3..2da8a825529 100644 --- a/dag/gunbc/auth/approval_device_enrolment_code.dag +++ b/dag/gunbc/auth/approval_device_enrolment_code.dag @@ -20,7 +20,7 @@ import gunbc.auth.approval_ntfy_access_readback { AclEstablished, AclRefused, approval_ntfy_runtime_reading, approval_ntfy_channel_verdict, approval_ntfy_acl_deviation_sentence, } -import gunbc.auth.approval_broker_endpoint { approval_broker_base_url } +import gunbc.auth.approval_broker_endpoint { approval_broker_serving_base_url, ApprovalBrokerAddress, ApprovalBrokerAddressResolved, ApprovalBrokerAddressRefused } import gunbc.auth.approval_request_submission { PublisherTokenRead, read_publisher_token, publish_notification, NotificationPublish, NotificationAttempted, NotificationPublisherTokenUnavailable } import gunbc.auth.approval_device_redemption { approval_device_store_root, issue_enrolment_code, @@ -94,11 +94,11 @@ fn enrolment_code_of_entropy(octets_b64: String) -> NonEmptyStr? { } // The one place the code is rendered into text: the notification body. Nothing else renders it. -fn enrolment_code_notification(code: NonEmptyStr, expires_at: Timestamp) -> NtfyMessage { +fn enrolment_code_notification(code: NonEmptyStr, expires_at: Timestamp, approval_url: NonEmptyStr) -> NtfyMessage { approval_notification( title: approval_enrolment_code_notification_title, body: ("Enter this code in Approve within " + to_string(second_count(s: approval_enrolment_code_window)) + " seconds: " + (code as String) + " (expires " + expires_at + ")") as NonEmptyStr, - approval_url: approval_broker_base_url(), + approval_url: approval_url, ) } @@ -146,23 +146,28 @@ fn enrolment_code_issuance_response(i: EnrolmentCodeIssuance) -> CliWireResponse // executed by a claim on the route it has (review 69755 of gunbc#12013): the slot write runs on // the wet route against a fresh root, and the publish -- with the receipt minted from the // published message and never from the code -- runs on the hermetic ntfy arm. -fn issue_and_deliver_enrolment_code_until(root: NonEmptyStr, now: Timestamp, expires_at: Timestamp, code: NonEmptyStr, publisher: PublisherTokenRead) -> EnrolmentCodeIssuance admit_callers: [ +fn issue_and_deliver_enrolment_code_until(root: NonEmptyStr, now: Timestamp, expires_at: Timestamp, code: NonEmptyStr, publisher: PublisherTokenRead, broker: ApprovalBrokerAddress) -> EnrolmentCodeIssuance admit_callers: [ decl_ref(module_path: "gunbc.auth.approval_device_enrolment_code", decl_name: "issue_device_enrolment_code"), decl_ref(module_path: "test.claim.approval_device_enrolment_code_wet_witness_test", decl_name: "the_code_slot_is_written_before_delivery_by_real_execution"), ] { match issue_enrolment_code(root: root, code: code, issued_at: now, expires_at: expires_at) { DeviceStoreSlotOccupied => EnrolmentCodeIssuanceRefused { cause: "the minted code's slot is already occupied; run again" } DeviceStoreRefused { detail: d } => EnrolmentCodeIssuanceRefused { cause: d as String } - DeviceStoreWritten => deliver_enrolment_code(code: code, expires_at: expires_at, publisher: publisher) + DeviceStoreWritten => deliver_enrolment_code(code: code, expires_at: expires_at, publisher: publisher, broker: broker) } } -fn deliver_enrolment_code(code: NonEmptyStr, expires_at: Timestamp, publisher: PublisherTokenRead) -> EnrolmentCodeIssuance admit_callers: [ +fn deliver_enrolment_code(code: NonEmptyStr, expires_at: Timestamp, publisher: PublisherTokenRead, broker: ApprovalBrokerAddress) -> EnrolmentCodeIssuance admit_callers: [ decl_ref(module_path: "gunbc.auth.approval_device_enrolment_code", decl_name: "issue_and_deliver_enrolment_code_until"), decl_ref(module_path: "test.claim.approval_device_enrolment_code_witness_test", decl_name: "the_delivered_receipt_comes_from_the_published_message_and_its_line_never_carries_the_code"), ] { - let message = enrolment_code_notification(code: code, expires_at: expires_at) - EnrolmentCodeIssued { topic: message.topic as NonEmptyStr, expires_at: expires_at, delivery: enrolment_code_delivery_of(publish: publish_notification(message: message, publisher: publisher)) } + match broker { + ApprovalBrokerAddressRefused { detail } => EnrolmentCodeIssuanceRefused { cause: detail as String } + ApprovalBrokerAddressResolved { value: url } => { + let message = enrolment_code_notification(code: code, expires_at: expires_at, approval_url: url) + EnrolmentCodeIssued { topic: message.topic as NonEmptyStr, expires_at: expires_at, delivery: enrolment_code_delivery_of(publish: publish_notification(message: message, publisher: publisher)) } + } + } } // No token means nothing was sent: that, like an explicit 4xx, establishes non-delivery. @@ -203,7 +208,7 @@ fn issue_device_enrolment_code(revision: String) -> CliWireResponse match mint_enrolment_code() { Absent => CliWireUnprintable { cause: "the entropy read did not yield the requested octets" } Present { value: code } => - enrolment_code_issuance_response(i: issue_and_deliver_enrolment_code_until(root: approval_device_store_root, now: now, expires_at: device_instant_plus(t: now, s: approval_enrolment_code_window), code: code, publisher: read_publisher_token())) + enrolment_code_issuance_response(i: issue_and_deliver_enrolment_code_until(root: approval_device_store_root, now: now, expires_at: device_instant_plus(t: now, s: approval_enrolment_code_window), code: code, publisher: read_publisher_token(), broker: approval_broker_serving_base_url())) } } } diff --git a/dag/gunbc/auth/approval_device_enrolment_code_issue.dag b/dag/gunbc/auth/approval_device_enrolment_code_issue.dag index e7b3955fcd7..ef44f5e9888 100644 --- a/dag/gunbc/auth/approval_device_enrolment_code_issue.dag +++ b/dag/gunbc/auth/approval_device_enrolment_code_issue.dag @@ -1,5 +1,6 @@ module gunbc.auth.approval_device_enrolment_code_issue +import gunbc.live_deploy.desired { prod_role_actuated_realization } import std.types { NonEmptyStr, String, List, Bool, CommitSha } import std.algebra { trim } import std.process { ProcessExit, ExitSuccess, ExitFailure, exit_failure } @@ -19,7 +20,9 @@ import gunbc.live_deploy.slice_bounds { approval_broker_slice_memory_max, approv import extdeps.tools.env { env_prefixed_argv } import v2.std.orchestration { EnvSet } import gunbc.cli_run_workspace_root_scaffold { gunbc_workspace_root_env_name } -import gunbc.fleet_host_identity { operator_host_srv1 } +import gunbc.auth.approval_broker_placement { approval_broker_host_under, approval_broker_base_url_under, ApprovalBrokerHostResolved, ApprovalBrokerHostRefused } +import gunbc.auth.approval_broker_endpoint { approval_broker_base_url_env_name, ApprovalBrokerAddressResolved, ApprovalBrokerAddressRefused } +import gunbc.roadmap_dashboard_instance { ProdRoleRealization, prod_role_realization, prod_role_realization_refusal_detail } import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, fleet_converge_expected_host_env_name, read_expected_revision, ExpectedRevisionPresent, ExpectedRevisionAbsent, @@ -42,7 +45,7 @@ import gunbc.live_deploy.release_locus { // โ”€โ”€ THE OPERATOR'S CODE-ISSUING VERB, over the fleet SSH administrator edge โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // A FleetConvergeWorkflowMode step (ApprovalDeviceEnrolmentCodeIssue) run on the host-pinned // runner, exactly as gunbc.auth.approval_keyring_converge reaches srv1: the job prepares the fleet -// SSH context, connects to srv1 as the bootstrap principal, and elevates to the OPERATOR'S POSIX +// SSH context, connects to the broker's placed host as the bootstrap principal, and elevates to the OPERATOR'S POSIX // USER -- the owner of approval_device_store_root and a reader of the ntfy publisher token -- to // run gunbc.auth.approval_device_enrolment_code issue_device_enrolment_code from the broker's admitted // release tree. The remote verb writes the code's slot and delivers the code over the operator's @@ -58,9 +61,28 @@ import gunbc.live_deploy.release_locus { // missing binary and a loud remote failure, never a fallback to another tree. data approval_device_enrolment_code_issue_attempt_raw: String = "approval-device-enrolment-code-issue-ci" -fn enrolment_code_issue_ssh_target() -> SshTarget { +// THE BROKER'S HOST IS PLACEMENT'S, NOT A SPELLED srv1: the verb runs where the broker is installed, +// which follows the prod-role holder (gunbc.auth.approval_broker_placement). The ssh target, the +// dispatch host check and the base URL the verb is given all come from that one placement, and under +// a refused realization the step refuses before reaching any host. +type EnrolmentCodeIssuePlacement + = EnrolmentCodeIssuePlaced { host: NonEmptyStr, base_url: NonEmptyStr } + | EnrolmentCodeIssueUnplaced { detail: NonEmptyStr } + +fn enrolment_code_issue_placement_under(realization: ProdRoleRealization) -> EnrolmentCodeIssuePlacement { + match approval_broker_host_under(realization: realization) { + ApprovalBrokerHostRefused { cause } => + EnrolmentCodeIssueUnplaced { detail: ("approval device enrolment code: " + prod_role_realization_refusal_detail(cause: cause)) as NonEmptyStr } + ApprovalBrokerHostResolved { host } => match approval_broker_base_url_under(realization: realization) { + ApprovalBrokerAddressResolved { value: url } => EnrolmentCodeIssuePlaced { host: host, base_url: url } + ApprovalBrokerAddressRefused { detail } => EnrolmentCodeIssueUnplaced { detail: detail } + } + } +} + +fn enrolment_code_issue_ssh_target(host: NonEmptyStr) -> SshTarget { SshTarget { - endpoint: fleet_probe_endpoint_for(host: operator_host_srv1 as String) as NonEmptyStr, + endpoint: fleet_probe_endpoint_for(host: host as String) as NonEmptyStr, principal: executor_bootstrap_principal.login, } } @@ -102,12 +124,13 @@ fn enrolment_code_issue_scope_properties() -> List { // supplementary groups; systemd-run's own --uid/--gid did not, and every code was minted and then // refused delivery (fleet-converge run 37098981121). The drop sits inside the scope so the bound // still covers the whole verb. -fn enrolment_code_issue_remote_command(revision: ReleaseRevisionBinding) -> SystemdRunCommandReading { +fn enrolment_code_issue_remote_command(revision: ReleaseRevisionBinding, base_url: NonEmptyStr) -> SystemdRunCommandReading { let root = srv1_gunbc_approval_broker_root as String let release_dir = approval_broker_release_dir(root: root, revision: revision) let account = fleet_posix_operator_user.name let verb = env_prefixed_argv(bindings: [ EnvSet { name: gunbc_workspace_root_env_name, value: release_dir }, + EnvSet { name: approval_broker_base_url_env_name as String, value: base_url as String }, ], command_argv: [ approval_broker_release_binary_path(root: root, revision: revision), "run", @@ -121,26 +144,26 @@ fn enrolment_code_issue_remote_command(revision: ReleaseRevisionBinding) -> Syst systemd_run_system_scope_command(properties: enrolment_code_issue_scope_properties(), command_argv: as_account) } -fn enrolment_code_issue_refuses_off_srv1(host: String) -> ProcessExit { - if host == (operator_host_srv1 as String) { ExitSuccess } - else { exit_failure(reason: "approval device enrolment code: refuses host '" + host + "': the store lives on the machine that serves /approve, which is " + (operator_host_srv1 as String)) } +fn enrolment_code_issue_refuses_off_broker_host(host: String, broker_host: NonEmptyStr) -> ProcessExit { + if host == (broker_host as String) { ExitSuccess } + else { exit_failure(reason: "approval device enrolment code: refuses host '" + host + "': the store lives on the machine that serves /approve, which is " + (broker_host as String)) } } // The projection refuses a word the wire cannot carry, and the invocation the transport would have // run is never minted: the caller sees the refusal's reason, not a command that failed to run -- // because it IS one. -fn enrolment_code_issue_remote(revision: ReleaseRevisionBinding) -> CliWireResponse +fn enrolment_code_issue_remote(revision: ReleaseRevisionBinding, host: NonEmptyStr, base_url: NonEmptyStr) -> CliWireResponse uses net: Network { match prepare_fleet_ssh_agent_context(attempt_raw: approval_device_enrolment_code_issue_attempt_raw) { FleetSshContextRefused { cause: c } => CliWireUnprintable { cause: ("approval device enrolment code: " + c) as NonEmptyStr } FleetSshContextReady { context: context, receipt: _ } => - match enrolment_code_issue_remote_command(revision: revision) { + match enrolment_code_issue_remote_command(revision: revision, base_url: base_url) { SystemdRunCommandRefused { cause: why } => CliWireUnprintable { cause: ("approval device enrolment code: remote invocation refused: " + systemd_run_option_words_refused_cause_detail(cause: why)) as NonEmptyStr } SystemdRunCommandReady { command: command } => match typed_argv_exec_over_fleet_ssh( - target: enrolment_code_issue_ssh_target(), + target: enrolment_code_issue_ssh_target(host: host), context: context, argv: sudo_elevate_words(command: command), ) { @@ -158,21 +181,24 @@ fn sudo_elevate_words(command: ArgvCommand) -> List { concat([inv.bin_path], inv.args) } -// THE STEP ENTRY: one host per run, named by the dispatch and required to be srv1; one release, -// named by the dispatch. +// THE STEP ENTRY: one host per run, named by the dispatch and required to be the broker's placed host; +// one release, named by the dispatch. fn approval_device_enrolment_code_issue_wet() -> CliWireResponse uses net: Network { match actions_variable_read(name: fleet_converge_expected_host_env_name) { ActionsVariableAbsent { variable: v, detail: d } => CliWireUnprintable { cause: ("approval device enrolment code: " + v + " " + d + "; one host per run, named by the dispatch") as NonEmptyStr } - ActionsVariablePresent { value: host } => - match enrolment_code_issue_refuses_off_srv1(host: host) { - ExitSuccess => - match read_expected_revision() { - ExpectedRevisionAbsent { variable: v, detail: d } => CliWireUnprintable { cause: ("approval device enrolment code: " + v + " " + d + "; the release to run is named by the dispatch") as NonEmptyStr } - ExpectedRevisionPresent { revision: rev } => enrolment_code_issue_remote(revision: RevisionBoundAtEmission { revision: rev as CommitSha }) - } - ExitFailure { code: _, reason: why } => CliWireUnprintable { cause: why as NonEmptyStr } - } + ActionsVariablePresent { value: host } => match enrolment_code_issue_placement_under(realization: prod_role_actuated_realization()) { + EnrolmentCodeIssueUnplaced { detail } => CliWireUnprintable { cause: detail } + EnrolmentCodeIssuePlaced { host: broker_host, base_url } => + match enrolment_code_issue_refuses_off_broker_host(host: host, broker_host: broker_host) { + ExitSuccess => + match read_expected_revision() { + ExpectedRevisionAbsent { variable: v, detail: d } => CliWireUnprintable { cause: ("approval device enrolment code: " + v + " " + d + "; the release to run is named by the dispatch") as NonEmptyStr } + ExpectedRevisionPresent { revision: rev } => enrolment_code_issue_remote(revision: RevisionBoundAtEmission { revision: rev as CommitSha }, host: broker_host, base_url: base_url) + } + ExitFailure { code: _, reason: why } => CliWireUnprintable { cause: why as NonEmptyStr } + } + } } } diff --git a/dag/gunbc/auth/approval_gate.dag b/dag/gunbc/auth/approval_gate.dag index 3cb5de14324..67a74077526 100644 --- a/dag/gunbc/auth/approval_gate.dag +++ b/dag/gunbc/auth/approval_gate.dag @@ -1,5 +1,6 @@ module gunbc.auth.approval_gate +import std.decl_ref { decl_ref } import std.types { String, Bool, Int, NonEmptyStr, List, Timestamp } import std.measure { Second, second_count } import std.optional { Present, Absent } @@ -25,6 +26,8 @@ import gunbc.auth.approval_decision_store { grant_from_approved_decision, GrantD import gunbc.auth.approval_keyring_pins { approval_submission_mac_key_path_env } import gunbc.auth.approval_request_client { sign_stored_request, approval_submit_url, approval_status_url, approval_poll_standing, + approval_origin_admission_here, approval_origin_standing_detail, DialableApprovalOrigin, + ApprovalOriginDialable, ApprovalOriginNotAdmitted, SubmissionMacReady, SubmissionMacKeyNotHex, ApprovalPollStanding, PollApproved, PollDenied, PollExpired, PollStillPending, PollNotFiled, PollUnreadable, } @@ -193,8 +196,14 @@ fn approval_standing_read(escalation_id: NonEmptyStr, expires_at: Timestamp) -> match approval_instant_now() { Absent => StandingClockUnreadable Present { value: observed } => { - let got = http.Client.GetLocalhostBounded(url: approval_status_url(escalation_id: escalation_id)) - StandingReadAt { standing: approval_poll_standing(status: approval_status_from_get(success: got.success, body: got.body), observed_at: observed, expires_at: expires_at), observed_at: observed } + match approval_origin_admission_here() { + ApprovalOriginNotAdmitted { standing } => + StandingReadAt { standing: approval_poll_standing(status: StatusUnreadable { detail: approval_origin_standing_detail(standing: standing) }, observed_at: observed, expires_at: expires_at), observed_at: observed } + ApprovalOriginDialable { dial } => { + let got = http.Client.GetLocalhostBounded(url: approval_status_url(dial: dial, escalation_id: escalation_id)) + StandingReadAt { standing: approval_poll_standing(status: approval_status_from_get(success: got.success, body: got.body), observed_at: observed, expires_at: expires_at), observed_at: observed } + } + } } } } @@ -255,6 +264,15 @@ type ApprovalFiling fn approval_file_stored_request(stored: StoredApprovalRequest, body_file_name: String, submit_timeout: Second) -> ApprovalFiling { + match approval_origin_admission_here() { + ApprovalOriginDialable { dial } => approval_file_stored_request_at(dial: dial, stored: stored, body_file_name: body_file_name, submit_timeout: submit_timeout) + ApprovalOriginNotAdmitted { standing } => ApprovalFilingRefused { reason: approval_origin_standing_detail(standing: standing) as String } + } +} + +fn approval_file_stored_request_at(dial: DialableApprovalOrigin, stored: StoredApprovalRequest, body_file_name: String, submit_timeout: Second) -> ApprovalFiling admit_callers: [ + decl_ref(module_path: "gunbc.auth.approval_gate", decl_name: "approval_file_stored_request"), +] { let sub_path = actions_variable_trimmed(name: approval_submission_mac_key_path_env) if sub_path == "" { ApprovalFilingRefused { reason: join([approval_submission_mac_key_path_env as String, " is unset; the request cannot be filed"], "") } @@ -276,7 +294,7 @@ fn approval_file_stored_request(stored: StoredApprovalRequest, body_file_name: S if !written.success { ApprovalFilingRefused { reason: join(["could not write the submit body: ", written.error], "") } } else { - let posted = http.Client.PostJsonFromFile(url: approval_submit_url(), request_body_file: body_path as NonEmptyStr, max_seconds: to_string(second_count(s: submit_timeout)) as NonEmptyStr) + let posted = http.Client.PostJsonFromFile(url: approval_submit_url(dial: dial), request_body_file: body_path as NonEmptyStr, max_seconds: to_string(second_count(s: submit_timeout)) as NonEmptyStr) let _wiped = Filesystem.Delete(path: body_path) if submission_body_admits_poll(body: posted.body) { ApprovalFiled } else { ApprovalFilingRefused { reason: join(["POST /approvals refused: ", posted.body], "") } } } diff --git a/dag/gunbc/auth/approval_request_client.dag b/dag/gunbc/auth/approval_request_client.dag index f395f4b43fd..75fcad027c3 100644 --- a/dag/gunbc/auth/approval_request_client.dag +++ b/dag/gunbc/auth/approval_request_client.dag @@ -1,6 +1,8 @@ module gunbc.auth.approval_request_client +import gunbc.live_deploy.desired { prod_role_actuated_realization } import std.types { Bool, NonEmptyStr, String, Timestamp, Secret } +import std.decl_ref { decl_ref } import std.scoped_authorization { AuthorizationRequest } import extdeps.crypto.mac { MacKey, MacSigned, MacKeyMaterialNotHex, HmacSha256, MacKeyId, @@ -16,7 +18,12 @@ import gunbc.auth.approval_request_submission { approval_submission_mac_key_id, sign_submission, } import gunbc.fleet_posix_accounts { fleet_operator_email } -import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } +import gunbc.hostname_read { hostname_short_read, HostnameShortCapture, HostnameShortCaptured, HostnameShortCaptureRefused } +import gunbc.host_effect { LocalShell } +import gunbc.roadmap_dashboard_instance { + HostDashboardInstance, ProdRoleRealization, ProdRoleRealizedBy, ProdRoleRealizationRefused, ProdRoleRealizationRefusal, + prod_role_realization, prod_role_realization_refusal_detail, +} import gunbc.auth.approval_writer_authority { approval_writer_authority, approval_writer_process, ApprovalServingProcess, RoadmapProcess, BrokerProcess, } @@ -33,30 +40,126 @@ import gunbc.auth.approval_broker_endpoint { // disagree; the broker's address and paths come from gunbc.auth.approval_broker_endpoint. data approval_request_submit_path: NonEmptyStr = approval_broker_submit_path -fn approval_loopback_origin_for(writer: ApprovalServingProcess) -> NonEmptyStr { - match writer { - RoadmapProcess => { - let instance = srv1_live_dashboard_instance() - concat( - concat("http://", instance.listen_host as String), - concat(":", to_string(instance.listen_port)), - ) as NonEmptyStr - } - BrokerProcess => approval_broker_loopback_base_url(port: approval_broker_listen_port) +// AND THE WRITER LIVES ON THE PROD-ROLE HOLDER'S DEPLOYMENT (gunbc.deployment_risk +// prod_role_selection). Whichever process the authority row names, it is served by the holder's +// instance -- the roadmap at that instance's listen bind, the broker on that host's loopback -- so both +// arms resolve through gunbc.roadmap_dashboard_instance prod_role_realization. Under NoProdRole neither +// is installed anywhere and the origin REFUSES with the realization's typed cause; it never falls back +// to srv1. +// THE ORIGIN CARRIES ITS HOST. A loopback origin means something only on the host it is loopback ON, +// so the resolved arm names the holder instance's host beside the endpoint; dropping it would let an +// executor on another host dial a stale writer there instead of refusing. +type ApprovalLoopbackOrigin + = ApprovalLoopbackOriginResolved { host: NonEmptyStr, origin: NonEmptyStr } + | ApprovalLoopbackOriginRefused { cause: ProdRoleRealizationRefusal } + +fn approval_loopback_origin_of(instance: HostDashboardInstance) -> NonEmptyStr { + concat( + concat("http://", instance.listen_host as String), + concat(":", to_string(instance.listen_port)), + ) as NonEmptyStr +} + +fn approval_loopback_origin_for(writer: ApprovalServingProcess, realization: ProdRoleRealization) -> ApprovalLoopbackOrigin { + match realization { + ProdRoleRealizationRefused { cause } => ApprovalLoopbackOriginRefused { cause: cause } + ProdRoleRealizedBy { holder } => + match writer { + RoadmapProcess => ApprovalLoopbackOriginResolved { host: holder.instance.host_identity, origin: approval_loopback_origin_of(instance: holder.instance) } + BrokerProcess => ApprovalLoopbackOriginResolved { host: holder.instance.host_identity, origin: approval_broker_loopback_base_url(port: approval_broker_listen_port) } + } + } +} + +fn approval_loopback_origin_under(realization: ProdRoleRealization) -> ApprovalLoopbackOrigin { + approval_loopback_origin_for(writer: approval_writer_process(authority: approval_writer_authority), realization: realization) +} + +fn approval_loopback_origin() -> ApprovalLoopbackOrigin { + approval_loopback_origin_under(realization: prod_role_actuated_realization()) +} + +fn approval_loopback_origin_refusal_detail(cause: ProdRoleRealizationRefusal) -> NonEmptyStr { + join(["approval broker origin unresolved: ", prod_role_realization_refusal_detail(cause: cause)], "") as NonEmptyStr +} + +// โ”€โ”€ THE EXECUTOR MUST BE THE ORIGIN'S HOST, BEFORE ANY LOCAL HTTP โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// CLASSIFIER VERSUS CAPABILITY, the split gunbc.live_deploy.role_singleton_convergence uses for its +// admission. classify_approval_origin is pure and fixture-callable and returns a STANDING, never a +// token. The token -- DialableApprovalOrigin, the only thing a URL can be built from -- is minted by +// one sealed function, reachable only from approval_origin_admission_here, which reads this +// executor's own short hostname itself (gunbc.hostname_read, the established local-host read) and +// is in turn sealed to the two effectful approval_gate compositions. So no caller can hand in a +// hostname, and no URL exists for an executor that is not the prod-role holder's host. +type DialableApprovalOrigin sole_constructor { + host: NonEmptyStr + origin: NonEmptyStr +} + +type ApprovalOriginStanding + = ApprovalOriginWouldDial { host: NonEmptyStr, origin: NonEmptyStr } + | ApprovalOriginUnresolved { cause: ProdRoleRealizationRefusal } + | ApprovalExecutorNotTheOriginHost { origin_host: NonEmptyStr, executor_host: String } + | ApprovalExecutorHostUnobserved { reason: String } + +fn classify_approval_origin(origin: ApprovalLoopbackOrigin, executor: HostnameShortCapture) -> ApprovalOriginStanding { + match origin { + ApprovalLoopbackOriginRefused { cause } => ApprovalOriginUnresolved { cause: cause } + ApprovalLoopbackOriginResolved { host, origin: o } => + match executor { + HostnameShortCaptureRefused { reason } => ApprovalExecutorHostUnobserved { reason: reason } + HostnameShortCaptured { value } => + if value == (host as String) { + ApprovalOriginWouldDial { host: host, origin: o } + } else { + ApprovalExecutorNotTheOriginHost { origin_host: host, executor_host: value } + } + } } } -fn approval_loopback_origin() -> NonEmptyStr { - approval_loopback_origin_for(writer: approval_writer_process(authority: approval_writer_authority)) +type ApprovalOriginAdmission + = ApprovalOriginDialable { dial: DialableApprovalOrigin } + | ApprovalOriginNotAdmitted { standing: ApprovalOriginStanding } + +fn mint_dialable_approval_origin(host: NonEmptyStr, origin: NonEmptyStr) -> DialableApprovalOrigin admit_callers: [ + decl_ref(module_path: "gunbc.auth.approval_request_client", decl_name: "approval_origin_admission_here"), +] { + DialableApprovalOrigin { host: host, origin: origin } +} + +// The live admission: the actuated origin against this process's own host, read here and nowhere else. +fn approval_origin_admission_here() -> ApprovalOriginAdmission admit_callers: [ + decl_ref(module_path: "gunbc.auth.approval_gate", decl_name: "approval_standing_read"), + decl_ref(module_path: "gunbc.auth.approval_gate", decl_name: "approval_file_stored_request"), +] { + let standing = classify_approval_origin(origin: approval_loopback_origin(), executor: hostname_short_read(transport: LocalShell)) + match standing { + ApprovalOriginWouldDial { host, origin } => ApprovalOriginDialable { dial: mint_dialable_approval_origin(host: host, origin: origin) } + ApprovalOriginUnresolved { cause: _ } => ApprovalOriginNotAdmitted { standing: standing } + ApprovalExecutorNotTheOriginHost { origin_host: _, executor_host: _ } => ApprovalOriginNotAdmitted { standing: standing } + ApprovalExecutorHostUnobserved { reason: _ } => ApprovalOriginNotAdmitted { standing: standing } + } +} + +fn approval_origin_standing_detail(standing: ApprovalOriginStanding) -> NonEmptyStr { + match standing { + ApprovalOriginWouldDial { host: _, origin: _ } => "approval origin admitted" as NonEmptyStr + ApprovalOriginUnresolved { cause } => approval_loopback_origin_refusal_detail(cause: cause) + ApprovalExecutorNotTheOriginHost { origin_host, executor_host } => + join(["approval writer is loopback on ", origin_host as String, " (the prod-role holder's host) but this executor is ", executor_host, "; refusing before any request"], "") as NonEmptyStr + ApprovalExecutorHostUnobserved { reason } => + join(["this executor's host could not be read (", reason, "); refusing before any request to the approval writer"], "") as NonEmptyStr + } } -fn approval_submit_url() -> NonEmptyStr { - concat(approval_loopback_origin() as String, approval_request_submit_path as String) as NonEmptyStr +fn approval_submit_url(dial: DialableApprovalOrigin) -> NonEmptyStr { + concat(dial.origin as String, approval_request_submit_path as String) as NonEmptyStr } -fn approval_status_url(escalation_id: NonEmptyStr) -> NonEmptyStr { +fn approval_status_url(dial: DialableApprovalOrigin, escalation_id: NonEmptyStr) -> NonEmptyStr { concat( - concat(approval_loopback_origin() as String, concat(approval_request_submit_path as String, "/")), + concat(dial.origin as String, concat(approval_request_submit_path as String, "/")), escalation_id as String, ) as NonEmptyStr } diff --git a/dag/gunbc/auth/approval_request_submission.dag b/dag/gunbc/auth/approval_request_submission.dag index fdf623c4b08..baaecb8161a 100644 --- a/dag/gunbc/auth/approval_request_submission.dag +++ b/dag/gunbc/auth/approval_request_submission.dag @@ -14,7 +14,7 @@ import gunbc.auth.approval_capability { ProposedDecision, ProposeApprove, Propos import gunbc.auth.approval_broker { SubmissionAdmission, SubmissionAdmitted, SubmissionRequesterNotEnrolled, SubmissionAuthorityNotPermittedForRequester, admit_submission, BrokerApproved, BrokerDenied } import gunbc.auth.approval_notification { approval_notification } import gunbc.auth.approval_ntfy_publish -import gunbc.auth.approval_broker_endpoint { approval_broker_base_url, approval_broker_confirm_path_prefix } +import gunbc.auth.approval_broker_endpoint { approval_broker_confirm_path_prefix, ApprovalBrokerAddress, ApprovalBrokerAddressResolved, ApprovalBrokerAddressRefused } import std.algebra { trim } import gunbc.auth.approval_status_wire { FiledApprovalStatus, StatusNotFiled, StatusPending, StatusApproved, StatusDenied, StatusUnreadable, @@ -200,7 +200,7 @@ fn admit_parsed_submission(tag_hex: NonEmptyStr, request: StoredApprovalRequest, // that sentence false while leaving the STRING it produced byte-identical, because the front door // keeps the public host and port. A grounding that has gone false while its output stayed right is // exactly the fact nothing reads back, so the derivation moved to the authority that now owns the -// endpoint (gunbc.auth.approval_broker_endpoint approval_broker_base_url) rather than being left +// endpoint (gunbc.auth.approval_broker_endpoint approval_broker_serving_base_url) rather than being left // here with its reason rewritten. The undeclared-domain `.invalid` arm still lives with // gunbc.fleet_tailnet tailnet_base_url_from and is not restated at either site. @@ -253,6 +253,7 @@ type SubmissionOutcome | SubmissionFiledNotNotified { escalation_id: NonEmptyStr, publish: NtfyPublishOutcome } | SubmissionFiledPublisherTokenUnavailable { escalation_id: NonEmptyStr, detail: String } | SubmissionFiledLinksNotIssued { escalation_id: NonEmptyStr, key_id: MacKeyId } + | SubmissionFiledBrokerAddressRefused { escalation_id: NonEmptyStr, detail: NonEmptyStr } | SubmissionAlreadyFiled { escalation_id: NonEmptyStr } | SubmissionStoreRefused { detail: NonEmptyStr } | SubmissionRefused { refusal: SubmissionRefusal } @@ -264,6 +265,7 @@ fn submission_outcome_admits_poll(o: SubmissionOutcome) -> Bool { SubmissionFiledNotNotified { escalation_id: _, publish: _ } => true SubmissionFiledPublisherTokenUnavailable { escalation_id: _, detail: _ } => true SubmissionFiledLinksNotIssued { escalation_id: _, key_id: _ } => false + SubmissionFiledBrokerAddressRefused { escalation_id: _, detail: _ } => false SubmissionAlreadyFiled { escalation_id: _ } => true SubmissionFiledCapabilityKeyUnavailable { escalation_id: _, cause: _ } => false SubmissionStoreRefused { detail: _ } => false @@ -281,6 +283,7 @@ fn submission_outcome_reason(o: SubmissionOutcome) -> String { SubmissionFiledNotNotified { escalation_id: e, publish: p } => join(["filed ", e as String, " but the notification is not confirmed published (a re-submission re-notifies the same request): ", ntfy_publish_outcome_sentence(o: p)], "") SubmissionFiledPublisherTokenUnavailable { escalation_id: e, detail: d } => join(["filed ", e as String, " but the publisher token is unavailable: ", d], "") SubmissionFiledLinksNotIssued { escalation_id: e, key_id: k } => join(["filed ", e as String, " but no link could be issued under key ", k as String], "") + SubmissionFiledBrokerAddressRefused { escalation_id: e, detail: d } => join(["filed ", e as String, " but no link could be issued: ", d as String], "") SubmissionAlreadyFiled { escalation_id: e } => join(["escalation ", e as String, " is already filed"], "") SubmissionStoreRefused { detail: d } => join(["approval store refused the filing: ", d as String], "") SubmissionRefused { refusal: v } => match v { @@ -306,10 +309,13 @@ fn notify_filed( capability_keyring: ApprovalKeyringRead, publisher: PublisherTokenRead, request: StoredApprovalRequest, + broker: ApprovalBrokerAddress, ) -> SubmissionOutcome { match capability_keyring { ApprovalKeyringRefused { cause: c } => SubmissionFiledCapabilityKeyUnavailable { escalation_id: request.escalation_id, cause: c } - ApprovalKeyLoaded { key: key } => match issue_links(capability_key: key, base: approval_broker_base_url(), request: request) { + ApprovalKeyLoaded { key: key } => match broker { + ApprovalBrokerAddressRefused { detail } => SubmissionFiledBrokerAddressRefused { escalation_id: request.escalation_id, detail: detail } + ApprovalBrokerAddressResolved { value: base } => match issue_links(capability_key: key, base: base, request: request) { LinksNotIssued { key_id: k } => SubmissionFiledLinksNotIssued { escalation_id: request.escalation_id, key_id: k } links => match approval_message(request: request, links: links) { Absent => SubmissionFiledLinksNotIssued { escalation_id: request.escalation_id, key_id: key.id } @@ -324,6 +330,7 @@ fn notify_filed( } } } + } } } @@ -392,16 +399,17 @@ fn file_and_notify( capability_keyring: ApprovalKeyringRead, publisher: PublisherTokenRead, request: StoredApprovalRequest, + broker: ApprovalBrokerAddress, ) -> SubmissionOutcome { match file_approval_request(root: root, request: request) { StoreSlotOccupied => match occupied_slot_standing(slot: observe_escalation(root: root, escalation_id: request.escalation_id), request: request) { - OccupiedRenotifiable => notify_filed(capability_keyring: capability_keyring, publisher: publisher, request: request) + OccupiedRenotifiable => notify_filed(capability_keyring: capability_keyring, publisher: publisher, request: request, broker: broker) OccupiedConflict => SubmissionAlreadyFiled { escalation_id: request.escalation_id } OccupiedUnreadable { detail: d } => SubmissionStoreRefused { detail: d } } StoreRefused { detail: d } => SubmissionStoreRefused { detail: d } - StoreWritten => notify_filed(capability_keyring: capability_keyring, publisher: publisher, request: request) + StoreWritten => notify_filed(capability_keyring: capability_keyring, publisher: publisher, request: request, broker: broker) } } @@ -419,9 +427,10 @@ fn submit_parsed_submission( publisher: PublisherTokenRead, tag_hex: NonEmptyStr, request: StoredApprovalRequest, + broker: ApprovalBrokerAddress, ) -> SubmissionOutcome { match admit_parsed_submission(tag_hex: tag_hex, request: request, keyring: submission_keyring) { - SubmissionVerified { request: r } => file_and_notify(root: root, capability_keyring: capability_keyring, publisher: publisher, request: r) + SubmissionVerified { request: r } => file_and_notify(root: root, capability_keyring: capability_keyring, publisher: publisher, request: r, broker: broker) SubmissionNotAdmitted { refusal: refused } => SubmissionRefused { refusal: refused } } } diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index bd95ae2f46a..617bfb86c8d 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -1,6 +1,6 @@ module gunbc.fabric_event_log -import gunbc.fabric_storage_binding { fabric_storage_binding_for } +import gunbc.fabric_storage_binding { fabric_storage_binding_for, fabric_storage_placement } import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat, nat_range_inclusive } @@ -14,7 +14,6 @@ import std.fabric_storage { FabricHeadAdvanced, FabricHeadMoved, FabricHeadAdvanceRefused, FabricClosureRead, FabricClosureBudgetExhausted, FabricClosureRefused, FabricStoredObject, } -import gunbc.fabric_storage_placement { fabric_storage_placement } import gunbc.fabric_storage_client { FabricStorageBinding, fabric_storage_head, fabric_storage_put, fabric_storage_advance, fabric_storage_closure } import product.capacity.pool { Pool, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at, PoolOutcome, PoolAdvanced, PoolRefused, pool_release, pool_refusal_wire } import product.capacity.event_chain { diff --git a/dag/gunbc/fabric/fabric_storage_binding.dag b/dag/gunbc/fabric/fabric_storage_binding.dag index e83f400349c..934d6d1cb74 100644 --- a/dag/gunbc/fabric/fabric_storage_binding.dag +++ b/dag/gunbc/fabric/fabric_storage_binding.dag @@ -1,17 +1,27 @@ module gunbc.fabric_storage_binding +import gunbc.live_deploy.desired { prod_role_actuated_realization } import std.types { String, NonEmptyStr } import gunbc.fabric_storage_address { fabric_storage_state_suffix, fabric_storage_route_prefix } -import gunbc.fabric_storage_placement { fabric_storage_placement, FabricStoragePlacement, FabricStoragePlaced, FabricStorageUnplaced, fabric_storage_door_socket } +import gunbc.fabric_storage_placement { fabric_storage_placement_under, FabricStoragePlacement, FabricStoragePlaced, FabricStorageUnplaced, FabricStoragePlacementRefused } +import gunbc.roadmap_dashboard_instance { prod_role_realization_refusal_detail } import product.host_identity { HostIdentity, host_identity_eq } import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageBindingUnplaced, FabricStorageBindingRefused, FabricStorageDoorSocket, FabricStorageServed } +// THE STORE'S RUNTIME PLACEMENT, over the prod role as something that can be actuated +// (gunbc.live_deploy.desired prod_role_actuated_realization): a holder with no desired deployment row +// refuses here rather than placing the store on a host nothing installs it on. +fn fabric_storage_placement() -> FabricStoragePlacement { + fabric_storage_placement_under(realization: prod_role_actuated_realization()) +} + fn fabric_storage_binding_for(placement: FabricStoragePlacement, executor: HostIdentity) -> FabricStorageBinding { match placement { FabricStorageUnplaced => FabricStorageBindingUnplaced - FabricStoragePlaced { host: h, store_root: _, endpoint: e } => + FabricStoragePlacementRefused { cause } => FabricStorageBindingRefused { detail: prod_role_realization_refusal_detail(cause: cause) as NonEmptyStr } + FabricStoragePlaced { host: h, store_root: _, endpoint: e, door_socket: k } => if host_identity_eq(a: h, b: executor) { - FabricStorageDoorSocket { socket: fabric_storage_door_socket(), route_prefix: fabric_storage_route_prefix as NonEmptyStr } + FabricStorageDoorSocket { socket: k, route_prefix: fabric_storage_route_prefix as NonEmptyStr } } else { FabricStorageServed { endpoint: e } } } } @@ -23,7 +33,8 @@ fn fabric_storage_binding_for(placement: FabricStoragePlacement, executor: HostI fn fabric_state_storage_binding() -> FabricStorageBinding { match fabric_storage_placement() { FabricStorageUnplaced => FabricStorageBindingUnplaced - FabricStoragePlaced { host: _, store_root: _, endpoint } => FabricStorageServed { + FabricStoragePlacementRefused { cause } => FabricStorageBindingRefused { detail: prod_role_realization_refusal_detail(cause: cause) as NonEmptyStr } + FabricStoragePlaced { host: _, store_root: _, endpoint, door_socket: _ } => FabricStorageServed { endpoint: concat(endpoint as String, fabric_storage_state_suffix) as NonEmptyStr, } } diff --git a/dag/gunbc/fabric/fabric_storage_placement.dag b/dag/gunbc/fabric/fabric_storage_placement.dag index bebf0d3a505..05498f3cdd2 100644 --- a/dag/gunbc/fabric/fabric_storage_placement.dag +++ b/dag/gunbc/fabric/fabric_storage_placement.dag @@ -5,8 +5,10 @@ import gunbc.fabric_storage_address { fabric_storage_route_prefix, fabric_storag import std.types { String, NonEmptyStr, FilePath, Int, Bool, Port, List } import product.host_identity { HostIdentity } import gunbc.fleet_intent_network { fleet_intent_network } -import gunbc.fleet_host_identity { operator_host_srv1 } -import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_fabric_storage_root, srv1_live_dashboard_instance, dashboard_instance_child } +import gunbc.roadmap_dashboard_instance { + HostDashboardInstance, dashboard_instance_fabric_storage_root, dashboard_instance_child, dashboard_ownership_marker_in, + ProdRoleRealization, ProdRoleRealizedBy, ProdRoleRealizationRefused, ProdRoleRealizationRefusal, prod_role_realization, +} import std.effect_grant { Read, Write, Execute } import extdeps.tailscale.serve { tailscale_serve_unix_proxy_peer } import gunbc.ownership { Ensured } @@ -16,12 +18,16 @@ import gunbc.managed_directory { } import gunbc.fleet_posix_accounts { fleet_posix_ci_runner_user } import gunbc.fabric_storage_file_store { FabricStorageFileRoot, fabric_storage_file_root, fabric_storage_file_root_with } -import extdeps.access.posix { FileMode, file_mode_bits } +import extdeps.access.posix { PosixUser, FileMode, PermissionBits, file_mode_bits, file_mode_octal, file_mode_of_octal_text } +import std.algebra { trim } // WHERE THE FABRIC DB IS: one host holds the store's files and answers the endpoint. This is the // POLICY fact of the three (DESIGN 3) -- std.fabric_storage is the interface, the file store and the -// served handler are its realization, and which host and which root is this row. Moving the store is -// a row change here; nothing downstream names the host. +// served handler are its realization. WHICH deployment holds it is not a row here: the store is the +// prod deployment's state, so the placement FOLLOWS gunbc.deployment_risk prod_role_selection through +// gunbc.roadmap_dashboard_instance prod_role_realization. Moving prod moves the store; nothing here or +// downstream names a host. Under NoProdRole the placement REFUSES with the realization's typed cause +// (FabricStoragePlacementRefused); it never falls back to srv1. // // ONE PLACEMENT, NOT A REPLICA SET (operator approval 2026-09-19): a head needs exactly one // linearization point, and an unreachable placement refuses acquisitions rather than failing over. @@ -29,7 +35,8 @@ import extdeps.access.posix { FileMode, file_mode_bits } type FabricStoragePlacement = FabricStorageUnplaced - | FabricStoragePlaced { host: HostIdentity, store_root: NonEmptyStr, endpoint: NonEmptyStr } + | FabricStoragePlacementRefused { cause: ProdRoleRealizationRefusal } + | FabricStoragePlaced { host: HostIdentity, store_root: NonEmptyStr, endpoint: NonEmptyStr, door_socket: NonEmptyStr } // The endpoint is the placed host's tailnet name on the store's own HTTPS listener, derived from the // fleet network's tailnet domain rather than spelled: an unset domain leaves the store unplaced @@ -51,12 +58,12 @@ data fabric_storage_https_port: Port = 10000 // admits exactly the writers declared below. tailscale serve proxies to it as root // (extdeps.tailscale.serve tailscale_serve_unix_target), so a proxied request's peer is root and a // local writer's peer is itself -- the two populations the door tells apart. -fn fabric_storage_door_directory_path() -> NonEmptyStr { - dashboard_instance_child(root: fabric_storage_placed_instance().instance_root, child: "fabric-storage-door") as String as NonEmptyStr +fn fabric_storage_door_directory_path(instance: HostDashboardInstance) -> NonEmptyStr { + dashboard_instance_child(root: instance.instance_root, child: "fabric-storage-door") as String as NonEmptyStr } -fn fabric_storage_door_socket() -> NonEmptyStr { - join([fabric_storage_door_directory_path() as String, "/door.sock"], "") as NonEmptyStr +fn fabric_storage_door_socket(instance: HostDashboardInstance) -> NonEmptyStr { + join([fabric_storage_door_directory_path(instance: instance) as String, "/door.sock"], "") as NonEmptyStr } // The peer the tailscale proxy connects as. tailscaled runs as root, and since v1.98.9 only root may @@ -68,18 +75,122 @@ data fabric_storage_door_proxy_peer: NonEmptyStr = tailscale_serve_unix_proxy_pe // The store root is the placed instance's fabric DB root -- one authority with the deployment that // creates it (gunbc.roadmap_dashboard_instance dashboard_instance_fabric_storage_root). -fn fabric_storage_placed_instance() -> HostDashboardInstance { - srv1_live_dashboard_instance() +fn fabric_storage_store_root(instance: HostDashboardInstance) -> NonEmptyStr { + dashboard_instance_fabric_storage_root(instance: instance) as String as NonEmptyStr +} + +// THE STORE'S OWNERSHIP MARKER, by the instance ownership-marker rule +// (gunbc.roadmap_dashboard_instance dashboard_ownership_marker_in) applied to the store's CONTROL +// directory, not to the store root. The store root, auth state and door directories are Ensured and no +// retract removes them, so a unit file alone cannot say whether a host holds a store; the marker can, +// and the role-singleton observation reads it (gunbc.live_deploy.role_singleton_convergence) as the +// deploy job principal. The store root's mode is derived from its one dependent, the service, and +// grants nobody else traversal, so a marker there is unreadable to that observer and every assessment +// would refuse as unobservable; widening the store root to admit the observer would hand it the store. +fn fabric_storage_ownership_marker_path(instance: HostDashboardInstance) -> NonEmptyStr { + dashboard_ownership_marker_in(root: fabric_storage_control_directory_path(instance: instance) as String as FilePath, instance_id: instance.instance_id) as String as NonEmptyStr +} + +fn fabric_storage_control_directory_path(instance: HostDashboardInstance) -> NonEmptyStr { + dashboard_instance_child(root: instance.instance_root, child: "fabric-storage-control") as String as NonEmptyStr +} + +// WHO READS THE MARKER: the principal the role-singleton observation runs as, the deploy job principal +// (gunbc.fleet_posix_accounts srv1_ci_deploy_runner_principal job_user). It already traverses the +// instance root to reach the door (fabric_storage_door_dependents), so the control directory is the +// only new grant, and it is read and traverse, never write. +data fabric_storage_marker_observer: PosixUser = fleet_posix_ci_runner_user + +// THE CONTROL DIRECTORY holds only the marker. The service writes it; the observer is its group and +// reads it. Its mode is derived from those two dependents (gunbc.managed_directory +// managed_directory_permissions), so the observer's access is a declared dependency, not a number. +fn fabric_storage_control_directory(instance: HostDashboardInstance) -> ManagedDirectory { + let service = srv1_dashboard_service_principal() + ManagedDirectory { + member: "fabric-storage-control", + path: fabric_storage_control_directory_path(instance: instance) as String as FilePath, + owner: service, + group_principal: fabric_storage_marker_observer, + dependents: [ + DirectoryDependent { who: service, needs: [Read, Write, Execute] }, + DirectoryDependent { who: fabric_storage_marker_observer, needs: [Read, Execute] }, + ], + ownership: Ensured, + entries: EntriesUnmanaged, + } +} + +fn fabric_storage_store_root_directory(instance: HostDashboardInstance) -> ManagedDirectory { + fabric_storage_directory(member: "fabric-storage-root", path: fabric_storage_store_root(instance: instance), writes: false) +} + +// THE MARKER'S CONTENT names whose store it is: the deployment and the host. +fn fabric_storage_ownership_marker_content(instance: HostDashboardInstance) -> NonEmptyStr { + join(["deployment=", instance.deployment.name as String, "\nhost=", instance.host_identity as String, "\n"], "") as NonEmptyStr } -fn fabric_storage_store_root() -> NonEmptyStr { - dashboard_instance_fabric_storage_root(instance: fabric_storage_placed_instance()) as String as NonEmptyStr +// THE MARKER'S ONE AUTHORITY: path, exact content, mode, owner and group, in one row derived from the +// instance. The holder spec installs it from this row (owner, group and mode applied, not defaulted), +// the observer reads every field back and admits a marker only on an exact match, and the refusal for +// an unmarked realization quotes this row -- so the install, the readback and the operator's one-time +// disposition cannot disagree. The owner and group are the store's service principal, the one writer +// of the store (fabric_storage_directory). +type FabricStorageOwnershipMarkerSpec { + path: NonEmptyStr + content: NonEmptyStr + mode: FileMode + owner: NonEmptyStr + group: NonEmptyStr } -// WHETHER AN INSTANCE HOLDS THE PLACEMENT: the deployment owns the store root and the endpoint only -// there. Identity is the instance's own id, not a path comparison. -fn fabric_storage_placed_on(instance: HostDashboardInstance) -> Bool { - (instance.instance_id as String) == (fabric_storage_placed_instance().instance_id as String) +data fabric_storage_ownership_marker_mode: FileMode = FileMode { + owner: PermissionBits { read: true, write: true, execute: false }, + group: PermissionBits { read: true, write: false, execute: false }, + other: PermissionBits { read: true, write: false, execute: false }, + setuid: false, + setgid: false, + sticky: false, +} + +fn fabric_storage_ownership_marker_spec(instance: HostDashboardInstance) -> FabricStorageOwnershipMarkerSpec { + FabricStorageOwnershipMarkerSpec { + path: fabric_storage_ownership_marker_path(instance: instance), + content: fabric_storage_ownership_marker_content(instance: instance), + mode: fabric_storage_ownership_marker_mode, + owner: srv1_dashboard_service_principal().name, + group: srv1_dashboard_service_principal().name, + } +} + +// WHAT A READBACK OF THE MARKER SAW, field for field, and whether it is exactly the row. Absent is not +// a reading (the observer reports it as no marker); a reading that differs in ANY field is a mismatch, +// named by its first differing field, and is never admitted as the marker. +type FabricStorageMarkerReading { + path: NonEmptyStr + content: String + mode: String + owner: String + group: String +} + +type FabricStorageMarkerVerdict + = FabricStorageMarkerExact + | FabricStorageMarkerMismatched { field: NonEmptyStr, expected: String, observed: String } + +fn fabric_storage_marker_mode_matches(spec: FileMode, observed: String) -> Bool { + match file_mode_of_octal_text(text: observed) { + Present { value: m } => file_mode_octal(mode: m) == file_mode_octal(mode: spec) + Absent => false + } +} + +fn fabric_storage_marker_verdict(spec: FabricStorageOwnershipMarkerSpec, reading: FabricStorageMarkerReading) -> FabricStorageMarkerVerdict { + if !((reading.path as String) == (spec.path as String)) { FabricStorageMarkerMismatched { field: "path", expected: spec.path as String, observed: reading.path as String } } + else if !(reading.content == (spec.content as String)) { FabricStorageMarkerMismatched { field: "content", expected: spec.content as String, observed: reading.content } } + else if !fabric_storage_marker_mode_matches(spec: spec.mode, observed: reading.mode) { FabricStorageMarkerMismatched { field: "mode", expected: file_mode_octal(mode: spec.mode), observed: reading.mode } } + else if !(trim(s: reading.owner) == (spec.owner as String)) { FabricStorageMarkerMismatched { field: "owner", expected: spec.owner as String, observed: reading.owner } } + else if !(trim(s: reading.group) == (spec.group as String)) { FabricStorageMarkerMismatched { field: "group", expected: spec.group as String, observed: reading.group } } + else { FabricStorageMarkerExact } } fn fabric_storage_endpoint_for(host: HostIdentity) -> NonEmptyStr? { @@ -89,13 +200,29 @@ fn fabric_storage_endpoint_for(host: HostIdentity) -> NonEmptyStr? { } } -fn fabric_storage_placement() -> FabricStoragePlacement { - match fabric_storage_endpoint_for(host: operator_host_srv1) { - Absent => FabricStorageUnplaced - Present { value: e } => FabricStoragePlaced { host: operator_host_srv1, store_root: fabric_storage_store_root(), endpoint: e } +fn fabric_storage_placement_under(realization: ProdRoleRealization) -> FabricStoragePlacement { + match realization { + ProdRoleRealizationRefused { cause } => FabricStoragePlacementRefused { cause: cause } + ProdRoleRealizedBy { holder } => { + let instance = holder.instance + let host = instance.host_identity as String as HostIdentity + match fabric_storage_endpoint_for(host: host) { + Absent => FabricStorageUnplaced + Present { value: e } => FabricStoragePlaced { + host: host, + store_root: fabric_storage_store_root(instance: instance), + endpoint: e, + door_socket: fabric_storage_door_socket(instance: instance), + } + } + } } } +// THE RUNTIME PLACEMENT (fabric_storage_placement, over the ACTUATED realization) lives in +// gunbc.fabric_storage_binding: this module is imported by gunbc.live_deploy.spec, so it cannot see +// the desired roster that decides whether the holder can be actuated. + // โ”€โ”€ WHO WRITES THE STORE, AND SO WHAT ITS DIRECTORIES ARE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // // ONE PRINCIPAL WRITES THIS STORE'S FILES: the served endpoint (gunbc.live_deploy.emit @@ -144,24 +271,28 @@ fn fabric_storage_directory(member: NonEmptyStr, path: NonEmptyStr, writes: Bool // twice, by the kernel and by the handler. The service owns it; the CI runner (fleet-converge jobs on // the placed host) is its group and needs only to traverse to connect: the socket itself is opened // to every class by `gunbc serve --unix-socket`, so its directory is the whole filesystem wall. -fn fabric_storage_door_directory() -> ManagedDirectory { +fn fabric_storage_door_dependents() -> List { let service = srv1_dashboard_service_principal() + [ + DirectoryDependent { who: service, needs: [Read, Write, Execute] }, + DirectoryDependent { who: fleet_posix_ci_runner_user, needs: [Execute] }, + ] +} + +fn fabric_storage_door_directory(instance: HostDashboardInstance) -> ManagedDirectory { ManagedDirectory { member: "fabric-storage-door", - path: fabric_storage_door_directory_path() as String as FilePath, - owner: service, + path: fabric_storage_door_directory_path(instance: instance) as String as FilePath, + owner: srv1_dashboard_service_principal(), group_principal: fleet_posix_ci_runner_user, - dependents: [ - DirectoryDependent { who: service, needs: [Read, Write, Execute] }, - DirectoryDependent { who: fleet_posix_ci_runner_user, needs: [Execute] }, - ], + dependents: fabric_storage_door_dependents(), ownership: Ensured, entries: EntriesUnmanaged, } } fn fabric_storage_door_local_writers() -> List { - map(fabric_storage_door_directory().dependents, d => d.who.name) + map(fabric_storage_door_dependents(), d => d.who.name) } // ONE ENTRY MODE FOR THE STORE, OR A REFUSAL. Both writable areas derive it from their own @@ -185,8 +316,8 @@ fn fabric_storage_entry_mode_of(objects: ManagedDirectory, heads: ManagedDirecto } } -fn fabric_storage_entry_mode() -> FabricStorageEntryMode { - let areas = fabric_storage_file_root(root: fabric_storage_store_root()) +fn fabric_storage_entry_mode(store_root: NonEmptyStr) -> FabricStorageEntryMode { + let areas = fabric_storage_file_root(root: store_root) fabric_storage_entry_mode_of( objects: fabric_storage_directory(member: "fabric-storage-objects", path: areas.objects, writes: true), heads: fabric_storage_directory(member: "fabric-storage-heads", path: areas.heads, writes: true), @@ -201,23 +332,27 @@ type FabricStoragePlacedRoot | FabricStoragePlacedRootRefused { detail: NonEmptyStr } fn fabric_storage_placed_file_root(store_root: NonEmptyStr) -> FabricStoragePlacedRoot { - match fabric_storage_entry_mode() { + match fabric_storage_entry_mode(store_root: store_root) { FabricStorageEntryModeRefused { detail } => FabricStoragePlacedRootRefused { detail: detail } FabricStorageEntryModeDerived { mode } => FabricStoragePlacedRootReady { root: fabric_storage_file_root_with(root: store_root, entry_mode: Present { value: mode }) } } } -fn fabric_storage_store_directories() -> List { - let root = fabric_storage_store_root() +// THE PLACED INSTANCE'S STORE DIRECTORIES. Which instance is placed is the caller's holder token +// (gunbc.live_deploy.spec deployment_role_singleton_directories), so the directories land on the +// same target as the unit and route. +fn fabric_storage_store_directories_for(instance: HostDashboardInstance) -> List { + let root = fabric_storage_store_root(instance: instance) let areas = fabric_storage_file_root(root: root) [ - managed_directory_admit(d: fabric_storage_directory(member: "fabric-storage-root", path: root, writes: false)), + managed_directory_admit(d: fabric_storage_store_root_directory(instance: instance)), managed_directory_admit(d: fabric_storage_directory(member: "fabric-storage-objects", path: areas.objects, writes: true)), managed_directory_admit(d: fabric_storage_directory(member: "fabric-storage-heads", path: areas.heads, writes: true)), managed_directory_admit(d: fabric_storage_state_directory(member: "fabric-state-root", path: fabric_storage_state_root(store_root: root))), managed_directory_admit(d: fabric_storage_state_directory(member: "fabric-state-objects", path: fabric_storage_state_file_root(store_root: root).objects)), managed_directory_admit(d: fabric_storage_state_directory(member: "fabric-state-heads", path: fabric_storage_state_file_root(store_root: root).heads)), - managed_directory_admit(d: fabric_storage_door_directory()), + managed_directory_admit(d: fabric_storage_door_directory(instance: instance)), + managed_directory_admit(d: fabric_storage_control_directory(instance: instance)), ] } diff --git a/dag/gunbc/fabric/fabric_storage_serve.dag b/dag/gunbc/fabric/fabric_storage_serve.dag index 7012a3fd47b..123ccfc8479 100644 --- a/dag/gunbc/fabric/fabric_storage_serve.dag +++ b/dag/gunbc/fabric/fabric_storage_serve.dag @@ -1,5 +1,6 @@ module gunbc.fabric_storage_serve +import gunbc.fabric_storage_binding { fabric_storage_placement } import gunbc.fabric_storage_protocol { fabric_storage_closure_bound_cap, text_response, ok, words, trim_line, word, head_name, malformed, fabric_storage_respond, AdvanceRequest, advance_request } import gunbc.fabric_storage_state_serve { fabric_storage_state_serve_over } @@ -19,7 +20,8 @@ import gunbc.fabric_storage_file_store { } import gunbc.fabric_storage_wire { FabricDoorPrincipal, DoorProxied, DoorLocalPeer, DoorUnattested, fabric_storage_wire_head, fabric_storage_wire_put, fabric_storage_wire_advance, fabric_storage_wire_closure, fabric_storage_wire_identity } import gunbc.fabric_writer_roster { fabric_door_writer_admission_over, fabric_door_principal_of, fabric_served_writer_roster, fabric_door_local_roster, FabricDoorWriterAdmitted, FabricDoorWriterRefused } -import gunbc.fabric_storage_placement { fabric_storage_placement, FabricStoragePlaced, FabricStorageUnplaced, fabric_storage_route_prefix, fabric_storage_placed_file_root, FabricStoragePlacedRootReady, FabricStoragePlacedRootRefused } +import gunbc.roadmap_dashboard_instance { prod_role_realization_refusal_detail } +import gunbc.fabric_storage_placement { FabricStoragePlaced, FabricStorageUnplaced, FabricStoragePlacementRefused, fabric_storage_route_prefix, fabric_storage_placed_file_root, FabricStoragePlacedRootReady, FabricStoragePlacedRootRefused } // THE FABRIC DB ENDPOINT: the store side of the served handler bound to std.fabric_storage. One // `gunbc serve` process on the placed host, compiled once, answers each request by running the @@ -121,7 +123,8 @@ fn fabric_storage_serve_handle_over(root: FabricStorageFileRoot, rosters: Fabric fn fabric_storage_serve_handle(method: String, path: String, body: String, tailscale_identity: String, peer_user: String, release_revision: CommitSha) -> ServeWireResponse { match fabric_storage_placement() { FabricStorageUnplaced => text_response(status: 503, body: "fabric_storage/1 the fabric DB is unplaced\n") - FabricStoragePlaced { host: _, store_root: r, endpoint: _ } => + FabricStoragePlacementRefused { cause } => text_response(status: 503, body: join(["fabric_storage/1 ", prod_role_realization_refusal_detail(cause: cause), "\n"], "")) + FabricStoragePlaced { host: _, store_root: r, endpoint: _, door_socket: _ } => match fabric_storage_placed_file_root(store_root: r) { FabricStoragePlacedRootRefused { detail } => text_response(status: 503, body: join(["fabric_storage/1 the store's entry mode is refused: ", detail as String, "\n"], "")) FabricStoragePlacedRootReady { root } => diff --git a/dag/gunbc/fabric/fabric_writer_identity_observe.dag b/dag/gunbc/fabric/fabric_writer_identity_observe.dag index ba27523f296..133c6806ec0 100644 --- a/dag/gunbc/fabric/fabric_writer_identity_observe.dag +++ b/dag/gunbc/fabric/fabric_writer_identity_observe.dag @@ -1,6 +1,6 @@ module gunbc.fabric_writer_identity_observe -import gunbc.fabric_storage_binding { fabric_storage_binding_for } +import gunbc.fabric_storage_binding { fabric_storage_binding_for, fabric_storage_placement } import std.types { String, NonEmptyStr, List, Bool } import std.process { ProcessExit, ExitSuccess, exit_failure } @@ -8,7 +8,7 @@ import std.fabric_storage { fabric_storage_fault_wire } import extdeps.filesystem.filesystem_io { Filesystem } import extdeps.tailscale.identity { TailnetIdentity, TailnetUser, TailnetIdentityAbsent } import product.host_identity { HostIdentity, host_identity_eq } -import gunbc.fabric_storage_placement { fabric_storage_placement, FabricStoragePlaced, FabricStorageUnplaced } +import gunbc.fabric_storage_placement { FabricStoragePlaced, FabricStorageUnplaced } import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageLocalFiles, FabricStorageDoorSocket, FabricStorageServed, FabricStorageBindingUnplaced, FabricStorageBindingRefused, fabric_storage_presented_identity } import gunbc.fabric_storage_wire { FabricIdentityRead, FabricIdentityPresented, FabricIdentityUnproxied, FabricIdentityReadRefused, FabricIdentityLocalPeer, FabricIdentityUnattested } import gunbc.hostname_read { hostname_short_read, HostnameShortCaptured, HostnameShortCaptureRefused } diff --git a/dag/gunbc/fabric/fabric_writer_roster.dag b/dag/gunbc/fabric/fabric_writer_roster.dag index b65e0821f2d..3f02a02a782 100644 --- a/dag/gunbc/fabric/fabric_writer_roster.dag +++ b/dag/gunbc/fabric/fabric_writer_roster.dag @@ -1,12 +1,13 @@ module gunbc.fabric_writer_roster -import gunbc.fabric_storage_binding { fabric_storage_binding_for } +import gunbc.fabric_storage_binding { fabric_storage_binding_for, fabric_storage_placement } import std.types { String, NonEmptyStr, List, Bool } import extdeps.tailscale.identity { TailnetIdentity, TailnetUser, TailnetIdentityAbsent, tailnet_identity_of } import product.host_identity { HostIdentity } import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageLocalFiles } import gunbc.fabric_storage_file_store { fabric_storage_file_root } -import gunbc.fabric_storage_placement { fabric_storage_placement, fabric_storage_store_root, fabric_storage_door_local_writers, fabric_storage_door_proxy_peer } +import gunbc.fabric_storage_placement { fabric_storage_store_root, fabric_storage_door_local_writers, fabric_storage_door_proxy_peer } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import std.fabric_storage { FabricStoreUnreachable, FabricWriterRefusal, WriterLoginAbsent, WriterLoginUnrostered, WriterPeerUnrostered, WriterPeerUnattested } import gunbc.fabric_storage_wire { FabricIdentityPresented, FabricIdentityUnproxied, FabricIdentityReadRefused, FabricDoorPrincipal, DoorProxied, DoorLocalPeer, DoorUnattested } import gunbc.fabric_writer_identity_observe { @@ -51,10 +52,12 @@ fn door_unresolved() -> NonEmptyStr { "could not resolve the placed host's tailnet name (curl exit 6)" as NonEmptyStr } +// RECEIPTS NAME WHAT WAS OBSERVED, not who holds the prod role now: the 2026-09-27 srv1 read was of +// srv1-live's store root, so the row names that instance rather than following prod_role_selection. fn fabric_writer_identity_observations() -> List { [ FabricWriterIdentityObservation { run: "36347411725" as NonEmptyStr, receipt: FabricWriterIdentityReceipt { - host: "srv1" as HostIdentity, binding: FabricStorageLocalFiles { root: fabric_storage_file_root(root: fabric_storage_store_root()) }, + host: "srv1" as HostIdentity, binding: FabricStorageLocalFiles { root: fabric_storage_file_root(root: fabric_storage_store_root(instance: srv1_live_dashboard_instance())) }, read: FabricIdentityUnproxied, observed_at: "2026-09-27T20:29Z" as NonEmptyStr } }, FabricWriterIdentityObservation { run: "36347414620" as NonEmptyStr, receipt: FabricWriterIdentityReceipt { host: "srv3" as HostIdentity, binding: recorded_binding(host: "srv3"), diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index f47ec149dcc..767ebef83e1 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -1,5 +1,6 @@ module gunbc.fleet_converge_workflow +import gunbc.live_deploy.desired { prod_role_actuated_realization } import extdeps.github.expressions { Expression, ContextAccess, Github, StringLiteral, Equals, LogicalOr, LogicalAnd, interpolate } import gunbc.fleet_converge_scope { FleetConvergeScope, FullHost, LaunchEnvironment, FabricAllocationStoreOnly, WorkspaceSlotCommissioningOnly, LiveDeploy } import gunbc.fleet_plan_artifact_location { fleet_converge_plan_artifact_dir } @@ -63,6 +64,10 @@ import gunbc.workflow_capability_closure { capability_closure_is_closed, } import v2.std.algebra { list_map } +import v2.std.diagnostic { Outcome, Accepted, Rejected } +import v2.std.node { Node } +import v2.workflow.bash_emit { bash_emit_stmts } +import v2.extdeps.languages.bash_build { bash_build_with_redir, bash_build_command_with_words, bash_build_word_lit, bash_build_exit } import gunbc.fleet_workflow_steps { ci_prelude_steps_with_checkout, ci_prelude_annotations, @@ -124,7 +129,8 @@ import gunbc.fleet_host_identity { operator_host_mtcollins1 } import gunbc.auth.gcp_iam_converge_federation { gcp_iam_converge_environment, gcp_iam_observe_wif_provider_resource } import gunbc.auth.gcp_iam_converge { iam_observe_service_account, iam_converge_service_account, iam_converge_token_lifetime } import gunbc.auth.gcp_iam_converge_run { gcp_iam_converge_step_timeout_minutes, iam_converge_token_env, gcp_iam_converge_approved_output } -import gunbc.auth.approval_broker_endpoint { approval_broker_host_identity } +import gunbc.auth.approval_broker_placement { approval_broker_host_under, ApprovalBrokerHostResolved, ApprovalBrokerHostRefused } +import gunbc.roadmap_dashboard_instance { ProdRoleRealization, ProdRoleRealizationRefusal, prod_role_realization, prod_role_realization_refusal_detail } import gunbc.auth.managed_host_boot_federation { ManagedHostBootFederationStanding, BootFederationProvisioned, BootFederationUnprovisioned, boot_federation_for, boot_federation_provider_resource, @@ -2978,7 +2984,7 @@ fn fleet_converge_microvm_runner_group_ensure_receipt_upload_step() -> Step { fn fleet_converge_approval_keyring_converge_step() -> Step { RunStep { - name: Present { value: "Materialize approval MAC keys on srv1 (SM pinned -> sha256 prefix -> /etc/gunbc-roadmap 0640)" }, + name: Present { value: "Materialize approval MAC keys on the approval broker host (SM pinned -> sha256 prefix -> /etc/gunbc-roadmap 0640)" }, id: Present { value: "approval_keyring_converge" }, run: gunbc_ci_approval_keyring_converge_invoke(), shell: none, @@ -2994,14 +3000,14 @@ fn fleet_converge_approval_keyring_converge_step() -> Step { } // THE ENROLMENT-CODE STEP: one remote invocation of gunbc.auth.approval_device_enrolment_code -// issue_device_enrolment_code on srv1 as the operator's user, from the release the dispatch names; +// issue_device_enrolment_code on the approval broker host as the operator's user, from the release the dispatch names; // the step prints the delivery receipt and NEVER the code (decision msg_a1d8d1b9 -- the code goes // over the operator's ntfy topic, whose deny-by-default policy is DECLARED and not yet read back: // gunbc.auth.approval_ntfy_deployment's access rows are NoDischargeSiteModeled). No WIF token: the verb needs no secret the host does not // already hold. fn fleet_converge_approval_device_enrolment_code_issue_step() -> Step { RunStep { - name: Present { value: "Issue a device enrolment code on srv1 and deliver it to the operator channel (receipt only in this log)" }, + name: Present { value: "Issue a device enrolment code on the approval broker host and deliver it to the operator channel (receipt only in this log)" }, id: Present { value: "approval_device_enrolment_code_issue" }, run: gunbc_ci_approval_device_enrolment_code_issue_invoke(), shell: none, @@ -3569,7 +3575,7 @@ data fleet_converge_approval_broker_dark_install_step_if: String = fleet_converg fn fleet_converge_approval_broker_dark_install_step() -> Step { RunStep { - name: Present { value: "Install the approval broker dark on srv1 (additive unit + slice; does not restart gunbc-roadmap.service)" }, + name: Present { value: "Install the approval broker dark on the approval broker host (additive unit + slice; does not restart gunbc-roadmap.service)" }, id: Present { value: "approval_broker_dark_install" }, run: gunbc_ci_approval_broker_dark_install_invoke(), shell: none, @@ -4910,11 +4916,105 @@ fn fleet_converge_gcp_iam_apply_step() -> Step { } } +// THE JOB RUNS WHERE THE APPROVAL BROKER RUNS, which follows the prod role +// (gunbc.auth.approval_broker_endpoint approval_broker_host_under). Under a refused realization it is +// still EMITTED, so a dispatch of this mode is not silently absent, but as a refused job whose only +// step fails with the typed cause: no federation, no approval filing, no host it could be pinned to. fn fleet_converge_gcp_iam_converge_job() -> Job { + fleet_converge_gcp_iam_converge_job_under(realization: prod_role_actuated_realization()) +} + +fn fleet_converge_gcp_iam_converge_job_under(realization: ProdRoleRealization) -> Job { + match approval_broker_host_under(realization: realization) { + ApprovalBrokerHostResolved { host } => fleet_converge_gcp_iam_converge_job_on(host: host) + ApprovalBrokerHostRefused { cause } => fleet_converge_gcp_iam_converge_refused_job(cause: cause) + } +} + +// THE REFUSED JOB'S SCRIPT IS EMITTED, NOT SPELLED: an echo of the typed cause to stderr and exit 1, +// as v2.extdeps.languages.bash_build nodes through v2.workflow.bash_emit bash_emit_stmts. A script +// bash_emit rejects has no text here; the generation gate fleet_converge_gcp_iam_refusal_script_admitted +// refuses the whole workflow instead of emitting a step with no script. +fn gcp_iam_converge_refusal_statements(cause: ProdRoleRealizationRefusal) -> List { + [ + bash_build_with_redir( + inner: bash_build_command_with_words(words: [ + bash_build_word_lit(text: "echo"), + bash_build_word_lit(text: join(["REFUSED gunbc.fleet_converge_workflow gcp-iam-converge: ", prod_role_realization_refusal_detail(cause: cause)], "")), + ]), + redir: ^bash_production_fold_redir_stdout_to_stderr, + ), + bash_build_exit(code: "1"), + ] +} + +fn gcp_iam_converge_refusal_script(cause: ProdRoleRealizationRefusal) -> Outcome { + bash_emit_stmts(statements: gcp_iam_converge_refusal_statements(cause: cause), set_e: true) +} + +fn gcp_iam_converge_refusal_script_text(outcome: Outcome) -> String { + match outcome { + Accepted { value: v, diagnostics: _ } => v + Rejected { diagnostics: _ } => "" + } +} + +// Admitted when the live realization resolves (no refused job is emitted) or bash_emit accepts the +// refusal script for its cause. +fn fleet_converge_gcp_iam_refusal_script_admitted() -> Bool { + match approval_broker_host_under(realization: prod_role_actuated_realization()) { + ApprovalBrokerHostResolved { host: _ } => true + ApprovalBrokerHostRefused { cause } => + match gcp_iam_converge_refusal_script(cause: cause) { + Accepted { value: _, diagnostics: _ } => true + Rejected { diagnostics: _ } => false + } + } +} + +fn fleet_converge_gcp_iam_converge_refused_job(cause: ProdRoleRealizationRefusal) -> Job { + Job { + id: fleet_converge_mode_job_id(mode: GcpIamConverge) as String, + name: none, + runner: SelfHosted { labels: ["self-hosted", "linux", "arm64"] }, + environment: none, + steps: [ + RunStep { + name: Present { value: "REFUSED: the approval broker has no host (prod role unresolved)" }, + id: none, + run: gcp_iam_converge_refusal_script_text(outcome: gcp_iam_converge_refusal_script(cause: cause)), + shell: none, + env: none, + working_directory: none, + if_condition: none, + continue_on_error: none, + timeout_minutes: Present { value: 1 } + }, + ], + needs: fleet_converge_job_needs(id: fleet_converge_mode_job_id(mode: GcpIamConverge)), + env: none, + outputs: none, + if_condition: Present { value: fleet_converge_gcp_iam_converge_step_if }, + timeout_minutes: Present { value: 5 }, + continue_on_error: none, + concurrency: none, + permissions: Present { + value: WorkflowPermissions { + contents: Present { value: PermRead }, + pull_requests: none, + issues: none, + actions: none, + id_token: none, + } + } + } +} + +fn fleet_converge_gcp_iam_converge_job_on(host: NonEmptyStr) -> Job { Job { id: fleet_converge_mode_job_id(mode: GcpIamConverge) as String, name: none, - runner: SelfHosted { labels: ["self-hosted", "linux", "arm64", approval_broker_host_identity() as String] }, + runner: SelfHosted { labels: ["self-hosted", "linux", "arm64", host as String] }, environment: Present { value: gcp_iam_converge_environment as String }, steps: [ fleet_converge_event_sha_checkout_step(), @@ -5201,6 +5301,10 @@ fn fleet_converge_yml_after_toolchain_admission() -> FleetConvergeYamlGeneration } } else if !fleet_converge_boot_federation_admitted() { FleetConvergeYamlGenerationRefused { reason: fleet_converge_boot_federation_unprovisioned_refusal } + } else if !fleet_converge_gcp_iam_refusal_script_admitted() { + FleetConvergeYamlGenerationRefused { + reason: "REFUSED gunbc.fleet_converge_workflow: v2.workflow.bash_emit refused the gcp-iam-converge refusal script, so no workflow is emitted rather than a step with no script", + } } else if !release_bins_scripts_admitted() { FleetConvergeYamlGenerationRefused { reason: "REFUSED gunbc.fleet_converge_workflow: v2.workflow.bash_emit refused a release-bins runner (gunbc.fleet_release_bins_key), so no workflow is emitted rather than a step with no script", diff --git a/dag/gunbc/fleet/fleet_tailnet.dag b/dag/gunbc/fleet/fleet_tailnet.dag index 3475cc5ed92..7e46efb72c7 100644 --- a/dag/gunbc/fleet/fleet_tailnet.dag +++ b/dag/gunbc/fleet/fleet_tailnet.dag @@ -4,7 +4,7 @@ import std.types { NonEmptyStr, String, Port } // THE FLEET'S TAILNET DOMAIN, as a leaf. gunbc.fleet_intent_network fleet_intent_network carries it // as its topology's tailnet_domain, and the approval broker composes its public address from it -// (gunbc.auth.approval_broker_endpoint approval_broker_base_url); a reader of the domain does not +// (gunbc.auth.approval_broker_endpoint approval_broker_base_url_on); a reader of the domain does not // import the topology's assets, segments and endpoints. data fleet_tailnet_domain: NonEmptyStr? = Present { value: "tailecbe08.ts.net" } @@ -13,7 +13,7 @@ data fleet_tailnet_domain: NonEmptyStr? = Present { value: "tailecbe08.ts.net" } // "https://srv1.tailecbe08.ts.net:8443", which re-minted two facts the corpus owns elsewhere -- the // tailnet domain (fleet_tailnet_domain above) and the host's identity, which each caller passes as // `host` from its own authority (gunbc.fleet_host_identity operator_host_srv1 for ntfy, -// gunbc.auth.approval_broker_endpoint approval_broker_host_identity for the broker). Renaming the +// the serving host for the broker, gunbc.auth.approval_broker_endpoint approval_broker_serving_base_url). Renaming the // tailnet or the host would have left that string stale while every modeled consumer moved. // // The address a CLIENT dials is a different fact from the address the SERVER listens on diff --git a/dag/gunbc/host/host_effect.dag b/dag/gunbc/host/host_effect.dag index 2bb2d0f3b7a..008d69867da 100644 --- a/dag/gunbc/host/host_effect.dag +++ b/dag/gunbc/host/host_effect.dag @@ -26,6 +26,7 @@ import gunbc.runner_lifecycle { RunnerReplacementCause } import gunbc.auth.github_apps { gunbai_ci_declared } import gunbc.live_deploy.candidate_identity { CandidateRelease } import gunbc.live_deploy.member_identity { ReleaseEffectPlan } +import gunbc.live_deploy.apply_warrant { AdmittedLiveDeployApply } import std.effects { EffectShape, KeySource, @@ -76,7 +77,8 @@ type HostEffect = | Srv3BootOnceReadBackSleep | Srv3ReceiptEmit { lines: List } | HostIdentityShortHostnameRead - | LiveDeployApply { spec: DeploymentSpec, candidate: CandidateRelease, plan: ReleaseEffectPlan } + | LiveDeployApply { warrant: AdmittedLiveDeployApply } + | LiveDeployApplyReported { spec: DeploymentSpec, candidate: CandidateRelease } | LiveDeployRetract { spec: DeploymentSpec } | LiveDeployApprovalBrokerDarkInstall { spec: DeploymentSpec, candidate: CandidateRelease } | LiveDeployMicrovmControllerInstall { candidate: CandidateRelease } diff --git a/dag/gunbc/host/host_effect_realize.dag b/dag/gunbc/host/host_effect_realize.dag index e9235cc03f9..70ba8a5e1e0 100644 --- a/dag/gunbc/host/host_effect_realize.dag +++ b/dag/gunbc/host/host_effect_realize.dag @@ -27,6 +27,7 @@ import gunbc.host_effect { Srv3ReceiptEmit, HostIdentityShortHostnameRead, LiveDeployApply, + LiveDeployApplyReported, LiveDeployRetract, LiveDeployApprovalBrokerDarkInstall, LiveDeployMicrovmControllerInstall, @@ -537,8 +538,10 @@ fn resolve_host_os_effect_cell_after_direct_arms(n: ComputeHost, effect: HostEff CodexSupervisedWorkerTurnOnHost { node: n, intent: i, lease_key: k } HostIdentityShortHostnameRead => HostnameReadOnHost { node: n } - LiveDeployApply { spec: s, candidate: c, plan: p } => - LiveDeployApplyOnHost { node: n, spec: s, candidate: c, plan: p } + LiveDeployApply { warrant: w } => + LiveDeployApplyOnHost { node: n, spec: w.spec, candidate: w.candidate, plan: w.plan } + LiveDeployApplyReported { spec: _, candidate: _ } => + IncompatibleCell { reason: "host_effect: LiveDeployApplyReported is the warrant-free report of a live-deploy apply that already ran; it is not an effect and realizes nowhere." } LiveDeployRetract { spec: s } => LiveDeployRetractOnHost { node: n, spec: s } LiveDeployApprovalBrokerDarkInstall { spec: s, candidate: c } => @@ -619,7 +622,9 @@ fn resolve_bmc_controller_effect_cell_after_direct_arms(n: ComputeHost, effect: IncompatibleCell { reason: "host_effect: SudoNopasswdGrantListProbe is in-band (typed sudo grant-list probe) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } HostIdentityShortHostnameRead => IncompatibleCell { reason: "host_effect: HostIdentityShortHostnameRead is in-band (shell hostname -s) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } - LiveDeployApply { spec: _, candidate: _, plan: _ } => + LiveDeployApplyReported { spec: _, candidate: _ } => + IncompatibleCell { reason: "host_effect: LiveDeployApplyReported is the warrant-free report of a live-deploy apply that already ran; it is not an effect and realizes nowhere." } + LiveDeployApply { warrant: _ } => IncompatibleCell { reason: "host_effect: LiveDeployApply is in-band (actuator host shell mutation) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } LiveDeployRetract { spec: _ } => IncompatibleCell { reason: "host_effect: LiveDeployRetract is in-band (actuator host shell mutation) and cannot target BmcController (out-of-band Redfish only); target HostOs instead. Typed mismatch via total fold, DESIGN section 5." } @@ -683,7 +688,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res Srv3BootOnceReadBackSleep => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } Srv3ReceiptEmit { lines: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } HostIdentityShortHostnameRead => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } - LiveDeployApply { spec: _, candidate: _, plan: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } + LiveDeployApply { warrant: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } + LiveDeployApplyReported { spec: _, candidate: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } LiveDeployRetract { spec: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } LiveDeployApprovalBrokerDarkInstall { spec: _, candidate: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } LiveDeployMicrovmControllerInstall { candidate: _ } => IncompatibleCell { reason: "host_effect: RunnerLocalHost admits only ProbeToolchainHomeInterference; fleet-addressed effects require HostOs or BmcController." } @@ -721,8 +727,10 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res BmcwebSessionLoginOnHost { node: n } Srv3BmcwebTokenExtract => BmcwebTokenExtractOnHost { node: n } - LiveDeployApply { spec: s, candidate: c, plan: p } => - LiveDeployApplyOnHost { node: n, spec: s, candidate: c, plan: p } + LiveDeployApply { warrant: w } => + LiveDeployApplyOnHost { node: n, spec: w.spec, candidate: w.candidate, plan: w.plan } + LiveDeployApplyReported { spec: _, candidate: _ } => + IncompatibleCell { reason: "host_effect: LiveDeployApplyReported is the warrant-free report of a live-deploy apply that already ran; it is not an effect and realizes nowhere." } LiveDeployRetract { spec: s } => LiveDeployRetractOnHost { node: n, spec: s } LiveDeployApprovalBrokerDarkInstall { spec: s, candidate: c } => @@ -787,7 +795,8 @@ fn resolve_host_effect_cell(target: NodeControlPlane, effect: HostEffect) -> Res Srv3BootOnceReadBackSleep => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) Srv3ReceiptEmit { lines: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) HostIdentityShortHostnameRead => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) - LiveDeployApply { spec: _, candidate: _, plan: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) + LiveDeployApply { warrant: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) + LiveDeployApplyReported { spec: _, candidate: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) LiveDeployRetract { spec: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) LiveDeployApprovalBrokerDarkInstall { spec: _, candidate: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) LiveDeployMicrovmControllerInstall { candidate: _ } => resolve_bmc_controller_effect_cell_after_direct_arms(n: n, effect: effect) @@ -1631,7 +1640,8 @@ fn realize_provision_codex_runtime( Srv3BootOnceReadBackSleep => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) Srv3ReceiptEmit { lines: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) HostIdentityShortHostnameRead => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) - LiveDeployApply { spec: _, candidate: _, plan: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) + LiveDeployApply { warrant: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) + LiveDeployApplyReported { spec: _, candidate: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) LiveDeployRetract { spec: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) LiveDeployApprovalBrokerDarkInstall { spec: _, candidate: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) LiveDeployMicrovmControllerInstall { candidate: _ } => realize_provision_codex_runtime_wrong_effect(effect_intent: effect_intent) @@ -3454,7 +3464,8 @@ fn realize_provision_build_cache( Srv3BootOnceReadBackSleep => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) Srv3ReceiptEmit { lines: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) HostIdentityShortHostnameRead => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) - LiveDeployApply { spec: _, candidate: _, plan: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) + LiveDeployApply { warrant: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) + LiveDeployApplyReported { spec: _, candidate: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) LiveDeployRetract { spec: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) LiveDeployApprovalBrokerDarkInstall { spec: _, candidate: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) LiveDeployMicrovmControllerInstall { candidate: _ } => realize_provision_build_cache_wrong_effect(effect_intent: effect_intent) diff --git a/dag/gunbc/instruments/approval_store_live_probe.dag b/dag/gunbc/instruments/approval_store_live_probe.dag index 2334f9db08c..d34d674b2f3 100644 --- a/dag/gunbc/instruments/approval_store_live_probe.dag +++ b/dag/gunbc/instruments/approval_store_live_probe.dag @@ -1,5 +1,6 @@ module tools.approval_store_live_probe +import gunbc.auth.approval_broker_endpoint { approval_broker_serving_base_url } import std.types { Bool, NonEmptyStr, String, Timestamp, Secret } import std.process { ProcessExit, ExitSuccess, exit_failure } import std.content_hash { ContentHash, Sha256Hash, Sha256Digest } @@ -193,6 +194,7 @@ fn probe_submit(root: NonEmptyStr, signer: MacKey, request: StoredApprovalReques publisher: PublisherTokenUnavailable { path: "/etc/gunbc-ntfy/publisher-token", detail: "probe fixture: no message leaves this probe" }, tag_hex: t.hex, request: request, + broker: approval_broker_serving_base_url(), ) match outcome { SubmissionRefused { refusal: SubmissionNotEnrolled { detail: _ } } => ProbeNotEnrolled diff --git a/dag/gunbc/live_deploy/apply.dag b/dag/gunbc/live_deploy/apply.dag index b6e8cd72b3f..91a4be43717 100644 --- a/dag/gunbc/live_deploy/apply.dag +++ b/dag/gunbc/live_deploy/apply.dag @@ -1,5 +1,6 @@ module gunbc.live_deploy.apply +import std.decl_ref { decl_ref } import gunbc.workspace_commissioning_executor_input { workspace_executor_installed_adapter_matches } import gunbc.workspace_commissioning_executor_unit { workspace_executor_unit_instance_name } import gunbc.runner_microvm_slot_unit { microvm_controller_requires_ci_reservation_unit } @@ -23,7 +24,7 @@ import gunbc.live_deploy.spec { deployment_plan_host_identity, deployment_tailnet_door_socket, } -import gunbc.live_deploy.desired { deployment_spec_srv1 } +import gunbc.live_deploy.desired { deployment_spec_srv1, DesiredDeployment, srv1_production_desired_deployment, selected_deployment_spec } import std.types { String, CommitSha, NonEmptyStr } import gunbc.live_deploy.candidate { CandidateRelease, @@ -55,6 +56,7 @@ import v2.std.collection { Present, Absent } import std.access { AccessDecision, Permit, Deny } import gunbc.host_effect { HostEffect, + LiveDeployApplyReported, HostOs, NodeControlPlane, host_effect_identity_evidence, @@ -84,6 +86,12 @@ import gunbc.fabric_storage_binding { fabric_state_storage_binding } import gunbc.fabric_storage_client { FabricStorageBinding } import gunbc.live_deploy.member_observe { deployment_effect_plan, deployment_identity_readback, IdentityReadbackConverged, IdentityReadbackDrift, IdentityReadbackRefused } import gunbc.live_deploy.member_identity { EffectsDerived, EffectsRefused, decision_label } +import gunbc.live_deploy.apply_warrant { warrant_live_deploy_apply } +import gunbc.live_deploy.role_singleton_convergence { + assess_and_admit, role_singleton_refusal_detail, admitted_spec, admitted_decision, + RoleSingletonAdmission, RoleSingletonAdmitted, RoleSingletonNotAdmitted, + RoleSingletonNoop, RoleSingletonInstall, RoleSingletonRefused, RoleSingletonConvergence, +} import gunbc.auth.approval_broker_cutover { CutoverAdmission, StageAdmitted, StageRefused, dark_install_release_discharged, cutover_refusal_reason, @@ -142,6 +150,7 @@ import gunbc.fleet_converge_receipt { fleet_converge_plan_receipt_path, fleet_converge_apply_receipt_path, } import gunbc.live_deploy.apply_receipt { + LiveDeployRoleSingletonDisposition, RoleSingletonsNoop, RoleSingletonsInstalled, RoleSingletonsRefusedBeforeApply, LiveDeployApplyReceipt, LiveDeployFleetProvenance, LiveDeployApplyOutcome, LiveDeployReadinessOutcome, read_live_deploy_fleet_provenance, FleetProvenancePresent, FleetProvenanceAbsent, LiveDeployFleetProvenance, @@ -163,7 +172,8 @@ import gunbc.running_release_identity { // gated step exists to install. Every other effect installs nothing and is gated on the full roster. fn live_deploy_pre_mutation_access(effect: HostEffect, access: DeployAccess) -> DeployAccess { match effect { - LiveDeployApply { spec: _, candidate: _, plan: _ } => deploy_access_sudoers_install_scope(access: access) + LiveDeployApply { warrant: _ } => deploy_access_sudoers_install_scope(access: access) + LiveDeployApplyReported { spec: _, candidate: _ } => access LiveDeployRetract { spec: _ } => deploy_access_sudoers_install_scope(access: access) RedfishAction { action: _ } => access ConvergePlan { policy: _ } => access @@ -328,15 +338,32 @@ fn live_deploy_ensure_dependency( ) } +// THE MUTATION ACTS ON THE ADMITTED SPEC ONLY. live_deploy_apply_via_transport admits the roster-built +// spec against the role-singleton assessment first (gunbc.live_deploy.role_singleton_convergence +// assess_and_admit); this takes the admission token, so there is no route by which a spec the +// assessment did not admit -- one built before the role moved -- reaches member planning. +fn live_deploy_apply_mutation_via_transport( + admission: RoleSingletonAdmission, + candidate: CandidateRelease, + transport: HostEffectTransport, + access: DeployAccess +) -> Reconciliation admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_admitted_via_transport"), +] { + live_deploy_apply_member_plan_via_transport(spec: admitted_spec(admission: admission), candidate: candidate, transport: transport, access: access) +} + // THE PLAN IS DERIVED FROM THE HOST BEFORE THE MUTATION IS BUILT (gunbc.live_deploy.member_observe): // the same evaluation that decides also acts, and a plan that could not be derived refuses here with // every member's cause, installing nothing -- never the whole roster as a fallback. -fn live_deploy_apply_mutation_via_transport( +fn live_deploy_apply_member_plan_via_transport( spec: DeploymentSpec, candidate: CandidateRelease, transport: HostEffectTransport, access: DeployAccess -) -> Reconciliation { +) -> Reconciliation admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_mutation_via_transport"), +] { match deployment_effect_plan(spec: spec, candidate: candidate, transport: transport) { EffectsRefused { refusals } => NotConverged { @@ -344,17 +371,70 @@ fn live_deploy_apply_mutation_via_transport( applied: live_deploy_preflight_intent(access: access), } EffectsDerived { plan } => - live_deploy_fold( + live_deploy_apply_reported( + r: live_deploy_fold( + spec: spec, + effect: LiveDeployApply { warrant: warrant_live_deploy_apply(spec: spec, candidate: candidate, plan: plan) }, + transport: transport, + access: access, + fixture_actor: Absent, + fixture_host: Absent + ), spec: spec, - effect: LiveDeployApply { spec: spec, candidate: candidate, plan: plan }, - transport: transport, - access: access, - fixture_actor: Absent, - fixture_host: Absent + candidate: candidate, ) } } +// THE WARRANT DOES NOT LEAVE THE OPERATION THAT MINTED IT. The fold's intent carries the +// LiveDeployApply effect it realized, so returning it would hand every caller an +// AdmittedLiveDeployApply to re-wrap; the reported intent replaces that effect with the +// warrant-free LiveDeployApplyReported, which realizes nowhere. The re-wrap itself stays +// expressible until coproduct variant construction can be sealed (gunbc.recurring_failure_mode +// modifier_accepted_on_a_shape_its_check_cannot_see); this makes the warrant unobtainable. +fn live_deploy_apply_reported( + r: Reconciliation, + spec: DeploymentSpec, + candidate: CandidateRelease, +) -> Reconciliation admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_member_plan_via_transport"), +] { + match r { + Converged { evidence: e, applied: i } => + Converged { evidence: e, applied: live_deploy_reported_intent(i: i, spec: spec, candidate: candidate) } + NotConverged { reason: why, applied: i } => + NotConverged { reason: why, applied: live_deploy_reported_intent(i: i, spec: spec, candidate: candidate) } + } +} + +fn live_deploy_reported_intent(i: HostEffectIntent, spec: DeploymentSpec, candidate: CandidateRelease) -> HostEffectIntent { + HostEffectIntent { + target: i.target, + effect: LiveDeployApplyReported { spec: spec, candidate: candidate }, + evidence: i.evidence, + transport: i.transport, + privilege_denial: i.privilege_denial, + } +} + +fn live_deploy_apply_readiness_gate( + spec: DeploymentSpec, + candidate: CandidateRelease, + transport: HostEffectTransport, + access: DeployAccess, + mutation: Reconciliation +) -> Reconciliation { + live_deploy_apply_readiness_gate_with_readiness( + mutation: mutation, + readiness: live_deploy_poll_until_service_ready_for_deploy( + spec: spec, + candidate: candidate, + transport: transport, + access: access, + ), + ) +} + // One NotConverged, reached from both non-Ready arms. The Pending/Refused // distinction is NOT erased by the collapse: service_readiness_reason carries the // readiness value into the reason, and the retry decision that consumes the @@ -543,13 +623,38 @@ fn live_deploy_apply_identity_readback_gate( } fn live_deploy_apply_via_transport( - spec: DeploymentSpec, + deployment: DesiredDeployment, candidate: CandidateRelease, transport: HostEffectTransport, access: DeployAccess ) -> Reconciliation { + match assess_and_admit(deployment: deployment, own: transport) { + RoleSingletonNotAdmitted { cause } => + NotConverged { + reason: join(["live_deploy: ", role_singleton_refusal_detail(cause: cause)], ""), + applied: live_deploy_preflight_intent(access: access), + } + RoleSingletonAdmitted { admission } => live_deploy_apply_admitted_via_transport(admission: admission, candidate: candidate, transport: transport, access: access) + } +} + +// THE ROLE SINGLETONS ARE ASSESSED BEFORE ANYTHING ELSE: the desired (actuated) prod-role realization +// against an observation of every roster host, and the spec built from the deployment's bound roster +// row is admitted only when its holding agrees. A move, a removal, an unobservable host, a multiplicity, a partial realization or a stale +// spec stops the line above, before any member is planned. From here on every step reads the ADMITTED +// spec, which no caller supplied. +fn live_deploy_apply_admitted_via_transport( + admission: RoleSingletonAdmission, + candidate: CandidateRelease, + transport: HostEffectTransport, + access: DeployAccess +) -> Reconciliation admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_via_transport"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_transaction_decided"), +] { + let spec = admitted_spec(admission: admission) let mutation = live_deploy_apply_mutation_via_transport( - spec: spec, + admission: admission, candidate: candidate, transport: transport, access: access, @@ -654,12 +759,12 @@ fn live_deploy_access_preflight() -> Reconciliation Reconciliation { live_deploy_apply_via_transport( - spec: spec, + deployment: deployment, candidate: candidate, transport: access.transport, access: access, @@ -671,7 +776,7 @@ fn live_deploy_apply_srv1_with_access( candidate: CandidateRelease, ) -> Reconciliation { live_deploy_apply_with_access( - spec: deployment_spec_srv1(), + deployment: srv1_production_desired_deployment(), access: access, candidate: candidate, ) @@ -744,19 +849,19 @@ fn live_deploy_reconciliation_exit(r: Reconciliation ProcessExit { match repository_transition_admission(realization: realization) { LegacyGitFileSyncNotAdmitted { reason: why } => exit_failure(reason: why) - RepositoryTransitionAdmitted => live_deploy_admitted_wet_with_access(spec: spec, access: access) + RepositoryTransitionAdmitted => live_deploy_admitted_wet_with_access(deployment: deployment, access: access) } } -fn live_deploy_wet_with_access(spec: DeploymentSpec, access: DeployAccess) -> ProcessExit { +fn live_deploy_wet_with_access(deployment: DesiredDeployment, access: DeployAccess) -> ProcessExit { live_deploy_wet_with_access_for_realization( - spec: spec, + deployment: deployment, access: access, realization: deployed_tree_repository_transition, ) @@ -769,12 +874,12 @@ fn live_deploy_wet_with_access(spec: DeploymentSpec, access: DeployAccess) -> Pr // ancestry, on the same grounds the repository-transition admission above states for itself: a // wall that also blocked teardown would strand an operator with a host they could neither // converge nor take down. -fn live_deploy_admitted_wet_with_access(spec: DeploymentSpec, access: DeployAccess) -> ProcessExit { +fn live_deploy_admitted_wet_with_access(deployment: DesiredDeployment, access: DeployAccess) -> ProcessExit { match observe_candidate_release() { CandidateUnobservable { cause: why } => exit_failure(reason: candidate_unobservable_detail(cause: why)) CandidateObserved { candidate: candidate } => - live_deploy_candidate_wet_with_access(spec: spec, access: access, candidate: candidate) + live_deploy_candidate_wet_with_access(deployment: deployment, access: access, candidate: candidate) } } @@ -786,7 +891,7 @@ fn live_deploy_admitted_wet_with_access(spec: DeploymentSpec, access: DeployAcce // zero-mutation plan rather than a short circuit, and a host whose tree drifted beneath an // unchanged announcement is repaired rather than skipped. fn live_deploy_candidate_wet_with_access( - spec: DeploymentSpec, + deployment: DesiredDeployment, access: DeployAccess, candidate: CandidateRelease, ) -> ProcessExit { @@ -795,7 +900,7 @@ fn live_deploy_candidate_wet_with_access( live_deploy_reconciliation_exit(r: preflight) } else { live_deploy_reconciliation_exit( - r: live_deploy_apply_with_access(spec: spec, access: access, candidate: candidate) + r: live_deploy_apply_with_access(deployment: deployment, access: access, candidate: candidate) ) } } @@ -1128,18 +1233,18 @@ fn approval_broker_dark_install_srv1_wet() -> ProcessExit { } fn live_deploy_apply_srv1_wet() -> ProcessExit { - live_deploy_wet_with_access(spec: deployment_spec_srv1(), access: ci_deploy_srv1_access) + live_deploy_wet_with_access(deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_access) } fn live_deploy_apply_srv1_operator_wet() -> ProcessExit { - live_deploy_wet_with_access(spec: deployment_spec_srv1(), access: ci_deploy_srv1_operator_access) + live_deploy_wet_with_access(deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_operator_access) } // The converge verb's deploy mode: the same admitted path the CI entry runs, under the operator // principal on the host itself. Candidate observation, preflight, mutation and the readiness // gate are all shared; nothing here is a second deploy. fn live_deploy_apply_srv1_operator_local_wet() -> ProcessExit { - live_deploy_wet_with_access(spec: deployment_spec_srv1(), access: ci_deploy_srv1_operator_local_access) + live_deploy_wet_with_access(deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_operator_local_access) } fn live_deploy_retract_srv1_wet() -> ProcessExit { @@ -1192,6 +1297,22 @@ fn live_deploy_write_transaction_receipt(spec: DeploymentSpec, receipt: LiveDepl } } +// THE ADMITTED DECISION, AS THE RECEIPT RECORDS IT: Installed names the first install, Noop is +// everything else that was admitted, and a refusal before any member was planned is RefusedBeforeApply. +fn role_singleton_disposition_of(admission: RoleSingletonAdmission) -> LiveDeployRoleSingletonDisposition admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_transaction_decided"), +] { + role_singleton_disposition_of_decision(decision: admitted_decision(admission: admission)) +} + +fn role_singleton_disposition_of_decision(decision: RoleSingletonConvergence) -> LiveDeployRoleSingletonDisposition { + match decision { + RoleSingletonNoop => RoleSingletonsNoop + RoleSingletonInstall { holder } => RoleSingletonsInstalled { deployment: holder.instance.deployment.name as String, host: holder.instance.host_identity as String } + RoleSingletonRefused { cause } => RoleSingletonsRefusedBeforeApply { detail: role_singleton_refusal_detail(cause: cause) } + } +} + fn live_deploy_transaction_receipt_for( spec: DeploymentSpec, run: ActionsRunBinding, @@ -1199,6 +1320,7 @@ fn live_deploy_transaction_receipt_for( candidate: CandidateRelease, candidate_tree: String, apply_outcome: LiveDeployApplyOutcome, + role_singleton_disposition: LiveDeployRoleSingletonDisposition, readiness: LiveDeployReadinessOutcome, provenance: LiveDeployFleetProvenance, ) -> LiveDeployApplyReceipt { @@ -1211,6 +1333,7 @@ fn live_deploy_transaction_receipt_for( candidate_surface_identity: candidate.expected_surface_identity.digest as String, candidate_tree_oid: candidate_tree, apply_outcome: apply_outcome, + role_singleton_disposition: role_singleton_disposition, readiness: readiness, provenance: provenance, completed_at: clock_now_probed_at_or_unknown() as String, @@ -1218,7 +1341,7 @@ fn live_deploy_transaction_receipt_for( } fn live_deploy_transaction_decided( - spec: DeploymentSpec, + deployment: DesiredDeployment, access: DeployAccess, run: ActionsRunBinding, expected: String, @@ -1226,28 +1349,42 @@ fn live_deploy_transaction_decided( candidate_tree: String, provenance: LiveDeployFleetProvenance, ) -> ProcessExit { + let spec = selected_deployment_spec(row: deployment) let preflight = live_deploy_access_preflight_for(access: access) if !reconciliation_converged(r: preflight) { live_deploy_write_transaction_receipt( spec: spec, - receipt: live_deploy_transaction_receipt_for(spec: spec, run: run, expected: expected, candidate: candidate, candidate_tree: candidate_tree, apply_outcome: LiveDeployApplyNotConverged { reason: (match preflight { NotConverged { reason: why, applied: _ } => why Converged { evidence: _, applied: _ } => "preflight" }) }, readiness: LiveDeployReadinessRefused { detail: "not observed: preflight refused" }, provenance: provenance), + receipt: live_deploy_transaction_receipt_for(spec: spec, run: run, expected: expected, candidate: candidate, candidate_tree: candidate_tree, apply_outcome: LiveDeployApplyNotConverged { reason: (match preflight { NotConverged { reason: why, applied: _ } => why Converged { evidence: _, applied: _ } => "preflight" }) }, role_singleton_disposition: RoleSingletonsRefusedBeforeApply { detail: "not assessed: the access preflight refused first" }, readiness: LiveDeployReadinessRefused { detail: "not observed: preflight refused" }, provenance: provenance), exit: live_deploy_reconciliation_exit(r: preflight), ) } else { - let applied = live_deploy_apply_with_access(spec: spec, access: access, candidate: candidate) - let outcome = match applied { - Converged { evidence: _, applied: _ } => LiveDeployApplyConverged - NotConverged { reason: why, applied: _ } => LiveDeployApplyNotConverged { reason: why } + match assess_and_admit(deployment: deployment, own: access.transport) { + RoleSingletonNotAdmitted { cause } => { + let why = join(["live_deploy: ", role_singleton_refusal_detail(cause: cause)], "") + live_deploy_write_transaction_receipt( + spec: spec, + receipt: live_deploy_transaction_receipt_for(spec: spec, run: run, expected: expected, candidate: candidate, candidate_tree: candidate_tree, apply_outcome: LiveDeployApplyNotConverged { reason: why }, role_singleton_disposition: RoleSingletonsRefusedBeforeApply { detail: why }, readiness: LiveDeployReadinessRefused { detail: "not observed: the role-singleton assessment refused before any member was planned" }, provenance: provenance), + exit: exit_failure(reason: why), + ) + } + RoleSingletonAdmitted { admission } => { + let admitted = admitted_spec(admission: admission) + let applied = live_deploy_apply_admitted_via_transport(admission: admission, candidate: candidate, transport: access.transport, access: access) + let outcome = match applied { + Converged { evidence: _, applied: _ } => LiveDeployApplyConverged + NotConverged { reason: why, applied: _ } => LiveDeployApplyNotConverged { reason: why } + } + live_deploy_write_transaction_receipt( + spec: admitted, + receipt: live_deploy_transaction_receipt_for(spec: admitted, run: run, expected: expected, candidate: candidate, candidate_tree: candidate_tree, apply_outcome: outcome, role_singleton_disposition: role_singleton_disposition_of(admission: admission), readiness: live_deploy_post_apply_readiness(spec: admitted, access: access), provenance: provenance), + exit: live_deploy_reconciliation_exit(r: applied), + ) + } } - live_deploy_write_transaction_receipt( - spec: spec, - receipt: live_deploy_transaction_receipt_for(spec: spec, run: run, expected: expected, candidate: candidate, candidate_tree: candidate_tree, apply_outcome: outcome, readiness: live_deploy_post_apply_readiness(spec: spec, access: access), provenance: provenance), - exit: live_deploy_reconciliation_exit(r: applied), - ) } } -fn live_deploy_transaction_admitted(spec: DeploymentSpec, access: DeployAccess, run: ActionsRunBinding, expected: String, provenance: LiveDeployFleetProvenance) -> ProcessExit { +fn live_deploy_transaction_admitted(deployment: DesiredDeployment, access: DeployAccess, run: ActionsRunBinding, expected: String, provenance: LiveDeployFleetProvenance) -> ProcessExit { match observe_candidate_release() { CandidateUnobservable { cause: why } => exit_failure(reason: candidate_unobservable_detail(cause: why)) @@ -1260,7 +1397,7 @@ fn live_deploy_transaction_admitted(spec: DeploymentSpec, access: DeployAccess, exit_failure(reason: join(["live_deploy transaction: CandidateTreeUnobservable โ€” ", d], "")) CandidateTreeOidObserved { oid_hex: candidate_tree } => live_deploy_transaction_decided( - spec: spec, + deployment: deployment, access: access, run: run, expected: expected, @@ -1273,7 +1410,7 @@ fn live_deploy_transaction_admitted(spec: DeploymentSpec, access: DeployAccess, } } -fn live_deploy_transaction_admitted_wet_with_access(spec: DeploymentSpec, access: DeployAccess) -> ProcessExit { +fn live_deploy_transaction_admitted_wet_with_access(deployment: DesiredDeployment, access: DeployAccess) -> ProcessExit { match observe_actions_run_binding() { ActionsRunUnbound { variable: v, detail: d } => exit_failure(reason: join(["live_deploy transaction: RunUnbound โ€” ", v, " ", d, "; the transaction entry runs only inside the fleet-converge workflow"], "")) @@ -1305,7 +1442,7 @@ fn live_deploy_transaction_admitted_wet_with_access(spec: DeploymentSpec, access plan_receipt_identity: fleet_converge_plan_receipt_identity_hex(r: plan_receipt), apply_receipt_identity: fleet_converge_apply_receipt_identity_hex(r: fleet_apply_receipt), } - live_deploy_transaction_admitted(spec: spec, access: access, run: run, expected: expected, provenance: provenance) + live_deploy_transaction_admitted(deployment: deployment, access: access, run: run, expected: expected, provenance: provenance) } } } @@ -1319,24 +1456,24 @@ fn live_deploy_transaction_admitted_wet_with_access(spec: DeploymentSpec, access // Keeping realization injectable at this internal seam makes the refusing arm executable in a // hermetic witness while the production entry below remains constructed from the one bound row. fn live_deploy_transaction_wet_with_access_for_realization( - spec: DeploymentSpec, + deployment: DesiredDeployment, access: DeployAccess, realization: RepositoryTransitionRealization, ) -> ProcessExit { match repository_transition_admission(realization: realization) { LegacyGitFileSyncNotAdmitted { reason: why } => exit_failure(reason: why) - RepositoryTransitionAdmitted => live_deploy_transaction_admitted_wet_with_access(spec: spec, access: access) + RepositoryTransitionAdmitted => live_deploy_transaction_admitted_wet_with_access(deployment: deployment, access: access) } } -fn live_deploy_transaction_wet_with_access(spec: DeploymentSpec, access: DeployAccess) -> ProcessExit { +fn live_deploy_transaction_wet_with_access(deployment: DesiredDeployment, access: DeployAccess) -> ProcessExit { live_deploy_transaction_wet_with_access_for_realization( - spec: spec, + deployment: deployment, access: access, realization: deployed_tree_repository_transition, ) } fn live_deploy_apply_srv1_transaction_wet() -> ProcessExit { - live_deploy_transaction_wet_with_access(spec: deployment_spec_srv1(), access: ci_deploy_srv1_access) + live_deploy_transaction_wet_with_access(deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_access) } diff --git a/dag/gunbc/live_deploy/apply_receipt.dag b/dag/gunbc/live_deploy/apply_receipt.dag index be9079581b2..f4e9b4bd27e 100644 --- a/dag/gunbc/live_deploy/apply_receipt.dag +++ b/dag/gunbc/live_deploy/apply_receipt.dag @@ -47,7 +47,7 @@ import gunbc.actions_run_binding { // and '}'), so the candidate side and the deployed side // (gunbc.live_deploy.deployed_tree_observation deployed_tree_root_tree_argv) read the same object // the same way. -data live_deploy_apply_receipt_schema: String = "live-deploy-apply-receipt/v2" +data live_deploy_apply_receipt_schema: String = "live-deploy-apply-receipt/v3" fn live_deploy_apply_receipt_artifact_name_for(instance_id: String) -> String { join(["live-deploy-", instance_id, "-receipt"], "") @@ -120,6 +120,62 @@ type LiveDeployReadinessOutcome = LiveDeployReadinessObserved { revision: String, surface: String } | LiveDeployReadinessRefused { detail: String } +// WHAT THE ROLE-SINGLETON ASSESSMENT ADMITTED THIS APPLY TO DO (gunbc.live_deploy.role_singleton_convergence), +// carried in the receipt so a first install is visible and countable, +// and a refusal before any member was planned is told apart from a failed apply. +type LiveDeployRoleSingletonDisposition + = RoleSingletonsNoop + | RoleSingletonsInstalled { deployment: String, host: String } + | RoleSingletonsRefusedBeforeApply { detail: String } + +fn live_deploy_role_singleton_disposition_text(d: LiveDeployRoleSingletonDisposition) -> String { + match d { + RoleSingletonsNoop => "noop" + RoleSingletonsInstalled { deployment, host } => join(["installed:", deployment, "@", host], "") + RoleSingletonsRefusedBeforeApply { detail } => join(["refused:", detail], "") + } +} + +fn live_deploy_role_singleton_disposition_json(d: LiveDeployRoleSingletonDisposition) -> JsonValue { + match d { + RoleSingletonsNoop => json_object(members: [json_kv(key: "disposition", value: json_string(s: "noop"))]) + RoleSingletonsInstalled { deployment, host } => json_object(members: [json_kv(key: "disposition", value: json_string(s: "installed")), json_kv(key: "deployment", value: json_string(s: deployment)), json_kv(key: "host", value: json_string(s: host))]) + RoleSingletonsRefusedBeforeApply { detail } => json_object(members: [json_kv(key: "disposition", value: json_string(s: "refused")), json_kv(key: "detail", value: json_string(s: detail))]) + } +} + +type LiveDeployDispositionDecode + = DispositionDecoded { disposition: LiveDeployRoleSingletonDisposition } + | DispositionUnreadable { reason: String } + +fn live_deploy_role_singleton_disposition_of_member(doc: JsonValue) -> LiveDeployDispositionDecode { + match json_object_unique_member(v: doc, key: "role_singleton_disposition") { + JsonMemberAbsent => DispositionUnreadable { reason: "role_singleton_disposition is missing" } + JsonMemberDuplicated { count: c } => DispositionUnreadable { reason: join(["role_singleton_disposition appears ", to_string(c), " times; the member must be unique"], "") } + JsonMemberNotAnObject => DispositionUnreadable { reason: "the receipt document is not an object at role_singleton_disposition" } + JsonMemberFound { value: d } => match json_required_string_member(doc: d, key: "disposition") { + JsonStringMemberRefused { reason: r } => DispositionUnreadable { reason: r } + JsonStringMemberFound { value: kind } => + if kind == "noop" { DispositionDecoded { disposition: RoleSingletonsNoop } } + else if kind == "refused" { + match json_required_string_member(doc: d, key: "detail") { + JsonStringMemberRefused { reason: r } => DispositionUnreadable { reason: r } + JsonStringMemberFound { value: detail } => DispositionDecoded { disposition: RoleSingletonsRefusedBeforeApply { detail: detail } } + } + } else if kind == "installed" { + match json_required_string_member(doc: d, key: "deployment") { + JsonStringMemberRefused { reason: r } => DispositionUnreadable { reason: r } + JsonStringMemberFound { value: dep } => match json_required_string_member(doc: d, key: "host") { + JsonStringMemberRefused { reason: r } => DispositionUnreadable { reason: r } + JsonStringMemberFound { value: host } => + DispositionDecoded { disposition: RoleSingletonsInstalled { deployment: dep, host: host } } + } + } + } else { DispositionUnreadable { reason: join(["live-deploy receipt role_singleton_disposition is not one of noop|installed|refused: ", kind], "") } } + } + } +} + type LiveDeployApplyReceipt { run: ActionsRunBinding target: String @@ -129,6 +185,7 @@ type LiveDeployApplyReceipt { candidate_surface_identity: String candidate_tree_oid: String apply_outcome: LiveDeployApplyOutcome + role_singleton_disposition: LiveDeployRoleSingletonDisposition readiness: LiveDeployReadinessOutcome provenance: LiveDeployFleetProvenance completed_at: String @@ -153,6 +210,7 @@ fn live_deploy_apply_receipt_identity_text(r: LiveDeployApplyReceipt) -> String actions_run_binding_identity_text(binding: r.run), "|", r.target, "|", r.instance_id, "|", r.expected_revision, "|", r.candidate_revision, "|", r.candidate_surface_identity, "|", r.candidate_tree_oid, "|", live_deploy_apply_outcome_text(o: r.apply_outcome), "|", + live_deploy_role_singleton_disposition_text(d: r.role_singleton_disposition), "|", live_deploy_readiness_text(o: r.readiness), "|", r.provenance.plan_run_id, "|", r.provenance.apply_run_id, "|", r.provenance.plan_artifact_hash, "|", r.provenance.plan_receipt_identity, "|", r.provenance.apply_receipt_identity, "|", @@ -162,7 +220,7 @@ fn live_deploy_apply_receipt_identity_text(r: LiveDeployApplyReceipt) -> String fn live_deploy_apply_receipt_identity_hex(r: LiveDeployApplyReceipt) -> String { content_hash_tagged_structural( - tag: "live-deploy-apply-receipt-v2" as NonEmptyStr, + tag: "live-deploy-apply-receipt-v3" as NonEmptyStr, payload: content_hash_atom(value: live_deploy_apply_receipt_identity_text(r: r) as NonEmptyStr), ).digest as String } @@ -205,6 +263,7 @@ fn live_deploy_apply_receipt_json(r: LiveDeployApplyReceipt) -> JsonValue { json_kv(key: "candidate_surface_identity", value: json_string(s: r.candidate_surface_identity)), json_kv(key: "candidate_tree_oid", value: json_string(s: r.candidate_tree_oid)), json_kv(key: "apply_outcome", value: live_deploy_apply_outcome_json(o: r.apply_outcome)), + json_kv(key: "role_singleton_disposition", value: live_deploy_role_singleton_disposition_json(d: r.role_singleton_disposition)), json_kv(key: "readiness", value: live_deploy_readiness_json(o: r.readiness)), json_kv(key: "provenance", value: json_object(members: [ json_kv(key: "plan_run_id", value: json_string(s: r.provenance.plan_run_id)), @@ -357,6 +416,9 @@ fn live_deploy_apply_receipt_decode(raw: String) -> LiveDeployApplyReceiptDecode match live_deploy_apply_outcome_of_member(doc: doc) { OutcomeUnreadable { reason: r } => LiveDeployApplyReceiptUnreadable { reason: r } OutcomeDecoded { outcome: apply_outcome } => + match live_deploy_role_singleton_disposition_of_member(doc: doc) { + DispositionUnreadable { reason: r } => LiveDeployApplyReceiptUnreadable { reason: r } + DispositionDecoded { disposition: role_singleton_disposition } => match live_deploy_readiness_of_member(doc: doc) { ReadinessUnreadable { reason: r } => LiveDeployApplyReceiptUnreadable { reason: r } ReadinessDecoded { readiness: readiness } => @@ -375,6 +437,7 @@ fn live_deploy_apply_receipt_decode(raw: String) -> LiveDeployApplyReceiptDecode candidate_surface_identity: surface, candidate_tree_oid: tree_oid, apply_outcome: apply_outcome, + role_singleton_disposition: role_singleton_disposition, readiness: readiness, provenance: provenance, completed_at: completed_at, @@ -392,6 +455,7 @@ fn live_deploy_apply_receipt_decode(raw: String) -> LiveDeployApplyReceiptDecode } } } + } } } } diff --git a/dag/gunbc/live_deploy/apply_warrant.dag b/dag/gunbc/live_deploy/apply_warrant.dag new file mode 100644 index 00000000000..7085a9acc56 --- /dev/null +++ b/dag/gunbc/live_deploy/apply_warrant.dag @@ -0,0 +1,25 @@ +module gunbc.live_deploy.apply_warrant + +import std.decl_ref { decl_ref } +import gunbc.live_deploy.spec { DeploymentSpec } +import gunbc.live_deploy.candidate_identity { CandidateRelease } +import gunbc.live_deploy.member_identity { ReleaseEffectPlan } + +// THE ONLY WAY A DEPLOY REACHES THE HOST-EFFECT REALIZER. gunbc.host_effect LiveDeployApply carries this +// warrant rather than a raw (spec, candidate, plan), and the warrant is sole_constructor with one mint, +// sealed to the member-plan step of the admitted apply (gunbc.live_deploy.apply +// live_deploy_apply_member_plan_via_transport), which is itself sealed to the admitted mutation. So a +// spec the role-singleton assessment did not admit cannot be planned, warranted or realized: the +// realizer receives an apply only through the composition that assessed it. A leaf module, so the +// effect vocabulary (gunbc.host_effect) can name the warrant without importing the assessment. +type AdmittedLiveDeployApply sole_constructor { + spec: DeploymentSpec + candidate: CandidateRelease + plan: ReleaseEffectPlan +} + +fn warrant_live_deploy_apply(spec: DeploymentSpec, candidate: CandidateRelease, plan: ReleaseEffectPlan) -> AdmittedLiveDeployApply admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_member_plan_via_transport"), +] { + AdmittedLiveDeployApply { spec: spec, candidate: candidate, plan: plan } +} diff --git a/dag/gunbc/live_deploy/desired.dag b/dag/gunbc/live_deploy/desired.dag index 3eeb3ead16c..1bf8941ac8a 100644 --- a/dag/gunbc/live_deploy/desired.dag +++ b/dag/gunbc/live_deploy/desired.dag @@ -19,11 +19,19 @@ import gunbc.fleet_intent_network { import gunbc.fleet_host_identity { operator_host_srv2 } import gunbc.roadmap_dashboard_instance { HostDashboardInstance, + ProdRoleRealization, + ProdRoleRealizedBy, + ProdRoleRealizationRefused, + ProdRoleHolderUnactuatable, + prod_role_realization, + prod_role_realization_in, + public_builtin_dashboard_instances, srv1_live_dashboard_instance, srv1_lab_dashboard_instance, srv2_deploy_dashboard_instance, } import v2.std.collection { Present, Absent } +import gunbc.deployment_risk { ProdRoleSelection } import gunbc.live_deploy.spec { DeploymentSpec, DeploymentHostTarget, @@ -72,6 +80,7 @@ fn srv2_deploy_selected_identity() -> DeclarationRef { fn selected_deployment_spec(row: DesiredDeployment) -> DeploymentSpec { deployment_spec_for( + realization: prod_role_actuated_realization(), instance: row.instance, target: row.target, names: row.names, @@ -126,6 +135,32 @@ fn srv2_deploy_desired_deployment() -> DesiredDeployment { } } +// THE PROD ROLE AS SOMETHING THAT CAN BE ACTUATED. The holder resolves over the six dashboard +// instances (gunbc.roadmap_dashboard_instance prod_role_realization), but only a row of THIS roster +// has a target and a spec, so a holder outside it -- the MacBook, srv2-preview, srv2's lab -- names a +// deployment nothing can install. Joined here, against the one roster, it refuses with +// ProdRoleHolderUnactuatable instead of placing the role-following singletons nowhere. Every +// role-following reader takes this joined realization; there is no second list of eligible holders. +fn prod_role_actuated_realization_over(realization: ProdRoleRealization, rows: List) -> ProdRoleRealization { + match realization { + ProdRoleRealizedBy { holder } => + if any(rows, r => r.instance.deployment == holder.instance.deployment) { realization } + else { ProdRoleRealizationRefused { cause: ProdRoleHolderUnactuatable { deployment: holder.instance.deployment } } } + ProdRoleRealizationRefused { cause: _ } => realization + } +} + +fn prod_role_actuated_realization_in(selection: ProdRoleSelection) -> ProdRoleRealization { + prod_role_actuated_realization_over( + realization: prod_role_realization_in(selection: selection, instances: public_builtin_dashboard_instances()), + rows: desired_deployment_roster(), + ) +} + +fn prod_role_actuated_realization() -> ProdRoleRealization { + prod_role_actuated_realization_over(realization: prod_role_realization(), rows: desired_deployment_roster()) +} + fn desired_deployment_roster() -> List { [ srv1_production_desired_deployment(), diff --git a/dag/gunbc/live_deploy/emit.dag b/dag/gunbc/live_deploy/emit.dag index ddf5413f7a0..8242d1377a6 100644 --- a/dag/gunbc/live_deploy/emit.dag +++ b/dag/gunbc/live_deploy/emit.dag @@ -1,5 +1,6 @@ module gunbc.live_deploy.emit +import extdeps.access.posix { file_mode_octal } import gunbc.workspace_commissioning_dispatch_operations { workspace_dispatch_grant_path, workspace_dispatch_grant_content } import gunbc.live_deploy.release_environment { installed_release_environment, installed_release_unit } @@ -138,10 +139,12 @@ import gunbc.live_deploy.slice_bounds { repository_convergence_slice_memory_max, repository_convergence_slice_memory_high, repository_convergence_slice_memory_swap_max, } -import gunbc.fabric_storage_placement { fabric_storage_door_socket } +import gunbc.auth.approval_broker_placement { approval_broker_base_url_of_holder } +import gunbc.auth.approval_broker_endpoint { approval_broker_base_url_env_name } import gunbc.live_deploy.spec { + HoldsProdRoleSingletons, HoldsNoRoleSingleton, deployment_slice_unit_name, deployment_slice_unit_path, - deployment_fabric_storage_unit_name, deployment_fabric_storage_serve_endpoint, deployment_approval_broker_front_door_endpoints, + deployment_fabric_storage_unit_name, deployment_fabric_storage_serve_endpoint, deployment_fabric_storage_door_socket, deployment_approval_broker_front_door_endpoints, gunbc_approval_broker_unit_name, gunbc_approval_broker_unit_path, deployment_owns_the_host_approval_broker, gunbc_approval_broker_slice_unit_name, gunbc_approval_broker_slice_unit_path, @@ -204,6 +207,7 @@ import gunbc.membership_reconcile { } import gunbc.shell_command_text { shell_command_text_of_argv, shell_privileged_command_text_of_argv } import gunbc.live_deploy.operations { + install_file_owned_command, deploy_stage_dir_var, deploy_stage_write_command, deploy_stage_write_expansion_command, @@ -221,7 +225,6 @@ import gunbc.live_deploy.operations { rm_empty_directory_command, rm_force_recursive_command, install_directory_owned_command, - install_file_owned_command, } import gunbc.live_deploy.intent { live_deploy_unit_status_tail_command } import v2.std.orchestration { @@ -592,7 +595,7 @@ fn apply_tailnet_door_unit_write_step(spec: DeploymentSpec, revision: ReleaseRev // unbounded hold of the serial accept loop into a typed refusal exactly as they do there. The process // keeps no state between requests -- the store is the files under the ensured fabric DB directory -- // so Restart=on-failure loses nothing. -fn live_deploy_fabric_storage_unit_file(spec: DeploymentSpec, revision: ReleaseRevisionBinding) -> SystemdUnitFile { +fn live_deploy_fabric_storage_unit_file(spec: DeploymentSpec, revision: ReleaseRevisionBinding, door_socket: NonEmptyStr) -> SystemdUnitFile { let svc = spec.service ServiceUnitFile { unit: [ @@ -611,7 +614,7 @@ fn live_deploy_fabric_storage_unit_file(spec: DeploymentSpec, revision: ReleaseR svc.serve_binary as String, " serve --source-root dag --source-root src/v2", " --entry dag/gunbc/fabric/fabric_storage_serve.dag --function fabric_storage_serve_handle", - " --unix-socket ", fabric_storage_door_socket() as String, + " --unix-socket ", door_socket as String, " --release-revision ", release_revision_execstart_text(binding: revision), eval_budget_flag_text(flag: "--eval-budget-cpu-ms", limit: svc.serve_cpu_limit), eval_budget_flag_text(flag: "--eval-budget-wall-ms", limit: svc.serve_wall_limit), @@ -622,13 +625,21 @@ fn live_deploy_fabric_storage_unit_file(spec: DeploymentSpec, revision: ReleaseR } } -fn apply_fabric_storage_unit_write_step(spec: DeploymentSpec, revision: ReleaseRevisionBinding) -> PipelineStep { +fn apply_fabric_storage_unit_write_step(spec: DeploymentSpec, revision: ReleaseRevisionBinding, door_socket: NonEmptyStr) -> PipelineStep { deploy_raw(command: deploy_stage_write_command( stage_name: deployment_fabric_storage_unit_name(names: spec.names) as String, - content: render(doc: doc_text(text: serialize_systemd_unit_file(file: live_deploy_fabric_storage_unit_file(spec: spec, revision: revision))), proto: live_deploy_protocol), + content: render(doc: doc_text(text: serialize_systemd_unit_file(file: live_deploy_fabric_storage_unit_file(spec: spec, revision: revision, door_socket: door_socket))), proto: live_deploy_protocol), )) } +// A fabric storage step for a spec whose instance is not in the roster has no door socket to name; +// the marker is not valid shell, so the stage fails rather than serving an invented path. +data fabric_storage_door_unknown_instance_poison: String = "__GUNBC_DEPLOY_REFUSED__ fabric storage step reached with a deployment whose instance_id resolves to no HostDashboardInstance, so it has no door socket (gunbc.live_deploy.spec deployment_fabric_storage_door_socket). instance=" + +fn fabric_storage_door_unknown_step(spec: DeploymentSpec) -> PipelineStep { + deploy_raw(command: join([fabric_storage_door_unknown_instance_poison, spec.instance_id as String, "\n"], "")) +} + fn emit_systemd_unit_doc(spec: DeploymentSpec, revision: ReleaseRevisionBinding) -> Doc { doc_text(text: serialize_systemd_unit_file(file: live_deploy_serve_unit_file(spec: spec, revision: revision))) } @@ -715,6 +726,7 @@ fn live_deploy_approval_broker_unit_file_on( ServiceSlice { name: gunbc_approval_broker_slice_unit_name }, WorkingDirectory { path: release_dir as NonEmptyStr }, release_locus_workspace_root_environment(release_dir: release_dir), + approval_broker_base_url_environment(spec: spec), ExecStart { command: join([ approval_broker_release_binary_path(root: svc.repo_root as String, revision: revision), " serve --source-root dag --source-root src/v2", @@ -1483,6 +1495,10 @@ fn ensure_dependency_steps(dep: DeploymentDependencyStep, privileged: Bool) -> L EnsuredManagedHostDirectory { admission } => concat([deploy_raw(command: ensure_managed_directory_command(admission: admission))], map(managed_directory_entry_repair_commands(admission: admission), c => deploy_raw(command: c))) + EnsuredOwnershipMarker { marker } => [ + deploy_raw(command: deploy_stage_write_command(stage_name: "store-ownership-marker", content: marker.content as String)), + deploy_raw(command: deploy_stage_install_owned_command(stage_name: "store-ownership-marker", mode: file_mode_octal(mode: marker.mode) as NonEmptyStr, owner: marker.owner, group: marker.group, dest: marker.path as String)), + ] } } @@ -1860,6 +1876,21 @@ data approval_broker_release_retention_note: String = "The broker's release locu // from a directory that is not a git checkout, and a locus is not one -- measured on srv1 // 2026-09-21, where both the broker's serve unit and the microVM slot unit exited 101 in // workspace_root before any stage ran. One row feeds both units. +// THE BROKER IS GIVEN ITS PUBLIC BASE URL, built from the holder token the spec carries +// (gunbc.auth.approval_broker_placement approval_broker_base_url_of_holder), so the serving broker +// reads where it runs instead of resolving it. A spec holding no role never reaches a broker unit +// write (apply_approval_broker_unit_write_steps is empty for it); if one is rendered anyway the +// binding is a refusal marker the broker's address read rejects as a non-URL, never a guessed host. +data approval_broker_base_url_unbound_marker: String = "__GUNBC_DEPLOY_REFUSED__no-prod-role-holder" + +fn approval_broker_base_url_environment(spec: DeploymentSpec) -> SystemdServiceDirective { + let value = match spec.role_singletons { + HoldsProdRoleSingletons { holder } => approval_broker_base_url_of_holder(holder: holder) as String + HoldsNoRoleSingleton => approval_broker_base_url_unbound_marker + } + Environment { assignment: join([approval_broker_base_url_env_name as String, "=", value], "") as NonEmptyStr } +} + fn release_locus_workspace_root_environment(release_dir: String) -> SystemdServiceDirective { installed_release_environment(release_dir: release_dir) } @@ -2852,8 +2883,10 @@ fn emit_artifact_upsert(art: DeploymentArtifactStep, spec: DeploymentSpec, revis deploy_raw(command: "__GUNBC_FLEET_CONVERGE_REFUSED__ FleetConvergeTimerUnit retired from deployment membership โ€” timer cadence is on-demand via fleet-converge.yml plan/apply, not live_deploy apply (sleek-heron-218)"), ] TailscaleServeMapping => tailscale_serve_mapping_apply_steps(spec: spec) - FabricStorageServeUnit => [ - apply_fabric_storage_unit_write_step(spec: spec, revision: revision), + FabricStorageServeUnit => match deployment_fabric_storage_door_socket(spec: spec) { + Absent => [fabric_storage_door_unknown_step(spec: spec)] + Present { value: door_socket } => [ + apply_fabric_storage_unit_write_step(spec: spec, revision: revision, door_socket: door_socket), deploy_raw(command: deploy_stage_install_command( stage_name: deployment_fabric_storage_unit_name(names: spec.names) as String, mode: install_file_shared_mode, @@ -2862,12 +2895,16 @@ fn emit_artifact_upsert(art: DeploymentArtifactStep, spec: DeploymentSpec, revis deploy_effect(inv: systemctl_daemon_reload_effect()), deploy_effect(inv: systemctl_enable_effect(unit: deployment_fabric_storage_unit_name(names: spec.names))), deploy_effect(inv: systemctl_restart_effect(unit: deployment_fabric_storage_unit_name(names: spec.names))), - ] - FabricStorageServeMapping => [ - deploy_raw(command: shell_privileged_command_text_of_argv( - argv: concat(["tailscale"], tailscale_serve_enable_argv(endpoint: deployment_fabric_storage_serve_endpoint())), - )), - ] + ] + } + FabricStorageServeMapping => match deployment_fabric_storage_door_socket(spec: spec) { + Absent => [fabric_storage_door_unknown_step(spec: spec)] + Present { value: door_socket } => [ + deploy_raw(command: shell_privileged_command_text_of_argv( + argv: concat(["tailscale"], tailscale_serve_enable_argv(endpoint: deployment_fabric_storage_serve_endpoint(door_socket: door_socket))), + )), + ] + } TailnetDoorServeUnit => [ apply_tailnet_door_unit_write_step(spec: spec, revision: revision), deploy_raw(command: deploy_stage_install_command( @@ -2926,11 +2963,14 @@ fn emit_artifact_teardown(art: DeploymentArtifactStep, spec: DeploymentSpec) -> TailscaleServeMapping => [ tailscale_serve_teardown_step(spec: spec), ] - FabricStorageServeMapping => [ - deploy_raw(command: shell_privileged_command_text_of_argv( - argv: concat(["tailscale"], tailscale_serve_off_argv(endpoint: deployment_fabric_storage_serve_endpoint())), - )), - ] + FabricStorageServeMapping => match deployment_fabric_storage_door_socket(spec: spec) { + Absent => [fabric_storage_door_unknown_step(spec: spec)] + Present { value: door_socket } => [ + deploy_raw(command: shell_privileged_command_text_of_argv( + argv: concat(["tailscale"], tailscale_serve_off_argv(endpoint: deployment_fabric_storage_serve_endpoint(door_socket: door_socket))), + )), + ] + } ApprovalBrokerFrontDoor => map( deployment_approval_broker_front_door_endpoints(), e => deploy_raw(command: shell_privileged_command_text_of_argv(argv: concat(["tailscale"], tailscale_serve_off_argv(endpoint: e)))), @@ -3338,7 +3378,7 @@ fn approval_broker_dark_install_intent( // -- a stage that reports success having installed nothing, which is the absorbing fallback DESIGN // section 5 forbids at exactly the seam where the operator is being told a stage completed. The // marker is not valid shell, so the stage fails and names the owner. -data approval_broker_dark_install_not_owner_poison: String = "__GUNBC_DEPLOY_REFUSED__ approval-broker DARK INSTALL was reached with a deployment that does not own this host's broker unit. gunbc-approval-broker.service is a HOST singleton owned by gunbc.live_deploy.spec gunbc_approval_broker_owning_instance; a twin apply installing it would point the host's broker at the twin's tree and restart it over production's process (review 68094). This marker is not valid deploy shell, so the stage fails loudly rather than emitting a script that installs nothing and reports success. instance=" +data approval_broker_dark_install_not_owner_poison: String = "__GUNBC_DEPLOY_REFUSED__ approval-broker DARK INSTALL was reached with a deployment that does not own this host's broker unit. gunbc-approval-broker.service is a HOST singleton owned by the spec that holds the prod role (gunbc.live_deploy.spec RoleSingletonHolding over gunbc.deployment_risk prod_role_selection), and is owned by nobody under NoProdRole; a twin apply installing it would point the host's broker at the twin's tree and restart it over production's process (review 68094). This marker is not valid deploy shell, so the stage fails loudly rather than emitting a script that installs nothing and reports success. instance=" data approval_broker_dark_install_emit_refused_poison: String = "__GUNBC_ORCH_EMIT_REFUSED__ approval-broker DARK INSTALL intent emission was rejected by v2.compiler.emit_orchestration; this marker is not valid deploy shell, so stage 1 of the cutover fails loudly rather than a hand-spelled fallback masking the refusal (DESIGN section 5: refuse, never widen)\n" diff --git a/dag/gunbc/live_deploy/role_singleton_convergence.dag b/dag/gunbc/live_deploy/role_singleton_convergence.dag new file mode 100644 index 00000000000..2db9416237a --- /dev/null +++ b/dag/gunbc/live_deploy/role_singleton_convergence.dag @@ -0,0 +1,624 @@ +module gunbc.live_deploy.role_singleton_convergence + +import std.types { String, NonEmptyStr, List, Bool, Int } +import v2.std.collection { Present, Absent } +import gunbc.deployment_risk { DeploymentId } +import std.decl_ref { decl_ref } +import gunbc.roadmap_dashboard_instance { + ProdRoleHolder, + ProdRoleRealization, + ProdRoleRealizedBy, + ProdRoleRealizationRefused, + ProdRoleRealizationRefusal, + ProdRoleUnheld, + ProdRoleHolderUnrealized, + ProdRoleHolderAmbiguous, + ProdRoleHolderUnactuatable, + prod_role_realization_refusal_detail, +} +import gunbc.host_effect { HostEffectTransport } +import gunbc.live_deploy.spec { + DeploymentSpec, + HoldsProdRoleSingletons, + HoldsNoRoleSingleton, + deployment_fabric_storage_unit_path, + gunbc_approval_broker_unit_path, +} +import gunbc.fabric_storage_placement { FabricStorageOwnershipMarkerSpec, FabricStorageMarkerReading, FabricStorageMarkerExact, FabricStorageMarkerMismatched, fabric_storage_ownership_marker_spec, fabric_storage_marker_verdict } +import extdeps.access.posix { file_mode_octal } +import extdeps.shell +import std.algebra { trim } +import std.list { distinct_by_key } +import std.types { FilePath } +import gunbc.live_deploy.desired { DesiredDeployment, desired_deployment_roster, prod_role_actuated_realization, selected_deployment_spec, bind_desired_selected_identity, DesiredSelectedIdentityBound, DesiredSelectedIdentityUnknownInstance, DesiredSelectedIdentityRosterRefused, DesiredSelectedIdentityMismatch, instance_id_text } +import gunbc.live_deploy.member_observe { + ObservationArm, + OnTheHostItself, + OverTransport, + Leg, + LegOut, + LegRefused, + remote_leg, + unit_document_leg, + observation_arm, + entry_presence, + EntryPresence, + EntryPresent, + EntryAbsent, + EntryPresenceIndeterminate, +} + +// THE ROLE-FOLLOWING SINGLETONS AS A CONVERGENCE SUBJECT. The approval broker and the fabric store +// (unit, route, socket, root, directories and ownership marker -- one subject, gunbc.live_deploy.spec +// RoleSingletonHolding) are installed only by the spec that holds the prod role, and a spec's apply +// reconciles its owned members against `observed: []`. So a role move, or NoProdRole, would leave the +// old singleton running and, on a move, start a second EMPTY one beside it -- the store's state does not +// travel with a unit. This module assesses the DESIRED realization against an INDEPENDENT OBSERVATION +// of every roster host (DESIGN 3d) and, only when that assessment admits it, mints the one token an +// apply may act on (RoleSingletonAdmission). +// +// ONLY FIRST INSTALL IS IMPLEMENTED. Noop when exactly one complete realization is observed and it is +// the desired one on its own target (or nothing is desired and nothing is observed); Install when a +// valid holder is desired and NOTHING is observed anywhere. Every other transition REFUSES with a typed +// cause -- including a realization that predates the store ownership marker (RoleSingletonUnmarked), +// which waits for the operator's one-time disposition named in its own refusal text (gunbc#13124: +// manual resume, no self-repair outside an operator disposition). Move and Remove are +// refusal causes today and become arms of RoleSingletonConvergence when their actuation lands; neither +// renames the arms that exist. +// +// THE HOST UNIVERSE IS THE ROSTER'S TARGETS (desired_deployment_roster), derived, not listed. A +// singleton on a host outside the roster is INVISIBLE to this observation: a bounded residual whose +// trigger is a roster row for that host, stated in docs/plans/deployment-risk-conformance.md. + +// โ”€โ”€ What can be seen โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// THE MEMBERS a realization is made of. The broker unit is host-global and names no deployment; the +// fabric store unit and the store's ownership marker each name exactly one roster deployment. The +// marker (in the store's control directory, read by the deploy job principal) is read because the +// store root, auth state and door directories are Ensured and no retract +// removes them: an old store with no unit is still a store. +type RoleSingletonMember + = BrokerUnitMember + | FabricUnitMember + | StoreMarkerMember + +// ONE SIGHTING, KEPT AT (host, deployment, member) GRAIN until multiplicity is adjudicated -- never +// deduplicated by DeploymentId first, which is how the same deployment on two hosts, or a deployment +// realized on a host it does not target, would read as one ordinary holder. +type RoleSingletonSighting + = SightedDeploymentMember { host: NonEmptyStr, deployment: DeploymentId, member: RoleSingletonMember } + | SightedBroker { host: NonEmptyStr } + | SightedMismatchedMarker { host: NonEmptyStr, deployment: DeploymentId, field: NonEmptyStr, expected: String, observed: String } + +type HostRoleSingletonReading + = HostRoleSingletonsRead { host: NonEmptyStr, sightings: List } + | HostRoleSingletonsUnobservable { host: NonEmptyStr, cause: String } + +// THE READING SOURCE IS AN INTERFACE. Cross-host observation is host self-report (operator ruling +// 2026-10-05; gunbc#13339 realizes it). Until that lands the production source is +// local_only_reading_source: the applying host is read over the apply's own route, and every other +// roster host is unobservable by construction. The adjudication below is indifferent to which. +type RoleSingletonReadingSource { + read: fn(NonEmptyStr, List) -> HostRoleSingletonReading +} + +// โ”€โ”€ What is refused โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +type RoleSingletonRefusal + = RoleSingletonMoveUnimplemented { observed: DeploymentId, desired: DeploymentId } + | RoleSingletonRemovalUnimplemented { observed: DeploymentId } + | RoleSingletonDesiredRealizationRefused { cause: ProdRoleRealizationRefusal } + | RoleSingletonHostUnobservable { host: NonEmptyStr, cause: String } + | RoleSingletonHoldersAmbiguous { observed: List } + | RoleSingletonRealizedOnTwoHosts { deployment: DeploymentId, hosts: List } + | RoleSingletonMisplaced { deployment: DeploymentId, target: NonEmptyStr, observed_on: NonEmptyStr } + | RoleSingletonTwoOnOneHost { host: NonEmptyStr, deployments: List } + | RoleSingletonBrokerWithoutStore { host: NonEmptyStr } + | RoleSingletonPartial { deployment: DeploymentId, host: NonEmptyStr, missing: List } + | RoleSingletonUnmarked { deployment: DeploymentId, host: NonEmptyStr, marker: FabricStorageOwnershipMarkerSpec } + | RoleSingletonMarkerMismatched { deployment: DeploymentId, host: NonEmptyStr, field: NonEmptyStr, expected: String, observed: String } + | RoleSingletonStaleSpec { instance_id: NonEmptyStr, detail: String } + | RoleSingletonDeploymentUnrostered { instance_id: String, detail: String } + +// โ”€โ”€ What is decided โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +type RoleSingletonConvergence + = RoleSingletonNoop + | RoleSingletonInstall { holder: ProdRoleHolder } + | RoleSingletonRefused { cause: RoleSingletonRefusal } + +// โ”€โ”€ The observation โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +type ObservedRealization { + deployment: DeploymentId + host: NonEmptyStr + members: List +} + +type RoleSingletonObservation + = RoleSingletonsObserved { realizations: List } + | RoleSingletonObservationRefused { cause: RoleSingletonRefusal } + +fn member_eq(a: RoleSingletonMember, b: RoleSingletonMember) -> Bool { + match a { + BrokerUnitMember => match b { BrokerUnitMember => true FabricUnitMember => false StoreMarkerMember => false } + FabricUnitMember => match b { BrokerUnitMember => false FabricUnitMember => true StoreMarkerMember => false } + StoreMarkerMember => match b { BrokerUnitMember => false FabricUnitMember => false StoreMarkerMember => true } + } +} + +fn distinct_ids(ids: List) -> List { + distinct_by_key(xs: ids, key: d => d.name as String) +} + +fn distinct_hosts(hosts: List) -> List { + distinct_by_key(xs: hosts, key: h => h as String) +} + +fn deployment_sightings(readings: List) -> List { + readings |> flat_map(r => match r { + HostRoleSingletonsRead { host: _, sightings } => sightings |> filter(s => match s { + SightedDeploymentMember { host: _, deployment: _, member: _ } => true + SightedBroker { host: _ } => false + SightedMismatchedMarker { host: _, deployment: _, field: _, expected: _, observed: _ } => false + }) + HostRoleSingletonsUnobservable { host: _, cause: _ } => [] + }) +} + +fn sighting_host(s: RoleSingletonSighting) -> NonEmptyStr { + match s { + SightedDeploymentMember { host, deployment: _, member: _ } => host + SightedBroker { host } => host + SightedMismatchedMarker { host, deployment: _, field: _, expected: _, observed: _ } => host + } +} + +fn sighting_deployment(s: RoleSingletonSighting) -> DeploymentId? { + match s { + SightedDeploymentMember { host: _, deployment, member: _ } => Present { value: deployment } + SightedBroker { host: _ } => none + SightedMismatchedMarker { host: _, deployment, field: _, expected: _, observed: _ } => Present { value: deployment } + } +} + +fn sighting_member(s: RoleSingletonSighting) -> RoleSingletonMember { + match s { + SightedDeploymentMember { host: _, deployment: _, member } => member + SightedBroker { host: _ } => BrokerUnitMember + SightedMismatchedMarker { host: _, deployment: _, field: _, expected: _, observed: _ } => StoreMarkerMember + } +} + +fn broker_hosts(readings: List) -> List { + readings |> flat_map(r => match r { + HostRoleSingletonsRead { host: _, sightings } => sightings |> flat_map(s => match s { + SightedBroker { host } => [host] + SightedMismatchedMarker { host: _, deployment: _, field: _, expected: _, observed: _ } => [] + SightedDeploymentMember { host: _, deployment: _, member: _ } => [] + }) + HostRoleSingletonsUnobservable { host: _, cause: _ } => [] + }) +} + +fn sighted_ids(sightings: List) -> List { + distinct_ids(ids: sightings |> flat_map(s => match sighting_deployment(s: s) { Present { value: d } => [d] Absent => [] })) +} + +fn hosts_of(sightings: List, deployment: DeploymentId) -> List { + distinct_hosts(hosts: sightings |> filter(s => match sighting_deployment(s: s) { Present { value: x } => x == deployment Absent => false }) |> map(s => sighting_host(s: s))) +} + +fn target_host_of(rows: List, deployment: DeploymentId) -> NonEmptyStr? { + match rows |> filter(r => r.instance.deployment == deployment) |> first { + Present { value: r } => Present { value: r.target.ssh_host } + Absent => none + } +} + +// THE ADJUDICATION, over per-host readings and the roster, with NO deduplication before the +// multiplicity checks. In order: an unobservable host; one deployment realized on two hosts; one +// deployment realized off its own target; two deployments realized on one host; two deployments +// realized anywhere; a broker on a host where no store is realized. Only then are realizations formed, +// each (deployment, host) with the members seen for it, broker included from the same host. +fn role_singleton_observation_of(readings: List, rows: List) -> RoleSingletonObservation { + let unobservable = readings |> flat_map(r => match r { + HostRoleSingletonsUnobservable { host, cause } => [RoleSingletonHostUnobservable { host: host, cause: cause }] + HostRoleSingletonsRead { host: _, sightings: _ } => [] + }) + let mismatched = readings |> flat_map(r => match r { + HostRoleSingletonsRead { host: _, sightings: ss } => ss |> flat_map(x => match x { + SightedMismatchedMarker { host, deployment, field, expected, observed } => [RoleSingletonMarkerMismatched { deployment: deployment, host: host, field: field, expected: expected, observed: observed }] + SightedDeploymentMember { host: _, deployment: _, member: _ } => [] + SightedBroker { host: _ } => [] + }) + HostRoleSingletonsUnobservable { host: _, cause: _ } => [] + }) + let sightings = deployment_sightings(readings: readings) + let ids = sighted_ids(sightings: sightings) + let two_hosts = ids |> flat_map(d => if count(hosts_of(sightings: sightings, deployment: d)) > 1 { [RoleSingletonRealizedOnTwoHosts { deployment: d, hosts: hosts_of(sightings: sightings, deployment: d) }] } else { [] }) + let misplaced = ids |> flat_map(d => match target_host_of(rows: rows, deployment: d) { + Absent => [] + Present { value: t } => hosts_of(sightings: sightings, deployment: d) |> flat_map(h => if (h as String) == (t as String) { [] } else { [RoleSingletonMisplaced { deployment: d, target: t, observed_on: h }] }) + }) + let store_hosts = distinct_hosts(hosts: sightings |> map(s => sighting_host(s: s))) + let two_on_one = store_hosts |> flat_map(h => { + let here = distinct_ids(ids: sightings |> filter(s => (sighting_host(s: s) as String) == (h as String)) |> flat_map(s => match sighting_deployment(s: s) { Present { value: d } => [d] Absent => [] })) + if count(here) > 1 { [RoleSingletonTwoOnOneHost { host: h, deployments: here }] } else { [] } + }) + let brokers = broker_hosts(readings: readings) + let broker_alone = brokers |> flat_map(h => if any(store_hosts, s => (s as String) == (h as String)) { [] } else { [RoleSingletonBrokerWithoutStore { host: h }] }) + let first_refusal = concat(concat(concat(concat(unobservable, mismatched), two_hosts), concat(misplaced, two_on_one)), concat(if count(ids) > 1 { [RoleSingletonHoldersAmbiguous { observed: ids }] } else { [] }, broker_alone)) |> first + match first_refusal { + Present { value: c } => RoleSingletonObservationRefused { cause: c } + Absent => RoleSingletonsObserved { realizations: ids |> flat_map(d => hosts_of(sightings: sightings, deployment: d) |> map(h => ObservedRealization { + deployment: d, + host: h, + members: concat( + sightings |> filter(s => (sighting_host(s: s) as String) == (h as String)) |> map(s => sighting_member(s: s)), + if any(brokers, b => (b as String) == (h as String)) { [BrokerUnitMember] } else { [] }, + ), + })) } + } +} + +fn missing_members(r: ObservedRealization) -> List { + [BrokerUnitMember, FabricUnitMember, StoreMarkerMember] |> filter(m => !any(r.members, x => member_eq(a: x, b: m))) +} + +// โ”€โ”€ The decision โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +fn desired_is_no_prod(cause: ProdRoleRealizationRefusal) -> Bool { + match cause { + ProdRoleUnheld => true + ProdRoleHolderUnrealized { deployment: _ } => false + ProdRoleHolderAmbiguous { deployment: _, count: _ } => false + ProdRoleHolderUnactuatable { deployment: _ } => false + } +} + +// ONLY ProdRoleUnheld MEANS "NO PROD". An unrealized, ambiguous or unactuatable holder is a refusal of +// the desired side, never read as the no-prod path -- which is how a typo in the selection would +// otherwise turn into a removal of the real prod singletons. +fn role_singleton_convergence(desired: ProdRoleRealization, observed: RoleSingletonObservation, rows: List) -> RoleSingletonConvergence { + match observed { + RoleSingletonObservationRefused { cause } => RoleSingletonRefused { cause: cause } + RoleSingletonsObserved { realizations } => match desired { + ProdRoleRealizationRefused { cause } => + if desired_is_no_prod(cause: cause) { + match realizations |> first { + Absent => RoleSingletonNoop + Present { value: r } => RoleSingletonRefused { cause: RoleSingletonRemovalUnimplemented { observed: r.deployment } } + } + } else { + RoleSingletonRefused { cause: RoleSingletonDesiredRealizationRefused { cause: cause } } + } + ProdRoleRealizedBy { holder } => match realizations |> first { + Absent => RoleSingletonInstall { holder: holder } + Present { value: r } => + if !(r.deployment == holder.instance.deployment) { + RoleSingletonRefused { cause: RoleSingletonMoveUnimplemented { observed: r.deployment, desired: holder.instance.deployment } } + } else { + let missing = missing_members(r: r) + if count(missing) == 0 { RoleSingletonNoop } + else if count(missing) == 1 && all(missing, m => member_eq(a: m, b: StoreMarkerMember)) { + RoleSingletonRefused { cause: RoleSingletonUnmarked { + deployment: r.deployment, + host: r.host, + marker: fabric_storage_ownership_marker_spec(instance: holder.instance), + } } + } else { + RoleSingletonRefused { cause: RoleSingletonPartial { deployment: r.deployment, host: r.host, missing: missing } } + } + } + } + } + } +} + +// โ”€โ”€ The admission: the decision bound to the spec that acts โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// THE APPLY CONSUMES THIS TOKEN, NEVER THE CALLER'S SPEC. sole_constructor, and its one mint +// (mint_role_singleton_admission) is sealed to assess_and_admit_rostered, reached only from assess_and_admit over a bound roster row. The spec inside is the one the assessment was made against, and it is admitted only when +// its own holding agrees with the desired realization: a spec built before the role moved, a holding +// spec under NoProdRole, or a non-holder spec while the holder's first install is pending all refuse +// before any member is planned. +type RoleSingletonAdmission sole_constructor { + spec: DeploymentSpec + decision: RoleSingletonConvergence +} + +type RoleSingletonAdmissionResult + = RoleSingletonAdmitted { admission: RoleSingletonAdmission } + | RoleSingletonNotAdmitted { cause: RoleSingletonRefusal } + +// THE PURE CLASSIFICATION a fixture may run: whether a spec WOULD be admitted for a decision. It +// returns a standing, never a usable token -- the only route to a RoleSingletonAdmission is the sealed +// mint below, reached only from the observing assessor. +type RoleSingletonAdmissionStanding + = RoleSingletonAdmissible { decision: RoleSingletonConvergence } + | RoleSingletonInadmissible { cause: RoleSingletonRefusal } + +// THE TOKEN'S READERS ARE SEALED TOO, to the apply steps that consume it inside one assess-and-act +// composition, so a token cannot be carried out of the apply it was minted for and replayed later. +fn admitted_spec(admission: RoleSingletonAdmission) -> DeploymentSpec admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_admitted_via_transport"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_mutation_via_transport"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "role_singleton_disposition_of"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_transaction_decided"), +] { + admission.spec +} + +fn admitted_decision(admission: RoleSingletonAdmission) -> RoleSingletonConvergence admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_admitted_via_transport"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_mutation_via_transport"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "role_singleton_disposition_of"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_transaction_decided"), +] { + admission.decision +} + +fn stale(spec: DeploymentSpec, detail: String) -> RoleSingletonAdmissionStanding { + RoleSingletonInadmissible { cause: RoleSingletonStaleSpec { instance_id: spec.instance_id, detail: detail } } +} + +fn classify_role_singleton_admission(spec: DeploymentSpec, desired: ProdRoleRealization, decision: RoleSingletonConvergence) -> RoleSingletonAdmissionStanding { + let spec_holder = match spec.role_singletons { + HoldsProdRoleSingletons { holder } => Present { value: holder.instance.deployment } + HoldsNoRoleSingleton => none + } + let desired_holder = match desired { + ProdRoleRealizedBy { holder } => Present { value: holder.instance } + ProdRoleRealizationRefused { cause: _ } => none + } + let desired_is_this_spec = match desired_holder { Present { value: i } => (i.instance_id as String) == (spec.instance_id as String) Absent => false } + let agrees = match spec_holder { + Present { value: d } => match desired_holder { Present { value: i } => i.deployment == d Absent => false } + Absent => !desired_is_this_spec + } + let admissible = RoleSingletonAdmissible { decision: decision } + let stale_holding = stale(spec: spec, detail: "the spec's role-singleton holding disagrees with the prod role resolved now; it was built against a different selection") + match decision { + RoleSingletonRefused { cause } => RoleSingletonInadmissible { cause: cause } + RoleSingletonNoop => if agrees { admissible } else { stale_holding } + RoleSingletonInstall { holder: _ } => + if !agrees { stale_holding } + else if desired_is_this_spec { admissible } + else { stale(spec: spec, detail: "the holder's first install is pending; no other spec applies until it lands") } + } +} + +// THE ONLY MINT, sealed to the observing assessor: no caller can hand it a fabricated decision or a +// noncanonical desired realization, because no other caller may reach it. +fn mint_role_singleton_admission(spec: DeploymentSpec, decision: RoleSingletonConvergence) -> RoleSingletonAdmission admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.role_singleton_convergence", decl_name: "assess_and_admit_rostered"), +] { + RoleSingletonAdmission { spec: spec, decision: decision } +} + +fn role_singleton_refusal_detail(cause: RoleSingletonRefusal) -> String { + match cause { + RoleSingletonMoveUnimplemented { observed, desired } => + join(["role singletons refused: the prod role moved from ", observed.name as String, " to ", desired.name as String, " while the old singletons are still observed; moving their persistent state is not implemented"], "") + RoleSingletonRemovalUnimplemented { observed } => + join(["role singletons refused: no deployment holds the prod role but ", observed.name as String, "'s singletons are still observed; removal is not implemented"], "") + RoleSingletonDesiredRealizationRefused { cause: c } => + join(["role singletons refused: the desired realization is not a holder and not NoProdRole: ", prod_role_realization_refusal_detail(cause: c)], "") + RoleSingletonHostUnobservable { host, cause: c } => + join(["role singletons refused: host ", host as String, " could not be observed (", c, "); unobserved is not absent"], "") + RoleSingletonHoldersAmbiguous { observed } => + join(["role singletons refused: ", to_string(count(observed)), " deployments are observed holding them: ", join(map(observed, d => d.name as String), ", ")], "") + RoleSingletonRealizedOnTwoHosts { deployment, hosts } => + join(["role singletons refused: ", deployment.name as String, " is realized on ", join(map(hosts, h => h as String), ", ")], "") + RoleSingletonMisplaced { deployment, target, observed_on } => + join(["role singletons refused: ", deployment.name as String, " targets ", target as String, " but is realized on ", observed_on as String], "") + RoleSingletonTwoOnOneHost { host, deployments } => + join(["role singletons refused: ", host as String, " carries the singletons of ", join(map(deployments, d => d.name as String), ", ")], "") + RoleSingletonBrokerWithoutStore { host } => + join(["role singletons refused: the approval broker is present on ", host as String, " but no store is realized there"], "") + RoleSingletonPartial { deployment, host, missing } => + join(["role singletons refused: ", deployment.name as String, " on ", host as String, " is partial (missing ", to_string(count(missing)), " member(s)); an interrupted install or a store without its broker is not adopted"], "") + RoleSingletonUnmarked { deployment, host, marker } => + join(["role singletons refused: ", deployment.name as String, " on ", host as String, " has its broker and fabric store unit but no store ownership marker, so whether that store is this holder's cannot be established. The operator's one-time disposition: on ", host as String, ", in the store's control directory (created first with the mode, owner and group gunbc.fabric_storage_placement fabric_storage_control_directory derives), write ", marker.path as String, " with mode ", file_mode_octal(mode: marker.mode), ", owner ", marker.owner as String, ", group ", marker.group as String, " and exactly this content: ", marker.content as String], "") + RoleSingletonMarkerMismatched { deployment, host, field, expected, observed } => + join(["role singletons refused: the store ownership marker for ", deployment.name as String, " on ", host as String, " does not match its row: ", field as String, " is '", observed, "', expected '", expected, "'; a marker that differs in any field is not the marker"], "") + RoleSingletonStaleSpec { instance_id, detail } => + join(["role singletons refused: spec ", instance_id as String, " is not admitted: ", detail], "") + RoleSingletonDeploymentUnrostered { instance_id, detail } => + join(["role singletons refused: deployment ", instance_id, " does not bind to its desired_deployment_roster row: ", detail], "") + } +} + +// โ”€โ”€ Reading one host โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// EVERY roster deployment's members are probed on the host read -- probing only where a deployment is +// targeted is what made a realization on the wrong host invisible. A host whose legs refuse is +// unobservable and stops the line; absence is only ever a successful listing. +type MemberProbe { + deployment: DeploymentId? + member: RoleSingletonMember + presence: EntryPresence +} + +fn presence_seen(p: EntryPresence) -> Bool { + match p { + EntryPresent => true + EntryAbsent => false + EntryPresenceIndeterminate { cause: _ } => false + } +} + +fn presence_cause(p: EntryPresence) -> String? { + match p { + EntryPresenceIndeterminate { cause } => Present { value: cause } + EntryPresent => none + EntryAbsent => none + } +} + +// THE MARKER IS READ BACK, NOT ONLY SEEN: its bytes, its mode, its owner and its group, each through +// the same observation arm, against the one marker row (gunbc.fabric_storage_placement +// fabric_storage_ownership_marker_spec). Only an exact match is the marker; any differing field is a +// distinct sighting the adjudication refuses, and a field that cannot be read makes the host +// unobservable rather than reading as either. +// WHICH STAT FIELD A LEG READS, as a closed choice: each arm names its own shell.Stat operation and its +// own format, so there is no string to mistype and no arm that a wrong value falls through to. +type MarkerStatField = MarkerStatMode | MarkerStatOwner | MarkerStatGroup + +fn marker_stat_format(field: MarkerStatField) -> String { + match field { + MarkerStatMode => "%a" + MarkerStatOwner => "%U" + MarkerStatGroup => "%G" + } +} + +fn stat_leg(arm: ObservationArm, path: String, field: MarkerStatField, leg: String) -> Leg { + match arm { + OnTheHostItself => { + let r = match field { + MarkerStatMode => shell.Stat.ModeOf(path: path as FilePath) + MarkerStatOwner => shell.Stat.OwnerOf(path: path as FilePath) + MarkerStatGroup => shell.Stat.GroupOf(path: path as FilePath) + } + if r.success { LegOut { stdout: r.stdout } } else { LegRefused { cause: join([leg, " failed: ", r.stderr], "") } } + } + OverTransport { transport } => remote_leg(transport: transport, argv: ["stat", "-c", marker_stat_format(field: field), "--", path], leg: leg) + } +} + +type MarkerRead + = MarkerReadBack { reading: FabricStorageMarkerReading } + | MarkerReadRefused { cause: String } + +fn read_marker(arm: ObservationArm, path: NonEmptyStr) -> MarkerRead { + match unit_document_leg(arm: arm, path: path as String) { + LegRefused { cause } => MarkerReadRefused { cause: cause } + LegOut { stdout: content } => match stat_leg(arm: arm, path: path as String, field: MarkerStatMode, leg: "reading the marker's mode") { + LegRefused { cause } => MarkerReadRefused { cause: cause } + LegOut { stdout: mode } => match stat_leg(arm: arm, path: path as String, field: MarkerStatOwner, leg: "reading the marker's owner") { + LegRefused { cause } => MarkerReadRefused { cause: cause } + LegOut { stdout: owner } => match stat_leg(arm: arm, path: path as String, field: MarkerStatGroup, leg: "reading the marker's group") { + LegRefused { cause } => MarkerReadRefused { cause: cause } + LegOut { stdout: group } => MarkerReadBack { reading: FabricStorageMarkerReading { path: path, content: content, mode: trim(s: mode), owner: owner, group: group } } + } + } + } + } +} + +type MarkerSighting + = MarkerSightingAbsent + | MarkerSightingOf { sighting: RoleSingletonSighting } + | MarkerSightingUnobservable { cause: String } + +fn marker_sighting(host: NonEmptyStr, r: DesiredDeployment, arm: ObservationArm) -> MarkerSighting { + let spec = fabric_storage_ownership_marker_spec(instance: r.instance) + marker_sighting_of( + host: host, + r: r, + presence: entry_presence(arm: arm, path: spec.path as String), + read_back: path => read_marker(arm: arm, path: path), + ) +} + +// THE THREE OUTCOMES, decided from what the observer's own legs returned: the marker exactly as its row +// (a sighting), no marker (a successful listing of the control directory without it), or a marker the +// observer cannot read (a listing or readback that failed -- permission denied included), which makes +// the host unobservable and is never read as absent. +fn marker_sighting_of(host: NonEmptyStr, r: DesiredDeployment, presence: EntryPresence, read_back: fn(NonEmptyStr) -> MarkerRead) -> MarkerSighting { + let spec = fabric_storage_ownership_marker_spec(instance: r.instance) + match presence { + EntryPresenceIndeterminate { cause } => MarkerSightingUnobservable { cause: cause } + EntryAbsent => MarkerSightingAbsent + EntryPresent => match read_back(spec.path) { + MarkerReadRefused { cause } => MarkerSightingUnobservable { cause: cause } + MarkerReadBack { reading } => match fabric_storage_marker_verdict(spec: spec, reading: reading) { + FabricStorageMarkerExact => MarkerSightingOf { sighting: SightedDeploymentMember { host: host, deployment: r.instance.deployment, member: StoreMarkerMember } } + FabricStorageMarkerMismatched { field, expected, observed } => MarkerSightingOf { sighting: SightedMismatchedMarker { host: host, deployment: r.instance.deployment, field: field, expected: expected, observed: observed } } + } + } + } +} + +fn host_role_singleton_reading(host: NonEmptyStr, rows: List, arm: ObservationArm) -> HostRoleSingletonReading { + let probes = concat( + [MemberProbe { deployment: none, member: BrokerUnitMember, presence: entry_presence(arm: arm, path: gunbc_approval_broker_unit_path() as String) }], + rows |> map(r => MemberProbe { deployment: Present { value: r.instance.deployment }, member: FabricUnitMember, presence: entry_presence(arm: arm, path: deployment_fabric_storage_unit_path(names: r.names) as String) }), + ) + let markers = rows |> map(r => marker_sighting(host: host, r: r, arm: arm)) + let causes = concat( + probes |> flat_map(p => match presence_cause(p: p.presence) { Present { value: c } => [c] Absent => [] }), + markers |> flat_map(m => match m { MarkerSightingUnobservable { cause } => [cause] MarkerSightingAbsent => [] MarkerSightingOf { sighting: _ } => [] }), + ) + match causes |> first { + Present { value: cause } => HostRoleSingletonsUnobservable { host: host, cause: cause } + Absent => HostRoleSingletonsRead { + host: host, + sightings: concat( + probes |> filter(p => presence_seen(p: p.presence)) |> map(p => match p.deployment { + Present { value: d } => SightedDeploymentMember { host: host, deployment: d, member: p.member } + Absent => SightedBroker { host: host } + }), + markers |> flat_map(m => match m { MarkerSightingOf { sighting } => [sighting] MarkerSightingAbsent => [] MarkerSightingUnobservable { cause: _ } => [] }), + ), + } + } +} + +fn roster_hosts(rows: List) -> List { + distinct_hosts(hosts: rows |> map(r => r.target.ssh_host)) +} + +// THE D2 INTERIM SOURCE FAILS CLOSED BY CONSTRUCTION. The applying host is read over the apply's own +// route -- the route the apply already holds to its target, not a cross-host grant. Every other roster +// host is HostRoleSingletonsUnobservable without any transport being built, so no credential, granted +// or not, can make a nonlocal host observable here: this module names no ssh transport at all. gunbc#13339 +// replaces this source with host self-report. +data nonlocal_host_unobservable_cause: String = "self-report has not landed (gunbc#13339): a roster host other than the applying host is not observed by the applying host" + +fn local_only_reading_source(applying: DeploymentSpec, own: HostEffectTransport) -> RoleSingletonReadingSource { + RoleSingletonReadingSource { + read: (host, rows) => + if (host as String) == (applying.target.ssh_host as String) { + host_role_singleton_reading(host: host, rows: rows, arm: observation_arm(transport: own)) + } else { + HostRoleSingletonsUnobservable { host: host, cause: nonlocal_host_unobservable_cause } + }, + } +} + +fn observe_role_singletons_with(source: RoleSingletonReadingSource, rows: List) -> RoleSingletonObservation { + let read = source.read + role_singleton_observation_of(readings: map(roster_hosts(rows: rows), h => read(h, rows)), rows: rows) +} + +// THE CANONICAL ASSESSOR, AND THE ONLY ROUTE TO A TOKEN. It reads the desired realization itself +// (gunbc.live_deploy.desired prod_role_actuated_realization -- the real selection, never a caller's), +// observes every roster host through the ssh probe, decides, classifies the spec, and only then mints. +fn assess_and_admit(deployment: DesiredDeployment, own: HostEffectTransport) -> RoleSingletonAdmissionResult admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_apply_via_transport"), + decl_ref(module_path: "gunbc.live_deploy.apply", decl_name: "live_deploy_transaction_decided"), +] { + match bind_desired_selected_identity(desired: deployment) { + DesiredSelectedIdentityUnknownInstance { instance_id } => + RoleSingletonNotAdmitted { cause: RoleSingletonDeploymentUnrostered { instance_id: instance_id, detail: "no roster row carries this instance id" } } + DesiredSelectedIdentityRosterRefused { refusal: _ } => + RoleSingletonNotAdmitted { cause: RoleSingletonDeploymentUnrostered { instance_id: instance_id_text(id: deployment.instance.instance_id), detail: "the desired roster itself refused" } } + DesiredSelectedIdentityMismatch { carried: _, rostered: _ } => + RoleSingletonNotAdmitted { cause: RoleSingletonDeploymentUnrostered { instance_id: instance_id_text(id: deployment.instance.instance_id), detail: "the carried selected identity is not the rostered one" } } + DesiredSelectedIdentityBound { row } => assess_and_admit_rostered(spec: selected_deployment_spec(row: row), own: own) + } +} + +// THE SPEC IS BUILT, NEVER SUPPLIED: the only caller is assess_and_admit above, which derives it from +// the bound roster row, so an altered DeploymentSpec has no route to the observation or the mint. +fn assess_and_admit_rostered(spec: DeploymentSpec, own: HostEffectTransport) -> RoleSingletonAdmissionResult admit_callers: [ + decl_ref(module_path: "gunbc.live_deploy.role_singleton_convergence", decl_name: "assess_and_admit"), +] { + let rows = desired_deployment_roster() + let desired = prod_role_actuated_realization() + let decision = role_singleton_convergence( + desired: desired, + observed: observe_role_singletons_with(source: local_only_reading_source(applying: spec, own: own), rows: rows), + rows: rows, + ) + match classify_role_singleton_admission(spec: spec, desired: desired, decision: decision) { + RoleSingletonInadmissible { cause } => RoleSingletonNotAdmitted { cause: cause } + RoleSingletonAdmissible { decision: d } => RoleSingletonAdmitted { admission: mint_role_singleton_admission(spec: spec, decision: d) } + } +} diff --git a/dag/gunbc/live_deploy/spec.dag b/dag/gunbc/live_deploy/spec.dag index 1bc03c1b494..b082851527b 100644 --- a/dag/gunbc/live_deploy/spec.dag +++ b/dag/gunbc/live_deploy/spec.dag @@ -42,10 +42,17 @@ import gunbc.host_layout { srv1_dispatch_attempt_state_root, srv1_devboot_artifact_store_root, } -import gunbc.fabric_storage_placement { fabric_storage_store_directories, fabric_storage_placed_on, fabric_storage_https_port, fabric_storage_door_socket } +import gunbc.fabric_storage_placement { fabric_storage_store_directories_for, FabricStorageOwnershipMarkerSpec, fabric_storage_ownership_marker_spec, fabric_storage_https_port, fabric_storage_door_socket } import gunbc.managed_directory { ManagedDirectoryAdmission, managed_directory_admission_path } import gunbc.roadmap_dashboard_instance { HostDashboardInstance, + DashboardInstanceForId, + DashboardInstanceIdUnknown, + dashboard_instance_for_id, + ProdRoleRealization, + ProdRoleHolder, + ProdRoleRealizedBy, + ProdRoleRealizationRefused, dashboard_instance_dispatch_worktree_root, dashboard_instance_dispatch_attempt_state_root, dashboard_instance_compute_root, @@ -116,6 +123,7 @@ type EnsuredDependencyKind = | DevbootArtifactStoreDirectory | CodexRuntimeReleaseDirectory | FabricStorageStoreDirectory + | FabricStorageOwnershipMarker | ProviderStateRootDirectory // ONE CONCEPT -- AN ENSURED HOST RESOURCE -- WITH N KINDS, not a second ensured mechanism beside @@ -141,12 +149,14 @@ type EnsuredSubject = EnsuredPackage { package: NonEmptyStr } | EnsuredHostDirectory { path: NonEmptyStr, owner: NonEmptyStr } | EnsuredManagedHostDirectory { admission: ManagedDirectoryAdmission } + | EnsuredOwnershipMarker { marker: FabricStorageOwnershipMarkerSpec } fn ensured_subject_identity(subject: EnsuredSubject) -> NonEmptyStr { match subject { EnsuredPackage { package } => package EnsuredHostDirectory { path, owner } => path EnsuredManagedHostDirectory { admission } => managed_directory_admission_path(a: admission) as String as NonEmptyStr + EnsuredOwnershipMarker { marker } => marker.path } } @@ -414,20 +424,15 @@ data approval_broker_serve_entry_file: NonEmptyStr = "dag/gunbc/auth/approval_br // // SO OWNERSHIP IS DECLARED AND THE EMITTER READS IT. One instance per host owns the broker unit; // every other spec emits no broker step at all, writes nothing and restarts nothing. -// THE INSTANCE THAT OWNS THE HOST'S BROKER. It is srv1-live because that is where the approval -// store is: gunbc.auth.approval_decision_store approval_decision_store_root is one path on one -// host, and the whole point of a single-writer store is that exactly one process opens it. +// THE INSTANCE THAT OWNS THE HOST'S BROKER IS THE REALIZATION OF THE PROD-ROLE HOLDER +// (gunbc.deployment_risk prod_role_selection, resolved by gunbc.roadmap_dashboard_instance +// prod_role_realization). The approval store is the prod deployment's state -- one single-writer +// path that exactly one process opens -- so the broker follows the role, not a host or an +// instance_id literal. Moving prod is an edit to the selection row and to nothing here. // -// IT IS A DECLARED OWNER RATHER THAN A DERIVATION FROM THE INSTANCE because "which deployment runs -// the approval loop" is not recoverable from any field an instance carries -- srv1-live and -// srv1-lab are the same shape on the same host, and the only thing that distinguishes them for this -// purpose is that one of them was chosen. Deriving it would mean inventing a rule that happens to -// select production today. -// -// A SECOND HOST SERVING APPROVALS IS WHAT RETIRES THIS ROW, at which point the owner becomes a -// per-host lookup and the store root becomes host-derived with it. Until then a single row is the -// honest shape: there is one approval loop, and it has one home. -data gunbc_approval_broker_owning_instance: NonEmptyStr = "srv1-live" +// UNDER NoProdRole NOBODY OWNS IT. The realization refuses with a typed cause and every spec emits +// no broker step; there is no fallback to srv1. A second host serving approvals still retires the +// single-owner shape, at which point ownership becomes per-host. data gunbc_approval_broker_unit_name: NonEmptyStr = "gunbc-approval-broker.service" @@ -446,18 +451,33 @@ fn gunbc_repository_convergence_slice_unit_path() -> NonEmptyStr { join(["/etc/systemd/system/", gunbc_repository_convergence_slice_unit_name as String], "") as NonEmptyStr } -// A SPEC THAT DOES NOT OWN THE BROKER EMITS NO BROKER STEP. This is the predicate the emitter -// consults; it is not a validation standing beside a write that would otherwise happen, because the -// steps are CONSTRUCTED from it -- a non-owning apply produces an empty list, so there is nothing to -// skip and nothing to get wrong. -fn deployment_owns_the_host_approval_broker(spec: DeploymentSpec) -> Bool { - instance_owns_the_host_approval_broker(instance_id: spec.instance_id) +// THE ROLE-FOLLOWING SINGLETONS A SPEC HOLDS -- the approval broker unit and front door, the fabric +// storage unit, route, socket, root and directories -- as ONE subject, carried on the spec as a +// holder token (gunbc.roadmap_dashboard_instance ProdRoleHolder, sole_constructor: only a successful +// resolution of prod_role_selection mints one). Every singleton effect is BUILT FROM the token's +// instance, never from a caller-supplied instance_id, so no spec can take ownership by spelling one. +// Which spec carries the token is decided by the role model's own identity (DeploymentId, a +// structural record), and under every refused realization no spec holds anything. +type RoleSingletonHolding + = HoldsProdRoleSingletons { holder: ProdRoleHolder } + | HoldsNoRoleSingleton + +fn role_singleton_holding_under(realization: ProdRoleRealization, instance: HostDashboardInstance) -> RoleSingletonHolding { + match realization { + ProdRoleRealizedBy { holder } => + if holder.instance.deployment == instance.deployment { HoldsProdRoleSingletons { holder: holder } } else { HoldsNoRoleSingleton } + ProdRoleRealizationRefused { cause: _ } => HoldsNoRoleSingleton + } } -// THE ONE OWNERSHIP COMPARISON, read by the spec-level predicate above and by the front-door member, -// which is built before a spec exists and so holds only the instance. -fn instance_owns_the_host_approval_broker(instance_id: NonEmptyStr) -> Bool { - (instance_id as String) == (gunbc_approval_broker_owning_instance as String) +// A SPEC THAT DOES NOT HOLD THE ROLE EMITS NO BROKER STEP. This is the predicate the emitter +// consults; the steps are CONSTRUCTED from it -- a non-holding apply produces an empty list, so there +// is nothing to skip and nothing to get wrong. +fn deployment_owns_the_host_approval_broker(spec: DeploymentSpec) -> Bool { + match spec.role_singletons { + HoldsProdRoleSingletons { holder: _ } => true + HoldsNoRoleSingleton => false + } } fn gunbc_approval_broker_unit_path() -> NonEmptyStr { @@ -638,7 +658,7 @@ fn deployment_belt_unit_stem(names: DeploymentNames) -> String { } // THE FABRIC DB ENDPOINT'S UNIT, derived like the belt's: one per deployment identity, and emitted -// only by the deployment whose instance holds the fabric DB placement (fabric_storage_placed_on). It is +// only by the deployment whose instance holds the fabric DB placement (RoleSingletonHolding). It is // deployment state -- a retract removes the unit and the tailscale mapping -- while the store's files // are an ensured host directory no retract touches (FabricStorageStoreDirectory). fn deployment_fabric_storage_unit_name(names: DeploymentNames) -> NonEmptyStr { @@ -649,11 +669,20 @@ fn deployment_fabric_storage_unit_path(names: DeploymentNames) -> NonEmptyStr { join(["/etc/systemd/system/", deployment_fabric_storage_unit_name(names: names) as String], "") as NonEmptyStr } -fn deployment_fabric_storage_serve_endpoint() -> TailscaleServeEndpoint { +// THE DOOR SOCKET OF THE HOLDER THIS SPEC CARRIES, built from the token's instance like every other +// role singleton. A spec holding no role has no socket, and the emitter refuses rather than invent one. +fn deployment_fabric_storage_door_socket(spec: DeploymentSpec) -> NonEmptyStr? { + match spec.role_singletons { + HoldsProdRoleSingletons { holder } => Present { value: fabric_storage_door_socket(instance: holder.instance) } + HoldsNoRoleSingleton => none + } +} + +fn deployment_fabric_storage_serve_endpoint(door_socket: NonEmptyStr) -> TailscaleServeEndpoint { TailscaleServeEndpoint { https_port: fabric_storage_https_port, mount: ServeRootMount, - backend: tailscale_serve_unix_target(path: fabric_storage_door_socket()), + backend: tailscale_serve_unix_target(path: door_socket), } } @@ -982,6 +1011,7 @@ type DeploymentSpec { approval_broker: DeploymentServiceConfig names: DeploymentNames instance_id: NonEmptyStr + role_singletons: RoleSingletonHolding } // TWO CONSUMERS, ONE SCOPE-FREE ROUTE LABEL (KEYING-0 follow-on, node://adhoc-907f1da5-c20): @@ -1049,6 +1079,7 @@ type DeploymentSpec { fn deployment_owned_steps( instance: HostDashboardInstance, names: DeploymentNames, + holding: RoleSingletonHolding, ) -> List { concat([ DeploymentArtifactStep { kind: ServeBinary, path: instance.serve_binary as String as NonEmptyStr }, @@ -1072,8 +1103,8 @@ fn deployment_owned_steps( path: tailscale_serve_route_member_path(endpoint: names.tailscale_serve_endpoint), }, ], concat( - deployment_fabric_storage_steps(instance: instance, names: names), - deployment_approval_broker_front_door_steps(instance: instance), + deployment_fabric_storage_steps(holding: holding, names: names), + deployment_approval_broker_front_door_steps(holding: holding), )))) } @@ -1094,13 +1125,14 @@ fn deployment_owned_steps( // One member, two mappings: its identity is the /approve mount's route key and its apply and // teardown realize every endpoint approval_broker_front_door_endpoints returns, the way // GunbcSourceTree's one member owns its sync unit and env file. It is emitted only by the instance -// that owns the host's broker (instance_owns_the_host_approval_broker). +// that owns the host's broker (RoleSingletonHolding). fn deployment_approval_broker_front_door_endpoints() -> List { approval_broker_front_door_endpoints(https_port: tailscale_default_https_port, backend_port: approval_broker_listen_port) } -fn deployment_approval_broker_front_door_steps(instance: HostDashboardInstance) -> List { - if instance_owns_the_host_approval_broker(instance_id: instance.instance_id) && approval_front_door_mounts_installed() { +fn deployment_approval_broker_front_door_steps(holding: RoleSingletonHolding) -> List { + let holds = match holding { HoldsProdRoleSingletons { holder: _ } => true HoldsNoRoleSingleton => false } + if holds && approval_front_door_mounts_installed() { [ DeploymentArtifactStep { kind: ApprovalBrokerFrontDoor, @@ -1116,14 +1148,31 @@ fn deployment_approval_broker_front_door_steps(instance: HostDashboardInstance) // references the binary and tree installed above) and then its tailscale route (routes to a live // service, the same reason the dashboard's mapping is last). Empty on every instance that does not // hold the placement. -fn deployment_fabric_storage_steps(instance: HostDashboardInstance, names: DeploymentNames) -> List { - if fabric_storage_placed_on(instance: instance) { - [ +fn deployment_fabric_storage_steps(holding: RoleSingletonHolding, names: DeploymentNames) -> List { + match holding { + HoldsProdRoleSingletons { holder } => [ DeploymentArtifactStep { kind: FabricStorageServeUnit, path: deployment_fabric_storage_unit_path(names: names) }, - DeploymentArtifactStep { kind: FabricStorageServeMapping, path: tailscale_serve_route_member_path(endpoint: deployment_fabric_storage_serve_endpoint()) }, + DeploymentArtifactStep { kind: FabricStorageServeMapping, path: tailscale_serve_route_member_path(endpoint: deployment_fabric_storage_serve_endpoint(door_socket: fabric_storage_door_socket(instance: holder.instance))) }, ] - } else { - [] + HoldsNoRoleSingleton => [] + } +} + +// THE FABRIC STORE'S DIRECTORIES TRAVEL WITH ITS UNIT AND ROUTE. They used to be a member of the srv1 +// target's ensured directories, so moving the role moved the unit to srv2 and left the directories +// on srv1. Derived from the same holding, they land on exactly the target that gets the unit. +fn deployment_role_singleton_directories(holding: RoleSingletonHolding) -> List { + match holding { + HoldsProdRoleSingletons { holder } => concat(map(fabric_storage_store_directories_for(instance: holder.instance), a => DeploymentDependencyStep { + kind: FabricStorageStoreDirectory, + subject: EnsuredManagedHostDirectory { admission: a }, + }), [ + DeploymentDependencyStep { + kind: FabricStorageOwnershipMarker, + subject: EnsuredOwnershipMarker { marker: fabric_storage_ownership_marker_spec(instance: holder.instance) }, + }, + ]) + HoldsNoRoleSingleton => [] } } @@ -1172,7 +1221,7 @@ fn deployment_ensured_packages() -> List { // // AND AN AREA THE WRITER CANNOT WRITE IS NOT A STORE EITHER. The three directories are managed: // their owner, group and mode come from the writers gunbc.fabric_storage_placement -// fabric_storage_store_directories declares (the service principal and the CI job principal), not +// fabric_storage_store_directories_for declares (the service principal and the CI job principal), not // from gunbc_service_user at the shared 0755 -- which is what refused fleet-converge run // 36139624393's claim permission_denied. install -d re-applies owner, group and mode to a directory // that already exists, so the next apply repairs the areas #12126 created. @@ -1185,10 +1234,7 @@ fn deployment_ensured_srv1_host_directories() -> List owner: gunbc_service_user, }, }, - ], concat(map(fabric_storage_store_directories(), a => DeploymentDependencyStep { - kind: FabricStorageStoreDirectory, - subject: EnsuredManagedHostDirectory { admission: a }, - }), [ + ], [ DeploymentDependencyStep { kind: CodexRuntimeReleaseDirectory, subject: EnsuredHostDirectory { @@ -1196,7 +1242,7 @@ fn deployment_ensured_srv1_host_directories() -> List owner: gunbc_service_user, }, }, - ])) + ]) } fn deployment_ensured_steps(target: DeploymentHostTarget) -> List { @@ -1285,11 +1331,12 @@ fn deployment_steps_apply_order( instance: HostDashboardInstance, target: DeploymentHostTarget, names: DeploymentNames, + holding: RoleSingletonHolding, ) -> List { concat( - map(deployment_ensured_steps(target: target), s => Dependency { step: s }), + map(concat(deployment_ensured_steps(target: target), deployment_role_singleton_directories(holding: holding)), s => Dependency { step: s }), map( - deployment_owned_steps(instance: instance, names: names), + deployment_owned_steps(instance: instance, names: names, holding: holding), s => ArtifactStep { step: s }, ) ) @@ -1395,14 +1442,21 @@ fn deployment_service_config( // two machines shares instance and identity and differs in target. Neither case needs a new // function, which is the property that was missing. + +// THE REALIZATION IS SUPPLIED, never resolved here: the spec cannot see the desired roster that decides +// whether the holder is actuatable, so gunbc.live_deploy.desired resolves it +// (prod_role_actuated_realization) and every caller passes it in. A witness passes a moved one. fn deployment_spec_for( + realization: ProdRoleRealization, instance: HostDashboardInstance, target: DeploymentHostTarget, names: DeploymentNames, ) -> DeploymentSpec { + let holding = role_singleton_holding_under(realization: realization, instance: instance) DeploymentSpec { target: target, - steps: deployment_steps_apply_order(instance: instance, target: target, names: names), + steps: deployment_steps_apply_order(instance: instance, target: target, names: names, holding: holding), + role_singletons: holding, service: deployment_service_config( instance: instance, names: names, diff --git a/dag/gunbc/managed_directory.dag b/dag/gunbc/managed_directory.dag index 0e0738d9efb..7b49481e0dd 100644 --- a/dag/gunbc/managed_directory.dag +++ b/dag/gunbc/managed_directory.dag @@ -217,6 +217,18 @@ fn managed_directory_permissions(d: ManagedDirectory) -> FilePermissions { } } +// WHETHER A DIRECTORY'S DERIVED MODE LETS A PRINCIPAL LIST AND TRAVERSE IT, by the class that principal +// falls in. The fabric store's marker witness asks it of the store's control directory and of its store root with the real observer. +fn managed_directory_admits_reader(d: ManagedDirectory, who: PosixUser) -> Bool { + let mode = managed_directory_permissions(d: d).mode + let bits = match dependent_class_of(owner: d.owner, group_gid: d.group_principal.gid, who: who) { + OwnerClass => mode.owner + GroupClass => mode.group + OtherClass => mode.other + } + bits.read && bits.execute +} + // โ”€โ”€ THE ENTRIES' MODE, DERIVED FROM THE SAME DEPENDENTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // // An entry is owned by the dependent that CREATED it, not by the directory's owner, and its group is diff --git a/dag/gunbc/recurring_failure_mode/a_test_deployment_acts_on_the_prod_deployments_state.dag b/dag/gunbc/recurring_failure_mode/a_test_deployment_acts_on_the_prod_deployments_state.dag index 6a0a44dc22d..0c4a81cc7d8 100644 --- a/dag/gunbc/recurring_failure_mode/a_test_deployment_acts_on_the_prod_deployments_state.dag +++ b/dag/gunbc/recurring_failure_mode/a_test_deployment_acts_on_the_prod_deployments_state.dag @@ -22,11 +22,20 @@ data a_test_deployment_acts_on_the_prod_deployments_state: RecurringFailureMode "CLIMB STARTED BY THE MODEL PR: the class is derived, the bindings resolve per exact class, and test.claim.deployment_risk_witness_test carries the discriminating RED (a TestRisk deployment refuses rather than resolving the prod topic and app) beside its positive control. That covers the two attachments the binding set carries today and none of the four receipts above, so the row's rung is unchanged by it.", - "NEXT-RUNG TRIGGER: every pin that names the prod deployment directly (the approval broker owner, fabric storage placement, the approval client/store/trust, the serve and dispatch entry points that call srv1_live_dashboard_instance, the apply's prod peer, fleet_reach_endpoint) resolves through prod_role_selection, AND every attachment the srv1 lab shares with prod today (ntfy, Codex account and home, admitted ref, publication identity, R2/GCP/workload identity, OIDC) resolves through DeploymentExternalBindingSet or refuses. One pin or one binding is not sufficient: the class is the route, and any remaining named route is the next receipt.", + "PINS REPOINTED BY DELIVERABLE 2: the role-following singletons follow gunbc.roadmap_dashboard_instance prod_role_realization -- one lookup, the instance roster filtered by its own deployment field -- which REFUSES with a typed ProdRoleRealizationRefusal under NoProdRole, an unrealized holder or an ambiguous one, and mints the only holder token (ProdRoleHolder, sole_constructor). A spec carries that token as gunbc.live_deploy.spec RoleSingletonHolding and every singleton member -- the approval broker unit and front door, the fabric store unit, route, socket, root and directories -- is built from it, never from a caller-spelled instance_id, so they move between targets as one subject. The apply first assesses the desired realization against an observation of every roster host (gunbc.live_deploy.role_singleton_convergence): Noop, Install only when nothing is observed anywhere, and a typed refusal for a move, a removal, an unobservable host, two holders or an ownerless broker. The approval client origin, the GCP IAM converge job's runner (gunbc.fleet_converge_workflow fleet_converge_gcp_iam_converge_job_under, refused under NoProdRole), the approvals app's host (gunbc.auth.approval_broker_placement) and the apply's protected siblings (dashboard_protected_siblings) follow the same realization; broker-side modules receive their host at serve time rather than resolving the role. The REDs are in test.claim.deployment_risk_witness_test at the spec, convergence and emitted-job boundaries.", + + "OPERATOR DISPOSITION OWED, ONE-TIME: today's prod-role holder (srv1-daily-workspace, realized on srv1) predates the fabric store ownership marker, so the role-singleton assessment refuses it as RoleSingletonUnmarked until the operator creates, on srv1, the control directory /opt/gunbc/fabric-storage-control (mode 2750, owner briansrls, group ghrunner, as gunbc.fabric_storage_placement fabric_storage_control_directory derives) and writes in it /opt/gunbc/fabric-storage-control/.gunbc-dashboard-instance-srv1-live (mode 0644, owner briansrls) containing exactly 'deployment=srv1-daily-workspace' newline 'host=srv1' newline -- the content gunbc.fabric_storage_placement fabric_storage_ownership_marker_content derives and the refusal text quotes. No automated adoption exists (gunbc#13124: manual resume).", + + "HONEST NON-PINS, not remaining debt: (a) the nullary serve, belt, dispatch and launch entry points that call srv1_live_dashboard_instance are srv1-live's OWN entry points (HostDashboardInstance.serve_function names roadmap_serve_handle for srv1-live and roadmap_serve_handle_ for every other instance, a binding test.claim.serve_function_binding_witness_test now checks roster-wide), so they must NOT follow the role -- following it would make srv1-live act on the new holder's state; (b) gunbc.fleet_reach_endpoint fleet_probe_remote_user_for's srv1/srv2 test is which hosts carry the gunbc-fleet account, a host-shared fact. NOT CLAIMED: that an unbound test deployment refuses everywhere -- roadmap_belt_actuate belt_dispatch_instance_cli still defaults an absent instance env to srv1-live, which stays with the belt owner's lane.", + + "NEXT-RUNG TRIGGER: (1) the role-singleton convergence gains its Move and Remove actuations, so a role change converges the old singleton away instead of refusing; (2) every attachment the srv1 lab shares with prod today (ntfy, Codex account and home, admitted ref, publication identity, R2/GCP/workload identity, OIDC) resolves through DeploymentExternalBindingTable or refuses (deliverable 3). One binding is not sufficient: the class is the route, and any remaining named route is the next receipt.", ], evidence: [ DeclarationRef { module_path: "gunbc.deployment_risk", decl_name: "resolve_deployment_bindings", field: WholeDeclaration }, DeclarationRef { module_path: "test.claim.deployment_risk_witness_test", decl_name: "a_test_deployment_refuses_rather_than_resolving_the_prod_bindings", field: WholeDeclaration }, + DeclarationRef { module_path: "gunbc.roadmap_dashboard_instance", decl_name: "prod_role_realization_in", field: WholeDeclaration }, + DeclarationRef { module_path: "test.claim.deployment_risk_witness_test", decl_name: "moving_the_prod_role_moves_every_role_singleton_member_to_the_new_holders_spec", field: WholeDeclaration }, + DeclarationRef { module_path: "gunbc.live_deploy.role_singleton_convergence", decl_name: "role_singleton_convergence", field: WholeDeclaration }, ], } diff --git a/dag/gunbc/recurring_failure_mode/entry_scoped_typecheck_rss_tracks_the_name_census.dag b/dag/gunbc/recurring_failure_mode/entry_scoped_typecheck_rss_tracks_the_name_census.dag index d749dfc0acf..24c8103eeb2 100644 --- a/dag/gunbc/recurring_failure_mode/entry_scoped_typecheck_rss_tracks_the_name_census.dag +++ b/dag/gunbc/recurring_failure_mode/entry_scoped_typecheck_rss_tracks_the_name_census.dag @@ -17,6 +17,10 @@ data entry_scoped_typecheck_rss_tracks_the_name_census: RecurringFailureMode = R "BOUNDED AGAINST THREE NEIGHBOURS. It is NOT `absent_reads_identically_to_never_looked`: that class is that census-only modules are never judged; this class is that they are still paid in RSS. It is NOT a hub re-export of `required_floor` by floor2_prepared_subject โ€” that import is absent; the wire that failed is the hub, and splitting it would not drop an isolated `--entry` of floor2 below the census cost. It is NOT the 2026-08-17 per-claim CPU shrink-only ruling: memory admission is a different subject, and a raise still needs measured srv4 headroom beside this peak. Isolated 9.73 GiB already consumes most of the 16 GiB class; there is no srv4 RSS sample before this typecheck on which to claim headroom. Raising is therefore the pattern that ruling names, with the cost-shape unfixed.", + "RECEIPT, THE SAME CLASS INSIDE A CLAIM (deployment-risk D2, #13217, 2026-10-05): every `compile_dag_diagnostic_census` probe in test.claim.deployment_risk_witness_test peaked at 17.6-20.9 GiB VmHWM run alone under a 20 GiB cgroup -- the outside-warrant-mint probe PASSED at 17.6 GiB with 607 eval_steps, the warrant-literal probe at 18.6 GiB with 914, and five others were OOM-killed at the bind -- and the peak did not move with the fixture: a probe importing only gunbc.live_deploy.role_singleton_convergence cost what a probe importing the whole apply tree cost. The census rows name v2.compiler.* modules the fixture never reaches, so the probe pays the source roots' census, not its closure. The eval steps are the claim's subject and are trivial; the compile is the whole cost. Measured by the per-claim VmHWM sampling run in that PR; the class's trigger above is the remedy, and narrowing a fixture's imports is not.", + + "RECEIPT, THE PEAK MOVES WITH MAIN, NOT WITH THE CLAIM (deployment-risk D2, #13217, 2026-10-06): the same seven deployment_risk compile probes in one process peaked at 22.7 GiB on 25dc63765f and were OOM-killed at a 22 GiB bind at 23.0 GiB after main was merged (bfb2ee9164) with no change to the claims or their fixtures; a sibling lane (gunbc#13339) saw the same kill on its own merge. The cost tracks the source roots' census, which grows with every landing, so a claim's memory budget is not a fact of the claim.", + "RUNG FOUND AT: mitigatable. The stall refuses typed and located. The invalid cost-shape remains writable.", "ATTAINABLE CEILING: mechanically preventable. Whether typecheck RSS is charged to the closure or to the census is a fact of `compile_emission` / `tree_bare_census_for_root`; a check can refuse an `--entry` whose M/N ratio and peak disagree with the closure grain, or the construction can stop forcing the whole-root census on an Entry subject. It is not structurally impossible: indexing the pool for bare-name parity is a legitimate construction; the defect is charging its resident set to a 16-module typecheck as if it were that module's work.", diff --git a/dag/gunbc/recurring_failure_mode/modifier_accepted_on_a_shape_its_check_cannot_see.dag b/dag/gunbc/recurring_failure_mode/modifier_accepted_on_a_shape_its_check_cannot_see.dag index ec8ccd0319d..5face44b334 100644 --- a/dag/gunbc/recurring_failure_mode/modifier_accepted_on_a_shape_its_check_cannot_see.dag +++ b/dag/gunbc/recurring_failure_mode/modifier_accepted_on_a_shape_its_check_cannot_see.dag @@ -22,6 +22,8 @@ data modifier_accepted_on_a_shape_its_check_cannot_see: RecurringFailureMode = R "**THE NEIGHBOUR IS `prose_asserts_an_enforcement_no_operation_can_produce` and the boundary is the carrier:** there the false claim is in prose a machine cannot read, here it is in SYNTAX THE COMPILER ACCEPTED, which is strictly more convincing and therefore strictly more dangerous. Against `discriminating_arm_built_but_never_enrolled`: there a real check exists and nothing calls it; here the check is called, runs, and answers the wrong question because its key does not match the shape.", "**RUNG AND CEILING FOR THE CLASS AS A CLASS**, which is all this row may speak to: the general mismatch between a modifier's accepted shapes and its enforcing pass's covered shapes is decidable per modifier and undecidable as a blanket property, so it stays a per-modifier obligation discharged where the modifier is minted. The specimen's own rung, ceiling and trigger are the parse site's to state and are not restated here.)", + + "**A CONSUMER THAT PAYS FOR THE SPECIMEN, not a second specimen (deployment-risk D2).** gunbc.host_effect `LiveDeployApply { warrant: AdmittedLiveDeployApply }` is a coproduct variant, so a warrant extracted from a HostEffect could be re-wrapped into a fresh LiveDeployApply anywhere and no compile refusal is authorable for it. D2 therefore makes the warrant UNOBTAINABLE rather than un-rewrappable: it is minted and realized inside one sealed operation and never returned or stored in an observable value, and the outside call of that operation is the compile RED. The residual re-wrap is this specimen, owned by the parse-site declaration above `v1.compiler.parse` `alias_rhs_is_anonymous_record`; its trigger is that declaration's capability -- coproduct variant construction wired through sole_constructor_construction_diags -- and when it lands, AdmittedLiveDeployApply's variant is sealed and the re-wrap probe becomes enrollable.", ], evidence: [], diff --git a/dag/gunbc/roadmap/roadmap_dashboard_instance.dag b/dag/gunbc/roadmap/roadmap_dashboard_instance.dag index 7e72e070ab6..1dbde7f8a32 100644 --- a/dag/gunbc/roadmap/roadmap_dashboard_instance.dag +++ b/dag/gunbc/roadmap/roadmap_dashboard_instance.dag @@ -2,14 +2,17 @@ module gunbc.roadmap_dashboard_instance import extdeps.tools.env { env_absolute_program } import std.types { String, NonEmptyStr, Int, Bool, List, FilePath, Port, GitRef } -import gunbc.srv1_dashboard_bind { srv1_dashboard_host_identity, srv1_dashboard_listen_host } import std.measure { ByteSize } +import v2.std.integer { integer_int_to_decimal_string } import gunbc.runner_slot_desired { gunbc_runner_slot_desired } import extdeps.tmux { TmuxServerScope, DefaultTmuxServer, NamedTmuxServer } import gunbc.ensure { EnsureObserveOnly } import gunbc.deployment_risk { DeploymentId, DeploymentRiskClass, + ProdRoleSelection, + NoProdRole, + ProdRoleHeldBy, deployment_risk_class, prod_role_selection, srv1_daily_workspace_deployment, @@ -392,9 +395,17 @@ fn dashboard_instance_dispatch_platform_executables( } fn dashboard_instance_ownership_marker_path(instance: HostDashboardInstance) -> FilePath { + dashboard_ownership_marker_in(root: instance.instance_root, instance_id: instance.instance_id) +} + +// THE OWNERSHIP MARKER'S ONE NAMING RULE: an empty file under a root, named for the instance that +// owns the root. The instance root carries it (above), and so does the prod-role holder's fabric +// store control directory (gunbc.fabric_storage_placement fabric_storage_ownership_marker_path), so an old store +// with no unit is still observably someone's state rather than reading as nothing. +fn dashboard_ownership_marker_in(root: FilePath, instance_id: NonEmptyStr) -> FilePath { dashboard_instance_child( - root: instance.instance_root, - child: join([".gunbc-dashboard-instance-", instance.instance_id as String], ""), + root: root, + child: join([".gunbc-dashboard-instance-", instance_id as String], ""), ) } @@ -781,6 +792,69 @@ fn dashboard_instance_for_id(instance_id: String) -> DashboardInstanceById { } } +// THE PROD-ROLE HOLDER'S REALIZATION -- the ONE lookup from the role's DeploymentId to the +// HostDashboardInstance that realizes it (gunbc.deployment_risk). It is the instance roster +// filtered by its own deployment field, so there is no second id-to-instance table to drift. +// Every role-following singleton (approval broker owner, fabric storage placement, the +// serve/dispatch entry points) asks THIS, never srv1 by name. Under NoProdRole it REFUSES with a +// typed cause; there is no arm that falls back to srv1, and a holder with zero or several +// realizations is its own refusal rather than "the first". +type ProdRoleRealizationRefusal + = ProdRoleUnheld + | ProdRoleHolderUnrealized { deployment: DeploymentId } + | ProdRoleHolderAmbiguous { deployment: DeploymentId, count: Int } + | ProdRoleHolderUnactuatable { deployment: DeploymentId } + +// THE HOLDER TOKEN. sole_constructor, so only this module -- and in it only prod_role_realization_in +// -- can mint one: a role-following singleton is built FROM a holder, and no caller can claim +// ownership by spelling an instance_id or by constructing a holder for an instance of its choosing. +type ProdRoleHolder sole_constructor { + instance: HostDashboardInstance +} + +type ProdRoleRealization + = ProdRoleRealizedBy { holder: ProdRoleHolder } + | ProdRoleRealizationRefused { cause: ProdRoleRealizationRefusal } + +fn prod_role_realization_in( + selection: ProdRoleSelection, + instances: List, +) -> ProdRoleRealization { + match selection { + NoProdRole => ProdRoleRealizationRefused { cause: ProdRoleUnheld } + ProdRoleHeldBy { deployment: holder } => { + let realizing = instances |> filter(i => i.deployment == holder) + match realizing |> first { + Absent => ProdRoleRealizationRefused { cause: ProdRoleHolderUnrealized { deployment: holder } } + Present { value: only } => + if count(realizing) == 1 { + ProdRoleRealizedBy { holder: ProdRoleHolder { instance: only } } + } else { + ProdRoleRealizationRefused { cause: ProdRoleHolderAmbiguous { deployment: holder, count: count(realizing) } } + } + } + } + } +} + +// The production question: the real switch over the real roster. +fn prod_role_realization() -> ProdRoleRealization { + prod_role_realization_in(selection: prod_role_selection, instances: public_builtin_dashboard_instances()) +} + +fn prod_role_realization_refusal_detail(cause: ProdRoleRealizationRefusal) -> String { + match cause { + ProdRoleUnheld => "prod role refused: prod_role_selection is NoProdRole; no deployment holds it" + ProdRoleHolderUnrealized { deployment } => + join(["prod role refused: holder ", deployment.name as String, " has no HostDashboardInstance"], "") + ProdRoleHolderAmbiguous { deployment, count: n } => + join(["prod role refused: holder ", deployment.name as String, " has ", integer_int_to_decimal_string(value: n), " instances"], "") + ProdRoleHolderUnactuatable { deployment } => + join(["prod role refused: holder ", deployment.name as String, " has no desired deployment row (gunbc.live_deploy.desired desired_deployment_roster), so nothing can actuate it"], "") + } +} + + data macbook_local_dashboard_id: NonEmptyStr = "macbook-local" as NonEmptyStr data macbook_operator_home_root: FilePath = "/Users/briansrls" as FilePath @@ -864,7 +938,7 @@ fn macbook_local_dashboard_instance() -> HostDashboardInstance { ), }, serve_binary: macbook_local_serve_binary, - serve_function: "roadmap_serve_handle" as NonEmptyStr, + serve_function: "roadmap_serve_handle_macbook_local" as NonEmptyStr, listen_host: "127.0.0.1" as NonEmptyStr, listen_port: 8081, tmux_server: dashboard_named_tmux_server(instance_id: macbook_local_dashboard_id), @@ -924,7 +998,7 @@ fn srv1_live_dashboard_instance() -> HostDashboardInstance { HostDashboardInstance { deployment: srv1_daily_workspace_deployment, instance_id: srv1_dashboard_live_id, - host_identity: srv1_dashboard_host_identity, + host_identity: "srv1" as NonEmptyStr, provision_parent_root: "/opt/gunbc" as FilePath, instance_root: "/opt/gunbc" as FilePath, local_origin_root: srv1_gunbc_repo_root, @@ -948,7 +1022,7 @@ fn srv1_live_dashboard_instance() -> HostDashboardInstance { }, serve_binary: srv1_gunbc_serve_binary, serve_function: "roadmap_serve_handle" as NonEmptyStr, - listen_host: srv1_dashboard_listen_host, + listen_host: "127.0.0.1" as NonEmptyStr, listen_port: 8080, tmux_server: DefaultTmuxServer, actuation: DashboardActuating, diff --git a/dag/gunbc/roadmap/roadmap_dashboard_instance_apply.dag b/dag/gunbc/roadmap/roadmap_dashboard_instance_apply.dag index b46a61a743a..da30e14c283 100644 --- a/dag/gunbc/roadmap/roadmap_dashboard_instance_apply.dag +++ b/dag/gunbc/roadmap/roadmap_dashboard_instance_apply.dag @@ -65,6 +65,7 @@ import gunbc.roadmap_dashboard_instance_observe { } import extdeps.uri { Uri, Http, uri_wire } import extdeps.languages.json.parse { json_text_parseable } +import gunbc.deployment_risk { ProdRoleSelection, ProdRisk, deployment_risk_class, prod_role_selection } import gunbc.roadmap_dashboard_instance { dashboard_instance_health_identity_prefix, DashboardProvisionTransport, @@ -91,7 +92,7 @@ import gunbc.roadmap_dashboard_instance { DashboardInstanceConverged, DashboardInstanceApply, DashboardInstanceRefused, - srv1_live_dashboard_instance, + public_builtin_dashboard_instances, srv1_lab_dashboard_instance, srv2_lab_dashboard_instance, dashboard_instance_provider_state_root, @@ -769,81 +770,90 @@ fn dashboard_production_snapshots_equal( && before.health == after.health } -// PRODUCTION IS A PEER ON THE APPLYING HOST, and until this type existed the parameter forced every -// apply to name one whether the host had one or not. Both uses assume locality without saying so: -// the isolation check compares owned paths on one filesystem, and the liveness read-back curls -// 127.0.0.1 at production's declared port. Passing a remote instance satisfies neither โ€” it -// compares this machine's paths against another machine's declarations, then probes a loopback port -// nobody here is serving.\n\nThat is exactly how srv2's first apply failed. srv1-live was passed as -// production because the parameter demanded an instance; its repo_root happened to exist here as a -// hand-made symlink so the git half of the probe SUCCEEDED, and the refusal surfaced only as a curl -// connection error against port 8080 โ€” a true message about a check that should never have run. The -// remedy for that refusal is not to start a server; it is to stop claiming srv1 is a peer of -// anything on srv2.\n\nNoProductionPeer is a DECLARATION, not a skipped check. A host with no -// production dashboard has nothing to be disjoint from and nothing whose liveness could be -// disturbed, so both obligations are discharged by the shape of the argument rather than silently -// bypassed inside the checks โ€” and the reading below stays type-distinct across before and after, -// so an apply cannot pair a real snapshot with an absent one and call the difference untouched. - -type DashboardProductionPeer - = ProductionPeer { instance: HostDashboardInstance } - | NoProductionPeer { host: NonEmptyStr } - -type DashboardProductionReading - = ProductionRead { snapshot: DashboardProductionSnapshot } - | ProductionUnpeered { host: NonEmptyStr } - -fn dashboard_peer_isolation_ok( +// THE APPLYING HOST'S PROTECTED SIBLINGS, DERIVED -- never passed by name. A sibling is another +// instance on the SAME host (host_identity), and it is protected exactly when its deployment derives +// ProdRisk through gunbc.deployment_risk prod_role_selection. Moving prod moves which sibling an +// apply must stay disjoint from and leave untouched; under NoProdRole no sibling is protected. +// +// Both obligations assume locality: the isolation check compares owned paths on one filesystem, and +// the liveness read-back curls 127.0.0.1 at the sibling's declared port. That is why the population +// is filtered by host rather than taken from the whole roster: srv2's first apply failed when +// srv1-live was passed as its "production peer" -- a hand-made symlink let the git half of the probe +// succeed and the refusal surfaced as a curl error against a port nobody on srv2 serves. A sibling +// on another host is not a sibling, and an empty population is a derived fact, not a skipped check. +// The readings are paired by instance_id across before and after, so an apply cannot pair a real +// snapshot with a missing one and call the difference untouched. +type DashboardProtectedSibling { + instance: HostDashboardInstance +} + +fn dashboard_protected_siblings_in( + selection: ProdRoleSelection, + instances: List, + instance: HostDashboardInstance, +) -> List { + instances + |> filter(s => + (s.host_identity as String) == (instance.host_identity as String) + && !((s.instance_id as String) == (instance.instance_id as String)) + && deployment_risk_class(selection: selection, deployment: s.deployment) == ProdRisk + ) + |> map(s => DashboardProtectedSibling { instance: s }) +} + +fn dashboard_protected_siblings(instance: HostDashboardInstance) -> List { + dashboard_protected_siblings_in( + selection: prod_role_selection, + instances: public_builtin_dashboard_instances(), + instance: instance, + ) +} + +type DashboardProtectedSiblingReading { + instance_id: NonEmptyStr + snapshot: DashboardProductionSnapshot +} + +fn dashboard_sibling_isolation_ok( instance: HostDashboardInstance, - peer: DashboardProductionPeer, + siblings: List, ) -> Bool { - match peer { - ProductionPeer { instance: production } => - dashboard_instances_have_disjoint_owned_paths(a: instance, b: production) - && !(instance.listen_port == production.listen_port) - NoProductionPeer { host: _ } => true - } + all(siblings, s => + dashboard_instances_have_disjoint_owned_paths(a: instance, b: s.instance) + && !(instance.listen_port == s.instance.listen_port) + ) } -fn dashboard_production_read(peer: DashboardProductionPeer) -> DashboardProductionReading { - match peer { - ProductionPeer { instance: production } => - ProductionRead { snapshot: dashboard_production_snapshot(production: production) } - NoProductionPeer { host } => ProductionUnpeered { host: host } - } +fn dashboard_protected_sibling_read(siblings: List) -> List { + map(siblings, s => DashboardProtectedSiblingReading { + instance_id: s.instance.instance_id, + snapshot: dashboard_production_snapshot(production: s.instance), + }) } -fn dashboard_production_reading_refused(reading: DashboardProductionReading) -> Bool { - match reading { - ProductionRead { snapshot } => !snapshot.observed - ProductionUnpeered { host: _ } => false - } +fn dashboard_protected_sibling_reading_refused(readings: List) -> Bool { + any(readings, r => !r.snapshot.observed) } -fn dashboard_production_reading_detail(reading: DashboardProductionReading) -> String { - match reading { - ProductionRead { snapshot } => snapshot.detail - ProductionUnpeered { host } => join(["no production peer on ", host as String], "") +fn dashboard_protected_sibling_reading_detail(readings: List) -> String { + if count(readings) == 0 { + "no protected sibling on this host" + } else { + join(map(readings, r => join([r.instance_id as String, ": ", r.snapshot.detail], "")), "; ") } } -fn dashboard_production_readings_equal( - before: DashboardProductionReading, - after: DashboardProductionReading, +fn dashboard_protected_sibling_readings_equal( + before: List, + after: List, ) -> Bool { - match before { - ProductionRead { snapshot: b } => - match after { - ProductionRead { snapshot: a } => - dashboard_production_snapshots_equal(before: b, after: a) - ProductionUnpeered { host: _ } => false - } - ProductionUnpeered { host: bh } => - match after { - ProductionRead { snapshot: _ } => false - ProductionUnpeered { host: ah } => bh == ah - } - } + count(before) == count(after) + && all(before, b => + count(after |> filter(a => + (a.instance_id as String) == (b.instance_id as String) + && dashboard_production_snapshots_equal(before: b.snapshot, after: a.snapshot) + )) == 1 + ) } fn dashboard_source_anchor_matches( @@ -1301,24 +1311,24 @@ fn dashboard_bootstrap_parent_decision( fn dashboard_instance_apply_preflight( instance: HostDashboardInstance, - production: DashboardProductionPeer, + siblings: List, ) -> DashboardApplyPreflightResult { if !dashboard_instance_paths_are_owned(instance: instance) - || !dashboard_peer_isolation_ok(instance: instance, peer: production) { + || !dashboard_sibling_isolation_ok(instance: instance, siblings: siblings) { dashboard_apply_preflight_refused( instance: instance, step: "isolation-preflight", detail: "instance path or port isolation failed", ) } else { - let production_read = dashboard_production_read(peer: production) - if dashboard_production_reading_refused(reading: production_read) { + let production_read = dashboard_protected_sibling_read(siblings: siblings) + if dashboard_protected_sibling_reading_refused(readings: production_read) { dashboard_apply_preflight_refused( instance: instance, step: "production-preflight", detail: join([ "production read-back refused: ", - dashboard_production_reading_detail(reading: production_read), + dashboard_protected_sibling_reading_detail(readings: production_read), ], ""), ) } else { @@ -1648,11 +1658,11 @@ fn dashboard_apply_refused( fn dashboard_instance_apply_after_preflight( instance: HostDashboardInstance, - production: DashboardProductionPeer, + siblings: List, desired_source_revision: CommitSha, ) -> DashboardInstanceApplyResult { if !dashboard_instance_paths_are_owned(instance: instance) - || !dashboard_peer_isolation_ok(instance: instance, peer: production) { + || !dashboard_sibling_isolation_ok(instance: instance, siblings: siblings) { dashboard_apply_refused( instance: instance, step: "isolation-preflight", @@ -1660,14 +1670,14 @@ fn dashboard_instance_apply_after_preflight( receipts: [], ) } else { - let production_before = dashboard_production_read(peer: production) - if dashboard_production_reading_refused(reading: production_before) { + let production_before = dashboard_protected_sibling_read(siblings: siblings) + if dashboard_protected_sibling_reading_refused(readings: production_before) { dashboard_apply_refused( instance: instance, step: "production-preflight", detail: join([ "production read-back refused before mutation: ", - dashboard_production_reading_detail(reading: production_before), + dashboard_protected_sibling_reading_detail(readings: production_before), ], ""), receipts: [], ) @@ -1725,8 +1735,8 @@ fn dashboard_instance_apply_after_preflight( receipts: applied.receipts, ) } else { - let production_after = dashboard_production_read(peer: production) - let untouched = dashboard_production_readings_equal( + let production_after = dashboard_protected_sibling_read(siblings: siblings) + let untouched = dashboard_protected_sibling_readings_equal( before: production_before, after: production_after, ) @@ -1830,11 +1840,11 @@ fn dashboard_apply_bootstrap_then_escalate( fn dashboard_instance_apply( instance: HostDashboardInstance, - production: DashboardProductionPeer, + siblings: List, ) -> DashboardInstanceApplyResult { match dashboard_instance_apply_preflight( instance: instance, - production: production, + siblings: siblings, ) { DashboardApplyPreflightRefused { instance_id: _, step, detail } => dashboard_apply_refused( @@ -1860,7 +1870,7 @@ fn dashboard_instance_apply( } => dashboard_instance_apply_after_preflight( instance: instance, - production: production, + siblings: siblings, desired_source_revision: source_revision, ) } @@ -1968,14 +1978,14 @@ fn srv2_deploy_ensure_roots_cli() -> ProcessExit { fn srv1_lab_dashboard_apply() -> DashboardInstanceApplyResult { dashboard_instance_apply( instance: srv1_lab_dashboard_instance(), - production: ProductionPeer { instance: srv1_live_dashboard_instance() }, + siblings: dashboard_protected_siblings(instance: srv1_lab_dashboard_instance()), ) } fn srv1_lab_dashboard_preflight() -> DashboardApplyPreflightResult { dashboard_instance_apply_preflight( instance: srv1_lab_dashboard_instance(), - production: ProductionPeer { instance: srv1_live_dashboard_instance() }, + siblings: dashboard_protected_siblings(instance: srv1_lab_dashboard_instance()), ) } @@ -2032,26 +2042,24 @@ fn srv1_lab_dashboard_apply_wet() -> ProcessExit { // witness: the mechanism was real, general, and complete, and it was simply never bound to the // second instance. An instance row declared without an actuation entry reads exactly like one that // has been provisioned and is refusing for a good reason, and the page cannot tell the operator -// which it is looking at.\n\nsrv1's lab passes srv1_live as its peer; srv2's lab passes -// NoProductionPeer, and the difference is not a convenience. An earlier draft of this very row -// asserted the opposite โ€” that srv1_live was the right peer for both labs because /opt/gunbc exists -// on this machine too โ€” and that reasoning was wrong in the way dashboard_production_peer_note now -// records: the paths here are hand-made symlinks, not a production dashboard, so the isolation -// check compared srv2's paths against srv1's declarations and the liveness probe curled a loopback -// port nobody serves. Being disjoint from something that is not there is not a property worth -// checking; declaring that it is not there is. +// which it is looking at.\n\nEach lab's protected siblings are DERIVED (dashboard_protected_siblings): +// today srv1's lab derives srv1-live, the prod-role holder on its host, and srv2's lab derives none. +// An earlier draft asserted srv1_live was the right peer for both labs because /opt/gunbc exists on +// srv2 too; the paths there are hand-made symlinks, not a dashboard, so the isolation check compared +// srv2's paths against srv1's declarations and the liveness probe curled a loopback port nobody +// serves. The host filter is what keeps that from recurring. fn srv2_lab_dashboard_apply() -> DashboardInstanceApplyResult { dashboard_instance_apply( instance: srv2_lab_dashboard_instance(), - production: NoProductionPeer { host: "srv2" as NonEmptyStr }, + siblings: dashboard_protected_siblings(instance: srv2_lab_dashboard_instance()), ) } fn srv2_lab_dashboard_preflight() -> DashboardApplyPreflightResult { dashboard_instance_apply_preflight( instance: srv2_lab_dashboard_instance(), - production: NoProductionPeer { host: "srv2" as NonEmptyStr }, + siblings: dashboard_protected_siblings(instance: srv2_lab_dashboard_instance()), ) } diff --git a/dag/gunbc/roadmap/srv1_dashboard_bind.dag b/dag/gunbc/roadmap/srv1_dashboard_bind.dag deleted file mode 100644 index a7cec1e6f91..00000000000 --- a/dag/gunbc/roadmap/srv1_dashboard_bind.dag +++ /dev/null @@ -1,11 +0,0 @@ -module gunbc.srv1_dashboard_bind - -import std.types { NonEmptyStr } - -// THE SRV1 DASHBOARD INSTANCE'S HOST AND BIND, as a leaf. gunbc.roadmap_dashboard_instance -// srv1_live_dashboard_instance reads both, and so do the approval modules that derive a standing -// from them (gunbc.auth.approval_decision_store approval_identity_trust, gunbc.auth.approval_broker_endpoint -// approval_broker_host_identity). They are facts of the instance row, kept in a module of their own so -// a reader of two strings does not import the instance row's provider, codex and host-layout closure. -data srv1_dashboard_host_identity: NonEmptyStr = "srv1" -data srv1_dashboard_listen_host: NonEmptyStr = "127.0.0.1" diff --git a/dag/gunbc/spark/pair_serving_d0_door.dag b/dag/gunbc/spark/pair_serving_d0_door.dag index 3dc39eb2e69..20496d8c5e6 100644 --- a/dag/gunbc/spark/pair_serving_d0_door.dag +++ b/dag/gunbc/spark/pair_serving_d0_door.dag @@ -17,10 +17,9 @@ import gunbc.auth.approval_gate { approval_instant_now, approval_expires_at, approval_standing_read, approval_poll_ticks, ApprovalStandingRead, StandingReadAt, StandingClockUnreadable, approval_file_stored_request, ApprovalFiled, ApprovalFilingRefused, } -import gunbc.auth.approval_request_client { PollStillPending, PollNotFiled, stored_request_from_authorization } +import gunbc.auth.approval_request_client { PollStillPending, PollNotFiled, stored_request_from_authorization, approval_loopback_origin, ApprovalLoopbackOrigin, ApprovalLoopbackOriginResolved, ApprovalLoopbackOriginRefused, approval_loopback_origin_refusal_detail } import gunbc.fabric_storage_client { FabricStorageBinding } import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, actions_variable_trimmed, fleet_converge_expected_host_env_name, read_expected_revision, ExpectedRevisionPresent, ExpectedRevisionAbsent } -import gunbc.fleet_host_identity { operator_host_srv1 } import extdeps.git import extdeps.git.inspect import extdeps.git.plumbing @@ -459,9 +458,10 @@ fn pair_serving_d0_wet(group: String, transaction: String, authorization: String // REVISION is the workflow's expected_revision (the ref the job checked out), frozen with the first // filing; a rerun that executes another revision refuses before polling or mutating anything. // -// THE EXECUTOR IS srv1, PROVED TWICE AND BEFORE ANY CREDENTIAL: the broker is loopback on the host -// that serves the dashboard. The dispatch's host input (the runner label the job was scheduled on) -// and the kernel's own hostname must both be srv1, and pair_serving_d0_admit_executor runs that +// THE EXECUTOR IS THE APPROVAL WRITER'S HOST, PROVED TWICE AND BEFORE ANY CREDENTIAL: the writer is +// loopback on the prod-role holder's host (gunbc.auth.approval_request_client approval_loopback_origin +// carries it). The dispatch's host input (the runner label the job was scheduled on) and the kernel's +// own hostname must both be that host -- under NoProdRole there is none and the mode refuses -- and pair_serving_d0_admit_executor runs that // admission as a credential-free step BEFORE the job acquires its WIF token, loads the fleet ssh key // or materializes the submission MAC key -- so a dispatch to another host refuses with no secret on // that host. The door's own entry re-runs the same admission after the credentials, from the same @@ -475,11 +475,20 @@ type D0ModeAdmission = D0ModeAdmitted { group: FabricGroup, transaction: NonEmptyStr, authorization: NonEmptyStr, execution_revision: NonEmptyStr } | D0ModeRefused { reason: String } -fn pair_serving_d0_mode_admission(declared_host: String, kernel_host: String, consent: String, execution_revision: String, checked_out_revision: String) -> D0ModeAdmission { - if declared_host != (operator_host_srv1 as String) { - D0ModeRefused { reason: join(["the dispatch names host ", declared_host, ": POST/GET /approvals is loopback on the machine that serves the dashboard, which is ", operator_host_srv1 as String], "") } - } else if kernel_host != (operator_host_srv1 as String) { - D0ModeRefused { reason: join(["the dispatch names ", operator_host_srv1 as String, " but this job runs on ", kernel_host, "; the broker is loopback on ", operator_host_srv1 as String], "") } +fn pair_serving_d0_mode_admission(writer: ApprovalLoopbackOrigin, declared_host: String, kernel_host: String, consent: String, execution_revision: String, checked_out_revision: String) -> D0ModeAdmission { + match writer { + ApprovalLoopbackOriginRefused { cause } => + D0ModeRefused { reason: approval_loopback_origin_refusal_detail(cause: cause) as String } + ApprovalLoopbackOriginResolved { host: writer_host, origin: _ } => + pair_serving_d0_mode_admission_on(writer_host: writer_host as String, declared_host: declared_host, kernel_host: kernel_host, consent: consent, execution_revision: execution_revision, checked_out_revision: checked_out_revision) + } +} + +fn pair_serving_d0_mode_admission_on(writer_host: String, declared_host: String, kernel_host: String, consent: String, execution_revision: String, checked_out_revision: String) -> D0ModeAdmission { + if declared_host != writer_host { + D0ModeRefused { reason: join(["the dispatch names host ", declared_host, ": POST/GET /approvals is loopback on the approval writer's host, the prod-role holder's, which is ", writer_host], "") } + } else if kernel_host != writer_host { + D0ModeRefused { reason: join(["the dispatch names ", writer_host, " but this job runs on ", kernel_host, "; the approval writer is loopback on ", writer_host], "") } } else if consent == "" { D0ModeRefused { reason: join([pair_serving_d0_consent_env as String, " is unset; the mode names no consent (escalation id), and a run must name the same one on every rerun"], "") } } else if execution_revision == "" { @@ -522,7 +531,7 @@ fn pair_serving_d0_kernel_host() -> String // holds any secret. fn pair_serving_d0_admit_executor() -> ProcessExit { - match pair_serving_d0_mode_admission(declared_host: pair_serving_d0_declared_host(), kernel_host: pair_serving_d0_kernel_host(), consent: actions_variable_trimmed(name: pair_serving_d0_consent_env), execution_revision: pair_serving_d0_execution_revision(), checked_out_revision: pair_serving_d0_checked_out_revision()) { + match pair_serving_d0_mode_admission(writer: approval_loopback_origin(), declared_host: pair_serving_d0_declared_host(), kernel_host: pair_serving_d0_kernel_host(), consent: actions_variable_trimmed(name: pair_serving_d0_consent_env), execution_revision: pair_serving_d0_execution_revision(), checked_out_revision: pair_serving_d0_checked_out_revision()) { D0ModeRefused { reason: r } => refuse(reason: r) D0ModeAdmitted { group: _, transaction: _, authorization: _, execution_revision: _ } => ExitSuccess } @@ -530,7 +539,7 @@ fn pair_serving_d0_admit_executor() -> ProcessExit fn pair_serving_d0_ci_wet() -> ProcessExit { - match pair_serving_d0_mode_admission(declared_host: pair_serving_d0_declared_host(), kernel_host: pair_serving_d0_kernel_host(), consent: actions_variable_trimmed(name: pair_serving_d0_consent_env), execution_revision: pair_serving_d0_execution_revision(), checked_out_revision: pair_serving_d0_checked_out_revision()) { + match pair_serving_d0_mode_admission(writer: approval_loopback_origin(), declared_host: pair_serving_d0_declared_host(), kernel_host: pair_serving_d0_kernel_host(), consent: actions_variable_trimmed(name: pair_serving_d0_consent_env), execution_revision: pair_serving_d0_execution_revision(), checked_out_revision: pair_serving_d0_checked_out_revision()) { D0ModeRefused { reason: r } => refuse(reason: r) D0ModeAdmitted { group: g, transaction: tx, authorization: auth, execution_revision: rev } => pair_serving_d0_wet(group: fabric_group_wire(g: g) as String, transaction: tx as String, authorization: auth as String, execution_revision: rev as String) diff --git a/dag/test/claim/approval_decision_store_witness_test.dag b/dag/test/claim/approval_decision_store_witness_test.dag index 3a40330014a..40c3157d545 100644 --- a/dag/test/claim/approval_decision_store_witness_test.dag +++ b/dag/test/claim/approval_decision_store_witness_test.dag @@ -5,7 +5,8 @@ import std.types { Bool, NonEmptyStr, String, Timestamp, Secret } import std.scoped_authorization { AttemptIdentity } import extdeps.crypto.mac { MacKey, HmacSha256, mac_verify, MacVerified, MacTagMismatch, mac_key_material_hex_length, mac_suite_key_size } import std.measure { byte_size_count } -import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } +import gunbc.auth.approval_broker_endpoint { approval_broker_listen_host } +import gunbc.live_deploy.desired { deployment_spec_srv1 } import extdeps.tailscale.identity { tailnet_identity_of } import extdeps.filesystem.filesystem_io { filesystem_exact_read } import gunbc.auth.approval_capability { ProposeApprove, ProposeDeny, approval_capability_signing_input } @@ -135,14 +136,16 @@ test fn fixture_tags_verify_over_the_stored_claims_and_not_across_decisions() -> // THE TRUST STANDING FOLLOWS THE INSTANCE ROW AND NOTHING ELSE: loopback opens the gate ONLY as // Unattributed (never as Established -- a loopback bind does not attribute the on-host peer, so // the record names the bearer), any other bind or an unparseable host keeps it closed, and the production row is -// exactly the derivation over srv1_live_dashboard_instance.listen_host -- so the gate cannot be +// exactly the derivation over the broker's bind row, the same value the emitted broker unit's --host +// carries -- so the gate cannot be // opened by editing a row beside the one the unit is emitted from. test fn identity_trust_is_derived_from_the_instance_bind() -> Bool { match approval_identity_trust_for(listen_host: "127.0.0.1") { IdentityHeaderUnattributed { residual: _ } => true _ => false } && match approval_identity_trust_for(listen_host: "127.0.0.1") { IdentityHeaderTrustEstablished { evidence: _ } => false _ => true } && match approval_identity_trust_for(listen_host: "0.0.0.0") { IdentityHeaderTrustNotEstablished { blocker: _ } => true _ => false } && match approval_identity_trust_for(listen_host: "srv1.internal") { IdentityHeaderTrustNotEstablished { blocker: _ } => true _ => false } - && approval_identity_trust == approval_identity_trust_for(listen_host: srv1_live_dashboard_instance().listen_host) + && approval_identity_trust == approval_identity_trust_for(listen_host: approval_broker_listen_host) + && approval_identity_trust == approval_identity_trust_for(listen_host: deployment_spec_srv1().approval_broker.listen.host as NonEmptyStr) } // AN UNTRUSTED IDENTITY HEADER REFUSES BEFORE THE SEGMENT IS EVEN PARSED -- with a well-formed diff --git a/dag/test/claim/approval_device_enrolment_code_wet_witness_test.dag b/dag/test/claim/approval_device_enrolment_code_wet_witness_test.dag index 7141ce711b9..33989bbb324 100644 --- a/dag/test/claim/approval_device_enrolment_code_wet_witness_test.dag +++ b/dag/test/claim/approval_device_enrolment_code_wet_witness_test.dag @@ -1,5 +1,6 @@ module test.claim.approval_device_enrolment_code_wet_witness_test +import gunbc.auth.approval_broker_endpoint { ApprovalBrokerAddressResolved } import std.logic { Bool } import std.types { String, NonEmptyStr, Timestamp, Int } import gunbc.auth.approval_device_enrolment_code { @@ -29,9 +30,9 @@ test fn the_code_slot_is_written_before_delivery_by_real_execution() -> Bool { let root_path = shell.Mktemp.DirWithTemplate(template: "/tmp/gunbc_enrol.XXXXXX").path let root = root_path as NonEmptyStr let publisher = PublisherTokenUnavailable { path: "/etc/gunbc-ntfy/publisher-token", detail: "absent" } - let first = issue_and_deliver_enrolment_code_until(root: root, now: "2026-09-21T20:00:00Z", expires_at: fx_expires, code: fx_code, publisher: publisher) + let first = issue_and_deliver_enrolment_code_until(root: root, now: "2026-09-21T20:00:00Z", expires_at: fx_expires, code: fx_code, publisher: publisher, broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) let slot = observe_enrolment_slot(root: root, code: fx_code) - let second = issue_and_deliver_enrolment_code_until(root: root, now: "2026-09-21T20:00:00Z", expires_at: fx_expires, code: fx_code, publisher: publisher) + let second = issue_and_deliver_enrolment_code_until(root: root, now: "2026-09-21T20:00:00Z", expires_at: fx_expires, code: fx_code, publisher: publisher, broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) let removed = shell.Remove.RecursiveForce(path: root_path) (match first { EnrolmentCodeIssued { topic: _, expires_at: x, delivery: DeliveryRefused { detail: _ } } => x == fx_expires diff --git a/dag/test/claim/approval_device_enrolment_code_witness_test.dag b/dag/test/claim/approval_device_enrolment_code_witness_test.dag index 1ee1be7d0bb..b43039c4c23 100644 --- a/dag/test/claim/approval_device_enrolment_code_witness_test.dag +++ b/dag/test/claim/approval_device_enrolment_code_witness_test.dag @@ -1,5 +1,8 @@ module test.claim.approval_device_enrolment_code_witness_test +import gunbc.auth.approval_broker_endpoint { approval_broker_base_url_env_name, ApprovalBrokerAddressResolved } +import gunbc.roadmap_dashboard_instance { prod_role_realization, prod_role_realization_in, public_builtin_dashboard_instances } +import gunbc.deployment_risk { ProdRoleHeldBy, NoProdRole, srv2_deploy_deployment } import std.logic { Bool } import std.types { String, NonEmptyStr, Timestamp, List, Secret, Int } import std.process { ExitSuccess } @@ -18,7 +21,7 @@ import gunbc.auth.approval_device_redemption { observe_enrolment_slot, Enrolment import extdeps.shell import gunbc.auth.approval_notification { approval_ntfy_topic } import extdeps.systemd.systemd_run { SystemdRunCommandReady, SystemdRunCommandRefused } -import gunbc.auth.approval_device_enrolment_code_issue { enrolment_code_issue_remote_command, sudo_elevate_words, enrolment_code_issue_refuses_off_srv1 } +import gunbc.auth.approval_device_enrolment_code_issue { enrolment_code_issue_remote_command, sudo_elevate_words, enrolment_code_issue_refuses_off_broker_host, enrolment_code_issue_placement_under, EnrolmentCodeIssuePlaced, EnrolmentCodeIssueUnplaced } import gunbc.live_deploy.release_locus { RevisionBoundAtEmission, } @@ -41,8 +44,8 @@ data fx_code: NonEmptyStr = "9f3ac1d20b7e44c1a5d3e8f60c2b91aa" data fx_expires: Timestamp = "2026-09-21T20:10:00Z" test fn the_delivered_receipt_comes_from_the_published_message_and_its_line_never_carries_the_code() -> Bool { - let message = enrolment_code_notification(code: fx_code, expires_at: fx_expires) - (match deliver_enrolment_code(code: fx_code, expires_at: fx_expires, publisher: PublisherTokenLoaded { token: "hermetic-publisher-token" as Secret }) { + let message = enrolment_code_notification(code: fx_code, expires_at: fx_expires, approval_url: "https://srv1.example.ts.net" as NonEmptyStr) + (match deliver_enrolment_code(code: fx_code, expires_at: fx_expires, publisher: PublisherTokenLoaded { token: "hermetic-publisher-token" as Secret }, broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) { EnrolmentCodeIssued { topic: t, expires_at: x, delivery: DeliveryAccepted { message_id: Present { value: m } } } => { let line = enrolment_code_delivery_line(r: EnrolmentCodeDeliveryReceipt { topic: t, message_id: Present { value: m }, expires_at: x }) (m as String) == hermetic_ntfy_publish_message_id @@ -54,7 +57,7 @@ test fn the_delivered_receipt_comes_from_the_published_message_and_its_line_neve } _ => false }) - && (match deliver_enrolment_code(code: fx_code, expires_at: fx_expires, publisher: PublisherTokenUnavailable { path: "/etc/gunbc-ntfy/publisher-token", detail: "absent" }) { + && (match deliver_enrolment_code(code: fx_code, expires_at: fx_expires, publisher: PublisherTokenUnavailable { path: "/etc/gunbc-ntfy/publisher-token", detail: "absent" }, broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) { EnrolmentCodeIssued { topic: _, expires_at: x, delivery: DeliveryRefused { detail: c } } => x == fx_expires && string_contains(s: c, pattern: "absent") _ => false }) @@ -79,9 +82,10 @@ test fn the_remote_argv_runs_the_release_verb_as_the_operator_user() -> Bool { let op = fleet_posix_operator_user.name as String let rev = "0123456789abcdef0123456789abcdef01234567" the_remote_verb_words_carry_the_release_shape( - reading: enrolment_code_issue_remote_command(revision: RevisionBoundAtEmission { revision: rev }), + reading: enrolment_code_issue_remote_command(revision: RevisionBoundAtEmission { revision: rev }, base_url: "https://srv1.example.ts.net" as NonEmptyStr), rev: rev, op: op, + base_url: "https://srv1.example.ts.net", ) } @@ -89,15 +93,15 @@ test fn the_remote_argv_runs_the_release_verb_as_the_operator_user() -> Bool { // string-built form rendered for the same arguments -- non-interactive sudo, a system-manager scope // bounded by the broker's slice rows, the account entered through setpriv --init-groups (never // systemd-run's --uid/--gid; run 37098981121), the release directory as the seed's checkout root -// inside the elevation, and the verb naming its module and function. The projection's rejected-word +// inside the elevation, the broker base URL the placement resolved, and the verb naming its module and function. The projection's rejected-word // refusals are unreachable for these compile-time shapes; the claim's red depends on the words, not // on a refusal arm. -fn the_remote_verb_words_carry_the_release_shape(reading: SystemdRunCommandReading, rev: String, op: String) -> Bool { +fn the_remote_verb_words_carry_the_release_shape(reading: SystemdRunCommandReading, rev: String, op: String, base_url: String) -> Bool { match reading { SystemdRunCommandRefused { cause: _ } => false SystemdRunCommandReady { command: command } => { let joined = join(sudo_elevate_words(command: command), " ") - string_contains(s: joined, pattern: "/usr/bin/sudo -n systemd-run --scope --property=MemoryMax=" + to_string(byte_size_count(b: approval_broker_slice_memory_max)) + " --property=MemoryHigh=" + to_string(byte_size_count(b: approval_broker_slice_memory_high)) + " -- setpriv --reuid=" + op + " --regid=" + op + " --init-groups -- /usr/bin/env " + gunbc_workspace_root_env_name + "=/opt/gunbc/approval-broker/releases/" + rev + " /opt/gunbc/approval-broker/releases/" + rev + "/gunbc run ") + string_contains(s: joined, pattern: "/usr/bin/sudo -n systemd-run --scope --property=MemoryMax=" + to_string(byte_size_count(b: approval_broker_slice_memory_max)) + " --property=MemoryHigh=" + to_string(byte_size_count(b: approval_broker_slice_memory_high)) + " -- setpriv --reuid=" + op + " --regid=" + op + " --init-groups -- /usr/bin/env " + gunbc_workspace_root_env_name + "=/opt/gunbc/approval-broker/releases/" + rev + " " + (approval_broker_base_url_env_name as String) + "=" + base_url + " /opt/gunbc/approval-broker/releases/" + rev + "/gunbc run ") && !string_contains(s: joined, pattern: "--uid=") && string_contains(s: joined, pattern: "/releases/" + rev + "/gunbc run ") && string_contains(s: joined, pattern: "--entry /opt/gunbc/approval-broker/releases/" + rev + "/dag/gunbc/auth/approval_device_enrolment_code.dag --function issue_device_enrolment_code --arg revision=" + rev) @@ -105,17 +109,27 @@ fn the_remote_verb_words_carry_the_release_shape(reading: SystemdRunCommandReadi } } -// The step runs on srv1 and nowhere else; the mode is rostered, wired, and consumes the fleet SSH -// key like its keyring sibling. -test fn the_mode_is_rostered_and_pinned_to_srv1() -> Bool { - (match enrolment_code_issue_refuses_off_srv1(host: operator_host_srv1 as String) { - ExitSuccess => true - _ => false +// The step runs on the broker's placed host and nowhere else -- srv1 under today's selection, srv2 +// when the role moves to srv2-deploy, and nowhere under NoProdRole; the mode is rostered, wired, and +// consumes the fleet SSH key like its keyring sibling. +test fn the_mode_is_rostered_and_pinned_to_the_brokers_placed_host() -> Bool { + let roster = public_builtin_dashboard_instances() + let today = enrolment_code_issue_placement_under(realization: prod_role_realization()) + let moved = enrolment_code_issue_placement_under(realization: prod_role_realization_in(selection: ProdRoleHeldBy { deployment: srv2_deploy_deployment }, instances: roster)) + let unheld = enrolment_code_issue_placement_under(realization: prod_role_realization_in(selection: NoProdRole, instances: roster)) + (match today { + EnrolmentCodeIssuePlaced { host, base_url: _ } => + (match enrolment_code_issue_refuses_off_broker_host(host: "srv1", broker_host: host) { ExitSuccess => true _ => false }) + && (match enrolment_code_issue_refuses_off_broker_host(host: "srv2", broker_host: host) { ExitSuccess => false _ => true }) + EnrolmentCodeIssueUnplaced { detail: _ } => false }) - && (match enrolment_code_issue_refuses_off_srv1(host: "srv2") { - ExitSuccess => false - _ => true + && (match moved { + EnrolmentCodeIssuePlaced { host, base_url } => (host as String) == "srv2" && string_contains(s: base_url as String, pattern: "srv2.") + && (match enrolment_code_issue_refuses_off_broker_host(host: "srv2", broker_host: host) { ExitSuccess => true _ => false }) + && (match enrolment_code_issue_refuses_off_broker_host(host: "srv1", broker_host: host) { ExitSuccess => false _ => true }) + EnrolmentCodeIssueUnplaced { detail: _ } => false }) + && (match unheld { EnrolmentCodeIssueUnplaced { detail: _ } => true _ => false }) && fleet_converge_workflow_mode_wire(mode: ApprovalDeviceEnrolmentCodeIssue) == "approval_device_enrolment_code_issue" && any(fleet_converge_workflow_modes, m => fleet_converge_workflow_mode_wire(mode: m) == "approval_device_enrolment_code_issue") && (match fleet_converge_mode_fleet_ssh_key_demand(mode: ApprovalDeviceEnrolmentCodeIssue) { diff --git a/dag/test/claim/approval_ntfy_access_readback_witness_test.dag b/dag/test/claim/approval_ntfy_access_readback_witness_test.dag index 768b1b8e13a..23d69f55ed2 100644 --- a/dag/test/claim/approval_ntfy_access_readback_witness_test.dag +++ b/dag/test/claim/approval_ntfy_access_readback_witness_test.dag @@ -1236,7 +1236,7 @@ test fn a_duplicated_principal_refuses() -> Bool { // โ”€โ”€ THE SEALS: no second root compiles โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // A fresh module reaching each effect leaf, and the runtime reader, from an unadmitted // declaration. Each call must be refused by admission, named by its subject. -data unadmitted_effect_caller_source: String = "module probe_enrolment_second_root\nimport gunbc.auth.approval_device_enrolment_code { mint_enrolment_code, issue_and_deliver_enrolment_code_until, deliver_enrolment_code, enrolment_code_notification }\nimport gunbc.auth.approval_device_redemption { issue_enrolment_code }\nimport gunbc.auth.approval_request_submission { read_publisher_token, publish_notification }\nimport gunbc.auth.approval_ntfy_access_readback { approval_ntfy_runtime_reading, approval_ntfy_run, ReadConfig }\nimport gunbc.live_deploy.emit { RevisionNotInstalled }\nfn second_root() -> Bool {\n let a = mint_enrolment_code()\n let t = read_publisher_token()\n let b = issue_enrolment_code(root: \"/tmp/x\", code: \"c\", issued_at: \"2026-09-21T20:00:00Z\", expires_at: \"2026-09-21T20:10:00Z\")\n let c = issue_and_deliver_enrolment_code_until(root: \"/tmp/x\", now: \"2026-09-21T20:00:00Z\", expires_at: \"2026-09-21T20:10:00Z\", code: \"c\", publisher: t)\n let d = deliver_enrolment_code(code: \"c\", expires_at: \"2026-09-21T20:10:00Z\", publisher: t)\n let e = publish_notification(message: enrolment_code_notification(code: \"c\", expires_at: \"2026-09-21T20:10:00Z\"), publisher: t)\n let f = approval_ntfy_runtime_reading(revision: RevisionNotInstalled)\n let g = approval_ntfy_run(read: ReadConfig, pid: \"1\", revision: RevisionNotInstalled)\n true\n}\n" +data unadmitted_effect_caller_source: String = "module probe_enrolment_second_root\nimport gunbc.auth.approval_device_enrolment_code { mint_enrolment_code, issue_and_deliver_enrolment_code_until, deliver_enrolment_code, enrolment_code_notification }\nimport gunbc.auth.approval_device_redemption { issue_enrolment_code }\nimport gunbc.auth.approval_request_submission { read_publisher_token, publish_notification }\nimport gunbc.auth.approval_ntfy_access_readback { approval_ntfy_runtime_reading, approval_ntfy_run, ReadConfig }\nimport gunbc.live_deploy.emit { RevisionNotInstalled }\nimport gunbc.auth.approval_broker_endpoint { ApprovalBrokerAddressRefused }\nfn second_root() -> Bool {\n let a = mint_enrolment_code()\n let t = read_publisher_token()\n let b = issue_enrolment_code(root: \"/tmp/x\", code: \"c\", issued_at: \"2026-09-21T20:00:00Z\", expires_at: \"2026-09-21T20:10:00Z\")\n let c = issue_and_deliver_enrolment_code_until(root: \"/tmp/x\", now: \"2026-09-21T20:00:00Z\", expires_at: \"2026-09-21T20:10:00Z\", code: \"c\", publisher: t, broker: ApprovalBrokerAddressRefused { detail: \"x\" })\n let d = deliver_enrolment_code(code: \"c\", expires_at: \"2026-09-21T20:10:00Z\", publisher: t, broker: ApprovalBrokerAddressRefused { detail: \"x\" })\n let e = publish_notification(message: enrolment_code_notification(code: \"c\", expires_at: \"2026-09-21T20:10:00Z\", approval_url: \"https://x\"), publisher: t)\n let f = approval_ntfy_runtime_reading(revision: RevisionNotInstalled)\n let g = approval_ntfy_run(read: ReadConfig, pid: \"1\", revision: RevisionNotInstalled)\n true\n}\n" data sealed_effect_leaves: List = ["mint_enrolment_code", "read_publisher_token", "issue_enrolment_code", "issue_and_deliver_enrolment_code_until", "deliver_enrolment_code", "publish_notification", "approval_ntfy_runtime_reading", "approval_ntfy_run"] diff --git a/dag/test/claim/approval_request_client_witness_test.dag b/dag/test/claim/approval_request_client_witness_test.dag index abd8f8e5f8c..e0bcee405ea 100644 --- a/dag/test/claim/approval_request_client_witness_test.dag +++ b/dag/test/claim/approval_request_client_witness_test.dag @@ -1,9 +1,13 @@ module test.claim.approval_request_client_witness_test import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } -import std.types { Bool, NonEmptyStr, String } +import std.types { Bool, NonEmptyStr, String, List, Int } +import v2.std.collection { Present, Absent } +import gunbc.guarantee_probe_corpus { census_blocking_count_for_class, census_blocking_count_for_subject } import gunbc.auth.approval_request_client { - approval_submit_url, approval_status_url, approval_loopback_origin, approval_loopback_origin_for, + approval_loopback_origin, approval_loopback_origin_for, approval_loopback_origin_of, approval_request_submit_path, + ApprovalLoopbackOrigin, ApprovalLoopbackOriginResolved, ApprovalLoopbackOriginRefused, + classify_approval_origin, ApprovalOriginStanding, ApprovalOriginWouldDial, ApprovalOriginUnresolved, ApprovalExecutorNotTheOriginHost, ApprovalExecutorHostUnobserved, approval_poll_standing, PollStillPending, PollApproved, PollExpired, PollUnreadable, StoredApprovalRequest, } @@ -11,8 +15,11 @@ import gunbc.auth.approval_status_wire { parse_filed_approval_status, filed_approval_status_text, submission_filed_text, submission_body_admits_poll, StatusNotFiled, StatusPending, StatusApproved, StatusDenied, StatusUnreadable, } -import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } -import gunbc.auth.approval_writer_authority { approval_writer_authority, approval_writer_process, RoadmapProcess, BrokerProcess } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv2_deploy_dashboard_instance, ProdRoleRealization, ProdRoleUnheld } +import gunbc.deployment_risk { ProdRoleHeldBy, NoProdRole, srv2_deploy_deployment } +import gunbc.hostname_read { HostnameShortCapture, HostnameShortCaptured, HostnameShortCaptureRefused } +import gunbc.live_deploy.desired { prod_role_actuated_realization, prod_role_actuated_realization_in } +import gunbc.auth.approval_writer_authority { approval_writer_authority, approval_writer_process, ApprovalServingProcess, RoadmapProcess, BrokerProcess } import gunbc.auth.approval_broker_endpoint { approval_broker_loopback_base_url, approval_broker_listen_port } import gunbc.auth.approval_request_submission { submission_envelope_json } import gunbc.auth.approval_decision_store { approval_mac_key_id } @@ -34,23 +41,142 @@ fn fixture_stored() -> StoredApprovalRequest { } } -test fn submit_and_status_urls_follow_the_srv1_instance_listen_bind_while_the_roadmap_writes() -> Bool { - let instance = srv1_live_dashboard_instance() - approval_loopback_origin_for(writer: RoadmapProcess) as String - == concat(concat("http://", instance.listen_host as String), concat(":", to_string(instance.listen_port))) +fn on_host(h: String) -> HostnameShortCapture { + HostnameShortCaptured { value: h } } -// THE REGRESSION run 37265259209 hit: with the broker as writer the client must dial the broker's -// listen port, never the roadmap's -- the two ports differ, so a client still on the roadmap is red. -test fn submit_and_status_urls_dial_the_declared_writer() -> Bool { - let origin = approval_loopback_origin() as String - let writer_origin = approval_loopback_origin_for(writer: approval_writer_process(authority: approval_writer_authority)) as String - let broker_origin = approval_broker_loopback_base_url(port: approval_broker_listen_port) as String - origin == writer_origin - && approval_loopback_origin_for(writer: BrokerProcess) as String == broker_origin - && broker_origin != approval_loopback_origin_for(writer: RoadmapProcess) as String - && approval_submit_url() as String == concat(origin, "/approvals") - && approval_status_url(escalation_id: "esc-1") as String == concat(origin, "/approvals/esc-1") +fn srv2_holds() -> ProdRoleRealization { + prod_role_actuated_realization_in(selection: ProdRoleHeldBy { deployment: srv2_deploy_deployment }) +} + +fn no_prod_role() -> ProdRoleRealization { + prod_role_actuated_realization_in(selection: NoProdRole) +} + +fn resolved_host_and_origin(o: ApprovalLoopbackOrigin) -> List { + match o { + ApprovalLoopbackOriginResolved { host, origin } => [host as String, origin as String] + ApprovalLoopbackOriginRefused { cause: _ } => [] + } +} + +fn would_dial(st: ApprovalOriginStanding) -> List { + match st { + ApprovalOriginWouldDial { host, origin } => [host as String, origin as String] + ApprovalOriginUnresolved { cause: _ } => [] + ApprovalExecutorNotTheOriginHost { origin_host: _, executor_host: _ } => [] + ApprovalExecutorHostUnobserved { reason: _ } => [] + } +} + +// The behavioural REDs run the pure classifier, which returns a STANDING and no dial token; the token +// is minted only on the sealed live route (the compile REDs below). + +// (5) POSITIVE CONTROL: today's selection, the broker as writer, an executor on srv1 -- would dial the +// broker's loopback on srv1. +test fn todays_broker_writer_on_the_srv1_holder_is_dialable_from_srv1() -> Bool { + would_dial(st: classify_approval_origin(origin: approval_loopback_origin_for(writer: BrokerProcess, realization: prod_role_actuated_realization()), executor: on_host(h: "srv1"))) + == ["srv1", approval_broker_loopback_base_url(port: approval_broker_listen_port) as String] +} + +// (1) The role on srv2-deploy, the broker as writer, an executor on srv2: would dial. +test fn a_broker_writer_on_a_srv2_holder_is_dialable_from_srv2() -> Bool { + would_dial(st: classify_approval_origin(origin: approval_loopback_origin_for(writer: BrokerProcess, realization: srv2_holds()), executor: on_host(h: "srv2"))) + == ["srv2", approval_broker_loopback_base_url(port: approval_broker_listen_port) as String] +} + +// (2) The role on srv2-deploy, the broker as writer, an executor on srv1: refuses before any request, +// naming both hosts -- it does not dial srv1's loopback, where a stale broker may still answer. +test fn a_broker_writer_on_a_srv2_holder_refuses_a_srv1_executor_before_any_request() -> Bool { + match classify_approval_origin(origin: approval_loopback_origin_for(writer: BrokerProcess, realization: srv2_holds()), executor: on_host(h: "srv1")) { + ApprovalExecutorNotTheOriginHost { origin_host, executor_host } => (origin_host as String) == "srv2" && executor_host == "srv1" + _ => false + } +} + +// (3) Switching the writer to the roadmap changes the endpoint and keeps the holder's host. +test fn the_roadmap_writer_changes_the_endpoint_and_keeps_the_holders_host() -> Bool { + let r = srv2_holds() + would_dial(st: classify_approval_origin(origin: approval_loopback_origin_for(writer: BrokerProcess, realization: r), executor: on_host(h: "srv2"))) + == ["srv2", approval_broker_loopback_base_url(port: approval_broker_listen_port) as String] + && would_dial(st: classify_approval_origin(origin: approval_loopback_origin_for(writer: RoadmapProcess, realization: r), executor: on_host(h: "srv2"))) + == ["srv2", approval_loopback_origin_of(instance: srv2_deploy_dashboard_instance()) as String] +} + +// (4) NoProdRole refuses on both writer arms, whatever the executor is. +fn unresolved_under_no_prod_role(w: ApprovalServingProcess) -> Bool { + match classify_approval_origin(origin: approval_loopback_origin_for(writer: w, realization: no_prod_role()), executor: on_host(h: "srv1")) { + ApprovalOriginUnresolved { cause: ProdRoleUnheld } => true + _ => false + } +} + +test fn no_prod_role_refuses_on_both_writer_arms() -> Bool { + unresolved_under_no_prod_role(w: RoadmapProcess) && unresolved_under_no_prod_role(w: BrokerProcess) +} + +// An executor that cannot read its own hostname does not dial either. +test fn an_executor_whose_host_is_unobserved_refuses() -> Bool { + match classify_approval_origin(origin: approval_loopback_origin_for(writer: BrokerProcess, realization: prod_role_actuated_realization()), executor: HostnameShortCaptureRefused { reason: "hostname -s exited unsuccessfully" }) { + ApprovalExecutorHostUnobserved { reason: _ } => true + _ => false + } +} + +// โ”€โ”€ THE DIAL TOKEN CANNOT BE FORGED, AND THE FILING STEP CANNOT BE REACHED AROUND IT โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// ONE probe source carries every forged route, one function each, so the floor compiles it once (the +// census is memoized by source; each compile costs the census, gunbc.recurring_failure_mode +// entry_scoped_typecheck_rss_tracks_the_name_census) and each claim reads ITS refusal by class AND +// subject -- the refused callee or type -- so no claim can pass on another route's refusal. +data forged_approval_routes_source: String = "module probe_forged_approval_routes\nimport gunbc.auth.approval_request_client { mint_dialable_approval_origin, DialableApprovalOrigin, approval_origin_admission_here, ApprovalOriginAdmission }\nimport gunbc.auth.approval_gate { approval_file_stored_request_at, ApprovalFiling }\nimport gunbc.auth.approval_decision_store { StoredApprovalRequest }\nimport std.measure { Second }\nfn forge_by_mint() -> DialableApprovalOrigin { mint_dialable_approval_origin(host: \"srv1\", origin: \"http://127.0.0.1:8085\") }\nfn forge_by_literal() -> DialableApprovalOrigin { DialableApprovalOrigin { host: \"srv1\", origin: \"http://127.0.0.1:8085\" } }\nfn obtain_admission() -> ApprovalOriginAdmission { approval_origin_admission_here() }\nfn bypass_filing(d: DialableApprovalOrigin, s: StoredApprovalRequest, t: Second) -> ApprovalFiling { approval_file_stored_request_at(dial: d, stored: s, body_file_name: \"x\", submit_timeout: t) }\n" + +data canonical_approval_routes_source: String = "module probe_canonical_approval_routes\nimport gunbc.auth.approval_gate { approval_file_stored_request, approval_standing_read, ApprovalFiling, ApprovalStandingRead }\nimport gunbc.auth.approval_decision_store { StoredApprovalRequest }\nimport std.measure { Second }\nimport std.types { Timestamp, NonEmptyStr }\nfn file(s: StoredApprovalRequest, t: Second) -> ApprovalFiling { approval_file_stored_request(stored: s, body_file_name: \"x\", submit_timeout: t) }\nfn read(e: NonEmptyStr, x: Timestamp) -> ApprovalStandingRead { approval_standing_read(escalation_id: e, expires_at: x) }\n" + +fn forged_route_refusals(class: String, subject: String) -> Int { + match census_blocking_count_for_subject(census: compile_dag_diagnostic_census(forged_approval_routes_source), wanted_class: class, wanted_subject: subject) { + Present { value: n } => n + Absent => 0 + } +} + +fn admission_refusals_in(source: String) -> Int { + match census_blocking_count_for_class(census: compile_dag_diagnostic_census(source), wanted: "ConstructorCallAdmissionRefused") { + Present { value: n } => n + Absent => 0 + } +} + +// (1) An outside module cannot mint the token: not through the mint, not as a literal, and not by +// calling the live admission (which is where the hostname is read). +test fn no_outside_module_can_mint_a_dial_token_through_the_mint() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "mint_dialable_approval_origin") >= 1 +} + +test fn no_outside_module_can_write_a_dial_token_literal() -> Bool { + forged_route_refusals(class: "SoleConstructorViolation", subject: "DialableApprovalOrigin") >= 1 +} + +test fn no_outside_module_can_call_the_live_origin_admission() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "approval_origin_admission_here") >= 1 +} + +// (2) The filing step that reads the key, writes the envelope and POSTs is reachable only from the +// filing composition that admitted the origin. +test fn no_outside_module_can_call_the_filing_step() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "approval_file_stored_request_at") >= 1 +} + +// (3) THE CONTROL: the canonical filing and standing-read routes compile with no admission refusal. +test fn the_canonical_filing_and_standing_read_routes_compile() -> Bool { + admission_refusals_in(source: canonical_approval_routes_source) == 0 +} + +// THE REGRESSION run 37265259209 hit: the live origin is the declared writer's, the broker's loopback +// when the broker writes, never the roadmap's port. +test fn the_live_origin_is_the_declared_writers() -> Bool { + let r = prod_role_actuated_realization() + resolved_host_and_origin(o: approval_loopback_origin()) == resolved_host_and_origin(o: approval_loopback_origin_for(writer: approval_writer_process(authority: approval_writer_authority), realization: r)) + && resolved_host_and_origin(o: approval_loopback_origin_for(writer: BrokerProcess, realization: r)) != resolved_host_and_origin(o: approval_loopback_origin_for(writer: RoadmapProcess, realization: r)) } test fn submit_body_nests_the_stored_request_as_a_json_string() -> Bool { diff --git a/dag/test/claim/approval_request_submission_witness_test.dag b/dag/test/claim/approval_request_submission_witness_test.dag index fbb5a59d7c3..c2b12e97647 100644 --- a/dag/test/claim/approval_request_submission_witness_test.dag +++ b/dag/test/claim/approval_request_submission_witness_test.dag @@ -1,5 +1,6 @@ module test.claim.approval_request_submission_witness_test +import gunbc.auth.approval_broker_endpoint { ApprovalBrokerAddressResolved } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import std.types { Bool, NonEmptyStr, String, Secret } import std.scoped_authorization { AttemptIdentity } @@ -181,11 +182,11 @@ test fn status_text_reports_not_filed_for_an_absent_slot_and_unreadable_off_the_ // the fixture message id; deleting the publish call, or the token arm, fails this control. The // token-unavailable arm is the same fold one input earlier, so both are read from one route. test fn a_filed_request_with_a_publisher_token_is_published_and_without_one_is_reported() -> Bool { - match notify_filed(capability_keyring: ApprovalKeyLoaded { key: capability_key() }, publisher: PublisherTokenLoaded { token: "hermetic-publisher-token" as Secret }, request: request()) { + match notify_filed(capability_keyring: ApprovalKeyLoaded { key: capability_key() }, publisher: PublisherTokenLoaded { token: "hermetic-publisher-token" as Secret }, request: request(), broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) { SubmissionFiledAndNotified { escalation_id: e, message_id: m } => e == "esc-mtc1-boot-1" && (m as String) == hermetic_ntfy_publish_message_id _ => false } - && match notify_filed(capability_keyring: ApprovalKeyLoaded { key: capability_key() }, publisher: PublisherTokenUnavailable { path: "/etc/gunbc-ntfy/publisher-token", detail: "absent" }, request: request()) { + && match notify_filed(capability_keyring: ApprovalKeyLoaded { key: capability_key() }, publisher: PublisherTokenUnavailable { path: "/etc/gunbc-ntfy/publisher-token", detail: "absent" }, request: request(), broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) { SubmissionFiledPublisherTokenUnavailable { escalation_id: _, detail: d } => d == "absent" _ => false } @@ -218,7 +219,7 @@ test fn an_occupied_slot_is_renotifiable_only_while_pending_with_the_same_bytes( // A CAPABILITY-KEY REFUSAL AFTER THE WRITE STILL SAYS WHICH ESCALATION WAS FILED, like every other // post-write arm, so the filer retries the same bytes instead of minting a new id (review 67295). test fn a_capability_key_refusal_after_filing_names_the_filed_escalation() -> Bool { - match notify_filed(capability_keyring: ApprovalKeyringRefused { cause: ApprovalKeyFileAbsent { path: "/etc/gunbc-roadmap/approval-mac-key" } }, publisher: PublisherTokenLoaded { token: "hermetic-publisher-token" as Secret }, request: request()) { + match notify_filed(capability_keyring: ApprovalKeyringRefused { cause: ApprovalKeyFileAbsent { path: "/etc/gunbc-roadmap/approval-mac-key" } }, publisher: PublisherTokenLoaded { token: "hermetic-publisher-token" as Secret }, request: request(), broker: ApprovalBrokerAddressResolved { value: "https://srv1.example.ts.net" as NonEmptyStr }) { SubmissionFiledCapabilityKeyUnavailable { escalation_id: e, cause: _ } => e == "esc-mtc1-boot-1" _ => false } diff --git a/dag/test/claim/approve_ios_project_witness_test.dag b/dag/test/claim/approve_ios_project_witness_test.dag index 2b69d0c5590..3280d20fbd3 100644 --- a/dag/test/claim/approve_ios_project_witness_test.dag +++ b/dag/test/claim/approve_ios_project_witness_test.dag @@ -11,7 +11,10 @@ import extdeps.apple.xcode { XcodeRelease, xcode_26_6, xcode_release_text } import extdeps.xcodegen.project_spec { xcodegen_last_upgrade_check } import gunbc.approve_ios_app { approve_ios_build_standing, NotExecutedByAnyRequiredLane, approve_ios_build_unexecuted_frontier } import gunbc.approve_ios_project { expected_approve_ios_project_yml, approve_ios_project_yml_from, ApproveIosProjectGeneration, ApproveIosProjectGenerated, ApproveIosProjectRefused } -import gunbc.auth.approval_broker_endpoint { approval_broker_base_url } +import gunbc.auth.approval_broker_endpoint { approval_broker_base_url_on } +import gunbc.auth.approval_broker_placement { approval_broker_host_under, ApprovalBrokerHostResolved, ApprovalBrokerHostRefused } +import gunbc.roadmap_dashboard_instance { prod_role_realization } +import gunbc.fleet_tailnet { fleet_tailnet_domain } import gunbc.fleet_host_identity { operator_host_srv1 } import gunbc.auth.approval_ntfy_publish { approval_ntfy_serve_port } @@ -238,10 +241,11 @@ fn generated_server_host(g: ApproveIosProjectGeneration) -> String? { } // The host the app dials and the base the approval links are issued against are one address: -// https:// plus the projected host IS gunbc.auth.approval_broker_endpoint approval_broker_base_url. +// https:// plus the projected host IS the broker's base URL on the host it is placed on +// (gunbc.auth.approval_broker_placement), i.e. what the serving broker issues its links against. test fn the_committed_projection_names_the_brokers_public_host() -> Bool { match generated_server_host(g: expected_approve_ios_project_yml()) { - Present { value: h } => concat("https://", h) == (approval_broker_base_url() as String) + Present { value: h } => concat("https://", h) == approval_broker_base_url_text() Absent => false } } @@ -251,7 +255,7 @@ test fn the_committed_projection_names_the_brokers_public_host() -> Bool { test fn changing_the_tailnet_domain_changes_the_generated_host() -> Bool { match generated_server_host(g: approve_ios_project_yml_from(domain: Present { value: "other-tailnet.example" as NonEmptyStr }, port: none)) { Present { value: h } => (h == join([operator_host_srv1 as String, ".other-tailnet.example"], "")) - && (concat("https://", h) != (approval_broker_base_url() as String)) + && (concat("https://", h) != approval_broker_base_url_text()) Absent => false } } @@ -272,3 +276,12 @@ test fn red_a_non_default_public_port_refuses_the_projection() -> Bool { ApproveIosProjectGenerated { content: _ } => false } } + +// The link base on the placed broker's host under today's selection; a refused placement is "" and +// fails every equality. +fn approval_broker_base_url_text() -> String { + match approval_broker_host_under(realization: prod_role_realization()) { + ApprovalBrokerHostResolved { host } => approval_broker_base_url_on(host: host, domain: fleet_tailnet_domain) as String + ApprovalBrokerHostRefused { cause: _ } => "" + } +} diff --git a/dag/test/claim/deployed_tree_remote_witness_test.dag b/dag/test/claim/deployed_tree_remote_witness_test.dag index 4e6040a7cf2..d9b00c19935 100644 --- a/dag/test/claim/deployed_tree_remote_witness_test.dag +++ b/dag/test/claim/deployed_tree_remote_witness_test.dag @@ -15,6 +15,7 @@ import extdeps.git { } import extdeps.github.github { github_https_clone_url } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } +import gunbc.live_deploy.desired { deployment_spec_srv1 } import gunbc.live_deploy.spec { OwnedArtifactKind, PublicationHelperTimerUnit, @@ -256,6 +257,7 @@ fn ordered_tags() -> List { deployment_owned_steps( instance: srv1_live_dashboard_instance(), names: srv1_live_deployment_names(instance: srv1_live_dashboard_instance()), + holding: deployment_spec_srv1().role_singletons, ), init: [], f: (acc, step) => { diff --git a/dag/test/claim/deployment_risk_witness_test.dag b/dag/test/claim/deployment_risk_witness_test.dag index 1cd9f422a0d..834d9d04e06 100644 --- a/dag/test/claim/deployment_risk_witness_test.dag +++ b/dag/test/claim/deployment_risk_witness_test.dag @@ -1,12 +1,13 @@ module test.claim.deployment_risk_witness_test import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } -import std.types { NonEmptyStr, List, Bool } +import std.types { NonEmptyStr, List, Bool, String, Int } import v2.std.collection { Present, Absent } import extdeps.ntfy.ntfy { NtfyTopic } import gunbc.auth.approval_notification { approval_ntfy_topic } import gunbc.auth.github_apps { gunbai_bot_declared, gunbai_ci_declared } import gunbc.deployment_risk { + deployment_id, DeploymentId, DeploymentRiskClass, TestRisk, @@ -26,6 +27,7 @@ import gunbc.deployment_risk { srv1_daily_workspace_deployment, srv1_lab_codex_feedback_deployment, srv2_deploy_deployment, + macbook_local_deployment, } import gunbc.recurring_failure_mode.a_test_deployment_acts_on_the_prod_deployments_state { a_test_deployment_acts_on_the_prod_deployments_state } import gunbc.roadmap_dashboard_instance { @@ -37,6 +39,114 @@ import gunbc.roadmap_dashboard_instance { srv2_preview_dashboard_instance, srv2_deploy_dashboard_instance, srv2_lab_dashboard_instance, + ProdRoleRealization, + ProdRoleRealizedBy, + ProdRoleRealizationRefused, + ProdRoleUnheld, + prod_role_realization_refusal_detail, + ProdRoleHolderUnrealized, + ProdRoleHolderUnactuatable, + prod_role_realization, + prod_role_realization_in, + public_builtin_dashboard_instances, +} +import gunbc.live_deploy.spec { + DeploymentSpec, + DeploymentStep, + ArtifactStep, + Dependency, + FabricStorageServeUnit, + FabricStorageServeMapping, + FabricStorageStoreDirectory, + EnsuredOwnershipMarker, + deployment_srv1_target, + srv1_live_deployment_names, + deployment_spec_for, + deployment_owns_the_host_approval_broker, + deployment_fabric_storage_door_socket, +} +import gunbc.live_deploy.desired { deployment_spec_srv1, srv1_production_desired_deployment, DesiredDeployment, desired_deployment_roster, deployment_srv2_target, prod_role_actuated_realization, prod_role_actuated_realization_in } +import v2.std.diagnostic { Accepted, Rejected } +import v2.workflow.bash_emit { bash_emit_stmts } +import v2.extdeps.languages.bash_build { bash_build_with_redir, bash_build_command_with_words, bash_build_word_lit, bash_build_exit } +import gunbc.guarantee_probe_corpus { census_blocking_count_for_class, census_blocking_count_for_subject } +import gunbc.live_deploy.role_singleton_convergence { + RoleSingletonConvergence, + RoleSingletonNoop, + RoleSingletonInstall, + RoleSingletonRefused, + RoleSingletonMoveUnimplemented, + RoleSingletonRemovalUnimplemented, + RoleSingletonDesiredRealizationRefused, + RoleSingletonHostUnobservable, + RoleSingletonHoldersAmbiguous, + RoleSingletonRealizedOnTwoHosts, + RoleSingletonMisplaced, + RoleSingletonTwoOnOneHost, + RoleSingletonBrokerWithoutStore, + RoleSingletonPartial, + RoleSingletonUnmarked, + RoleSingletonMarkerMismatched, + SightedMismatchedMarker, + role_singleton_refusal_detail, + RoleSingletonStaleSpec, + RoleSingletonDeploymentUnrostered, + RoleSingletonSighting, + SightedDeploymentMember, + SightedBroker, + FabricUnitMember, + StoreMarkerMember, + HostRoleSingletonReading, + HostRoleSingletonsRead, + HostRoleSingletonsUnobservable, + RoleSingletonAdmissionStanding, + RoleSingletonAdmissible, + RoleSingletonInadmissible, + classify_role_singleton_admission, + role_singleton_convergence, + role_singleton_observation_of, + local_only_reading_source, + nonlocal_host_unobservable_cause, + marker_sighting_of, + MarkerSighting, + MarkerSightingAbsent, + MarkerSightingOf, + MarkerSightingUnobservable, + MarkerRead, + MarkerReadBack, + MarkerReadRefused, +} +import gunbc.live_deploy.member_observe { EntryPresent, EntryAbsent, EntryPresenceIndeterminate } +import gunbc.host_effect { SshShell } +import gunbc.managed_directory { managed_directory_admits_reader, managed_directory_permissions } +import gunbc.fleet_posix_accounts { srv1_ci_deploy_runner_principal } +import gunbc.fleet_converge_workflow { fleet_converge_gcp_iam_converge_job_under } +import gunbc.live_deploy.emit { approval_broker_base_url_environment, approval_broker_base_url_unbound_marker } +import gunbc.auth.approval_broker_endpoint { approval_broker_base_url_env_name, approval_broker_serving_base_url_of, ApprovalBrokerAddressResolved, ApprovalBrokerAddressRefused } +import extdeps.systemd.unit_file { SystemdServiceDirective, Environment } +import extdeps.github.actions { SelfHosted, RunStep } +import gunbc.fabric_storage_placement { + FabricStoragePlacementRefused, + fabric_storage_placement_under, + fabric_storage_store_directories_for, + fabric_storage_door_socket, + fabric_storage_ownership_marker_path, + fabric_storage_ownership_marker_content, + fabric_storage_ownership_marker_spec, + fabric_storage_marker_verdict, + FabricStorageOwnershipMarkerSpec, + FabricStorageMarkerReading, + FabricStorageMarkerExact, + FabricStorageMarkerMismatched, + fabric_storage_control_directory, + fabric_storage_store_root_directory, + fabric_storage_marker_observer, +} +import gunbc.auth.approval_request_client { + ApprovalLoopbackOriginResolved, + ApprovalLoopbackOriginRefused, + approval_loopback_origin_under, + approval_loopback_origin_of, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -131,3 +241,613 @@ test fn exactly_one_dashboard_instance_derives_prod_risk_and_it_is_the_daily_wor test fn the_wrong_instance_failure_mode_row_cites_this_red() -> Bool { any(a_test_deployment_acts_on_the_prod_deployments_state.evidence, r => r.decl_name == "a_test_deployment_refuses_rather_than_resolving_the_prod_bindings") } + +// โ”€โ”€ The role-following singletons, at the TRANSITION boundary โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Every claim below reads what an apply would actually be handed: the roster's specs built under a +// supplied realization (deployment_spec_for), the convergence decision over supplied per-host +// readings, and the emitted GCP IAM job. The production entry points run the same functions over +// prod_role_selection and a real observation; the positive control at the end is over those. +fn fx_moved() -> ProdRoleRealization { + prod_role_actuated_realization_in(selection: ProdRoleHeldBy { deployment: srv2_deploy_deployment }) +} + +fn fx_unheld() -> ProdRoleRealization { + prod_role_actuated_realization_in(selection: NoProdRole) +} + +fn roster_specs_under(realization: ProdRoleRealization) -> List { + map(desired_deployment_roster(), r => deployment_spec_for(realization: realization, instance: r.instance, target: r.target, names: r.names)) +} + +fn spec_store_directory_count(spec: DeploymentSpec) -> Int { + count(spec.steps |> filter(s => match s { + Dependency { step } => match step.kind { FabricStorageStoreDirectory => true _ => false } + ArtifactStep { step: _ } => false + })) +} + +fn spec_store_artifact_count(spec: DeploymentSpec) -> Int { + count(spec.steps |> filter(s => match s { + ArtifactStep { step } => match step.kind { FabricStorageServeUnit => true FabricStorageServeMapping => true _ => false } + Dependency { step: _ } => false + })) +} + +fn spec_holds_no_singleton(spec: DeploymentSpec) -> Bool { + spec_store_directory_count(spec: spec) == 0 + && spec_store_artifact_count(spec: spec) == 0 + && !deployment_owns_the_host_approval_broker(spec: spec) + && match deployment_fabric_storage_door_socket(spec: spec) { Absent => true Present { value: _ } => false } +} + +fn is_srv2_deploy(spec: DeploymentSpec) -> Bool { + (spec.instance_id as String) == (srv2_deploy_dashboard_instance().instance_id as String) +} + +// THE DISCRIMINATING RED. Move the role to srv2-deploy: the srv2 spec carries EVERY store directory, +// the unit and route, the broker, and its own door socket; both srv1 specs carry none of them. A pin +// back to srv1 by name, or directories left on the srv1 target, fails it. +test fn moving_the_prod_role_moves_every_role_singleton_member_to_the_new_holders_spec() -> Bool { + let specs = roster_specs_under(realization: fx_moved()) + let srv2 = specs |> filter(s => is_srv2_deploy(spec: s)) + let others = specs |> filter(s => !is_srv2_deploy(spec: s)) + let want_dirs = count(fabric_storage_store_directories_for(instance: srv2_deploy_dashboard_instance())) + count(srv2) == 1 + && count(others) > 0 + && want_dirs > 0 + && all(srv2, s => + spec_store_directory_count(spec: s) == want_dirs + && spec_store_artifact_count(spec: s) == 2 + && deployment_owns_the_host_approval_broker(spec: s) + && match deployment_fabric_storage_door_socket(spec: s) { + Present { value: k } => (k as String) == (fabric_storage_door_socket(instance: srv2_deploy_dashboard_instance()) as String) + Absent => false + } + ) + && all(others, s => spec_holds_no_singleton(spec: s)) +} + +// UNDER NoProdRole NO TARGET GETS A DIRECTORY, A UNIT, A ROUTE OR THE BROKER, and every role-follower +// refuses with the realization's typed cause. +test fn under_no_prod_role_no_spec_holds_a_role_singleton_and_every_follower_refuses() -> Bool { + let unheld = fx_unheld() + let placement_refused = match fabric_storage_placement_under(realization: unheld) { + FabricStoragePlacementRefused { cause: ProdRoleUnheld } => true + _ => false + } + let origin_refused = match approval_loopback_origin_under(realization: unheld) { + ApprovalLoopbackOriginRefused { cause: ProdRoleUnheld } => true + _ => false + } + all(roster_specs_under(realization: unheld), s => spec_holds_no_singleton(spec: s)) + && placement_refused + && origin_refused +} + +// A HOLDER WITH NO REALIZATION REFUSES TOO, rather than resolving "the first instance". +test fn a_prod_role_holder_with_no_instance_refuses_as_unrealized() -> Bool { + let ghost = deployment_id(name: "no-such-deployment" as NonEmptyStr) + match prod_role_realization_in(selection: ProdRoleHeldBy { deployment: ghost }, instances: public_builtin_dashboard_instances()) { + ProdRoleRealizationRefused { cause: ProdRoleHolderUnrealized { deployment: d } } => d == ghost + _ => false + } +} + +// โ”€โ”€ The convergence transitions, over supplied per-host sightings โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Sightings are supplied at the adjudication boundary (DESIGN 3: a witness discriminates at one +// interface); the production source that produces them is local_only_reading_source, and the +// positive control below runs the real decision over the real realization. +fn full_on(host: String, d: DeploymentId) -> List { + [ + SightedDeploymentMember { host: host as NonEmptyStr, deployment: d, member: FabricUnitMember }, + SightedDeploymentMember { host: host as NonEmptyStr, deployment: d, member: StoreMarkerMember }, + SightedBroker { host: host as NonEmptyStr }, + ] +} + +fn marker_only_on(host: String, d: DeploymentId) -> List { + [SightedDeploymentMember { host: host as NonEmptyStr, deployment: d, member: StoreMarkerMember }] +} + +fn read_on(host: String, sightings: List) -> HostRoleSingletonReading { + HostRoleSingletonsRead { host: host as NonEmptyStr, sightings: sightings } +} + +fn empty_world() -> List { + [read_on(host: "srv1", sightings: []), read_on(host: "srv2", sightings: [])] +} + +fn srv1_holds() -> List { + [read_on(host: "srv1", sightings: full_on(host: "srv1", d: srv1_daily_workspace_deployment)), read_on(host: "srv2", sightings: [])] +} + +fn decide(desired: ProdRoleRealization, readings: List) -> RoleSingletonConvergence { + let rows = desired_deployment_roster() + role_singleton_convergence(desired: desired, observed: role_singleton_observation_of(readings: readings, rows: rows), rows: rows) +} + +fn refused_with(c: RoleSingletonConvergence, label: String) -> Bool { + match c { + RoleSingletonRefused { cause } => match cause { + RoleSingletonMoveUnimplemented { observed: _, desired: _ } => label == "move" + RoleSingletonRemovalUnimplemented { observed: _ } => label == "removal" + RoleSingletonDesiredRealizationRefused { cause: _ } => label == "desired" + RoleSingletonHostUnobservable { host: _, cause: _ } => label == "unobservable" + RoleSingletonHoldersAmbiguous { observed: _ } => label == "ambiguous" + RoleSingletonRealizedOnTwoHosts { deployment: _, hosts: _ } => label == "two-hosts" + RoleSingletonMisplaced { deployment: _, target: _, observed_on: _ } => label == "misplaced" + RoleSingletonTwoOnOneHost { host: _, deployments: _ } => label == "two-on-one" + RoleSingletonBrokerWithoutStore { host: _ } => label == "broker-alone" + RoleSingletonPartial { deployment: _, host: _, missing: _ } => label == "partial" + RoleSingletonUnmarked { deployment: _, host: _, marker: _ } => label == "unmarked" + RoleSingletonMarkerMismatched { deployment: _, host: _, field: _, expected: _, observed: _ } => label == "marker-mismatch" + RoleSingletonDeploymentUnrostered { instance_id: _, detail: _ } => label == "unrostered" + RoleSingletonStaleSpec { instance_id: _, detail: _ } => label == "stale" + } + RoleSingletonNoop => false + RoleSingletonInstall { holder: _ } => false + } +} + +test fn a_moved_role_with_the_old_holder_still_observed_refuses_as_a_move() -> Bool { + refused_with(c: decide(desired: fx_moved(), readings: srv1_holds()), label: "move") +} + +test fn no_prod_role_with_a_holder_observed_refuses_as_a_removal() -> Bool { + refused_with(c: decide(desired: fx_unheld(), readings: srv1_holds()), label: "removal") +} + +// UNOBSERVED IS NOT ABSENT: with srv1 unreadable and srv2 empty, the moved role must NOT install. +test fn an_unobservable_roster_host_refuses_rather_than_reading_as_absent() -> Bool { + refused_with(c: decide(desired: fx_moved(), readings: [HostRoleSingletonsUnobservable { host: "srv1" as NonEmptyStr, cause: "ssh refused" }, read_on(host: "srv2", sightings: [])]), label: "unobservable") +} + +test fn nothing_observed_and_a_valid_holder_installs_on_that_holder() -> Bool { + match decide(desired: fx_moved(), readings: empty_world()) { + RoleSingletonInstall { holder } => holder.instance.deployment == srv2_deploy_deployment + _ => false + } +} + +test fn exactly_one_complete_realization_of_the_desired_holder_on_its_target_is_a_noop() -> Bool { + match decide(desired: fx_moved(), readings: [read_on(host: "srv1", sightings: []), read_on(host: "srv2", sightings: full_on(host: "srv2", d: srv2_deploy_deployment))]) { + RoleSingletonNoop => true + _ => false + } +} + +// ONLY ProdRoleUnheld IS "NO PROD". An unrealized or ambiguous holder refuses as a desired-side +// refusal in an EMPTY world (it must not install) and in an OCCUPIED one (it must not remove). +fn fx_unrealized() -> ProdRoleRealization { + prod_role_realization_in(selection: ProdRoleHeldBy { deployment: deployment_id(name: "no-such-deployment" as NonEmptyStr) }, instances: public_builtin_dashboard_instances()) +} + +fn fx_ambiguous() -> ProdRoleRealization { + prod_role_realization_in(selection: prod_role_selection, instances: [srv1_live_dashboard_instance(), srv1_live_dashboard_instance()]) +} + +test fn an_unrealized_or_ambiguous_holder_refuses_as_desired_side_in_empty_and_occupied_worlds() -> Bool { + refused_with(c: decide(desired: fx_unrealized(), readings: empty_world()), label: "desired") + && refused_with(c: decide(desired: fx_unrealized(), readings: srv1_holds()), label: "desired") + && refused_with(c: decide(desired: fx_ambiguous(), readings: empty_world()), label: "desired") + && refused_with(c: decide(desired: fx_ambiguous(), readings: srv1_holds()), label: "desired") +} + +// MULTIPLICITY IS ADJUDICATED BEFORE ANY DEDUPLICATION: the same deployment on two hosts, a deployment +// realized off its own target, and two deployments' stores on one host each refuse. +test fn the_same_deployment_on_two_hosts_refuses() -> Bool { + refused_with(c: decide(desired: prod_role_actuated_realization(), readings: [ + read_on(host: "srv1", sightings: full_on(host: "srv1", d: srv1_daily_workspace_deployment)), + read_on(host: "srv2", sightings: full_on(host: "srv2", d: srv1_daily_workspace_deployment)), + ]), label: "two-hosts") +} + +test fn a_deployment_targeted_at_srv2_but_realized_on_srv1_refuses() -> Bool { + refused_with(c: decide(desired: fx_moved(), readings: [ + read_on(host: "srv1", sightings: full_on(host: "srv1", d: srv2_deploy_deployment)), + read_on(host: "srv2", sightings: []), + ]), label: "misplaced") +} + +test fn two_deployments_stores_on_one_host_refuse() -> Bool { + refused_with(c: decide(desired: prod_role_actuated_realization(), readings: [ + read_on(host: "srv1", sightings: concat(full_on(host: "srv1", d: srv1_daily_workspace_deployment), marker_only_on(host: "srv1", d: srv1_lab_codex_feedback_deployment))), + read_on(host: "srv2", sightings: []), + ]), label: "two-on-one") +} + +test fn two_observed_holders_or_a_broker_without_a_store_refuse() -> Bool { + refused_with(c: decide(desired: fx_moved(), readings: [ + read_on(host: "srv1", sightings: full_on(host: "srv1", d: srv1_daily_workspace_deployment)), + read_on(host: "srv2", sightings: full_on(host: "srv2", d: srv2_deploy_deployment)), + ]), label: "ambiguous") + && refused_with(c: decide(desired: fx_moved(), readings: [read_on(host: "srv1", sightings: [SightedBroker { host: "srv1" as NonEmptyStr }]), read_on(host: "srv2", sightings: [])]), label: "broker-alone") +} + +// AN ABSENT UNIT IS NOT AN ABSENT STORE. The marker keeps an old store visible: during a move and +// under NoProdRole it refuses; a store without its broker and an install interrupted after the +// directories (marker) but before the units are partial and refuse -- never adopted, never installed over. +test fn an_old_store_with_no_unit_or_a_partial_realization_refuses() -> Bool { + let old_store = [read_on(host: "srv1", sightings: marker_only_on(host: "srv1", d: srv1_daily_workspace_deployment)), read_on(host: "srv2", sightings: [])] + refused_with(c: decide(desired: fx_moved(), readings: old_store), label: "move") + && refused_with(c: decide(desired: fx_unheld(), readings: old_store), label: "removal") + && refused_with(c: decide(desired: fx_moved(), readings: [read_on(host: "srv1", sightings: []), read_on(host: "srv2", sightings: [ + SightedDeploymentMember { host: "srv2" as NonEmptyStr, deployment: srv2_deploy_deployment, member: FabricUnitMember }, + SightedDeploymentMember { host: "srv2" as NonEmptyStr, deployment: srv2_deploy_deployment, member: StoreMarkerMember }, + ])]), label: "partial") + && refused_with(c: decide(desired: fx_moved(), readings: [read_on(host: "srv1", sightings: []), read_on(host: "srv2", sightings: marker_only_on(host: "srv2", d: srv2_deploy_deployment))]), label: "partial") +} + + + + + + + + + + + + + + + + + +// โ”€โ”€ The admission: the decision is bound to the spec that acts โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +fn spec_of(realization: ProdRoleRealization, id: String) -> List { + roster_specs_under(realization: realization) |> filter(s => (s.instance_id as String) == id) +} + +fn admission_refused_as_stale(r: RoleSingletonAdmissionStanding) -> Bool { + match r { + RoleSingletonInadmissible { cause } => match cause { RoleSingletonStaleSpec { instance_id: _, detail: _ } => true _ => false } + RoleSingletonAdmissible { decision: _ } => false + } +} + +test fn a_stale_spec_is_refused_before_member_planning() -> Bool { + let live_id = srv1_live_dashboard_instance().instance_id as String + let lab_id = srv1_lab_dashboard_instance().instance_id as String + let srv2_id = srv2_deploy_dashboard_instance().instance_id as String + let stale_srv1 = spec_of(realization: prod_role_actuated_realization(), id: live_id) + let moved_install = decide(desired: fx_moved(), readings: empty_world()) + let unheld_noop = decide(desired: fx_unheld(), readings: empty_world()) + all(stale_srv1, s => admission_refused_as_stale(r: classify_role_singleton_admission(spec: s, desired: fx_moved(), decision: moved_install))) + && all(stale_srv1, s => admission_refused_as_stale(r: classify_role_singleton_admission(spec: s, desired: fx_unheld(), decision: unheld_noop))) + && all(spec_of(realization: fx_moved(), id: lab_id), s => admission_refused_as_stale(r: classify_role_singleton_admission(spec: s, desired: fx_moved(), decision: moved_install))) + && all(spec_of(realization: fx_moved(), id: srv2_id), s => match classify_role_singleton_admission(spec: s, desired: fx_moved(), decision: moved_install) { + RoleSingletonAdmissible { decision: _ } => (s.instance_id as String) == srv2_id + RoleSingletonInadmissible { cause: _ } => false + }) + && count(stale_srv1) == 1 +} + +// โ”€โ”€ Role eligibility: only a row of the one desired roster can hold an actuated role โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +test fn a_holder_with_no_desired_row_is_unactuatable_and_holds_nothing() -> Bool { + let mac = prod_role_actuated_realization_in(selection: ProdRoleHeldBy { deployment: macbook_local_deployment }) + (match mac { ProdRoleRealizationRefused { cause: ProdRoleHolderUnactuatable { deployment: d } } => d == macbook_local_deployment _ => false }) + && all(roster_specs_under(realization: mac), s => spec_holds_no_singleton(spec: s)) + && refused_with(c: decide(desired: mac, readings: empty_world()), label: "desired") +} + +// โ”€โ”€ The emitted GCP IAM job follows the holder โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +test fn the_gcp_iam_job_runs_on_the_holders_host_and_is_refused_under_no_prod_role() -> Bool { + let moved = fleet_converge_gcp_iam_converge_job_under(realization: fx_moved()) + let unheld = fleet_converge_gcp_iam_converge_job_under(realization: fx_unheld()) + let srv2_host = srv2_deploy_dashboard_instance().host_identity as String + let srv1_host = srv1_live_dashboard_instance().host_identity as String + let moved_labels = match moved.runner { SelfHosted { labels } => labels _ => [] } + let unheld_labels = match unheld.runner { SelfHosted { labels } => labels _ => [] } + let unheld_refuses = any(unheld.steps, st => match st { + RunStep { name: _, id: _, run, shell: _, env: _, working_directory: _, if_condition: _, continue_on_error: _, timeout_minutes: _ } => + string_contains(s: run, pattern: "REFUSED") && string_contains(s: run, pattern: "NoProdRole") + _ => false + }) + any(moved_labels, l => l == srv2_host) + && !any(moved_labels, l => l == srv1_host) + && !any(unheld_labels, l => l == srv1_host || l == srv2_host) + && count(unheld.steps) == 1 + && unheld_refuses +} + +// THE REFUSED JOB'S SCRIPT IS THE MODELED EMISSION, byte for byte: an echo of the typed cause redirected +// to stderr and exit 1, built here as bash_build nodes and emitted by v2.workflow.bash_emit -- so a +// hand-joined shell string in the job, or any drift from the modeled route, is red. +fn expected_gcp_iam_refusal_script(detail: String) -> String { + match bash_emit_stmts(statements: [ + bash_build_with_redir( + inner: bash_build_command_with_words(words: [ + bash_build_word_lit(text: "echo"), + bash_build_word_lit(text: join(["REFUSED gunbc.fleet_converge_workflow gcp-iam-converge: ", detail], "")), + ]), + redir: ^bash_production_fold_redir_stdout_to_stderr, + ), + bash_build_exit(code: "1"), + ], set_e: true) { + Accepted { value: v, diagnostics: _ } => v + Rejected { diagnostics: _ } => "" + } +} + +test fn the_refused_gcp_iam_job_script_is_the_bash_emit_emission() -> Bool { + let expected = expected_gcp_iam_refusal_script(detail: prod_role_realization_refusal_detail(cause: ProdRoleUnheld)) + let unheld = fleet_converge_gcp_iam_converge_job_under(realization: fx_unheld()) + expected != "" + && count(unheld.steps) == 1 + && all(unheld.steps, st => match st { + RunStep { name: _, id: _, run, shell: _, env: _, working_directory: _, if_condition: _, continue_on_error: _, timeout_minutes: _ } => run == expected + _ => false + }) +} + +// โ”€โ”€ THE POSITIVE CONTROL, over the REAL selection and roster โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Today the daily-workspace deployment (srv1-live) holds the role: its spec carries every singleton +// member, srv1's lab and srv2-deploy carry none, the GCP IAM job runs on srv1, the real realization +// with one complete srv1 realization observed is a Noop, and the srv1 spec built now is admitted. +test fn the_real_selection_places_every_singleton_on_its_holder_and_converges_as_noop() -> Bool { + let real = prod_role_actuated_realization() + let specs = roster_specs_under(realization: real) + let live_id = srv1_live_dashboard_instance().instance_id as String + let live = specs |> filter(s => (s.instance_id as String) == live_id) + let others = specs |> filter(s => !((s.instance_id as String) == live_id)) + let job_labels = match fleet_converge_gcp_iam_converge_job_under(realization: real).runner { SelfHosted { labels } => labels _ => [] } + let noop = decide(desired: real, readings: srv1_holds()) + let is_noop = match noop { RoleSingletonNoop => true _ => false } + count(live) == 1 + && all(live, s => spec_store_directory_count(spec: s) > 0 && spec_store_artifact_count(spec: s) == 2 && deployment_owns_the_host_approval_broker(spec: s) + && match classify_role_singleton_admission(spec: s, desired: real, decision: noop) { RoleSingletonAdmissible { decision: _ } => true RoleSingletonInadmissible { cause: _ } => false }) + && all(others, s => spec_holds_no_singleton(spec: s)) + && any(job_labels, l => l == (srv1_live_dashboard_instance().host_identity as String)) + && is_noop +} + +// โ”€โ”€ The broker is GIVEN its address, at the emitted-unit boundary โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// The broker unit's Environment= line is built from the spec's holder token: srv2's base URL when the +// role moves to srv2-deploy, srv1's under today's selection, and the refusal marker -- which the +// broker's own read rejects as not an https URL -- for a spec holding no role. +fn broker_env_assignment(spec: DeploymentSpec) -> String { + match approval_broker_base_url_environment(spec: spec) { + Environment { assignment } => assignment as String + _ => "" + } +} + +fn bound(value: String) -> String { + join([approval_broker_base_url_env_name as String, "=", value], "") +} + +test fn the_broker_unit_is_given_the_holders_base_url_and_a_non_holder_gets_a_refusal() -> Bool { + let moved = roster_specs_under(realization: fx_moved()) + let real = roster_specs_under(realization: prod_role_actuated_realization()) + let srv2_moved = moved |> filter(s => is_srv2_deploy(spec: s)) + let srv1_moved = moved |> filter(s => (s.instance_id as String) == (srv1_live_dashboard_instance().instance_id as String)) + let live_real = real |> filter(s => (s.instance_id as String) == (srv1_live_dashboard_instance().instance_id as String)) + count(srv2_moved) == 1 && count(srv1_moved) == 1 && count(live_real) == 1 + && all(srv2_moved, s => starts_with(s: broker_env_assignment(spec: s), prefix: bound(value: "https://srv2."))) + && all(live_real, s => starts_with(s: broker_env_assignment(spec: s), prefix: bound(value: "https://srv1."))) + && all(srv1_moved, s => broker_env_assignment(spec: s) == bound(value: approval_broker_base_url_unbound_marker)) + && match approval_broker_serving_base_url_of(bound: Present { value: approval_broker_base_url_unbound_marker }) { ApprovalBrokerAddressRefused { detail: _ } => true _ => false } + && match approval_broker_serving_base_url_of(bound: none) { ApprovalBrokerAddressRefused { detail: _ } => true _ => false } +} + +// โ”€โ”€ The unmarked legacy store waits for the operator โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// A realization that predates the store ownership marker -- today's srv1 holder, its broker and fabric +// unit present and the marker missing -- REFUSES with RoleSingletonUnmarked, and the refusal names +// exactly what the operator's one-time disposition writes: the marker path, its content (deployment, +// host) and the host. It is the same content the holder spec's own marker ensure writes, so the +// disposition and an install agree byte for byte. Nothing adopts it automatically (gunbc#13124). +fn units_without_marker_on(host: String, d: DeploymentId) -> List { + [SightedDeploymentMember { host: host as NonEmptyStr, deployment: d, member: FabricUnitMember }, SightedBroker { host: host as NonEmptyStr }] +} + +fn spec_marker_content(spec: DeploymentSpec) -> List { + spec.steps |> flat_map(s => match s { + Dependency { step } => match step.subject { + EnsuredOwnershipMarker { marker } => [marker.content as String] + _ => [] + } + ArtifactStep { step: _ } => [] + }) +} + +test fn an_unmarked_legacy_store_refuses_naming_the_operators_disposition() -> Bool { + let real = prod_role_actuated_realization() + let live = srv1_live_dashboard_instance() + let want_path = fabric_storage_ownership_marker_path(instance: live) as String + let want_content = fabric_storage_ownership_marker_content(instance: live) as String + let decision = decide(desired: real, readings: [read_on(host: "srv1", sightings: units_without_marker_on(host: "srv1", d: srv1_daily_workspace_deployment)), read_on(host: "srv2", sightings: [])]) + let names_it = match decision { + RoleSingletonRefused { cause } => match cause { + RoleSingletonUnmarked { deployment, host, marker } => + deployment == srv1_daily_workspace_deployment && (host as String) == "srv1" + && (marker.path as String) == want_path && (marker.content as String) == want_content + && (marker.owner as String) == "briansrls" && (marker.group as String) == "briansrls" + && string_contains(s: role_singleton_refusal_detail(cause: cause), pattern: want_path) + && string_contains(s: role_singleton_refusal_detail(cause: cause), pattern: want_content) + _ => false + } + _ => false + } + names_it + && string_contains(s: want_content, pattern: "deployment=srv1-daily-workspace") + && string_contains(s: want_content, pattern: "host=srv1") + && all(spec_of(realization: real, id: live.instance_id as String), s => spec_marker_content(spec: s) == [want_content]) + && refused_with(c: decide(desired: fx_moved(), readings: [read_on(host: "srv1", sightings: []), read_on(host: "srv2", sightings: units_without_marker_on(host: "srv2", d: srv2_deploy_deployment))]), label: "unmarked") +} + +// โ”€โ”€ A token cannot be forged or replayed โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// The mint is sealed to the observing assessor, the assessor to the two canonical apply compositions, +// and the token's readers and every admitted apply step to those same compositions. So an outside +// module can neither mint an admission from a fabricated decision, obtain one from the assessor, nor +// carry one into an apply step later -- each refused at compile time as a located diagnostic. +// ONE probe source carries every forged deploy route, one function each, so the floor compiles it +// once (the census is memoized by source) and each claim reads ITS refusal by class AND subject. +data forged_deploy_routes_source: String = "module probe_forged_deploy_routes\nimport gunbc.live_deploy.role_singleton_convergence { mint_role_singleton_admission, RoleSingletonNoop, RoleSingletonAdmission, assess_and_admit, assess_and_admit_rostered, RoleSingletonAdmissionResult, admitted_spec }\nimport gunbc.live_deploy.desired { deployment_spec_srv1, srv1_production_desired_deployment }\nimport gunbc.live_deploy.spec { DeploymentSpec }\nimport gunbc.host_effect { LocalShell, HostEffect, LiveDeployApply }\nimport gunbc.live_deploy.apply { live_deploy_apply_member_plan_via_transport }\nimport gunbc.live_deploy.apply_warrant { warrant_live_deploy_apply, AdmittedLiveDeployApply }\nimport gunbc.live_deploy.candidate_identity { CandidateRelease }\nimport gunbc.live_deploy.member_identity { ReleaseEffectPlan }\nimport gunbc.ci_deploy_access { DeployAccess }\nfn forge_admission() -> RoleSingletonAdmission { mint_role_singleton_admission(spec: deployment_spec_srv1(), decision: RoleSingletonNoop) }\nfn obtain_admission() -> RoleSingletonAdmissionResult { assess_and_admit(deployment: srv1_production_desired_deployment(), own: LocalShell) }\nfn replay_admission(a: RoleSingletonAdmission) -> DeploymentSpec { admitted_spec(admission: a) }\nfn alter_spec() -> RoleSingletonAdmissionResult { assess_and_admit_rostered(spec: deployment_spec_srv1(), own: LocalShell) }\nfn bypass_member_plan(c: CandidateRelease, a: DeployAccess) -> Bool { let r = live_deploy_apply_member_plan_via_transport(spec: deployment_spec_srv1(), candidate: c, transport: LocalShell, access: a)\n true }\nfn forge_warrant(c: CandidateRelease, p: ReleaseEffectPlan) -> AdmittedLiveDeployApply { warrant_live_deploy_apply(spec: deployment_spec_srv1(), candidate: c, plan: p) }\nfn forge_warrant_literal(c: CandidateRelease, p: ReleaseEffectPlan) -> HostEffect { LiveDeployApply { warrant: AdmittedLiveDeployApply { spec: deployment_spec_srv1(), candidate: c, plan: p } } }\n" + +data canonical_apply_source: String = "module probe_canonical_apply\nimport gunbc.live_deploy.apply { live_deploy_apply_with_access }\nimport gunbc.live_deploy.desired { srv1_production_desired_deployment }\nimport gunbc.live_deploy.candidate_identity { CandidateRelease }\nimport gunbc.ci_deploy_access { DeployAccess }\nfn act(c: CandidateRelease, a: DeployAccess) -> Bool { let r = live_deploy_apply_with_access(deployment: srv1_production_desired_deployment(), access: a, candidate: c)\n true }\n" + +data spec_to_production_apply_source: String = "module probe_spec_to_production_apply\nimport gunbc.live_deploy.apply { live_deploy_apply_with_access }\nimport gunbc.live_deploy.desired { deployment_spec_srv1 }\nimport gunbc.live_deploy.candidate_identity { CandidateRelease }\nimport gunbc.ci_deploy_access { DeployAccess }\nfn hand(c: CandidateRelease, a: DeployAccess) -> Bool { let r = live_deploy_apply_with_access(spec: deployment_spec_srv1(), access: a, candidate: c)\n true }\n" + +fn forged_route_refusals(class: String, subject: String) -> Int { + match census_blocking_count_for_subject(census: compile_dag_diagnostic_census(forged_deploy_routes_source), wanted_class: class, wanted_subject: subject) { + Present { value: n } => n + Absent => 0 + } +} + +fn admission_refusals_in(source: String) -> Int { + match census_blocking_count_for_class(census: compile_dag_diagnostic_census(source), wanted: "ConstructorCallAdmissionRefused") { + Present { value: n } => n + Absent => 0 + } +} + +fn class_refusals_in(source: String, wanted: String) -> Int { + match census_blocking_count_for_class(census: compile_dag_diagnostic_census(source), wanted: wanted) { + Present { value: n } => n + Absent => 0 + } +} + +// NO CALLER-SUPPLIED SPEC REACHES THE ASSESSOR. The production apply takes a DesiredDeployment and +// the spec is built from its bound roster row inside assess_and_admit; (a) handing a spec to the +// production entry does not type (it has no spec parameter), and (b) handing one to the +// spec-taking assessor step is refused at its seal, while (c) the canonical row-taking call compiles. +// THE ROUTE, NOT ONLY A REFUSAL. The probe hands a spec to the UNSEALED production entry +// (live_deploy_apply_with_access), so no seal can refuse it: the only refusal it can draw is the label +// wall's, because the entry has no spec parameter. Zero seal refusals is asserted beside it, so the +// claim cannot pass on a ConstructorCallAdmissionRefused standing in for the missing parameter -- which +// is what a probe aimed at the sealed assessor itself drew (measured: only ConstructorCallAdmissionRefused +// on assess_and_admit, never the label wall). One compile per claim. +test fn no_caller_supplied_spec_reaches_the_role_singleton_assessment() -> Bool { + class_refusals_in(source: spec_to_production_apply_source, wanted: "CallArgumentNameUnknown") >= 1 + && admission_refusals_in(source: spec_to_production_apply_source) == 0 +} + +test fn an_altered_spec_is_refused_at_the_assessor_seal() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "assess_and_admit_rostered") >= 1 +} + +test fn no_outside_module_can_mint_obtain_or_replay_an_admission() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "mint_role_singleton_admission") >= 1 + && forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "assess_and_admit") >= 1 + && forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "admitted_spec") >= 1 +} + + +// THE EFFECT DESTINATION IS CLOSED. (a) an outside call to the member-plan step and (b) an outside +// warrant -- minted or written as a literal inside a LiveDeployApply -- are each refused at compile +// time; (c) the canonical assess-and-act entry still compiles with no admission refusal. +test fn the_realizer_receives_an_apply_only_through_the_assessed_composition() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "live_deploy_apply_member_plan_via_transport") >= 1 +} + +test fn no_outside_module_can_mint_a_warrant() -> Bool { + forged_route_refusals(class: "ConstructorCallAdmissionRefused", subject: "warrant_live_deploy_apply") >= 1 +} + +test fn no_outside_module_can_write_a_warrant_literal() -> Bool { + forged_route_refusals(class: "SoleConstructorViolation", subject: "AdmittedLiveDeployApply") >= 1 +} + +// THE ONE CANONICAL-PATH CONTROL: the row-taking production apply compiles with no admission refusal, +// so the REDs above are not a compiler that refuses every call into the apply module. +test fn the_canonical_row_taking_apply_compiles_without_admission_refusal() -> Bool { + admission_refusals_in(source: canonical_apply_source) == 0 +} + +// โ”€โ”€ The marker has one authority, and only an exact readback is the marker โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Each field mutated alone -- path, content, mode, owner, group -- is a mismatch named by that field; +// the unmutated reading is exact; and a mismatched marker sighted on the holder's host refuses as +// RoleSingletonMarkerMismatched, distinct from an absent one (RoleSingletonUnmarked). +fn exact_reading(m: FabricStorageOwnershipMarkerSpec) -> FabricStorageMarkerReading { + FabricStorageMarkerReading { path: m.path, content: m.content as String, mode: "644", owner: m.owner as String, group: m.group as String } +} + +fn mismatched_on(m: FabricStorageOwnershipMarkerSpec, r: FabricStorageMarkerReading, field: String) -> Bool { + match fabric_storage_marker_verdict(spec: m, reading: r) { + FabricStorageMarkerMismatched { field: f, expected: _, observed: _ } => (f as String) == field + FabricStorageMarkerExact => false + } +} + +test fn every_marker_field_is_read_back_and_any_mismatch_refuses() -> Bool { + let m = fabric_storage_ownership_marker_spec(instance: srv1_live_dashboard_instance()) + let r = exact_reading(m: m) + let exact = match fabric_storage_marker_verdict(spec: m, reading: r) { FabricStorageMarkerExact => true _ => false } + let mismatch_world = [read_on(host: "srv1", sightings: [ + SightedDeploymentMember { host: "srv1" as NonEmptyStr, deployment: srv1_daily_workspace_deployment, member: FabricUnitMember }, + SightedBroker { host: "srv1" as NonEmptyStr }, + SightedMismatchedMarker { host: "srv1" as NonEmptyStr, deployment: srv1_daily_workspace_deployment, field: "owner", expected: "briansrls", observed: "root" }, + ]), read_on(host: "srv2", sightings: [])] + exact + && mismatched_on(m: m, r: FabricStorageMarkerReading { path: "/opt/gunbc/fabric-storage/elsewhere" as NonEmptyStr, content: r.content, mode: r.mode, owner: r.owner, group: r.group }, field: "path") + && mismatched_on(m: m, r: FabricStorageMarkerReading { path: r.path, content: "deployment=someone-else\nhost=srv1\n", mode: r.mode, owner: r.owner, group: r.group }, field: "content") + && mismatched_on(m: m, r: FabricStorageMarkerReading { path: r.path, content: r.content, mode: "600", owner: r.owner, group: r.group }, field: "mode") + && mismatched_on(m: m, r: FabricStorageMarkerReading { path: r.path, content: r.content, mode: r.mode, owner: "root", group: r.group }, field: "owner") + && mismatched_on(m: m, r: FabricStorageMarkerReading { path: r.path, content: r.content, mode: r.mode, owner: r.owner, group: "root" }, field: "group") + && refused_with(c: decide(desired: prod_role_actuated_realization(), readings: mismatch_world), label: "marker-mismatch") +} + +// โ”€โ”€ The marker is readable by the principal that observes it, from a directory that is not the store โ”€ +// THE READ AUTHORITY, with the real observer and the derived modes: the observer is the deploy job +// principal (srv1_ci_deploy_runner_principal job_user), the control directory's derived mode admits it +// to list and traverse, and the store root's derived mode does not -- which is why the marker is not in +// the store root, and why the store root was not widened to admit it. The observer is admitted as the +// directory's GROUP, so the derived mode grants the other class nothing: the marker is not published +// to every account on the host. +test fn the_marker_observer_reads_the_control_directory_and_not_the_store_root() -> Bool { + let i = srv1_live_dashboard_instance() + (fabric_storage_marker_observer.name as String) == (srv1_ci_deploy_runner_principal.job_user as String) + && managed_directory_admits_reader(d: fabric_storage_control_directory(instance: i), who: fabric_storage_marker_observer) + && !managed_directory_permissions(d: fabric_storage_control_directory(instance: i)).mode.other.read + && !managed_directory_permissions(d: fabric_storage_control_directory(instance: i)).mode.other.execute + && !managed_directory_admits_reader(d: fabric_storage_store_root_directory(instance: i), who: fabric_storage_marker_observer) +} + +// THREE OUTCOMES, NEVER TWO: the observer's exact readback is the holder's marker, a listing without it +// is absent, and a listing or readback the observer is denied is unobservable -- never absent. +fn exact_marker_read(path: NonEmptyStr) -> MarkerRead { + MarkerReadBack { reading: exact_reading(m: fabric_storage_ownership_marker_spec(instance: srv1_live_dashboard_instance())) } +} + +fn denied_marker_read(path: NonEmptyStr) -> MarkerRead { + MarkerReadRefused { cause: join([path as String, ": Permission denied"], "") } +} + +test fn the_marker_readback_has_three_distinct_outcomes() -> Bool { + let row = srv1_production_desired_deployment() + let exact = match marker_sighting_of(host: "srv1" as NonEmptyStr, r: row, presence: EntryPresent, read_back: exact_marker_read) { + MarkerSightingOf { sighting: SightedDeploymentMember { host: _, deployment: d, member: StoreMarkerMember } } => d == srv1_daily_workspace_deployment + _ => false + } + let absent = match marker_sighting_of(host: "srv1" as NonEmptyStr, r: row, presence: EntryAbsent, read_back: exact_marker_read) { + MarkerSightingAbsent => true + _ => false + } + let unlistable = match marker_sighting_of(host: "srv1" as NonEmptyStr, r: row, presence: EntryPresenceIndeterminate { cause: "listing /opt/gunbc/fabric-storage-control: Permission denied" }, read_back: exact_marker_read) { + MarkerSightingUnobservable { cause: _ } => true + _ => false + } + let unreadable = match marker_sighting_of(host: "srv1" as NonEmptyStr, r: row, presence: EntryPresent, read_back: denied_marker_read) { + MarkerSightingUnobservable { cause: _ } => true + _ => false + } + exact && absent && unlistable && unreadable +} + +// โ”€โ”€ No cross-host transport, whatever credential exists โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// THE D2 INTERIM IS FAIL-CLOSED BY CONSTRUCTION, not by an ungranted grant. The apply's own transport is +// handed a VALID-LOOKING ssh credential to srv2 itself, and srv2 -- a roster host that is not the +// applying host -- is still unobservable with the self-report cause: the source builds no transport for +// a nonlocal host, so no credential reaches it. Reading the nonlocal host is pure, so this claim touches +// no host. gunbc#13339 swaps this refusal for the self-report source. +test fn a_valid_ssh_credential_cannot_make_a_nonlocal_roster_host_observable() -> Bool { + let source = local_only_reading_source(applying: deployment_spec_srv1(), own: SshShell { ssh_host: "srv2" as NonEmptyStr }) + let read = source.read + match read("srv2" as NonEmptyStr, desired_deployment_roster()) { + HostRoleSingletonsUnobservable { host, cause } => (host as String) == "srv2" && cause == nonlocal_host_unobservable_cause + HostRoleSingletonsRead { host: _, sightings: _ } => false + } +} diff --git a/dag/test/claim/devboot_subject_identity_witness_test.dag b/dag/test/claim/devboot_subject_identity_witness_test.dag index 57b9f9a2948..79aab004393 100644 --- a/dag/test/claim/devboot_subject_identity_witness_test.dag +++ b/dag/test/claim/devboot_subject_identity_witness_test.dag @@ -15,6 +15,7 @@ import std.content_hash { import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.host_layout { srv1_devboot_artifact_store_root } import gunbc.live_deploy.spec { + EnsuredOwnershipMarker, DeploymentStep, Dependency, ArtifactStep, @@ -729,6 +730,7 @@ fn devboot_ensured_step_projects_layout(acc: Int, step: DeploymentDependencyStep if (path as String) == (srv1_devboot_artifact_store_root as String) { acc + 1 } else { acc } EnsuredPackage { package } => acc EnsuredManagedHostDirectory { admission: _ } => acc + EnsuredOwnershipMarker { marker: _ } => acc } } diff --git a/dag/test/claim/fabric/fabric_writer_roster_witness_test.dag b/dag/test/claim/fabric/fabric_writer_roster_witness_test.dag index cb9a014ad69..712d541a6f5 100644 --- a/dag/test/claim/fabric/fabric_writer_roster_witness_test.dag +++ b/dag/test/claim/fabric/fabric_writer_roster_witness_test.dag @@ -14,6 +14,7 @@ import gunbc.fabric_writer_roster { fabric_door_writer_admission_over, fabric_door_principal_of, fabric_door_local_roster, FabricDoorWriterAdmitted, FabricDoorWriterRefused, } import gunbc.fabric_storage_placement { fabric_storage_directory, fabric_storage_store_root, fabric_storage_door_directory } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import gunbc.managed_directory { ManagedDirectory, managed_directory_permissions, dependent_class_of, OwnerClass, GroupClass, OtherClass } import gunbc.fleet_posix_accounts { fleet_posix_operator_user, fleet_posix_ci_runner_user } @@ -116,7 +117,7 @@ fn class_writes(d: ManagedDirectory, who: PosixUser) -> Bool { } test fn the_door_admits_the_operator_and_the_ci_runner_locally_and_no_one_else() -> Bool { - let door = fabric_storage_door_directory() + let door = fabric_storage_door_directory(instance: srv1_live_dashboard_instance()) let roster = fabric_door_local_roster() class_traverses(d: door, who: fleet_posix_operator_user) && class_traverses(d: door, who: fleet_posix_ci_runner_user) && class_traverses(d: door, who: outsider()) == false @@ -126,7 +127,7 @@ test fn the_door_admits_the_operator_and_the_ci_runner_locally_and_no_one_else() } test fn the_placed_store_areas_admit_no_writer_but_the_service() -> Bool { - let areas = fabric_storage_file_root(root: fabric_storage_store_root()) + let areas = fabric_storage_file_root(root: fabric_storage_store_root(instance: srv1_live_dashboard_instance())) let objects = fabric_storage_directory(member: "fabric-storage-objects", path: areas.objects, writes: true) let heads = fabric_storage_directory(member: "fabric-storage-heads", path: areas.heads, writes: true) class_writes(d: objects, who: fleet_posix_operator_user) && class_writes(d: heads, who: fleet_posix_operator_user) diff --git a/dag/test/claim/live_deploy/apply_receipt_witness_test.dag b/dag/test/claim/live_deploy/apply_receipt_witness_test.dag index 541da29095d..8f04dfd56a6 100644 --- a/dag/test/claim/live_deploy/apply_receipt_witness_test.dag +++ b/dag/test/claim/live_deploy/apply_receipt_witness_test.dag @@ -20,6 +20,7 @@ fn fixture_dashboard_receipt() -> LiveDeployApplyReceipt { candidate_surface_identity: "0011223344556677", candidate_tree_oid: apply_receipt_fixture_tree, apply_outcome: LiveDeployApplyConverged, + role_singleton_disposition: RoleSingletonsNoop, readiness: LiveDeployReadinessObserved { revision: apply_receipt_fixture_r, surface: "0011223344556677" }, provenance: LiveDeployFleetProvenance { plan_run_id: "1001", apply_run_id: "1002", plan_artifact_hash: "a1b2c3d4e5f60718", plan_receipt_identity: "1111111111111111", apply_receipt_identity: "2222222222222222" }, completed_at: "2026-08-30T12:00:00Z", @@ -45,6 +46,7 @@ test fn witness_dashboard_receipt_not_converged_round_trips_its_reason() -> Bool run: base.run, target: base.target, instance_id: base.instance_id, expected_revision: base.expected_revision, candidate_revision: base.candidate_revision, candidate_surface_identity: base.candidate_surface_identity, candidate_tree_oid: base.candidate_tree_oid, apply_outcome: LiveDeployApplyNotConverged { reason: "terminal no-op: this run installed nothing" }, + role_singleton_disposition: RoleSingletonsNoop, readiness: LiveDeployReadinessRefused { detail: "not observed" }, provenance: base.provenance, completed_at: base.completed_at, } diff --git a/dag/test/claim/live_deploy/emit_test.dag b/dag/test/claim/live_deploy/emit_test.dag index d5693258c52..f7e207fcc28 100644 --- a/dag/test/claim/live_deploy/emit_test.dag +++ b/dag/test/claim/live_deploy/emit_test.dag @@ -10,7 +10,7 @@ import test.claim.live_deploy.apply_emission_projection { pipeline_steps_text, a import test.claim.live_deploy.deploy_release_fixture { deploy_witness_executable_digest, deploy_witness_release_revision, deploy_witness_other_release_revision, deploy_witness_full_install_plan, deploy_witness_release_target } import std.types { Bool, String, List, NonEmptyStr, Int } import std.os.types { Apt } -import gunbc.fabric_storage_placement { fabric_storage_door_socket, fabric_storage_door_directory_path, fabric_storage_https_port, fabric_storage_entry_mode, FabricStorageEntryModeDerived, FabricStorageEntryModeRefused } +import gunbc.fabric_storage_placement { fabric_storage_door_socket, fabric_storage_door_directory_path, fabric_storage_store_root, fabric_storage_https_port, fabric_storage_entry_mode, FabricStorageEntryModeDerived, FabricStorageEntryModeRefused } import extdeps.access.posix { file_mode_octal } import gunbc.fabric_storage_file_store { fabric_storage_file_root } import extdeps.systemd.unit_file { serialize_systemd_unit_file } @@ -27,6 +27,7 @@ import gunbc.live_deploy.spec { deployment_tailnet_door_steps, deployment_approval_broker_front_door_steps, deployment_fabric_storage_serve_endpoint, + deployment_fabric_storage_door_socket, EnsuredPackage, EnsuredManagedHostDirectory, TailscalePackage, @@ -62,7 +63,7 @@ import gunbc.live_deploy.spec { deployment_step_teardown_disposition, gunbc_service_user, } -import gunbc.live_deploy.desired { deployment_spec_srv1, srv1_lab_desired_deployment, selected_deployment_spec } +import gunbc.live_deploy.desired { deployment_spec_srv1, srv1_lab_desired_deployment, selected_deployment_spec, prod_role_actuated_realization } import gunbc.host_layout { srv1_dispatch_attempt_state_root, belt_spawn_workdir_env_var, srv1_attempt_state_directory } import gunbc.managed_directory { managed_directory_mode_octal, @@ -408,6 +409,7 @@ test fn witness_apply_pipeline_carries_the_preamble_after_its_frame() -> Bool { approval_broker: real.approval_broker, names: real.names, instance_id: real.instance_id, + role_singletons: real.role_singletons, } match ci_deploy_srv1_access_or_refusal() { DeployAccessReady { access: a } => @@ -883,9 +885,9 @@ fn spec_owned_paths(spec: DeploymentSpec) -> List { // a socket, which the twin's derived route does not, and the twin must own none. fn live_host_singleton_paths(live: DeploymentSpec) -> List { concat( - deployment_fabric_storage_steps(instance: srv1_live_dashboard_instance(), names: live.names), + deployment_fabric_storage_steps(holding: live.role_singletons, names: live.names), concat( - deployment_approval_broker_front_door_steps(instance: srv1_live_dashboard_instance()), + deployment_approval_broker_front_door_steps(holding: live.role_singletons), deployment_tailnet_door_steps(names: live.names), ), ) |> map(s => s.path as String) @@ -908,13 +910,13 @@ test fn a_twin_deployment_on_one_host_collides_with_nothing() -> Bool { && all(singletons, h => any(live_paths, l => l == h)) && all(live_instance_scoped, l => any(twin_steps, t => t.kind == l.kind)) && all(twin_steps, t => any(live_instance_scoped, l => l.kind == t.kind)) - && count(deployment_fabric_storage_steps(instance: srv1_lab_dashboard_instance(), names: twin.names)) == 0 - && count(deployment_approval_broker_front_door_steps(instance: srv1_lab_dashboard_instance())) == 0 + && count(deployment_fabric_storage_steps(holding: twin.role_singletons, names: twin.names)) == 0 + && count(deployment_approval_broker_front_door_steps(holding: twin.role_singletons)) == 0 && count(deployment_tailnet_door_steps(names: twin.names)) == 0 && all(twin_paths, t => !any(live_paths, l => l == t)) } -data placement_scoped_members_note: String = "THE TWIN MIRRORS PRODUCTION ONE-FOR-ONE EXCEPT FOR THE PLACEMENT'S OWN MEMBERS. The fabric DB endpoint -- its serve unit and its tailscale route -- is a host singleton owned by the one instance that holds the fabric DB placement (gunbc.fabric_storage_placement fabric_storage_placed_on), so production carries exactly those members more than the twin, and the twin none. The relation stays between two live rosters: the difference is named by the function that decides it, not typed as a number." +data placement_scoped_members_note: String = "THE TWIN MIRRORS PRODUCTION ONE-FOR-ONE EXCEPT FOR THE PLACEMENT'S OWN MEMBERS. The fabric DB endpoint -- its serve unit and its tailscale route -- is a host singleton owned by the one instance that holds the fabric DB placement (gunbc.live_deploy.spec RoleSingletonHolding), so production carries exactly those members more than the twin, and the twin none. The relation stays between two live rosters: the difference is named by the function that decides it, not typed as a number." data non_degeneracy_without_transcription_note: String = "THE COUNTS ABOVE STOPPED BEING LITERALS (review, 2026-07-30 โ€” the live-state count-transcription class ruled against on 2026-07-29 and swept in #7464). Five was read off the live owned roster, so authoring a correct sixth member reds a claim that has nothing to say about members, and the reviewer's habit becomes editing the number until the day the number is wrong for a real reason. #7478 adds exactly such a member.\\n\\nDeleting the conjunct outright would have been the wrong fix, which is why this is a replacement rather than a removal: `all` over an empty list is vacuously true, so without a non-degeneracy guard a twin whose owned set collapsed to nothing would satisfy the disjointness claim perfectly. The literal was doing real work in a way its shape disguised.\\n\\nWhat replaces it says the same thing without reading the roster's size into the test: the production set is non-empty, and the twin mirrors it one-for-one. Both survive a member being added โ€” the relation is between two live rosters rather than between one roster and a number a human typed โ€” and the second is strictly stronger than the literal pair, since it would also catch a twin that gained or lost a member production did not." @@ -935,6 +937,7 @@ test fn retract_is_the_reverse_of_what_this_spec_applies() -> Bool { instance: srv1_lab_dashboard_instance(), target: deployment_srv1_target(), names: derived_deployment_names(instance: srv1_lab_dashboard_instance()), + holding: twin.role_singletons, ) let applied_paths = applied |> flat_map(s => match s { ArtifactStep { step } => [step.path as String] @@ -1125,6 +1128,7 @@ fn preserve_removal_client_target() -> DeploymentHostTarget { fn spec_with_target(target: DeploymentHostTarget) -> DeploymentSpec { deployment_spec_for( + realization: prod_role_actuated_realization(), instance: srv1_live_dashboard_instance(), target: target, names: srv1_live_deployment_names(instance: srv1_live_dashboard_instance()), @@ -1153,7 +1157,7 @@ test fn fleet_converge_timer_not_installed_by_deploy_apply() -> Bool { let spec = deployment_spec_srv1() let live = apply_emitted_steps_text(spec: spec, plan: deploy_witness_full_install_plan(spec: spec)) !any( - deployment_owned_steps(instance: srv1_live_dashboard_instance(), names: spec.names), + deployment_owned_steps(instance: srv1_live_dashboard_instance(), names: spec.names, holding: spec.role_singletons), s => match s.kind { FleetConvergeTimerUnit => true _ => false } ) && !string_contains(s: live, pattern: "OnCalendar=*:0/15") @@ -1429,7 +1433,7 @@ test fn the_fabric_storage_areas_are_the_services_and_the_door_admits_the_claim_ let files = fabric_storage_file_root(root: dashboard_instance_fabric_storage_root(instance: srv1_live_dashboard_instance()) as String as NonEmptyStr) let objects = srv1_fabric_area_ensure(path: files.objects as String) let heads = srv1_fabric_area_ensure(path: files.heads as String) - let door = srv1_fabric_area_ensure(path: fabric_storage_door_directory_path() as String) + let door = srv1_fabric_area_ensure(path: fabric_storage_door_directory_path(instance: srv1_live_dashboard_instance()) as String) string_contains(s: objects, pattern: "0700") && string_contains(s: objects, pattern: "briansrls") && !string_contains(s: objects, pattern: "ghrunner") && !string_contains(s: objects, pattern: "REFUSED") && string_contains(s: heads, pattern: "0700") && !string_contains(s: heads, pattern: "ghrunner") && !string_contains(s: heads, pattern: "REFUSED") && string_contains(s: door, pattern: "2710") && string_contains(s: door, pattern: "ghrunner") && !string_contains(s: door, pattern: "REFUSED") @@ -1459,7 +1463,7 @@ test fn the_fabric_storage_area_ensure_leaves_the_services_own_entries_alone() - // the refusal when a directory grants o+x, are witnessed on synthetic principals in // test.claim.managed_directory_witness. test fn the_placed_fabric_store_derives_entry_mode_0400() -> Bool { - match fabric_storage_entry_mode() { + match fabric_storage_entry_mode(store_root: fabric_storage_store_root(instance: srv1_live_dashboard_instance())) { FabricStorageEntryModeDerived { mode } => file_mode_octal(mode: mode) == "0400" FabricStorageEntryModeRefused { detail: _ } => false } @@ -1851,14 +1855,17 @@ test fn the_receipt_command_substitutes_the_digest_rather_than_splicing_it() -> // port any local account could reach with a forged identity header), and its route is tailscale // serve's 10000 listener at the root mount, proxying to exactly that socket. test fn the_fabric_storage_unit_binds_its_door_socket_and_its_route_proxies_to_it() -> Bool { - let unit = serialize_systemd_unit_file(file: live_deploy_fabric_storage_unit_file(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision })) - let route = deployment_fabric_storage_serve_endpoint() + let socket = fabric_storage_door_socket(instance: srv1_live_dashboard_instance()) + let unit = serialize_systemd_unit_file(file: live_deploy_fabric_storage_unit_file(spec: deployment_spec_srv1(), revision: RevisionBoundAtEmission { revision: deploy_witness_release_revision }, door_socket: socket)) + let route = deployment_fabric_storage_serve_endpoint(door_socket: socket) + let derived = match deployment_fabric_storage_door_socket(spec: deployment_spec_srv1()) { Present { value: k } => (k as String) == (socket as String) Absent => false } unit.contains("--function fabric_storage_serve_handle") - && unit.contains(join(["--unix-socket ", fabric_storage_door_socket() as String], "")) + && derived + && unit.contains(join(["--unix-socket ", socket as String], "")) && !unit.contains("--host") && !unit.contains("--port") && route.https_port == fabric_storage_https_port - && (route.backend as String) == join(["unix:", fabric_storage_door_socket() as String], "") + && (route.backend as String) == join(["unix:", socket as String], "") && (match route.mount { ServeRootMount => true ServeSetPath { path: _ } => false }) } diff --git a/dag/test/claim/live_deploy_unit_emission_oracle_witness_test.dag b/dag/test/claim/live_deploy_unit_emission_oracle_witness_test.dag index 7739658cc4c..79ae3797580 100644 --- a/dag/test/claim/live_deploy_unit_emission_oracle_witness_test.dag +++ b/dag/test/claim/live_deploy_unit_emission_oracle_witness_test.dag @@ -780,14 +780,14 @@ test fn slice_unit_is_not_installable() -> Bool { fn spec_with_principal(wrong_user: NonEmptyStr) -> DeploymentSpec { let base = live_spec() match base { - DeploymentSpec { target: t, steps: s, names: n, service: svc, approval_broker: ab, instance_id: iid } => + DeploymentSpec { target: t, steps: s, names: n, service: svc, approval_broker: ab, instance_id: iid, role_singletons: rs } => match svc { DeploymentServiceConfig { unit_name: u, serve_binary: b, serve_entry: se, serve_function: f, provider_state_root: ps, instance_root: ir, repo_root: rr, service_uid: uid, listen: l, serve_cpu_limit: cpu, serve_wall_limit: wall } => DeploymentSpec { - target: t, steps: s, names: n, instance_id: iid, approval_broker: ab, + target: t, steps: s, names: n, instance_id: iid, approval_broker: ab, role_singletons: rs, service: DeploymentServiceConfig { unit_name: u, serve_binary: b, serve_entry: se, serve_function: f, provider_state_root: ps, @@ -819,14 +819,14 @@ fn divergent_repo_root() -> NonEmptyStr { fn spec_with_repo_root(wrong_root: NonEmptyStr) -> DeploymentSpec { match live_spec() { - DeploymentSpec { target: t, steps: s, names: n, service: svc, approval_broker: ab, instance_id: iid } => + DeploymentSpec { target: t, steps: s, names: n, service: svc, approval_broker: ab, instance_id: iid, role_singletons: rs } => match svc { DeploymentServiceConfig { unit_name: u, serve_binary: b, serve_entry: se, serve_function: f, provider_state_root: ps, instance_root: ir, service_user: su, service_uid: uid, listen: l, serve_cpu_limit: cpu, serve_wall_limit: wall } => DeploymentSpec { - target: t, steps: s, names: n, instance_id: iid, approval_broker: ab, + target: t, steps: s, names: n, instance_id: iid, approval_broker: ab, role_singletons: rs, service: DeploymentServiceConfig { unit_name: u, serve_binary: b, serve_entry: se, serve_function: f, provider_state_root: ps, @@ -867,14 +867,14 @@ test fn serve_unit_user_goes_red_under_different_user() -> Bool { fn spec_with_port(wrong_port: Port) -> DeploymentSpec { match live_spec() { - DeploymentSpec { target: t, steps: s, names: n, service: svc, approval_broker: ab, instance_id: iid } => + DeploymentSpec { target: t, steps: s, names: n, service: svc, approval_broker: ab, instance_id: iid, role_singletons: rs } => match svc { DeploymentServiceConfig { unit_name: u, serve_binary: b, serve_entry: se, serve_function: f, provider_state_root: ps, instance_root: ir, repo_root: rr, service_user: su, service_uid: uid, listen: l, serve_cpu_limit: cpu, serve_wall_limit: wall } => DeploymentSpec { - target: t, steps: s, names: n, instance_id: iid, approval_broker: ab, + target: t, steps: s, names: n, instance_id: iid, approval_broker: ab, role_singletons: rs, service: DeploymentServiceConfig { unit_name: u, serve_binary: b, serve_entry: se, serve_function: f, provider_state_root: ps, diff --git a/dag/test/claim/repository_transition_admission_witness_test.dag b/dag/test/claim/repository_transition_admission_witness_test.dag index 908f4b80e27..4c9317d103a 100644 --- a/dag/test/claim/repository_transition_admission_witness_test.dag +++ b/dag/test/claim/repository_transition_admission_witness_test.dag @@ -6,7 +6,7 @@ import gunbc.live_deploy.apply { live_deploy_wet_with_access_for_realization, live_deploy_transaction_wet_with_access_for_realization, } -import gunbc.live_deploy.desired { deployment_spec_srv1 } +import gunbc.live_deploy.desired { srv1_production_desired_deployment } import gunbc.ci_deploy_access { ci_deploy_srv1_access, ci_deploy_srv1_operator_access } import gunbc.live_deploy.deployed_tree_scope { RepositoryTransitionRealization, @@ -84,12 +84,12 @@ fn exit_refusal_reason(e: ProcessExit) -> String { test fn both_direct_apply_principals_consult_repository_transition_admission() -> Bool { let expected = refusal_reason(a: repository_transition_admission(realization: LegacyGitFileSync)) exit_refusal_reason(e: live_deploy_wet_with_access_for_realization( - spec: deployment_spec_srv1(), + deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_access, realization: LegacyGitFileSync, )) == expected && exit_refusal_reason(e: live_deploy_wet_with_access_for_realization( - spec: deployment_spec_srv1(), + deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_operator_access, realization: LegacyGitFileSync, )) == expected @@ -97,7 +97,7 @@ test fn both_direct_apply_principals_consult_repository_transition_admission() - test fn receipted_transaction_ingress_consults_repository_transition_admission() -> Bool { exit_refusal_reason(e: live_deploy_transaction_wet_with_access_for_realization( - spec: deployment_spec_srv1(), + deployment: srv1_production_desired_deployment(), access: ci_deploy_srv1_access, realization: LegacyGitFileSync, )) == refusal_reason(a: repository_transition_admission(realization: LegacyGitFileSync)) diff --git a/dag/test/claim/roadmap/roadmap_dashboard_instance_apply_witness_test.dag b/dag/test/claim/roadmap/roadmap_dashboard_instance_apply_witness_test.dag index 3d169d4d871..3ade749ada3 100644 --- a/dag/test/claim/roadmap/roadmap_dashboard_instance_apply_witness_test.dag +++ b/dag/test/claim/roadmap/roadmap_dashboard_instance_apply_witness_test.dag @@ -1,6 +1,7 @@ module test.claim.roadmap_dashboard_instance_apply_witness -import std.types { Bool, String, NonEmptyStr, FilePath, GitRef } +import std.types { Bool, String, NonEmptyStr, FilePath, GitRef, List } +import gunbc.deployment_risk { NoProdRole, ProdRoleHeldBy, prod_role_selection, srv2_deploy_deployment } import extdeps.git { shape_git_bundle_create_argv, shape_git_is_shallow_repository_argv, @@ -21,6 +22,9 @@ import gunbc.host_layout { srv1_codex_bundled_executable } import gunbc.roadmap_dashboard_instance { srv1_lab_dashboard_instance, srv1_live_dashboard_instance, + srv2_lab_dashboard_instance, + srv2_deploy_dashboard_instance, + public_builtin_dashboard_instances, dashboard_instance_provider_state_root, dashboard_instance_provider_executable, dashboard_instances_have_disjoint_owned_paths, @@ -28,18 +32,16 @@ import gunbc.roadmap_dashboard_instance { dashboard_instance_dispatch_platform_executables, } import gunbc.roadmap_dashboard_instance_apply { - DashboardProductionPeer, - ProductionPeer, - NoProductionPeer, - DashboardProductionReading, - ProductionRead, - ProductionUnpeered, + DashboardProtectedSibling, + DashboardProtectedSiblingReading, + dashboard_protected_siblings, + dashboard_protected_siblings_in, DashboardProductionSnapshot, - dashboard_peer_isolation_ok, - dashboard_production_read, - dashboard_production_reading_refused, - dashboard_production_reading_detail, - dashboard_production_readings_equal, + dashboard_sibling_isolation_ok, + dashboard_protected_sibling_read, + dashboard_protected_sibling_reading_refused, + dashboard_protected_sibling_reading_detail, + dashboard_protected_sibling_readings_equal, DashboardProvisionStepReceipt, DashboardInstanceApplyResult, DashboardApplyRefused, @@ -583,61 +585,59 @@ test fn dashboard_unreachable_root_refuses_before_choosing_a_parent() -> Bool { } } -// THE ARM THAT MATTERS IS THE MIXED ONE. An unpeered host has no production liveness to compare, so the tempting shortcut is to let an absent reading compare equal to anything and move on โ€” which would let an apply that STARTED with production served and ENDED with it unreachable report untouched, the precise reading the post-apply check exists to prevent. Equality is therefore required to agree on the arm as well as the contents: two reads compare their snapshots, two unpeered readings compare their hosts, and one of each is never equal regardless of what happened in between.\n\nThe isolation claim is the counterpart. NoProductionPeer discharges the disjointness obligation because there is no peer to overlap with, but it must not discharge the instance's OWN ownership requirement โ€” an unowned path is unowned whether or not a production dashboard shares the machine, and preflight checks the two separately for that reason. -fn witness_peered() -> DashboardProductionPeer { - ProductionPeer { instance: srv1_live_dashboard_instance() } +// THE MISSING READING IS THE ONE THAT MATTERS. A host with no protected sibling has no liveness to +// compare; the tempting shortcut is to let a missing reading compare equal to anything, which would +// let an apply that STARTED with the sibling served and ENDED with it unreachable report untouched. +// So readings pair by instance_id: two equal populations compare equal, two empty ones compare +// equal, and a sibling present on one side only is never equal.\n\nThe isolation claim is the +// counterpart. An empty sibling population discharges disjointness because there is nothing to +// overlap with, but it must not discharge the instance's OWN ownership requirement. +fn witness_sibling_reading() -> DashboardProtectedSiblingReading { + DashboardProtectedSiblingReading { + instance_id: srv1_live_dashboard_instance().instance_id, + snapshot: DashboardProductionSnapshot { observed: true, head: "abc123", health: "ok", detail: "" }, + } } -fn witness_unpeered() -> DashboardProductionPeer { - NoProductionPeer { host: "srv2" as NonEmptyStr } +test fn protected_sibling_readings_never_equate_absent_with_observed() -> Bool { + let read = [witness_sibling_reading()] + let empty: List = [] + dashboard_protected_sibling_readings_equal(before: read, after: read) + && dashboard_protected_sibling_readings_equal(before: empty, after: empty) + && !dashboard_protected_sibling_readings_equal(before: read, after: empty) + && !dashboard_protected_sibling_readings_equal(before: empty, after: read) } -fn witness_reading_read() -> DashboardProductionReading { - ProductionRead { - snapshot: DashboardProductionSnapshot { - observed: true, - head: "abc123", - health: "ok", - detail: "", - }, - } +test fn a_host_without_a_protected_sibling_refuses_nothing_and_reads_nothing() -> Bool { + let empty = dashboard_protected_sibling_read(siblings: dashboard_protected_siblings(instance: srv2_lab_dashboard_instance())) + count(empty) == 0 + && !dashboard_protected_sibling_reading_refused(readings: empty) + && string_contains(s: dashboard_protected_sibling_reading_detail(readings: empty), pattern: "no protected sibling") } -test fn production_reading_never_equates_absent_with_observed() -> Bool { - let read = witness_reading_read() - let unpeered = ProductionUnpeered { host: "srv2" as NonEmptyStr } - dashboard_production_readings_equal(before: read, after: read) - && dashboard_production_readings_equal(before: unpeered, after: unpeered) - && !dashboard_production_readings_equal(before: read, after: unpeered) - && !dashboard_production_readings_equal(before: unpeered, after: read) - && !dashboard_production_readings_equal( - before: unpeered, - after: ProductionUnpeered { host: "srv1" as NonEmptyStr }, - ) +// THE SIBLING IS DERIVED FROM THE SELECTION AND THE HOST. Today srv1's lab is protected from +// srv1-live and srv2's lab from nothing; move the role to srv2-deploy and the protection moves with +// it; under NoProdRole no lab has a protected sibling. A sibling list passed by name would fail the +// moved-selection conjuncts. +fn sibling_ids(xs: List) -> List { + map(xs, x => x.instance.instance_id as String) } -test fn unpeered_host_refuses_nothing_and_reads_nothing() -> Bool { - !dashboard_production_reading_refused(reading: dashboard_production_read(peer: witness_unpeered())) - && string_contains( - s: dashboard_production_reading_detail( - reading: dashboard_production_read(peer: witness_unpeered()), - ), - pattern: "no production peer on srv2", - ) +test fn protected_siblings_follow_the_prod_role_selection_within_the_host() -> Bool { + let roster = public_builtin_dashboard_instances() + let today = prod_role_selection + let moved = ProdRoleHeldBy { deployment: srv2_deploy_deployment } + sibling_ids(xs: dashboard_protected_siblings_in(selection: today, instances: roster, instance: srv1_lab_dashboard_instance())) == [srv1_live_dashboard_instance().instance_id as String] + && count(dashboard_protected_siblings_in(selection: today, instances: roster, instance: srv2_lab_dashboard_instance())) == 0 + && count(dashboard_protected_siblings_in(selection: moved, instances: roster, instance: srv1_lab_dashboard_instance())) == 0 + && sibling_ids(xs: dashboard_protected_siblings_in(selection: moved, instances: roster, instance: srv2_lab_dashboard_instance())) == [srv2_deploy_dashboard_instance().instance_id as String] + && count(dashboard_protected_siblings_in(selection: NoProdRole, instances: roster, instance: srv1_lab_dashboard_instance())) == 0 } -test fn peer_isolation_still_requires_the_instance_to_own_its_paths() -> Bool { - dashboard_peer_isolation_ok( - instance: srv1_lab_dashboard_instance(), - peer: witness_peered(), - ) - && dashboard_peer_isolation_ok( - instance: srv1_lab_dashboard_instance(), - peer: witness_unpeered(), - ) - && !dashboard_peer_isolation_ok( - instance: srv1_live_dashboard_instance(), - peer: witness_peered(), - ) +test fn sibling_isolation_still_requires_the_instance_to_own_its_paths() -> Bool { + let srv1_siblings = dashboard_protected_siblings(instance: srv1_lab_dashboard_instance()) + dashboard_sibling_isolation_ok(instance: srv1_lab_dashboard_instance(), siblings: srv1_siblings) + && dashboard_sibling_isolation_ok(instance: srv1_lab_dashboard_instance(), siblings: []) + && !dashboard_sibling_isolation_ok(instance: srv1_live_dashboard_instance(), siblings: srv1_siblings) } diff --git a/dag/test/claim/roadmap/roadmap_launch_deployment_receipt_witness_test.dag b/dag/test/claim/roadmap/roadmap_launch_deployment_receipt_witness_test.dag index 77055c1f69e..d44cb1bda39 100644 --- a/dag/test/claim/roadmap/roadmap_launch_deployment_receipt_witness_test.dag +++ b/dag/test/claim/roadmap/roadmap_launch_deployment_receipt_witness_test.dag @@ -1,5 +1,6 @@ module test.claim.roadmap.roadmap_launch_deployment_receipt_witness +import gunbc.live_deploy.apply_receipt { RoleSingletonsNoop } import gunbc.roadmap_launch_deployment_cli { rlm1_merge_commit } import gunbc.live_deploy.desired { deployment_spec_srv1 } import gunbc.actions_run_binding { @@ -234,6 +235,7 @@ fn rlm_dashboard(run: ActionsRunBinding, expected: String) -> LiveDeployApplyRec candidate_surface_identity: rlm_fixture_surface, candidate_tree_oid: rlm_fixture_tree, apply_outcome: LiveDeployApplyConverged, + role_singleton_disposition: RoleSingletonsNoop, readiness: LiveDeployReadinessObserved { revision: expected, surface: rlm_fixture_surface }, provenance: LiveDeployFleetProvenance { plan_run_id: "1001", apply_run_id: "1002", plan_artifact_hash: rlm_fixture_plan_hash, @@ -641,7 +643,7 @@ test fn witness_dashboard_citing_another_plan_receipt_is_incomplete_at_dashboard let d = LiveDeployApplyReceipt { run: base.run, target: base.target, instance_id: base.instance_id, expected_revision: base.expected_revision, candidate_revision: base.candidate_revision, candidate_surface_identity: base.candidate_surface_identity, candidate_tree_oid: base.candidate_tree_oid, - apply_outcome: base.apply_outcome, readiness: base.readiness, + apply_outcome: base.apply_outcome, role_singleton_disposition: base.role_singleton_disposition, readiness: base.readiness, provenance: LiveDeployFleetProvenance { plan_run_id: "1001", apply_run_id: "1002", plan_artifact_hash: rlm_fixture_plan_hash, plan_receipt_identity: "9999999999999999", apply_receipt_identity: rlm_apply_identity(), diff --git a/dag/test/claim/roadmap/serve_function_binding_witness_test.dag b/dag/test/claim/roadmap/serve_function_binding_witness_test.dag new file mode 100644 index 00000000000..16b0f7b1b01 --- /dev/null +++ b/dag/test/claim/roadmap/serve_function_binding_witness_test.dag @@ -0,0 +1,85 @@ +module test.claim.serve_function_binding_witness_test + +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.types { Bool, String, NonEmptyStr, List, CommitSha } +import extdeps.http.server { ServeWireResponse, ServeRequestHeader } +import gunbc.roadmap_dashboard_instance { + HostDashboardInstance, + public_builtin_dashboard_instances, + dashboard_instance_health_identity_prefix, +} +import gunbc.roadmap_serve { + roadmap_serve_handle, + roadmap_serve_handle_macbook_local, + roadmap_serve_handle_srv1_lab, + roadmap_serve_handle_srv2_preview, + roadmap_serve_handle_srv2_deploy, + roadmap_serve_handle_srv2_lab, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// EVERY INSTANCE'S serve_function SERVES THAT SAME INSTANCE. HostDashboardInstance.serve_function is +// the name its unit passes to `gunbc serve --function`, and each declared handler binds exactly one +// instance; a row naming another instance's handler serves that other instance's state under its own +// unit -- the MacBook row named srv1-live's roadmap_serve_handle while roadmap_serve_handle_macbook_local +// existed. The control runs each handler on /instance.json (its per-instance identity route; /healthz +// is release-only and identical for every instance) and requires the answer to START with +// the identity prefix of the instance whose row names it, so a handler bound to any other instance +// fails, and a row naming a handler this table does not carry fails as unbound. +type ServeHandlerRow { + name: String + handle: fn(String, String, String, String, List, CommitSha) -> ServeWireResponse +} + +fn declared_serve_handlers() -> List { + [ + ServeHandlerRow { name: "roadmap_serve_handle", handle: (m, p, b, t, h, r) => roadmap_serve_handle(method: m, path: p, body: b, tailscale_identity: t, request_headers: h, release_revision: r) }, + ServeHandlerRow { name: "roadmap_serve_handle_macbook_local", handle: (m, p, b, t, h, r) => roadmap_serve_handle_macbook_local(method: m, path: p, body: b, tailscale_identity: t, request_headers: h, release_revision: r) }, + ServeHandlerRow { name: "roadmap_serve_handle_srv1_lab", handle: (m, p, b, t, h, r) => roadmap_serve_handle_srv1_lab(method: m, path: p, body: b, tailscale_identity: t, request_headers: h, release_revision: r) }, + ServeHandlerRow { name: "roadmap_serve_handle_srv2_preview", handle: (m, p, b, t, h, r) => roadmap_serve_handle_srv2_preview(method: m, path: p, body: b, tailscale_identity: t, request_headers: h, release_revision: r) }, + ServeHandlerRow { name: "roadmap_serve_handle_srv2_deploy", handle: (m, p, b, t, h, r) => roadmap_serve_handle_srv2_deploy(method: m, path: p, body: b, tailscale_identity: t, request_headers: h, release_revision: r) }, + ServeHandlerRow { name: "roadmap_serve_handle_srv2_lab", handle: (m, p, b, t, h, r) => roadmap_serve_handle_srv2_lab(method: m, path: p, body: b, tailscale_identity: t, request_headers: h, release_revision: r) }, + ] +} + +fn instance_serves_itself(instance: HostDashboardInstance) -> Bool { + let bound = filter(declared_serve_handlers(), h => h.name == (instance.serve_function as String)) + count(bound) == 1 + && all(bound, h => { + let serve = h.handle + let answer = serve("GET", "/instance.json", "", "", [], "0000000000000000000000000000000000000000" as CommitSha) + answer.status == 200 + && starts_with(s: answer.body, prefix: dashboard_instance_health_identity_prefix(instance_id: instance.instance_id as String)) + }) +} + +// ONE CLAIM PER INSTANCE: the cases are independent, so one conjunction would force six full serves +// through one budget while losing no coverage by splitting (DESIGN 3: a witness discriminates at one +// interface). The roster claim keeps completeness: every instance's serve_function names exactly one +// declared handler, so a seventh instance cannot slip past the per-instance claims unbound. +fn the_instance(id: String) -> List { + public_builtin_dashboard_instances() |> filter(i => (i.instance_id as String) == id) +} + +fn serves_itself_by_id(id: String) -> Bool { + let found = the_instance(id: id) + count(found) == 1 && all(found, i => instance_serves_itself(instance: i)) +} + +test fn every_instance_names_exactly_one_declared_handler() -> Bool { + count(public_builtin_dashboard_instances()) == count(declared_serve_handlers()) + && all(public_builtin_dashboard_instances(), i => count(filter(declared_serve_handlers(), h => h.name == (i.serve_function as String))) == 1) +} + +test fn srv1_live_serves_itself() -> Bool { serves_itself_by_id(id: "srv1-live") } + +test fn macbook_local_serves_itself() -> Bool { serves_itself_by_id(id: "macbook-local") } + +test fn srv1_lab_serves_itself() -> Bool { serves_itself_by_id(id: "srv1-lab-codex-feedback-v0") } + +test fn srv2_preview_serves_itself() -> Bool { serves_itself_by_id(id: "srv2-preview-codex-feedback-v0") } + +test fn srv2_deploy_serves_itself() -> Bool { serves_itself_by_id(id: "srv2-deploy") } + +test fn srv2_lab_serves_itself() -> Bool { serves_itself_by_id(id: "srv2-lab-v1-deletion") } diff --git a/dag/test/claim/spark/pair_serving_d0_authorization_witness_test.dag b/dag/test/claim/spark/pair_serving_d0_authorization_witness_test.dag index 16380e39aac..01f65198539 100644 --- a/dag/test/claim/spark/pair_serving_d0_authorization_witness_test.dag +++ b/dag/test/claim/spark/pair_serving_d0_authorization_witness_test.dag @@ -9,6 +9,8 @@ import std.scoped_authorization { AuthorizationRequest, AuthorizationGranted } import gunbc.auth.approval_decision_store { request_revision_of } import gunbc.auth.approval_request_client { PollApproved, PollNotFiled, PollStillPending, PollDenied } import gunbc.auth.approval_gate { StandingReadAt, StandingClockUnreadable, ApprovalDenied } +import gunbc.auth.approval_request_client { ApprovalLoopbackOrigin, ApprovalLoopbackOriginResolved, ApprovalLoopbackOriginRefused } +import gunbc.roadmap_dashboard_instance { ProdRoleUnheld } import gunbc.spark.pair_serving_d0_door { d0_absent_claim_step, D0AbsentClaimClockUnreadable, D0AbsentClaimFileFrozen, D0AbsentClaimEnterPoll, D0AbsentClaimDecided, pair_serving_d0_mode_admission, D0ModeAdmitted, D0ModeRefused } import gunbc.spark.fabric_switch_observed { fabric_group_wire } import gunbc.spark.fabric_switch_observed { FabricGroupA } @@ -170,13 +172,32 @@ test fn a_frozen_filing_with_no_claim_files_polls_or_decides_live_by_the_read() // revision that is not the checked-out tree's HEAD each refuse. test fn the_mode_admits_only_srv1_group_a_and_a_named_consent_and_revision() -> Bool { let rev = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - (match pair_serving_d0_mode_admission(declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { + (match pair_serving_d0_mode_admission(writer: srv1_writer(), declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeAdmitted { group: g, transaction: tx, authorization: a, execution_revision: r } => (fabric_group_wire(g: g) as String) == "group-a" && (tx as String) == "d0-esc-1" && (a as String) == "d0-esc-1" && (r as String) == rev D0ModeRefused { reason: _ } => false }) - && (match pair_serving_d0_mode_admission(declared_host: "srv2", kernel_host: "srv2", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("srv2") _ => false }) - && (match pair_serving_d0_mode_admission(declared_host: "srv1", kernel_host: "srv2", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("runs on srv2") _ => false }) - && (match pair_serving_d0_mode_admission(declared_host: "srv1", kernel_host: "srv1", consent: "", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("GUNBC_D0_CONSENT") _ => false }) - && (match pair_serving_d0_mode_admission(declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: "", checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("expected_revision") _ => false }) - && (match pair_serving_d0_mode_admission(declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") { D0ModeRefused { reason: r } => r.contains("not the tree this job runs from") _ => false }) + && (match pair_serving_d0_mode_admission(writer: srv1_writer(), declared_host: "srv2", kernel_host: "srv2", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("srv2") _ => false }) + && (match pair_serving_d0_mode_admission(writer: srv1_writer(), declared_host: "srv1", kernel_host: "srv2", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("runs on srv2") _ => false }) + && (match pair_serving_d0_mode_admission(writer: srv1_writer(), declared_host: "srv1", kernel_host: "srv1", consent: "", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("GUNBC_D0_CONSENT") _ => false }) + && (match pair_serving_d0_mode_admission(writer: srv1_writer(), declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: "", checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("expected_revision") _ => false }) + && (match pair_serving_d0_mode_admission(writer: srv1_writer(), declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") { D0ModeRefused { reason: r } => r.contains("not the tree this job runs from") _ => false }) +} + +// The writer origin is supplied at the admission's interface (gunbc.auth.approval_request_client +// resolves it from the prod-role holder; test.claim.approval_request_client_witness_test runs that). +fn srv1_writer() -> ApprovalLoopbackOrigin { + ApprovalLoopbackOriginResolved { host: "srv1" as NonEmptyStr, origin: "http://127.0.0.1:8085" as NonEmptyStr } +} + +fn srv2_writer() -> ApprovalLoopbackOrigin { + ApprovalLoopbackOriginResolved { host: "srv2" as NonEmptyStr, origin: "http://127.0.0.1:8085" as NonEmptyStr } +} + +// THE EXECUTOR FOLLOWS THE WRITER'S HOST: with the role on srv2 the mode admits a srv2 dispatch on a +// srv2 kernel and refuses srv1 before any credential; with no prod role it refuses outright. +test fn the_d0_executor_is_the_approval_writers_host_and_no_prod_role_refuses() -> Bool { + let rev = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + (match pair_serving_d0_mode_admission(writer: srv2_writer(), declared_host: "srv2", kernel_host: "srv2", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeAdmitted { group: _, transaction: _, authorization: _, execution_revision: _ } => true D0ModeRefused { reason: _ } => false }) + && (match pair_serving_d0_mode_admission(writer: srv2_writer(), declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: r } => r.contains("srv2") _ => false }) + && (match pair_serving_d0_mode_admission(writer: ApprovalLoopbackOriginRefused { cause: ProdRoleUnheld }, declared_host: "srv1", kernel_host: "srv1", consent: "d0-esc-1", execution_revision: rev, checked_out_revision: rev) { D0ModeRefused { reason: _ } => true _ => false }) } diff --git a/docs/plans/deployment-risk-conformance.md b/docs/plans/deployment-risk-conformance.md index 0aaf1789486..ff3b1aea8e3 100644 --- a/docs/plans/deployment-risk-conformance.md +++ b/docs/plans/deployment-risk-conformance.md @@ -20,16 +20,39 @@ Operator direction 2026-10-03; vocabulary fixed the same day by a side-chat ruli 1. **Model** (this PR): `gunbc.deployment_risk`; `HostDashboardInstance.deployment` (required, all six constructors); `dashboard_instance_risk_class`; the failure-mode row `a_test_deployment_acts_on_the_prod_deployments_state`; RED `test.claim.deployment_risk_witness_test`. - Declared frontier (ยง3c): `resolve_deployment_bindings` and `dashboard_instance_risk_class` gain production consumers in deliverables 2 and 3.1. - - Deferred to deliverable 2 (review 74937): `DeploymentRiskInterface` and `DeploymentHarmMagnitudeBet` land with their first executing consumer, the operator-disposition gate (which actions on a deployment need an operator), and the bet's magnitude is grounded as an order of magnitude over `std.measure` `Duration`, not a bare `Int`. + - Declared frontier (review 74937; merry-tern-58 ruling 2026-10-04): `DeploymentRiskInterface`, `DeploymentHarmMagnitudeBet` and the operator-disposition gate are **deferred until an operator policy differs by class**. Today's policy (gunbc#13124: automatic repair is "no" for every deployment) is class-independent, so a gate consulting the class would decide nothing (ยง3c decoration), and a class-dependent policy is the operator's call. No carrier is minted before then. Design note for when the bet lands: its magnitude is an order of magnitude over `Duration` (today `std.types` `Duration`; std has no order-of-magnitude carrier yet), e.g. a decade exponent beside its unit, typed with `extdeps.external_authority` `CitedFigureStanding`, and it never decides the class. 1b. **Machine qualification migration**: one motion over machine_intake (re-census on main first; coordinate warm-crane-577's mtcollins1 untangle lane). eager-gull-22 handed it to this lane. 2. **Repoint the prod pins** through `prod_role_selection` (broker owner, fabric storage placement, approval client/store/trust, serve/dispatch entry points, apply's prod peer โ†’ `DashboardProtectedSibling`, `fleet_reach_endpoint`). RED: a fixture selection moves the broker owner and storage placement. + - Landed: one resolver, `gunbc.roadmap_dashboard_instance` `prod_role_realization` (the instance roster filtered by `deployment`; no second table), refusing with `ProdRoleRealizationRefusal` (`ProdRoleUnheld` | `ProdRoleHolderUnrealized` | `ProdRoleHolderAmbiguous` | `ProdRoleHolderUnactuatable`) and minting the only holder token, `ProdRoleHolder` (`sole_constructor`). **Actuatability join:** `gunbc.live_deploy.desired` `prod_role_actuated_realization` joins the holder to the one `desired_deployment_roster`; a holder with no row (MacBook, srv2-preview, srv2's lab) refuses as `ProdRoleHolderUnactuatable`. Every role follower reads the joined realization. A spec carries the token as `gunbc.live_deploy.spec` `RoleSingletonHolding`, and the singleton members -- broker unit and front door, plus fabric store unit, route, socket, root, directories and **ownership marker** -- are built from it and move between targets as one subject. The marker is the dashboard ownership-marker rule (`dashboard_ownership_marker_in`) applied to the store's control directory `fabric-storage-control` (not the store root, whose derived mode admits only the service), whose derived mode admits the observing deploy job principal to read it, so an old store with no unit stays observable. The broker is GIVEN its address (`GUNBC_APPROVAL_BROKER_BASE_URL`, from `gunbc.auth.approval_broker_placement`); broker-side modules never resolve the role (#13132's closure). Also following the realization: approval client origin, the GCP IAM job's runner, the approvals app host, `DashboardProtectedSibling`. + - Role-singleton convergence (`gunbc.live_deploy.role_singleton_convergence`): + - **Observation.** It sits behind `RoleSingletonReadingSource`: host self-report is the ruled realization (operator, 2026-10-05: each host's services report their own role-singleton state; no srv1โ†’srv2 ssh will be granted) and is the immediate follow-up to D2. Until it lands the production source is `local_only_reading_source`: the applying host is read over the apply's own route, and every other roster host is `HostRoleSingletonsUnobservable` (cause: self-report has not landed) without any transport being built -- the assessor module names no ssh transport, so no credential can make a nonlocal host observable (`a_valid_ssh_credential_cannot_make_a_nonlocal_roster_host_observable`). Sightings are kept at (host, deployment, member) until multiplicity is adjudicated, and every roster deployment's broker, fabric unit and marker are probed on every roster host. + - **Decision.** Possible outcomes: + - `RoleSingletonNoop`: exactly one complete realization of the desired holder on its target. + - `RoleSingletonInstall`: nothing observed anywhere. + - `RoleSingletonRefused`, with one of these causes: move, removal, `RoleSingletonDesiredRealizationRefused`, unmarked, unobservable, two-hosts, misplaced, two-on-one, ambiguous, broker-without-store, partial, stale-spec. + + Only `ProdRoleUnheld` means no prod. + - **Admission.** `RoleSingletonAdmission` (`sole_constructor`) binds the exact spec. Its mint is sealed (`admit_callers`) to the canonical assessor `assess_and_admit(deployment, own)`, which takes a `DesiredDeployment`, binds it to its `desired_deployment_roster` row (refusing `RoleSingletonDeploymentUnrostered` otherwise), builds the spec from that row itself -- no caller-supplied `DeploymentSpec` reaches it -- and reads the actuated desired realization and the observation through `RoleSingletonReadingSource` itself. The assessor, the token's readers (`admitted_spec`, `admitted_decision`) and every admitted apply step are in turn sealed to the two canonical assess-and-act compositions (`live_deploy_apply_via_transport`, `live_deploy_transaction_decided`). So a token can be neither forged, obtained elsewhere nor replayed later. Fixtures use the pure `classify_role_singleton_admission`. The apply receipt records `role_singleton_disposition` (Noop, Installed or RefusedBeforeApply). + - **Unmarked legacy store (no automated adoption).** A realization with its broker and fabric unit but no store ownership marker refuses as `RoleSingletonUnmarked`. Today's srv1 holder predates the marker, so it is the expected case. Under the manual-resume ruling (#13124) it waits for the operator, and the refusal text names exactly what to write. **Operator's one-time disposition:** + - on `srv1`, create the control directory `/opt/gunbc/fabric-storage-control` with the mode, owner and group derived by `gunbc.fabric_storage_placement` `fabric_storage_control_directory` (2750, owner briansrls, group ghrunner); + - in it, write `.gunbc-dashboard-instance-srv1-live`; + - mode 0644, owner briansrls; + - content exactly `deployment=srv1-daily-workspace` + newline + `host=srv1` + newline. + + This is the same path and content the holder spec's own marker ensure writes. After it, the next apply observes a complete realization and converges as Noop. + - **Next trigger.** Move and Remove actuations become arms beside these. + - Bounded residual: the host universe is the roster's targets (`desired_deployment_roster`), so a singleton on a host outside the roster is invisible to the observation. Trigger: a roster row for that host. + - Consequence until self-report lands: an apply assesses every roster host, and every roster host other than the applying host is unobservable, so the apply refuses before planning (fail-closed, the manual-resume posture of #13124). **The read authority is named,** for the local read and for any future reader: The observer is the deploy job principal (`gunbc.fabric_storage_placement` `fabric_storage_marker_observer` = `srv1_ci_deploy_runner_principal` job_user, ghrunner), and the marker lives in the control directory whose derived mode admits it to list and traverse; the store root's derived mode does not, and is not widened. A reader that lands as another account and is denied reads as unobservable and refuses, never as absent. + - **Warrant.** The member-plan step mints `AdmittedLiveDeployApply`, realizes it, and returns the intent with the warrant replaced by `LiveDeployApplyReported`, so no value returned or stored outside that sealed step carries a warrant. The residual -- re-wrapping an extracted warrant into a fresh `LiveDeployApply` -- is not authorable as a compile RED because coproduct variant construction cannot be sealed; it is recorded on `gunbc.recurring_failure_mode` `modifier_accepted_on_a_shape_its_check_cannot_see`, trigger: coproduct variant construction wired through `sole_constructor_construction_diags` (declared at `v1.compiler.parse` `alias_rhs_is_anonymous_record`). + - Honest non-pins (not debt): the nullary serve/belt/dispatch/launch entry points are srv1-live's *own* entry points (`serve_function`, now checked roster-wide by `test.claim.serve_function_binding_witness_test`), so they must not follow the role; `fleet_reach_endpoint` is a host-shared account fact. Not claimed: that an unbound test deployment refuses everywhere โ€” `belt_dispatch_instance_cli`'s absent-env default stays with the belt owner's lane. 3. **Split shared bindings**, one PR each: ntfy โ†’ Codex account/home โ†’ admitted ref โ†’ publication identity/GitHub App โ†’ R2/GCP โ†’ OIDC. Each adds a field to `DeploymentExternalBindingSet` and either a TestRisk resource (operator approval first) or a standing refusal. 4. **One spec per deployment**; retire the `live_deploy.desired` rung drop by its own trigger. Renames: `srv1_production_desired_deployment โ†’ srv1_primary_desired_deployment`, `roadmap_belt_production_spawn_mode โ†’ roadmap_belt_operational_spawn_mode`. + - Naming residue: `gunbc.host_layout` `srv1_gunbc_approval_broker_root` ("/opt/gunbc/approval-broker") is host-agnostic in value but srv1 in name; rename it with the per-deployment names (not a prod pin; D2 review, 2026-10-04). 5. **Conformance row** `deployment-risk conformance` in `gunbc.design_argument conformance_domains`, once the home is consumed. Widened scope (side chat): classify every convergence subject as deployment-bound (Spark serving/experiments/seat grants, fabric storage, approval broker, R2 keyed by class + `BucketPurpose`, IAM targets, DNS/routes/certs, runner workloads, mutable caches), host-shared (Spark Wi-Fi/SSH/admin grants, KVM, base networking, toolchains), or fleet-control (root IAM writer, account admin). Pure computation identity and content-addressed outputs gain no deployment key. Manual convergence requests take `DeploymentConvergence { deployment } | HostResourceConvergence { host, resource } | FleetControlConvergence { operation }`. -## Shared-binding table (as of deliverable 1) +## Shared-binding table (as of deliverable 2) | Binding | State | Reason | |---|---|---| @@ -38,4 +61,7 @@ Widened scope (side chat): classify every convergence subject as deployment-boun | Codex account and home | still shared | 3.2 | | admitted ref | still shared (labs on `refs/fleet/desired`) | 3.3 | | R2 / GCP / WIF | still shared | 3.5; coordinate royal-moth-86 (#13035) | +| approval broker unit and front door, fabric store unit/route/socket/root/directories (role singletons) | one subject on the holder's spec (`RoleSingletonHolding`); a role change refuses until Move/Remove land; NoProdRole โ†’ no target holds any | deliverable 2 | +| GCP IAM converge job runner | holder's host; NoProdRole โ†’ refused job | deliverable 2 | +| approval loopback origin / identity trust | follows the role; NoProdRole โ†’ origin refused, trust not established | deliverable 2 | | OIDC client | still shared, unmodeled | 3.6 |