diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index b26e9566f4d..39c243aa549 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -11,9 +11,9 @@ on: options: [srv1, srv2, srv3, srv4] type: choice mode: - description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction + description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; spark_v41_group_a_launch reads the production image back under its tag on every Group A host, reads its registry inside its digest and every host's occupancy, and only when Group A is suspended for this candidate with every host held and vacant stages the Engram manifest and applies the V4.1 four-rank arm as one transaction at the capacity measurement's shape -- the target names only the session host; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, spark_v41_group_a_launch, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -183,7 +183,7 @@ jobs: actions: read id-token: write concurrency: - group: ${{ github.event.inputs.mode == 'spark_serving_apply' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_native_serving_apply' && 'gunbc-arm-mutation-glm-group-b-native' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_encode' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_readback' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_engram_differential' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_build' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'pair_serving_d0' && 'gunbc-host-mutation-srv1' || format('gunbc-host-mutation-{0}', github.event.inputs.host) }} + group: ${{ github.event.inputs.mode == 'spark_serving_apply' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_native_serving_apply' && 'gunbc-arm-mutation-glm-group-b-native' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_encode' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_readback' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_engram_differential' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_build' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_group_a_launch' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'pair_serving_d0' && 'gunbc-host-mutation-srv1' || format('gunbc-host-mutation-{0}', github.event.inputs.host) }} cancel-in-progress: false steps: - name: Checkout @@ -253,7 +253,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_engram_differential' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'mtcollins1_census_member_readback' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_engram_differential' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'spark_v41_group_a_launch' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'mtcollins1_census_member_readback' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -1312,6 +1312,35 @@ jobs: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} if: github.event.inputs.mode == 'spark_v41_runtime_image_distribute' timeout-minutes: 120 + - name: Spark V4.1 Group A launch (the production image read back on srv5-8, its registry read inside its digest, occupancy read, then on admission the Engram manifest staged and the four-rank arm applied as one transaction at the capacity measurement's shape) + id: spark_v41_group_a_launch + run: |- + set -euo pipefail + umask 077 + CRED_FILE="$RUNNER_TEMP/spark-administrator-credential" + RESP_FILE="$RUNNER_TEMP/spark-administrator-response.json" + HDR_FILE="$RUNNER_TEMP/spark-administrator-auth-header" + trap 'rm -f "$CRED_FILE" "$RESP_FILE" "$HDR_FILE"' EXIT + printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE" + case "${SPARK_CONVERGE_TARGET:-}" in srv5) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv6) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv7) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv8) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv9) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv10) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv11) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv12) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; *) echo "SparkAdministratorCredentialHostUnrostered: SPARK_CONVERGE_TARGET='${SPARK_CONVERGE_TARGET:-}' names no host in the administrator credential roster; contacting NO host" >&2; exit 1 ;; esac + HTTP_CODE="$(curl -sS -o "$RESP_FILE" -w '%{http_code}' -H @"$HDR_FILE" "$SM_URL" || echo 000)" + if [ "$HTTP_CODE" = 403 ]; then echo "SparkAdministratorCredentialUnreadable: $SM_RES answered 403 to serviceAccount:fleet-cloud-convergence@gunbai-secrets.iam.gserviceaccount.com. Secret Manager returns 403 both when a version exists and is not readable AND when it does not exist, so this run cannot tell those apart -- REMEDY EITHER WAY: confirm the version exists and grant that principal roles/secretmanager.secretAccessor on it. Contacting NO host." >&2; exit 1; fi + if [ "$HTTP_CODE" = 404 ]; then echo "SparkAdministratorCredentialAbsent: $SM_RES answered 404; the modeled reference names a version Secret Manager does not hold. Contacting NO host." >&2; exit 1; fi + if [ "$HTTP_CODE" != 200 ]; then echo "SparkAdministratorCredentialFetchFailed: HTTP $HTTP_CODE from $SM_RES; contacting NO host." >&2; exit 1; fi + python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$CRED_FILE" + rm -f "$RESP_FILE" + chmod 600 "$CRED_FILE" + if [ ! -s "$CRED_FILE" ]; then echo "SparkAdministratorCredentialEmpty: $SM_RES returned 200 with an empty payload, which is an enrolled-but-blank secret and never a blank password; contacting NO host" >&2; exit 1; fi + export GUNBC_SPARK_BOOTSTRAP_CREDENTIAL_FILE="$CRED_FILE" + echo "spark-admin-credential: materialized from $SM_RES to a 0600 run-local file, consumed and removed within this one step" + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/spark/v41_group_a_launch.dag --function v41_group_a_launch_ci_wet + cat "$ROOT/target/v41-group-a-launch-receipt.txt" + env: + SPARK_CONVERGE_TARGET: ${{ github.event.inputs.target }} + WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} + if: github.event.inputs.mode == 'spark_v41_group_a_launch' + timeout-minutes: 60 - name: "Runner host files: observe the teardown drop-in, the needrestart deferral and the loaded teardown (no writes)" id: runner_host_file_observe run: |- diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 647c65943cf..b88f5a04785 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -12,6 +12,7 @@ import gunbc.spark.v41_row_store_encode_run { v41_encode_run_receipt_path } import gunbc.spark.v41_row_store_readback_run { v41_readback_run_receipt_path } import gunbc.spark.v41_engram_differential_run { v41_differential_receipt_path } import gunbc.spark.v41_runtime_image_converge { v41_distribution_receipt_path } +import gunbc.spark.v41_group_a_launch { v41_group_a_launch_receipt_path } import gunbc.ledger_row_coherence { heal_repair_declaration_artifact_path } import extdeps.cloud.gcp.secret_ref { SecretRef, secret_ref_access_url, secret_ref_version_resource } import gunbc.auth.ci_app_key_rotation { @@ -1053,6 +1054,14 @@ data gunbc_ci_spark_v41_runtime_image_distribute_target: GunbcRunStepTarget = Gu function: "v41_runtime_image_distribute_ci_wet", } +// THE V4.1 FOUR-RANK LAUNCH ON GROUP A (gunbc.spark.v41_group_a_launch): the held image read back on +// every host, the registry read inside its digest, occupancy read, and only on admission the Engram +// staging and the arm transaction. The same prelude: the session host is the dispatch's target. +data gunbc_ci_spark_v41_group_a_launch_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/spark/v41_group_a_launch.dag", + function: "v41_group_a_launch_ci_wet", +} + // THE PUBLISHED V4.1 CHECKPOINT ON ONE GROUP A SPARK (gunbc.spark.v41_checkpoint_materialize): the same // administrator credential as the runtime-image probe, because every leg runs privileged on the Spark. data gunbc_ci_spark_v41_checkpoint_materialize_target: GunbcRunStepTarget = GunbcRunStepTarget { @@ -2653,6 +2662,17 @@ fn gunbc_ci_spark_v41_runtime_image_distribute_invoke() -> String { ) } +fn gunbc_ci_spark_v41_group_a_launch_invoke() -> String { + gunbc_run_step_script_with_prelude( + prelude: gunbc_ci_spark_grant_install_credential_prelude(), + source_roots: witness_layer_roots, + entry: gunbc_ci_spark_v41_group_a_launch_target.entry, + function: gunbc_ci_spark_v41_group_a_launch_target.function, + claim_run: false, + receipt_rel: v41_group_a_launch_receipt_path + ) +} + fn gunbc_ci_spark_v41_runtime_image_build_invoke() -> String { gunbc_run_step_script_with_prelude( prelude: gunbc_ci_spark_grant_install_credential_prelude(), diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index af018e0b5cc..2ea497977fd 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -151,6 +151,7 @@ import gunbc.ci_spec { gunbc_ci_spark_v41_index_selection_invoke, gunbc_ci_spark_v41_runtime_image_build_invoke, gunbc_ci_spark_v41_runtime_image_distribute_invoke, + gunbc_ci_spark_v41_group_a_launch_invoke, gunbc_fleet_converge_plan_artifact_name, gunbc_ci_fleet_converge_plan_artifact_hash_env_name, gunbc_ci_live_deploy_transaction_invoke, @@ -259,6 +260,7 @@ type FleetConvergeWorkflowMode | SparkV41EngramDifferential | SparkV41RuntimeImageBuild | SparkV41RuntimeImageDistribute + | SparkV41GroupALaunch | DashboardDeploy | RlmLaunchDeploymentReceipt | HostResetReturn @@ -314,6 +316,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String SparkV41EngramDifferential => "spark_v41_engram_differential" SparkV41RuntimeImageBuild => "spark_v41_runtime_image_build" SparkV41RuntimeImageDistribute => "spark_v41_runtime_image_distribute" + SparkV41GroupALaunch => "spark_v41_group_a_launch" DashboardDeploy => "dashboard_deploy" RlmLaunchDeploymentReceipt => "rlm_launch_deployment_receipt" HostResetReturn => "host_reset_return" @@ -366,7 +369,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, SparkV41GroupALaunch, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // // EVERY MODE IS NAMED, AND THE WILDCARD IS DELIBERATELY ABSENT. A `_ => none` would read the same @@ -413,6 +416,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc SparkV41EngramDifferential => none SparkV41RuntimeImageBuild => none SparkV41RuntimeImageDistribute => none + SparkV41GroupALaunch => none DashboardDeploy => Present { value: LiveDeploy } ApprovalBrokerDarkInstall => none MicrovmControllerInstall => none @@ -542,6 +546,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> SparkV41EngramDifferential => FleetSshKeyConsumed SparkV41RuntimeImageBuild => FleetSshKeyConsumed SparkV41RuntimeImageDistribute => FleetSshKeyConsumed + SparkV41GroupALaunch => FleetSshKeyConsumed DashboardDeploy => FleetSshKeyConsumed ApprovalBrokerDarkInstall => FleetSshKeyConsumed MicrovmControllerInstall => FleetSshKeyConsumed @@ -1105,6 +1110,7 @@ fn fleet_converge_mode_mutation_domain(mode: FleetConvergeWorkflowMode) -> Fleet SparkV41EngramDifferential => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } SparkV41RuntimeImageBuild => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } SparkV41RuntimeImageDistribute => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } + SparkV41GroupALaunch => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } DashboardDeploy => ExecutorDomain RlmLaunchDeploymentReceipt => ExecutorDomain HostResetReturn => ExecutorDomain @@ -1166,6 +1172,7 @@ data fleet_converge_spark_v41_engram_differential_step_if: String = fleet_conver data fleet_converge_spark_v41_runtime_image_build_step_if: String = fleet_converge_mode_step_if(mode: SparkV41RuntimeImageBuild) data fleet_converge_spark_v41_runtime_image_distribute_step_if: String = fleet_converge_mode_step_if(mode: SparkV41RuntimeImageDistribute) +data fleet_converge_spark_v41_group_a_launch_step_if: String = fleet_converge_mode_step_if(mode: SparkV41GroupALaunch) data fleet_converge_dashboard_deploy_step_if: String = fleet_converge_mode_step_if(mode: DashboardDeploy) @@ -2945,6 +2952,31 @@ fn fleet_converge_spark_v41_runtime_image_distribute_step() -> Step { } } +// THE V4.1 FOUR-RANK LAUNCH ON GROUP A (gunbc.spark.v41_group_a_launch v41_group_a_launch_ci_wet): the +// production image read back under its tag on srv5-8, its registry read inside its digest, every host's +// occupancy read, and only on the launch admission (Group A suspended for this candidate, every host +// held and vacant) the Engram manifest staged and the four-rank arm applied as one transaction. The +// target is only the session host. Group A's arm domain serializes it with D0, the build and the +// distribution. +fn fleet_converge_spark_v41_group_a_launch_step() -> Step { + RunStep { + name: Present { value: "Spark V4.1 Group A launch (the production image read back on srv5-8, its registry read inside its digest, occupancy read, then on admission the Engram manifest staged and the four-rank arm applied as one transaction at the capacity measurement's shape)" }, + id: Present { value: "spark_v41_group_a_launch" }, + run: gunbc_ci_spark_v41_group_a_launch_invoke(), + shell: none, + env: Present { + value: [ + kv(key: "SPARK_CONVERGE_TARGET", value: yaml_string(s: "${{ github.event.inputs.target }}")), + kv(key: "WIF_ACCESS_TOKEN", value: yaml_string(s: "${{ steps.wif_auth.outputs.access_token }}")), + ] + }, + working_directory: none, + if_condition: Present { value: fleet_converge_spark_v41_group_a_launch_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_spark_serving_step_timeout_minutes } + } +} + // THE PUBLISHED V4.1 CHECKPOINT ON ONE GROUP A SPARK (gunbc.spark.v41_checkpoint_materialize): every // admitted file fetched at the pinned revision, digested against its manifest row before it is // published, then the storage-backed Engram spans read from the verified shards. Its domain is Group @@ -3310,6 +3342,7 @@ fn fleet_converge_job() -> Job { fleet_converge_spark_v41_engram_differential_step(), fleet_converge_spark_v41_runtime_image_build_step(), fleet_converge_spark_v41_runtime_image_distribute_step(), + fleet_converge_spark_v41_group_a_launch_step(), fleet_converge_runner_host_file_observe_step(), fleet_converge_runner_host_file_converge_step(), fleet_converge_runner_password_session_tool_converge_step(), @@ -3649,7 +3682,7 @@ data fleet_converge_dispatch_inputs: List = [ }, DispatchInput { name: "mode", - description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction" }, + description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; spark_v41_group_a_launch reads the production image back under its tag on every Group A host, reads its registry inside its digest and every host's occupancy, and only when Group A is suspended for this candidate with every host held and vacant stages the Engram manifest and applies the V4.1 four-rank arm as one transaction at the capacity measurement's shape -- the target names only the session host; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction" }, required: true, default: none, type: InputChoice { options: fleet_converge_mode_options }, diff --git a/dag/gunbc/spark/native_serving_apply.dag b/dag/gunbc/spark/native_serving_apply.dag index 70cdd28e394..4287471c083 100644 --- a/dag/gunbc/spark/native_serving_apply.dag +++ b/dag/gunbc/spark/native_serving_apply.dag @@ -1290,13 +1290,47 @@ fn spark_native_transaction_receipt(t: SparkNativeArmTransaction) -> String { } } -fn spark_native_transaction_exit(t: SparkNativeArmTransaction, body: String) -> ProcessExit { - match t { - NativeArmCommitted { receipt: _ } => ExitSuccess - NativeArmCommittedWithCleanupPending { receipt: _ } => exit_failure(reason: body) - NativeArmRefusedBeforeActuation { stage: _, receipt: _ } => exit_failure(reason: body) - NativeArmRollbackActuationAccepted { stage: _, dispositions: _, receipt: _ } => exit_failure(reason: body) - NativeArmDispositionUnestablished { stage: _, dispositions: _, receipt: _ } => exit_failure(reason: body) +// ONE ARM TRANSACTION OVER PLANNED STEPS, WHOEVER PLANNED THEM: the grant preflight on every host (a +// refusal touches no host), the transaction identity, then the bounded transaction body. Group B's +// entry below and the V4.1 Group A launch (gunbc.spark.v41_group_a_launch v41_group_a_launch_ci_wet) +// both run their plans through this, so the preflight, preserve, head-first apply, readback and +// commit-or-rollback are one realization with two callers. Only a clean commit is `committed`; +// cleanup-pending, rolled back and unestablished all read false, as the entry's exit always did. +type SparkNativeApplyRun { + body: String + committed: Bool +} + +fn spark_native_apply_transaction( + context: FleetSshExecutionContext, trust: FleetOpenSshTrustPolicy, path: SparkReachPath, administrator: Secret, + steps: List, at_word: String, +) -> SparkNativeApplyRun +{ + let grants = map(map(steps, st => st.host), h => spark_native_grant_preflight_host(context: context, host: h)) + let grant_lines = join(map(grants, p => spark_native_grant_preflight_wire(p: p)), "") + if spark_native_grant_preflight_refusals(ps: grants).length() > 0 { + SparkNativeApplyRun { + body: join(["spark_native_serving_apply preflight: the offer's grant is not in place on every host of the arm; NO HOST WAS TOUCHED.\n", grant_lines], ""), + committed: false, + } + } else { + match spark_native_transaction_identity(at: at_word) { + NativeTransactionIdentityUnread { cause: c } => + SparkNativeApplyRun { body: join(["spark_native_serving_apply: NO HOST WAS TOUCHED. ", c, "\n"], ""), committed: false } + NativeTransactionIdentified { id: transaction_id } => { + let outcome = spark_native_transaction_body(trust: trust, path: path, administrator: administrator, steps: steps, transaction_id: transaction_id) + SparkNativeApplyRun { + body: join([spark_native_transaction_headline(t: outcome), grant_lines, spark_native_transaction_receipt(t: outcome)], ""), + committed: match outcome { + NativeArmCommitted { receipt: _ } => true + NativeArmCommittedWithCleanupPending { receipt: _ } => false + NativeArmRefusedBeforeActuation { stage: _, receipt: _ } => false + NativeArmRollbackActuationAccepted { stage: _, dispositions: _, receipt: _ } => false + NativeArmDispositionUnestablished { stage: _, dispositions: _, receipt: _ } => false + }, + } + } + } } } @@ -1328,36 +1362,21 @@ fn spark_native_serving_apply_wet() -> ProcessExit match fleet_ssh_execution_context_of(outcome: materialization, credential: binding) { Absent => exit_failure(reason: "the execution context could not be sealed for the grant preflight") Present { value: preflight_context } => { - let grants = map(map(steps, st => st.host), h => spark_native_grant_preflight_host(context: preflight_context, host: h)) - let refusals = spark_native_grant_preflight_refusals(ps: grants) - if refusals.length() > 0 { - exit_failure(reason: join([ - "spark_native_serving_apply preflight: the offer's grant is not in place on every host of the arm; NO HOST WAS TOUCHED.\n", - join(map(grants, p => spark_native_grant_preflight_wire(p: p)), ""), - ], "")) + let run = spark_native_apply_transaction( + context: preflight_context, trust: trust, path: path, administrator: administrator as Secret, + steps: steps, at_word: probed_at_word(reading: now) as String) + let body = join([ + "spark native serving apply at ", probed_at_word(reading: now) as String, + " from ", executor, " over ", spark_reach_path_wire(p: path), "\n", + run.body, + ], "") + let wr = Filesystem.Write(path: spark_native_apply_receipt_path, content: body) + if !wr.success { + exit_failure(reason: join(["spark native apply receipt write failed (the transaction's own outcome is above and is NOT changed by this): ", wr.error, "\n", body], "")) + } else if run.committed { + ExitSuccess } else { - match spark_native_transaction_identity(at: probed_at_word(reading: now) as String) { - NativeTransactionIdentityUnread { cause: c } => - exit_failure(reason: join(["spark_native_serving_apply: NO HOST WAS TOUCHED. ", c], "")) - NativeTransactionIdentified { id: transaction_id } => { - let outcome = spark_native_transaction_body( - trust: trust, path: path, administrator: administrator as Secret, - steps: steps, transaction_id: transaction_id) - let body = join([ - "spark native serving apply at ", probed_at_word(reading: now) as String, - " from ", executor, " over ", spark_reach_path_wire(p: path), "\n", - spark_native_transaction_headline(t: outcome), - join(map(grants, p => spark_native_grant_preflight_wire(p: p)), ""), - spark_native_transaction_receipt(t: outcome), - ], "") - let wr = Filesystem.Write(path: spark_native_apply_receipt_path, content: body) - if !wr.success { - exit_failure(reason: join(["spark native apply receipt write failed (the transaction's own outcome is above and is NOT changed by this): ", wr.error, "\n", body], "")) - } else { - spark_native_transaction_exit(t: outcome, body: body) - } - } - } + exit_failure(reason: body) } } } diff --git a/dag/gunbc/spark/v41_group_a_launch.dag b/dag/gunbc/spark/v41_group_a_launch.dag index 9d520761f83..aaaea4fc2ee 100644 --- a/dag/gunbc/spark/v41_group_a_launch.dag +++ b/dag/gunbc/spark/v41_group_a_launch.dag @@ -1,11 +1,12 @@ module gunbc.spark.v41_group_a_launch -import std.types { String, Bool, List, NonEmptyStr, Port, EpochSecs, FilePath } +import std.types { String, Bool, Int, List, NonEmptyStr, Port, EpochSecs, FilePath } +import std.algebra { trim } import std.nat { Nat } import v2.std.optional { Present, Absent } import v2.std.algebra { zip_map } import std.process { ProcessExit, ExitSuccess, exit_failure } -import std.measure { byte_size } +import std.measure { byte_size, Kibibyte, kibibyte, basis_point_count } import std.content_hash { ContentHash, content_hash_equal, serialize_content_hash } import product.placement_supply { HostIdentity } import extdeps.filesystem.filesystem_io { Filesystem } @@ -15,16 +16,17 @@ import gunbc.file_access { FileTreeScope } import gunbc.remote_operation { PlannedRemoteStep, RemoteArgv, RemoteFileBytes, remote_operation_label } import gunbc.typed_remote_file_write { typed_remote_file_write_seal, TypedRemoteFileWriteAdmitted, TypedRemoteFileWriteAdmissionRefused, RemoteFileModeOwnerReadWriteWorldRead } import extdeps.docker.cli { DockerEnvAssignment, BindMount } +import extdeps.vllm.source_build { VllmContainerBuildRecipe } import extdeps.container.oci.digest { render_oci_content_digest_wire } import gunbc.compute.host_occupancy { HostOccupancyAdmission, HostOccupancyNotRead, host_occupancy_refusal } import gunbc.fabric_event_log_host { now_epoch_seconds } -import gunbc.spark.fabric_reach { ExecutorReachKnown, ExecutorReachUnknown, observe_executor_reach } +import gunbc.spark.fabric_reach { ExecutorReachKnown, ExecutorReachUnknown, observe_executor_reach, ReachManagementLan, spark_reach_endpoint } import gunbc.spark.fabric_switch_observed { FabricGroupA, fabric_group_hosts } import gunbc.spark.serving_subject { PairServingUnitOn } -import gunbc.spark.host_commitment { admit_host_held_by_subject_among, spark_serving_admissible_hosts } +import gunbc.spark.host_commitment { admit_host_held_by_subject, admit_host_held_by_subject_among, spark_serving_admissible_hosts } import gunbc.spark.runtime_architecture_registry_reading { ArchitectureRegistryReading } import gunbc.spark.serving_arm { - ServingRuntime, GunbcDeepseekV41EngramProduced, DeepseekV41FlashDba1be0a, FourSparkTp4, + gb10_unified_pool_gib, ServingRuntime, GunbcDeepseekV41EngramProduced, DeepseekV41FlashDba1be0a, FourSparkTp4, ArmBoundedExperiment, ServingLaunchProfile, deepseek_v41_measurement_profile, runtime_artifact_identity, PinnedByDigest, SourceRevisionEstablished, } @@ -37,7 +39,7 @@ import gunbc.spark.native_serving_realization { } import gunbc.spark.native_serving_apply { SparkNativeArmSteps, NativeArmSteps, NativeArmStepsRefused, SparkNativeHostStep, spark_native_arm_steps_of, - spark_native_preflight_plans, spark_native_apply_stage_plans, + spark_native_preflight_plans, spark_native_apply_stage_plans, spark_native_apply_transaction, } import gunbc.spark.pair_serving_realization { spark_pair_serve_port } import gunbc.spark.pair_serving_authority { @@ -49,10 +51,33 @@ import gunbc.spark.pair_serving_authority_log { CurrentAuthority, CurrentAuthorityRead, CurrentAuthorityUnread, current_pair_serving_authorities, current_authority_for, current_authority_wire, } -import gunbc.spark.v41_runtime_candidate { V41CandidateStanding, V41CandidateKeyed, V41CandidateUnestablished, V41CandidateRefused, v41_candidate } -import gunbc.spark.v41_runtime_realized { - V41ProducedRuntimeStanding, V41RuntimeProduced, V41RuntimeProductionUnestablished, V41RuntimeProductionRefused, -} +import gunbc.spark.v41_runtime_candidate { + V41CandidateStanding, V41CandidateKeyed, V41CandidateUnestablished, V41CandidateRefused, v41_candidate, + v41_candidate_recipe, V41IdentityStanding, V41IdentityKeyed, V41IdentityUnestablished, V41IdentityRefused, v41_runtime_artifact_key, v41_candidate_inputs, v41_candidate_partition, +} +import gunbc.spark.v41_runtime_image_converge { + V41ProducedImageHeld, v41_production_produced_image, v41_produced_image_subject, + V41ProbeJoinedToProducedImage, V41ProbeDescribesAnotherImage, V41ProbeIdentityUnreadable, V41ProbeIdentityNotRead, + v41_probe_identity_join, v41_probe_identity_line, +} +import gunbc.spark.v41_runtime_image_probe { V41ProbeOutcome, V41ProbeRead, V41ProbeRefused, v41_probe_image_in_place } +import gunbc.spark.vllm_runtime_image_build { + VllmImageDistributionSubject, VllmImageTargetRead, VllmImageTargetHolds, VllmImageTargetNeeds, VllmImageTargetRefused, + vllm_image_target_read, vllm_image_target_ctx, + with_spark_build_host, SparkBuildHostWork, SparkBuildHostRefused, SparkBuildHostRan, +} +import gunbc.spark.runtime_architecture_registry_reading { vllm_registry_instrument } +import gunbc.spark.host_occupancy_admission { spark_host_occupancy_admission, spark_ctx_argv_run } +import gunbc.spark.v41_engram_row_store { v41_row_store_spec, v41_live_head_sizes, V41RowStoreSpecAdmitted, V41RowStoreSpecRefused, v41_published_store_readings } +import gunbc.spark.v41_engram_store_manifest { V41EngramStoreManifest, V41StoreManifestDerived, V41StoreManifestRefused, v41_engram_store_manifest } +import extdeps.deepseek.deepseek_v4_1_flash { deepseek_v4_1_flash_engram, deepseek_v4_1_flash_engram_sharded_entries } +import gunbc.spark.remote_step_sequence { run_remote_step_sequence, remote_step_sequence_failed, remote_step_sequence_wire } +import gunbc.host_effect { FleetSsh } +import gunbc.fleet_known_hosts_anchor { FleetSshExecutionContext, SshTarget } +import gunbc.spark.managed_access_bootstrap { spark_managed_executor_login } +import gunbc.spark.model_snapshot_materialize { SparkMaterializeContext } +import gunbc.clock_read { clock_now_probed_at, probed_at_word } +import gunbc.fleet_multi_principal_probe { spark_converge_target_selection, SparkConvergeTargetSingle, SparkConvergeTargetAllObserveOnly, SparkConvergeTargetRefused } import gunbc.spark.v41_capacity_measurement { V41MeasurementLaunchShape, v41_measurement_launch_shape } import gunbc.spark.v41_row_store_encode { v41_row_store_output_directory } @@ -69,12 +94,13 @@ import gunbc.spark.v41_row_store_encode { v41_row_store_output_directory } // rollback). This module names V4.1's tuple once, the way gunbc.spark.glm_group_b_launch names Group B's, // and every V4.1 rendering projects from it. Ruled by proud-deer-538 on 2026-09-26 (DS4.1). // -// WHAT IT RENDERS AND WHAT IT DOES NOT. It plans, renders every host's unit and container line, and -// renders the Engram staging and the apply's preflight and apply scripts for all four hosts -- a DRY -// RUN. It has no wet entry: nothing here reaches a host except the read of the authority log. The first -// consumer that executes these plans is the capacity measurement's serve -// (gunbc.spark.v41_capacity_measurement, the wet runner it names as its frontier), and it must pass -// v41_group_a_launch_admission before it does. +// WHAT IT RENDERS AND WHAT IT RUNS. The dry run plans, renders every host's unit and container line, +// the Engram staging and the apply's preflight and apply scripts for all four hosts, and reaches no host +// except the read of the authority log. The wet entry (v41_group_a_launch_ci_wet, the fleet mode +// spark_v41_group_a_launch) reads the held image, its registry and every host's occupancy, and only on +// v41_group_a_launch_admission stages the Engram manifest and runs the arm transaction. It is one serve +// at the capacity measurement's shape; the staircase that loads it is that measurement's wet runner +// (gunbc.spark.v41_capacity_measurement v41_wet_runner_frontier). // ── WHAT THIS DEPLOYMENT IS CALLED, DISTINCT FROM EVERY OTHER ──────────────────────────────── // @@ -158,14 +184,14 @@ fn v41_group_a_deployment_env() -> List { // ── THE INPUTS A PLAN IS MADE OF, SUPPLIED RATHER THAN READ INSIDE ──────────────────────────── // -// EVERY INPUT THAT IS NOT A DECLARATION ON MAIN ARRIVES AS A VALUE: the produced image and its registry -// reading (the build lane's answer, read back at run time) and the manifest -// text (#12359's gunbc.spark.v41_engram_store_manifest). A witness supplies them; the live binding below -// reads what exists and names what does not. +// EVERY INPUT THAT IS NOT A DECLARATION ON MAIN ARRIVES AS A VALUE: the image as it is HELD on the +// launch's hosts, its registry reading (both read at run time by the wet entry below) and the derived +// Engram row-store manifest. A witness supplies them; the live bindings read what exists and name what +// does not. type V41GroupALaunchInputs { - produced: V41ProducedRuntimeStanding + held: V41HeldRuntimeStanding registry_reading: ArchitectureRegistryReading? - manifest_text: String? + manifest: V41EngramStoreManifest } type V41GroupALaunchPlan @@ -179,39 +205,135 @@ type V41GroupALaunchPlan | V41GroupALaunchUnestablished { obligations: List } | V41GroupALaunchRefused { causes: List } -// THE RUNTIME VARIANT IS THE PRODUCED IMAGE'S IDENTITY: its configuration digest (the only id a -// never-pushed image has, and the one gunbc.spark.v41_runtime_image_converge distributes by), the -// source tree the build observed, and that tree's worktree diff -- the patch set as it was applied. -fn v41_serving_runtime(produced: V41ProducedRuntimeStanding, registry_reading: ArchitectureRegistryReading?) -> ServingRuntime? { - match produced { - V41RuntimeProductionUnestablished { obligations: _ } => none - V41RuntimeProductionRefused { defects: _ } => none - V41RuntimeProduced { specification_runtime_key: k, image: i } => +// ── THE IMAGE A LAUNCH RUNS IS HELD, NOT PRODUCED ───────────────────────────────────────────── +// +// A DIFFERENT STANDING FROM THE BUILD'S, ON PURPOSE. gunbc.spark.v41_runtime_realized +// V41RuntimeProduced is minted only by the build leg that produced the bytes (V41ProducedRuntimeImage +// is sole_constructor, and an image found under a tag refuses as AdoptedUnderExpectedTag), so a launch +// running hours later on four other hosts cannot hold one and must not fake one. What it CAN +// establish is the join of two facts: the production receipt row (gunbc.spark.v41_runtime_image_converge +// v41_production_produced_image -- the digest, the tree the build observed, the run that produced it) +// and a readback on EVERY launch host that the image's tag names exactly that digest there. The tag is +// derived from the receipt's observed tree by the same fold the distribution uses +// (v41_produced_image_subject), so a host holds the subject only if the distribution put it there. +// Three further checks are the admission's own: the receipt's tree is the candidate recipe's revision; +// the candidate's runtime artifact keys (v41_runtime_artifact_key, computed by the caller and handed +// in, since it reads the kernel wheels), which is the specification key the admission compares with +// the candidate below; and no host holds another image under the tag. A host +// that holds nothing under the tag is an OBLIGATION naming the distribution mode, not a refusal: the fix +// is to deliver, and nothing is wrong with the image. +type V41HeldRuntimeStanding + = V41RuntimeHeldOnHosts { specification_runtime_key: ContentHash, image: V41ProducedImageHeld, subject: VllmImageDistributionSubject } + | V41RuntimeHeldUnestablished { obligations: List } + | V41RuntimeHeldRefused { defects: List } + +fn v41_host_image_read(host: HostIdentity, reads: List) -> VllmImageTargetRead? { + first(filter(reads, r => (r.host as String) == (host as String))) +} + +fn v41_held_obligations(hosts: List, reads: List, subject: VllmImageDistributionSubject) -> List { + flat_map(hosts, h => match v41_host_image_read(host: h, reads: reads) { + Absent => [join(["no image readback was taken on ", h as String, ", so whether it holds ", subject.tag as String, " is unknown"], "")] + Present { value: r } => match r.state { + VllmImageTargetHolds => [] as List + VllmImageTargetNeeds => [join([h as String, " holds nothing under ", subject.tag as String, "; deliver it with the distribution mode (gunbc.spark.v41_runtime_image_converge v41_runtime_image_distribute_ci_wet)"], "")] + VllmImageTargetRefused { step: _, cause: _ } => [] as List + } + }) +} + +fn v41_held_defects(hosts: List, reads: List) -> List { + flat_map(hosts, h => match v41_host_image_read(host: h, reads: reads) { + Absent => [] as List + Present { value: r } => match r.state { + VllmImageTargetHolds => [] as List + VllmImageTargetNeeds => [] as List + VllmImageTargetRefused { step: st, cause: why } => [join([h as String, " refused the image readback at ", st as String, ": ", why], "")] + } + }) +} + +fn v41_admit_held_runtime( + recipe: VllmContainerBuildRecipe, + artifact: V41IdentityStanding, + image: V41ProducedImageHeld, + hosts: List, + reads: List, +) -> V41HeldRuntimeStanding { + let subject = v41_produced_image_subject(p: image) + let defects = concat( + if (image.observed.revision_head as String) == (recipe.source.full_revision as String) { [] as List } else { + [join(["the production receipt's image was built from a tree at ", image.observed.revision_head as String, ", and this candidate's recipe names ", recipe.source.full_revision as String], "")] + }, + v41_held_defects(hosts: hosts, reads: reads)) + let obligations = v41_held_obligations(hosts: hosts, reads: reads, subject: subject) + match artifact { + V41IdentityRefused { defects: d } => V41RuntimeHeldRefused { defects: concat(map(d, x => x as String), defects) } + V41IdentityUnestablished { obligations: o } => + if length(defects) != 0 { V41RuntimeHeldRefused { defects: defects } } else { V41RuntimeHeldUnestablished { obligations: concat(map(o, x => x as String), obligations) } } + V41IdentityKeyed { key: spec } => + if length(defects) != 0 { + V41RuntimeHeldRefused { defects: defects } + } else if length(obligations) != 0 { + V41RuntimeHeldUnestablished { obligations: obligations } + } else { + V41RuntimeHeldOnHosts { specification_runtime_key: spec, image: image, subject: subject } + } + } +} + +// THE RUNTIME VARIANT IS THE HELD IMAGE'S IDENTITY: its configuration digest (the only id a +// never-pushed image has, and the one the distribution moves and the hosts read back), the source tree +// the build observed, and that tree's worktree diff -- the patch set as it was applied. +fn v41_serving_runtime(held: V41HeldRuntimeStanding, registry_reading: ArchitectureRegistryReading?) -> ServingRuntime? { + match held { + V41RuntimeHeldUnestablished { obligations: _ } => none + V41RuntimeHeldRefused { defects: _ } => none + V41RuntimeHeldOnHosts { specification_runtime_key: k, image: i, subject: sub } => Present { value: GunbcDeepseekV41EngramProduced { artifact: runtime_artifact_identity( - pin: PinnedByDigest { reference: render_oci_content_digest_wire(d: i.local_config_digest) as NonEmptyStr }, + pin: PinnedByDigest { reference: render_oci_content_digest_wire(d: i.config) as NonEmptyStr }, source: SourceRevisionEstablished { revision: i.observed.revision_head }, patch_set: [join(["worktree-diff-sha256:", i.observed.worktree_diff_sha256.hex as String], "") as NonEmptyStr], - provenance: join(["produced by the V4.1 build lane under tag ", i.tag as String, " with build key ", serialize_content_hash(hash: i.build_key) as String, ", and admitted against the candidate's specification runtime key ", serialize_content_hash(hash: k) as String], "") as NonEmptyStr, + provenance: join(["produced by fleet-converge run ", i.receipt_run as String, " on ", i.held_on as String, " under tag ", sub.tag as String, ", read back under that tag at this digest on every launch host, and admitted against the candidate's specification runtime key ", serialize_content_hash(hash: k) as String], "") as NonEmptyStr, ), registry_reading: registry_reading, } } } } -fn v41_produced_obligations(p: V41ProducedRuntimeStanding) -> List { - match p { - V41RuntimeProduced { specification_runtime_key: _, image: _ } => [] as List - V41RuntimeProductionUnestablished { obligations: o } => map(o, x => join(["the V4.1 image: ", x as String], "")) - V41RuntimeProductionRefused { defects: _ } => [] as List +fn v41_held_obligations_of(h: V41HeldRuntimeStanding) -> List { + match h { + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: _, subject: _ } => [] as List + V41RuntimeHeldUnestablished { obligations: o } => map(o, x => join(["the V4.1 image: ", x], "")) + V41RuntimeHeldRefused { defects: _ } => [] as List } } -fn v41_produced_defects(p: V41ProducedRuntimeStanding) -> List { - match p { - V41RuntimeProductionRefused { defects: d } => map(d, x => join(["the V4.1 image was refused: ", x as String], "")) - V41RuntimeProduced { specification_runtime_key: _, image: _ } => [] as List - V41RuntimeProductionUnestablished { obligations: _ } => [] as List +fn v41_held_defects_of(h: V41HeldRuntimeStanding) -> List { + match h { + V41RuntimeHeldRefused { defects: d } => map(d, x => join(["the V4.1 image was refused: ", x], "")) + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: _, subject: _ } => [] as List + V41RuntimeHeldUnestablished { obligations: _ } => [] as List + } +} + +// ── THE MANIFEST THE LAUNCH STAGES IS DERIVED, AS THE DIFFERENTIAL'S IS ────────────────────── +// +// The same derivation gunbc.spark.v41_engram_differential_run hands its overlay: the row-store spec +// over the live Engram population, then gunbc.spark.v41_engram_store_manifest over the published store +// readings. A refusal at either names its causes, and the plan carries them as refusals. +fn v41_group_a_manifest() -> V41EngramStoreManifest { + match v41_row_store_spec(population: map(deepseek_v4_1_flash_engram_sharded_entries, e => e.entry), partition: v41_candidate_partition, tables: deepseek_v4_1_flash_engram, head_sizes: v41_live_head_sizes(tables: deepseek_v4_1_flash_engram)) { + V41RowStoreSpecRefused { defects: d } => V41StoreManifestRefused { causes: map(d, x => join(["the row-store spec refused: ", x as String], "")) } + V41RowStoreSpecAdmitted { spec: spec } => v41_engram_store_manifest(spec: spec, readings: v41_published_store_readings) + } +} + +fn v41_manifest_defects(m: V41EngramStoreManifest) -> List { + match m { + V41StoreManifestDerived { text: _, stores: _ } => [] as List + V41StoreManifestRefused { causes: c } => map(c, x => join(["the Engram row-store manifest refused: ", x], "")) } } @@ -232,45 +354,43 @@ fn v41_group_a_profile(shape: V41MeasurementLaunchShape) -> ServingLaunchProfile // inputs reaches the arm launcher, and its own walls -- the arm admission (registry reading, fit), the // compatibility roster and the bounded-experiment intent -- still decide. fn v41_group_a_launch_plan(inputs: V41GroupALaunchInputs) -> V41GroupALaunchPlan { - let defects = v41_produced_defects(p: inputs.produced) - let obligations = concat( - v41_produced_obligations(p: inputs.produced), - match inputs.manifest_text { Present { value: _ } => [] as List Absent => ["the Engram row-store manifest text: gunbc.spark.v41_engram_store_manifest (#12359) has not landed, so there is no derived manifest to stage"] }) + let defects = concat(v41_held_defects_of(h: inputs.held), v41_manifest_defects(m: inputs.manifest)) + let obligations = v41_held_obligations_of(h: inputs.held) if length(defects) != 0 { V41GroupALaunchRefused { causes: defects } } else if length(obligations) != 0 { V41GroupALaunchUnestablished { obligations: obligations } } else { - match inputs.produced { - V41RuntimeProductionUnestablished { obligations: _ } => V41GroupALaunchUnestablished { obligations: obligations } - V41RuntimeProductionRefused { defects: _ } => V41GroupALaunchRefused { causes: defects } - V41RuntimeProduced { specification_runtime_key: key, image: _ } => - match v41_serving_runtime(produced: inputs.produced, registry_reading: inputs.registry_reading) { - Absent => V41GroupALaunchRefused { causes: ["unreachable: a produced image always has a runtime identity"] } + match inputs.held { + V41RuntimeHeldUnestablished { obligations: _ } => V41GroupALaunchUnestablished { obligations: obligations } + V41RuntimeHeldRefused { defects: _ } => V41GroupALaunchRefused { causes: defects } + V41RuntimeHeldOnHosts { specification_runtime_key: key, image: _, subject: _ } => + match v41_serving_runtime(held: inputs.held, registry_reading: inputs.registry_reading) { + Absent => V41GroupALaunchRefused { causes: ["unreachable: a held image always has a runtime identity"] } Present { value: runtime } => - match inputs.manifest_text { - Absent => V41GroupALaunchUnestablished { obligations: obligations } - Present { value: manifest } => { - let profile = v41_group_a_profile(shape: v41_measurement_launch_shape()) - match plan_arm_launch_for( - checkpoint: DeepseekV41FlashDba1be0a, - runtime: runtime, - topology: FourSparkTp4, - group: FabricGroupA, - intent: ArmBoundedExperiment, - profile: profile, - deployment_env: v41_group_a_deployment_env(), - container: v41_group_a_arm_container, - ) { - ArmLaunchRefused { cause: c } => V41GroupALaunchRefused { causes: [c as String] } - ArmLaunchPlanned { lines: ls, master: m } => - V41GroupALaunchPlanned { - lines: ls, profile: profile, specification_runtime_key: key, manifest_text: manifest, - master_endpoint: glm_native_rendezvous_endpoint(master: m, port: v41_group_a_master_port), - } + match inputs.manifest { + V41StoreManifestRefused { causes: c } => V41GroupALaunchRefused { causes: c } + V41StoreManifestDerived { text: manifest, stores: _ } => { + let profile = v41_group_a_profile(shape: v41_measurement_launch_shape()) + match plan_arm_launch_for( + checkpoint: DeepseekV41FlashDba1be0a, + runtime: runtime, + topology: FourSparkTp4, + group: FabricGroupA, + intent: ArmBoundedExperiment, + profile: profile, + deployment_env: v41_group_a_deployment_env(), + container: v41_group_a_arm_container, + ) { + ArmLaunchRefused { cause: c } => V41GroupALaunchRefused { causes: [c as String] } + ArmLaunchPlanned { lines: ls, master: m } => + V41GroupALaunchPlanned { + lines: ls, profile: profile, specification_runtime_key: key, manifest_text: manifest, + master_endpoint: glm_native_rendezvous_endpoint(master: m, port: v41_group_a_master_port), } - } } + } + } } } } @@ -508,20 +628,20 @@ fn v41_group_a_dry_run_report(plan: V41GroupALaunchPlan, admission: V41LaunchAdm join(["V4.1 Group A TP4 launch -- DRY RUN (no host is touched)\n", body, "\nadmission: ", verdict, "\n"], "") } -// THE LIVE INPUTS, AND WHY EACH IS WHAT IT IS ON MAIN TODAY. No produced-image receipt is carried in -// the tree -- the build lane's admission is a value its own run reads back -- so the live standing names -// that obligation rather than transcribing a digest; no registry reading exists inside that image; the -// manifest is #12359, still open. -fn v41_group_a_live_inputs() -> V41GroupALaunchInputs { +// THE DRY RUN'S INPUTS. The manifest is handed in (the entry derives it with v41_group_a_manifest, as +// the wet entry does). The held +// image and its registry reading are read on the hosts, and the dry run reaches no host, so it names +// both as obligations rather than transcribing either. +fn v41_group_a_live_inputs(manifest: V41EngramStoreManifest) -> V41GroupALaunchInputs { V41GroupALaunchInputs { - produced: V41RuntimeProductionUnestablished { obligations: ["no produced-image admission (gunbc.spark.v41_runtime_realized v41_admit_produced_runtime) is carried on main; the dry run is handed one by the converge that produced and distributed it (#12353)" as NonEmptyStr] }, + held: V41RuntimeHeldUnestablished { obligations: ["the image is read back on every Group A host only by the wet launch (v41_group_a_launch_ci_wet); the dry run reaches no host"] }, registry_reading: none, - manifest_text: none, + manifest: manifest, } } // THE DRY RUN REACHES NO HOST, SO IT HAS NO OCCUPANCY READING, and says so per host rather than -// assuming vacancy. The wet runner reads each host through gunbc.spark.host_occupancy_admission +// assuming vacancy. The wet entry reads each host through gunbc.spark.host_occupancy_admission // spark_host_occupancy_admission and supplies the verdicts here. fn v41_dry_run_occupancy() -> List { map(fabric_group_hosts(g: FabricGroupA), h => V41HostOccupancy { @@ -530,24 +650,30 @@ fn v41_dry_run_occupancy() -> List { }) } -data v41_group_a_dry_run_path: String = "target/v41-group-a-launch-dry-run.txt" - -// THE DRY-RUN ENTRY. It reads the authority log (read-only) and the held hosts, renders, writes the -// report, and exits zero only when the plan rendered AND admission holds -- a report of a plan nobody -// may start is written all the same, and the exit says it may not start. -fn v41_group_a_launch_dry_run() -> ProcessExit { - let plan = v41_group_a_launch_plan(inputs: v41_group_a_live_inputs()) +// THE ADMISSION AGAINST THE LIVE AUTHORITY LOG, read-only, shared by the dry run and the wet entry: the +// executor names the log, the clock the lease standing, and each is a refusal when unread. +fn v41_group_a_launch_admission_live(plan: V41GroupALaunchPlan, occupancy: List) -> V41LaunchAdmission { let held = spark_serving_admissible_hosts(subject: PairServingUnitOn { group: FabricGroupA }) let candidate = v41_candidate().standing - let admission = match observe_executor_reach() { + match observe_executor_reach() { ExecutorReachUnknown { cause: c } => V41LaunchAdmissionRefused { causes: [join(["the executor could not be identified, so Group A's authority log could not be read: ", c], "")] } ExecutorReachKnown { short_hostname: executor, path: _ } => match now_epoch_seconds() { Absent => V41LaunchAdmissionRefused { causes: ["the clock could not be read, so the authority's lease standing cannot be derived"] } Present { value: at } => - v41_group_a_launch_admission_current(plan: plan, current: current_pair_serving_authorities(short_hostname: executor, at: at), candidate: candidate, held: held, occupancy: v41_dry_run_occupancy()) + v41_group_a_launch_admission_current(plan: plan, current: current_pair_serving_authorities(short_hostname: executor, at: at), candidate: candidate, held: held, occupancy: occupancy) } } +} + +data v41_group_a_dry_run_path: String = "target/v41-group-a-launch-dry-run.txt" + +// THE DRY-RUN ENTRY. It reads the authority log (read-only) and the held hosts, renders, writes the +// report, and exits zero only when the plan rendered AND admission holds -- a report of a plan nobody +// may start is written all the same, and the exit says it may not start. +fn v41_group_a_launch_dry_run() -> ProcessExit { + let plan = v41_group_a_launch_plan(inputs: v41_group_a_live_inputs(manifest: v41_group_a_manifest())) + let admission = v41_group_a_launch_admission_live(plan: plan, occupancy: v41_dry_run_occupancy()) let report = v41_group_a_dry_run_report(plan: plan, admission: admission) let w = Filesystem.Write(path: v41_group_a_dry_run_path, content: report) if !w.success { @@ -559,3 +685,225 @@ fn v41_group_a_launch_dry_run() -> ProcessExit { } } } + +// ── THE WET LAUNCH ─────────────────────────────────────────────────────────────────────────── +// +// THE REGISTRY READING IS TAKEN INSIDE THE HELD DIGEST, BY THE PROBE THE BUILD USES. The arm wall +// (gunbc.spark.serving_arm serving_arm_admits_under) admits a runtime only on a registry reading taken +// inside its own digest, and the production image has none on main. The build's capability probe +// (gunbc.spark.v41_runtime_image_probe v41_probe_image_in_place) already prints the whole registry +// (REGISTRY_COUNT, then one REGISTERED line per architecture) from inside the digest it is handed, so +// the wet entry runs that probe on a launch host and folds its output into the reading. The fold is +// complete or nothing: the probe's identity must join the receipt's digest +// (gunbc.spark.v41_runtime_image_converge v41_probe_identity_join), the count line must be present and +// positive, and the REGISTERED lines must number exactly that count; anything else is no reading, and +// the arm wall then refuses with its own cause. +fn v41_registered_architectures(stdout: String) -> List { + flat_map(split(stdout, "\n"), l => { + let t = trim(s: l) + if starts_with(s: t, prefix: "REGISTERED ") && length(t) > 11 { [substring(s: t, start: 11, end: length(t)) as NonEmptyStr] } else { [] as List } + }) +} + +fn v41_registry_count(stdout: String) -> Int? { + first(flat_map(split(stdout, "\n"), l => { + let t = trim(s: l) + if starts_with(s: t, prefix: "REGISTRY_COUNT ") { + match parse_int(s: substring(s: t, start: 15, end: length(t))) { Present { value: n } => [n] Absent => [] as List } + } else { [] as List } + })) +} + +fn v41_registry_reading_of(probe: V41ProbeOutcome, image: V41ProducedImageHeld, host: HostIdentity, observed_at: String) -> ArchitectureRegistryReading? { + match v41_probe_identity_join(produced: image.config, o: probe) { + V41ProbeDescribesAnotherImage { produced: _, observed: _ } => none + V41ProbeIdentityUnreadable { observed: _ } => none + V41ProbeIdentityNotRead { step: _, cause: _ } => none + V41ProbeJoinedToProducedImage { digest: d } => + match probe { + V41ProbeRefused { step: _, cause: _ } => none + V41ProbeRead { reference: _, observed_config_id: _, stdout: out, readings: _ } => { + let archs = v41_registered_architectures(stdout: out) + match v41_registry_count(stdout: out) { + Absent => none + Present { value: n } => + if n <= 0 || length(archs) != n { + none + } else { + Present { value: ArchitectureRegistryReading { + image_reference: d, + host: host as NonEmptyStr, + observed_at: observed_at as NonEmptyStr, + instrument: vllm_registry_instrument, + architectures: archs, + } } + } + } + } + } + } +} + +fn v41_registry_line(probe: V41ProbeOutcome, image: V41ProducedImageHeld, reading: ArchitectureRegistryReading?) -> String { + join([ + v41_probe_identity_line(j: v41_probe_identity_join(produced: image.config, o: probe)), + match reading { + Present { value: r } => join(["registry reading: ", to_string(length(r.architectures)), " architectures inside ", r.image_reference as String, " on ", r.host as String, "\n"], "") + Absent => "registry reading: NONE (the probe did not join the receipt's digest, or its REGISTRY_COUNT and REGISTERED lines did not agree)\n" + }, + ], "") +} + +// THE OCCUPANCY NEED IS THE ENGINE'S OWN REQUEST: the measurement's gpu-memory-utilization share of a +// GB10's unified pool (gunbc.spark.serving_arm gb10_unified_pool_gib), which vLLM asks the device for +// at startup. A host with less available than that cannot start the rank; the Engram page cache lives +// in the share the engine leaves, which the arm wall already charges. +fn v41_group_a_memory_need() -> Kibibyte { + kibibyte(count: basis_point_count(bp: v41_measurement_launch_shape().gpu_memory_utilization) * gb10_unified_pool_gib * 1048576 / 10000) +} + +fn v41_host_image_read_line(r: VllmImageTargetRead) -> String { + join(["image readback host=", r.host as String, " ", match r.state { + VllmImageTargetHolds => "HOLDS the receipt's digest under its tag" + VllmImageTargetNeeds => "holds NOTHING under the tag" + VllmImageTargetRefused { step: st, cause: why } => join(["REFUSED at ", st as String, ": ", why], "") + }, "\n"], "") +} + +fn v41_host_ssh_target(host: HostIdentity) -> SshTarget? { + match spark_reach_endpoint(host: host, path: ReachManagementLan) { + Absent => none + Present { value: e } => Present { value: SshTarget { endpoint: e as NonEmptyStr, principal: spark_managed_executor_login as NonEmptyStr } } + } +} + +// THE ENGRAM STAGING ON EVERY HOST, IN ORDER, HALTING AT THE FIRST HOST THAT FAILS. It runs before the +// apply because the apply's preflight refuses a container whose mount sources are missing; a failed +// staging therefore stops the launch with no serving unit touched. +type V41StagingOutcome + = V41StagedEverywhere { receipt: String } + | V41StagingStopped { receipt: String } + +fn v41_stage_engram_on(hosts: List, manifest_text: String, context: FleetSshExecutionContext) -> V41StagingOutcome { + match v41_engram_staging_steps(manifest_text: manifest_text) { + Absent => V41StagingStopped { receipt: "engram staging REFUSED: the manifest write could not be sealed to its directory; no host was staged\n" } + Present { value: steps } => + fold(hosts, init: V41StagedEverywhere { receipt: "" }, f: fn(acc, h) { + match acc { + V41StagingStopped { receipt: _ } => acc + V41StagedEverywhere { receipt: so_far } => + match v41_host_ssh_target(host: h) { + Absent => V41StagingStopped { receipt: join([so_far, "engram staging host=", h as String, " REFUSED: no management-LAN endpoint\n"], "") } + Present { value: target } => { + let standings = run_remote_step_sequence(steps: steps, transport: FleetSsh { target: target, context: context }) + let line = join(["engram staging host=", h as String, "\n", remote_step_sequence_wire(standings: standings)], "") + if remote_step_sequence_failed(standings: standings) { + V41StagingStopped { receipt: join([so_far, line], "") } + } else { + V41StagedEverywhere { receipt: join([so_far, line], "") } + } + } + } + } + }) + } +} + +// THE SESSION: everything the launch reads, then -- only on admission -- the staging and the arm +// transaction (gunbc.spark.native_serving_apply spark_native_apply_transaction, the same preflight, +// preserve, head-first apply, readback and commit-or-rollback Group B's relaunch runs). Every read is +// in the receipt whatever the verdict, so a refused run reports the whole distance to a launch. +fn v41_group_a_launch_session(now_word: String, context: FleetSshExecutionContext, ctx: SparkMaterializeContext) -> SparkBuildHostWork { + let hosts = fabric_group_hosts(g: FabricGroupA) + let image = v41_production_produced_image + let subject = v41_produced_image_subject(p: image) + let reads = map(hosts, h => vllm_image_target_read(source_ctx: ctx, host: h, image: subject)) + let held = v41_admit_held_runtime(recipe: v41_candidate_recipe(), artifact: v41_runtime_artifact_key(i: v41_candidate_inputs.runtime, partition: v41_candidate_partition), image: image, hosts: hosts, reads: reads) + let probe = match held { + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: _, subject: _ } => + v41_probe_image_in_place(ctx: ctx, reference: render_oci_content_digest_wire(d: image.config) as NonEmptyStr) + V41RuntimeHeldUnestablished { obligations: _ } => V41ProbeRefused { step: "registry-probe" as NonEmptyStr, cause: "the image is not established on every launch host, so it was not probed" } + V41RuntimeHeldRefused { defects: _ } => V41ProbeRefused { step: "registry-probe" as NonEmptyStr, cause: "the held image was refused, so it was not probed" } + } + let registry = v41_registry_reading_of(probe: probe, image: image, host: ctx.host, observed_at: now_word) + let plan = v41_group_a_launch_plan(inputs: V41GroupALaunchInputs { held: held, registry_reading: registry, manifest: v41_group_a_manifest() }) + let occupancy = map(hosts, h => V41HostOccupancy { + host: h, + admission: spark_host_occupancy_admission(run: spark_ctx_argv_run(ctx: vllm_image_target_ctx(source_ctx: ctx, host: h)), need: v41_group_a_memory_need()), + }) + let admission = v41_group_a_launch_admission_live(plan: plan, occupancy: occupancy) + let reads_receipt = join([ + join(["image receipt run=", image.receipt_run as String, " tag=", subject.tag as String, " config_digest=", render_oci_content_digest_wire(d: image.config), "\n"], ""), + join(map(reads, r => v41_host_image_read_line(r: r)), ""), + v41_registry_line(probe: probe, image: image, reading: registry), + v41_group_a_dry_run_report(plan: plan, admission: admission), + ], "") + match admission { + V41LaunchAdmissionRefused { causes: _ } => SparkBuildHostWork { receipt: join([reads_receipt, "NOT LAUNCHED: admission refused; no host was staged and no serving unit was touched\n"], ""), converged: false } + V41LaunchAdmitted => + match plan { + V41GroupALaunchUnestablished { obligations: _ } => SparkBuildHostWork { receipt: join([reads_receipt, "NOT LAUNCHED: unreachable, an admitted plan is planned\n"], ""), converged: false } + V41GroupALaunchRefused { causes: _ } => SparkBuildHostWork { receipt: join([reads_receipt, "NOT LAUNCHED: unreachable, an admitted plan is planned\n"], ""), converged: false } + V41GroupALaunchPlanned { lines: _, profile: _, specification_runtime_key: _, manifest_text: manifest, master_endpoint: _ } => + match v41_group_a_steps(p: plan) { + NativeArmStepsRefused { cause: c } => SparkBuildHostWork { receipt: join([reads_receipt, "NOT LAUNCHED: the units did not render: ", c as String, "\n"], ""), converged: false } + NativeArmSteps { steps: steps, master_endpoint: _ } => + match v41_stage_engram_on(hosts: map(steps, st => st.host), manifest_text: manifest, context: context) { + V41StagingStopped { receipt: r } => SparkBuildHostWork { receipt: join([reads_receipt, r, "NOT LAUNCHED: the Engram staging stopped; no serving unit was touched\n"], ""), converged: false } + V41StagedEverywhere { receipt: r } => { + let run = spark_native_apply_transaction(context: context, trust: ctx.trust, path: ctx.path, administrator: ctx.admin_credential, steps: steps, at_word: now_word) + SparkBuildHostWork { receipt: join([reads_receipt, r, run.body], ""), converged: run.committed } + } + } + } + } + } +} + +data v41_group_a_launch_attempt_raw: String = "v41-group-a-launch" +data v41_group_a_launch_receipt_path: String = "target/v41-group-a-launch-receipt.txt" + +// THE MODE ENTRY. The hosts are Group A's, the image is the production receipt's and the shape is the +// measurement's, so nothing an operator types changes what it launches. SPARK_CONVERGE_TARGET names only +// the SESSION host -- where the registry probe runs, and the host the administrator credential step +// resolves for -- and it must be a host Group A's serving subject holds; unset refuses rather than +// choosing. Every other host is reached over the same privileged edge. Exit zero means the arm +// COMMITTED: every rank read back and the head's front door serving this arm's model. +fn v41_group_a_launch_ci_wet() -> ProcessExit +{ + let now = clock_now_probed_at() + let now_word = probed_at_word(reading: now) as String + match spark_converge_target_selection(execute_apply: true) { + SparkConvergeTargetRefused { reason: why } => exit_failure(reason: why) + SparkConvergeTargetAllObserveOnly => + exit_failure(reason: "SPARK_CONVERGE_TARGET is unset: the launch needs one Group A host to open its session on, and it refuses rather than choosing one") + SparkConvergeTargetSingle { host: session_host } => + match admit_host_held_by_subject(raw: session_host, subject: PairServingUnitOn { group: FabricGroupA }) { + Present { value: why } => exit_failure(reason: why) + Absent => + match with_spark_build_host( + host: session_host, + attempt_raw: v41_group_a_launch_attempt_raw, + identity_file_name: "gunbc-v41-group-a-launch-admin-password" as NonEmptyStr, + use: fn(target, context, ctx) { v41_group_a_launch_session(now_word: now_word, context: context, ctx: ctx) }, + ) { + SparkBuildHostRefused { cause: c } => exit_failure(reason: c) + SparkBuildHostRan { work: work, endpoint: endpoint, executor: executor, trust_receipt: trust_receipt } => { + let body = join([ + join(["v41 group A launch probed_at=", now_word, " session_host=", session_host as String, " endpoint=", endpoint as String, " executor=", executor, "\n"], ""), + trust_receipt, + work.receipt, + ], "") + let w = Filesystem.Write(path: v41_group_a_launch_receipt_path, content: body) + if !w.success { + exit_failure(reason: join(["v41 group A launch receipt write failed (the launch's own outcome is below and is NOT changed by this): ", w.error, "\n", body], "")) + } else if work.converged { + ExitSuccess + } else { + exit_failure(reason: join(["the V4.1 Group A launch did not commit:\n", body], "")) + } + } + } + } + } +} diff --git a/dag/test/claim/spark/v41_group_a_launch_witness_test.dag b/dag/test/claim/spark/v41_group_a_launch_witness_test.dag index e1ade7489cc..7fad629cea2 100644 --- a/dag/test/claim/spark/v41_group_a_launch_witness_test.dag +++ b/dag/test/claim/spark/v41_group_a_launch_witness_test.dag @@ -3,7 +3,7 @@ module test.claim.spark.v41_group_a_launch_witness import std.types { Bool, String, List, NonEmptyStr } import std.nat { Nat } import v2.std.optional { Present, Absent } -import std.measure { token_count, basis_point, kibibyte, mebibyte } +import std.measure { token_count, basis_point, kibibyte, kibibyte_count, mebibyte } import extdeps.systemd { Active } import gunbc.compute.host_occupancy { HostUnoccupied, HostOccupiedBy, ServingUnitActive } import std.content_hash { ContentHash, Sha256Hash, sha256_hex_digest } @@ -14,22 +14,27 @@ import extdeps.vllm.engine_args { PrefixCachingExplicit } import gunbc.spark.fabric_switch_observed { FabricGroupA, fabric_group_hosts } import gunbc.spark.released_baseline { QuiescentReservedBaseline } import gunbc.spark.runtime_architecture_registry_reading { ArchitectureRegistryReading } -import gunbc.spark.vllm_runtime_image_build { BuiltByThisRun } +import gunbc.spark.vllm_runtime_image_build { VllmImageDistributionSubject, VllmImageTargetRead, VllmImageTargetHolds, VllmImageTargetNeeds, VllmImageTargetRefused } import gunbc.spark.v41_source_patch_converge { v41_observed_source_identity_of } -import gunbc.spark.v41_runtime_realized { V41ProducedRuntimeStanding, V41RuntimeProduced, v41_produced_runtime_image } +import gunbc.spark.v41_runtime_image_converge { V41ProducedImageHeld, v41_production_produced_image, v41_produced_image_subject } +import gunbc.spark.v41_runtime_image_probe { V41ProbeRead, V41ProbeRefused } +import gunbc.spark.v41_engram_store_manifest { V41StoreManifestDerived } +import gunbc.spark.v41_runtime_candidate { v41_candidate_recipe, V41IdentityStanding, V41IdentityKeyed, V41IdentityUnestablished } import gunbc.spark.v41_runtime_candidate { V41CandidateStanding, V41CandidateKeyed } import gunbc.spark.pair_serving_authority { PairServingGroupAuthority, PairServingActive, SuspendedForAuthorizedSuccessor, PairRealizationKeyed, PairRealizationUnestablished, } import gunbc.spark.serving_arm_launch { ArmNodeLaunch } -import gunbc.spark.serving_arm { serving_arm_admits, serving_arm_admits_under, deepseek_v41_measurement_profile, ServingArmAdmitted, ServingArmRefused, DeepseekV41FlashDba1be0a, FourSparkTp4 } +import gunbc.spark.serving_arm { gb10_unified_pool_gib, serving_arm_admits, serving_arm_admits_under, deepseek_v41_measurement_profile, ServingArmAdmitted, ServingArmRefused, DeepseekV41FlashDba1be0a, FourSparkTp4 } import gunbc.spark.native_serving_realization { GlmNativeUnitRender, GlmNativeUnitRendered, GlmNativeUnitRenderRefused } import gunbc.spark.v41_group_a_launch { V41GroupALaunchInputs, V41HostOccupancy, v41_dry_run_occupancy, V41GroupALaunchPlan, V41GroupALaunchPlanned, V41GroupALaunchUnestablished, V41GroupALaunchRefused, V41LaunchAdmission, V41LaunchAdmitted, V41LaunchAdmissionRefused, v41_group_a_launch_plan, v41_group_a_live_inputs, v41_group_a_unit_renders, v41_serving_runtime, + V41HeldRuntimeStanding, V41RuntimeHeldOnHosts, V41RuntimeHeldUnestablished, V41RuntimeHeldRefused, + v41_admit_held_runtime, v41_registry_reading_of, v41_group_a_memory_need, v41_group_a_launch_admission, v41_group_a_dry_run_report, v41_group_a_container_name, v41_engram_manifest_env_name, v41_engram_touch_receipt_env_name, v41_engram_manifest_dir, v41_engram_touch_receipt_dir, @@ -56,21 +61,21 @@ fn with_w_hashes(scenario: fn(ContentHash, ContentHash) -> Bool) -> Bool { } } -fn w_produced(runtime_key: ContentHash) -> V41ProducedRuntimeStanding? { +fn w_held(runtime_key: ContentHash) -> V41HeldRuntimeStanding? { match oci_sha256_content_digest(hex: w_config_hex) { Absent => none Present { value: d } => match sha256_hex_digest(hex: "4444444444444444444444444444444444444444444444444444444444444444") { Absent => none - Present { value: diff } => Present { value: V41RuntimeProduced { + Present { value: diff } => Present { value: V41RuntimeHeldOnHosts { specification_runtime_key: runtime_key, - image: v41_produced_runtime_image( - tag: "gunbc-vllm-v41-gb10:witness" as NonEmptyStr, - local_config_digest: d, - build_key: runtime_key, + image: V41ProducedImageHeld { observed: v41_observed_source_identity_of(revision_head: "d2d649e674c75425d2d6975c87eb89fd4d55fff8" as NonEmptyStr, worktree_diff_sha256: diff), - provenance: BuiltByThisRun, - ), + config: d, + held_on: "srv8" as HostIdentity, + receipt_run: "witness" as NonEmptyStr, + }, + subject: VllmImageDistributionSubject { tag: "gunbc-vllm-v41-gb10:witness" as NonEmptyStr, config: d }, } } } } @@ -89,10 +94,10 @@ fn w_registry() -> ArchitectureRegistryReading { data w_manifest: String = "{\"magic_hex\": \"witness\", \"stores\": []}" fn w_inputs(runtime_key: ContentHash, registry: ArchitectureRegistryReading?) -> V41GroupALaunchInputs? { - match w_produced(runtime_key: runtime_key) { + match w_held(runtime_key: runtime_key) { Absent => none Present { value: p } => Present { value: V41GroupALaunchInputs { - produced: p, registry_reading: registry, manifest_text: Present { value: w_manifest }, + held: p, registry_reading: registry, manifest: V41StoreManifestDerived { text: w_manifest, stores: 0 }, } } } } @@ -206,9 +211,9 @@ test fn w_an_unprobed_image_is_refused_by_the_arm_wall() -> Bool { // cannot hold -- the refusal names the allowance, so a wall that charged zero would not produce it. test fn w_the_engram_page_cache_allowance_is_charged_to_the_pool() -> Bool { with_w_hashes(scenario: fn(rk, ck) { - match w_produced(runtime_key: rk) { + match w_held(runtime_key: rk) { Absent => false - Present { value: p } => match v41_serving_runtime(produced: p, registry_reading: Present { value: w_registry() }) { + Present { value: p } => match v41_serving_runtime(held: p, registry_reading: Present { value: w_registry() }) { Absent => false Present { value: r } => { let half = deepseek_v41_measurement_profile( @@ -229,9 +234,9 @@ test fn w_the_engram_page_cache_allowance_is_charged_to_the_pool() -> Bool { // this checkpoint rather than fitting it on weights alone. test fn w_without_a_profile_the_allowance_is_unknown_and_refuses() -> Bool { with_w_hashes(scenario: fn(rk, ck) { - match w_produced(runtime_key: rk) { + match w_held(runtime_key: rk) { Absent => false - Present { value: p } => match v41_serving_runtime(produced: p, registry_reading: Present { value: w_registry() }) { + Present { value: p } => match v41_serving_runtime(held: p, registry_reading: Present { value: w_registry() }) { Absent => false Present { value: r } => match serving_arm_admits(checkpoint: DeepseekV41FlashDba1be0a, runtime: r, topology: FourSparkTp4) { ServingArmRefused { cause: c } => string_contains(s: c as String, pattern: "no profile was supplied") @@ -242,15 +247,15 @@ test fn w_without_a_profile_the_allowance_is_unknown_and_refuses() -> Bool { }) } -// THE PAIRING OBLIGATION: THE LIVE ROUTE. Main carries no produced image and no -// manifest, and the real live inputs must reach the planner and name all three -- the route, not just -// the fact of a refusal. +// THE PAIRING OBLIGATION: THE LIVE ROUTE. The dry run's real inputs reach the planner and name the held +// image as the wet entry's reading -- the route, not just the fact of a refusal. The manifest is supplied +// here; its derivation is gunbc.spark.v41_engram_store_manifest's subject. test fn w_the_live_route_reaches_the_planner_and_names_every_gap() -> Bool { - match v41_group_a_launch_plan(inputs: v41_group_a_live_inputs()) { + match v41_group_a_launch_plan(inputs: v41_group_a_live_inputs(manifest: V41StoreManifestDerived { text: w_manifest, stores: 0 })) { V41GroupALaunchUnestablished { obligations: o } => - any(o, x => string_contains(s: x, pattern: "the V4.1 image")) - && !any(o, x => string_contains(s: x, pattern: "#12371")) - && any(o, x => string_contains(s: x, pattern: "#12359")) + length(o) == 1 + && any(o, x => string_contains(s: x, pattern: "the V4.1 image") && string_contains(s: x, pattern: "v41_group_a_launch_ci_wet")) + && !any(o, x => string_contains(s: x, pattern: "manifest")) V41GroupALaunchPlanned { lines: _, profile: _, specification_runtime_key: _, manifest_text: _, master_endpoint: _ } => false V41GroupALaunchRefused { causes: _ } => false } @@ -395,3 +400,103 @@ test fn w_the_dry_run_renders_all_four_hosts() -> Bool { && string_contains(s: report, pattern: "admission: ADMITTED") }) } + +// ── THE HELD IMAGE: THE PRODUCTION RECEIPT JOINED TO EVERY HOST'S READBACK ───────────────────── +// +// These run the real admission over the real receipt row and candidate inputs, with each host's +// readback supplied -- the readback is the only part that needs a host. + +fn w_reads(state_of: fn(HostIdentity) -> VllmImageTargetRead) -> List { + map(fabric_group_hosts(g: FabricGroupA), h => state_of(h)) +} + +// THE CANDIDATE'S RUNTIME KEY IS SUPPLIED: computing it reads the kernel wheels, and it is the build's +// admission's subject (gunbc.spark.v41_runtime_realized), not this one's. +fn w_artifact() -> V41IdentityStanding { + match sha256_hex_digest(hex: "1111111111111111111111111111111111111111111111111111111111111111") { + Absent => V41IdentityUnestablished { obligations: ["witness digest did not parse" as NonEmptyStr] } + Present { value: d } => V41IdentityKeyed { key: Sha256Hash(d) } + } +} + +fn w_admit(reads: List) -> V41HeldRuntimeStanding { + v41_admit_held_runtime(recipe: v41_candidate_recipe(), artifact: w_artifact(), image: v41_production_produced_image, hosts: fabric_group_hosts(g: FabricGroupA), reads: reads) +} + +// THE POSITIVE CONTROL: every host holds the receipt's digest under its tag, so the image is held and +// its subject is the distribution's own (the tag re-derived from the receipt's tree). +test fn w_every_host_holding_the_receipt_digest_is_held() -> Bool { + match w_admit(reads: w_reads(state_of: fn(h) { VllmImageTargetRead { host: h, state: VllmImageTargetHolds } })) { + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: i, subject: sub } => + (i.receipt_run as String) == (v41_production_produced_image.receipt_run as String) + && (sub.tag as String) == (v41_produced_image_subject(p: v41_production_produced_image).tag as String) + V41RuntimeHeldUnestablished { obligations: _ } => false + V41RuntimeHeldRefused { defects: _ } => false + } +} + +// ONE HOST HOLDING NOTHING UNDER THE TAG IS AN OBLIGATION NAMING THAT HOST AND THE DISTRIBUTION MODE, +// never a refusal and never a pass. +test fn w_a_host_holding_nothing_is_an_obligation_to_distribute() -> Bool { + match w_admit(reads: w_reads(state_of: fn(h) { VllmImageTargetRead { host: h, state: if (h as String) == "srv7" { VllmImageTargetNeeds } else { VllmImageTargetHolds } } })) { + V41RuntimeHeldUnestablished { obligations: o } => + length(o) == 1 && any(o, x => string_contains(s: x, pattern: "srv7") && string_contains(s: x, pattern: "v41_runtime_image_distribute_ci_wet")) + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: _, subject: _ } => false + V41RuntimeHeldRefused { defects: _ } => false + } +} + +// A HOST HOLDING ANOTHER IMAGE UNDER THE TAG REFUSES: loading over it is not this launch's to do. +test fn w_a_host_holding_another_image_refuses() -> Bool { + match w_admit(reads: w_reads(state_of: fn(h) { VllmImageTargetRead { host: h, state: if (h as String) == "srv6" { VllmImageTargetRefused { step: "deliver-digest-pre" as NonEmptyStr, cause: "the target holds sha256:ffff under the tag" } } else { VllmImageTargetHolds } } })) { + V41RuntimeHeldRefused { defects: d } => any(d, x => string_contains(s: x, pattern: "srv6") && string_contains(s: x, pattern: "sha256:ffff")) + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: _, subject: _ } => false + V41RuntimeHeldUnestablished { obligations: _ } => false + } +} + +// A HOST WITH NO READBACK AT ALL IS AN OBLIGATION, NEVER READ AS HOLDING THE IMAGE. +test fn w_a_host_with_no_readback_is_an_obligation() -> Bool { + match w_admit(reads: filter(w_reads(state_of: fn(h) { VllmImageTargetRead { host: h, state: VllmImageTargetHolds } }), r => (r.host as String) != "srv5")) { + V41RuntimeHeldUnestablished { obligations: o } => any(o, x => string_contains(s: x, pattern: "no image readback was taken on srv5")) + V41RuntimeHeldOnHosts { specification_runtime_key: _, image: _, subject: _ } => false + V41RuntimeHeldRefused { defects: _ } => false + } +} + +// ── THE REGISTRY READING, FOLDED FROM THE PROBE'S OWN OUTPUT ───────────────────────────────── + +data w_probe_stdout: String = "COMPUTE_CAPABILITY_MAJOR 12\nREGISTRY_COUNT 2\nREGISTERED DeepseekV41ForCausalLM\nREGISTERED DeepseekV4ForCausalLM\nVERDICT architecture-registered=present\n" + +fn w_probe_of(stdout: String, observed: String) -> V41ProbeRead { + V41ProbeRead { reference: "witness" as NonEmptyStr, observed_config_id: observed as NonEmptyStr, stdout: stdout, readings: [] } +} + +fn w_production_digest() -> String { + "sha256:5de3c35577e5c8f3026c143d3d5b2d66cc527119281369f4651d969f581829d3" +} + +// A PROBE INSIDE THE RECEIPT'S DIGEST, WITH A COUNT ITS LINES AGREE WITH, IS A READING OF EXACTLY THOSE +// ARCHITECTURES AT THAT DIGEST. +test fn w_a_joined_complete_probe_is_a_registry_reading() -> Bool { + match v41_registry_reading_of(probe: w_probe_of(stdout: w_probe_stdout, observed: w_production_digest()), image: v41_production_produced_image, host: "srv5" as HostIdentity, observed_at: "2026-09-27") { + Absent => false + Present { value: r } => + (r.image_reference as String) == w_production_digest() + && length(r.architectures) == 2 + && any(r.architectures, a => (a as String) == "DeepseekV41ForCausalLM") + } +} + +// NO READING FROM A PROBE OF ANOTHER IMAGE, FROM A COUNT THE LINES DISAGREE WITH, OR FROM A REFUSED PROBE. +test fn w_an_unjoined_or_incomplete_probe_is_no_reading() -> Bool { + let other = v41_registry_reading_of(probe: w_probe_of(stdout: w_probe_stdout, observed: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), image: v41_production_produced_image, host: "srv5" as HostIdentity, observed_at: "2026-09-27") + let short = v41_registry_reading_of(probe: w_probe_of(stdout: "REGISTRY_COUNT 3\nREGISTERED DeepseekV41ForCausalLM\n", observed: w_production_digest()), image: v41_production_produced_image, host: "srv5" as HostIdentity, observed_at: "2026-09-27") + let refused = v41_registry_reading_of(probe: V41ProbeRefused { step: "probe-run" as NonEmptyStr, cause: "witness" }, image: v41_production_produced_image, host: "srv5" as HostIdentity, observed_at: "2026-09-27") + match other { Present { value: _ } => false Absent => match short { Present { value: _ } => false Absent => match refused { Present { value: _ } => false Absent => true } } } +} + +// THE OCCUPANCY NEED IS THE MEASUREMENT'S 0.80 SHARE OF THE UNIFIED POOL. +test fn w_the_occupancy_need_is_the_engine_share_of_the_pool() -> Bool { + kibibyte_count(k: v41_group_a_memory_need()) == 8000 * gb10_unified_pool_gib * 1048576 / 10000 +}