diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 4179400ba25..86544b70447 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -11,9 +11,9 @@ on: options: [srv1, srv2, srv3, srv4] type: choice mode: - description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction + description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_runtime_image_build, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -182,7 +182,7 @@ jobs: actions: read id-token: write concurrency: - group: ${{ github.event.inputs.mode == 'spark_serving_apply' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_native_serving_apply' && 'gunbc-arm-mutation-glm-group-b-native' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_encode' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_readback' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_build' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'pair_serving_d0' && 'gunbc-host-mutation-srv1' || format('gunbc-host-mutation-{0}', github.event.inputs.host) }} + group: ${{ github.event.inputs.mode == 'spark_serving_apply' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_native_serving_apply' && 'gunbc-arm-mutation-glm-group-b-native' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_encode' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_row_store_readback' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_build' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' && 'gunbc-host-mutation-srv1' || github.event.inputs.mode == 'pair_serving_d0' && 'gunbc-host-mutation-srv1' || format('gunbc-host-mutation-{0}', github.event.inputs.host) }} cancel-in-progress: false steps: - name: Checkout @@ -252,7 +252,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -1235,6 +1235,35 @@ jobs: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} if: github.event.inputs.mode == 'spark_v41_runtime_image_build' timeout-minutes: 180 + - name: Spark V4.1 runtime image distribute (move the produced image, named by its configuration digest, from the host its production receipt names to the selected Group A Spark, and read the digest back there) + id: spark_v41_runtime_image_distribute + run: |- + set -euo pipefail + umask 077 + CRED_FILE="$RUNNER_TEMP/spark-administrator-credential" + RESP_FILE="$RUNNER_TEMP/spark-administrator-response.json" + HDR_FILE="$RUNNER_TEMP/spark-administrator-auth-header" + trap 'rm -f "$CRED_FILE" "$RESP_FILE" "$HDR_FILE"' EXIT + printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE" + case "${SPARK_CONVERGE_TARGET:-}" in srv5) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv6) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv7) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv8) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv9) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv10) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv11) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; srv12) SM_URL="https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/spark-administrator-password/versions/1:access"; SM_RES="projects/gunbai-secrets/secrets/spark-administrator-password/versions/1" ;; *) echo "SparkAdministratorCredentialHostUnrostered: SPARK_CONVERGE_TARGET='${SPARK_CONVERGE_TARGET:-}' names no host in the administrator credential roster; contacting NO host" >&2; exit 1 ;; esac + HTTP_CODE="$(curl -sS -o "$RESP_FILE" -w '%{http_code}' -H @"$HDR_FILE" "$SM_URL" || echo 000)" + if [ "$HTTP_CODE" = 403 ]; then echo "SparkAdministratorCredentialUnreadable: $SM_RES answered 403 to serviceAccount:fleet-cloud-convergence@gunbai-secrets.iam.gserviceaccount.com. Secret Manager returns 403 both when a version exists and is not readable AND when it does not exist, so this run cannot tell those apart -- REMEDY EITHER WAY: confirm the version exists and grant that principal roles/secretmanager.secretAccessor on it. Contacting NO host." >&2; exit 1; fi + if [ "$HTTP_CODE" = 404 ]; then echo "SparkAdministratorCredentialAbsent: $SM_RES answered 404; the modeled reference names a version Secret Manager does not hold. Contacting NO host." >&2; exit 1; fi + if [ "$HTTP_CODE" != 200 ]; then echo "SparkAdministratorCredentialFetchFailed: HTTP $HTTP_CODE from $SM_RES; contacting NO host." >&2; exit 1; fi + python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$CRED_FILE" + rm -f "$RESP_FILE" + chmod 600 "$CRED_FILE" + if [ ! -s "$CRED_FILE" ]; then echo "SparkAdministratorCredentialEmpty: $SM_RES returned 200 with an empty payload, which is an enrolled-but-blank secret and never a blank password; contacting NO host" >&2; exit 1; fi + export GUNBC_SPARK_BOOTSTRAP_CREDENTIAL_FILE="$CRED_FILE" + echo "spark-admin-credential: materialized from $SM_RES to a 0600 run-local file, consumed and removed within this one step" + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/spark/v41_runtime_image_converge.dag --function v41_runtime_image_distribute_ci_wet + cat "$ROOT/target/v41-runtime-image-distribute-receipt.txt" + env: + SPARK_CONVERGE_TARGET: ${{ github.event.inputs.target }} + WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} + if: github.event.inputs.mode == 'spark_v41_runtime_image_distribute' + timeout-minutes: 120 - name: "Runner host files: observe the teardown drop-in, the needrestart deferral and the loaded teardown (no writes)" id: runner_host_file_observe run: |- diff --git a/dag/extdeps/docker/cli.dag b/dag/extdeps/docker/cli.dag index 173252607eb..16c7f976c05 100644 --- a/dag/extdeps/docker/cli.dag +++ b/dag/extdeps/docker/cli.dag @@ -472,6 +472,95 @@ fn docker_image_inspect_id_from_stdout(stdout: String) -> DockerImageInspectIdRe } } +// THE IMAGE'S SIZE, READ OFF THE SAME INSPECT DOCUMENT AS ITS Id. `Size` is the image's total +// uncompressed layer bytes (docs.docker.com/reference/api/engine/version/v1.47/#tag/Image/operation/ImageInspect: +// "Total size of the image including all layers it is composed of"), a JSON integer. It is what a +// `docker save` tar of the image carries, give or take the tar's own headers and the config and +// manifest members, and what a `docker load` on a host holding none of its layers writes -- so it is +// the byte requirement a byte transport between daemons states before moving anything. A member that +// is absent, duplicated, not an integer or negative is Unreadable, never a zero. +type DockerImageInspectSizeRead + = DockerImageInspectSize { bytes: Int } + | DockerImageInspectSizeUnreadable { cause: String } + +fn docker_image_inspect_size_from_stdout(stdout: String) -> DockerImageInspectSizeRead { + match parse_json_document(s: trim(s: stdout)) { + JsonDocumentUnreadable { gap: g } => + DockerImageInspectSizeUnreadable { cause: json_document_gap_text(gap: g) } + JsonDocumentParsed { value: v } => + match docker_inspect_json_first_of_one(v: v) { + Absent => DockerImageInspectSizeUnreadable { cause: "docker image inspect JSON is not a one-element array" } + Present { value: obj } => + match json_object_unique_member(v: obj, key: "Size") { + JsonMemberAbsent => DockerImageInspectSizeUnreadable { cause: "docker image inspect JSON names no Size" } + JsonMemberNotAnObject => DockerImageInspectSizeUnreadable { cause: "docker image inspect JSON element is not an object" } + JsonMemberDuplicated { count: n } => + DockerImageInspectSizeUnreadable { cause: join(["docker image inspect JSON names Size ", to_string(n), " times"], "") } + JsonMemberFound { value: sv } => + match sv { + JsonNumber { lexeme: l } => + match parse_int(s: l) { + Absent => DockerImageInspectSizeUnreadable { cause: join(["docker image inspect Size is not an integer: ", l], "") } + Present { value: n } => + if n < 0 { + DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is negative" } + } else { + DockerImageInspectSize { bytes: n } + } + } + JsonNull => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" } + JsonBool { value: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" } + JsonString { value: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" } + JsonArray { elements: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" } + JsonObject { members: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" } + } + } + } + } +} + +// THE DAEMON'S DATA ROOT, WHERE `docker load` WRITES LAYERS, READ FROM THE DAEMON. dockerd's +// --data-root defaults to /var/lib/docker (docs.docker.com/reference/cli/dockerd/) and daemon.json can +// move it, so the default is not a reading: a free-space read of the default path on a host whose +// root moved answers for the wrong filesystem. `docker info` with no template prints the server's +// state as text, one ` Docker Root Dir: ` line among them +// (docs.docker.com/reference/cli/docker/system/info/); no --format, for the portable-word reason +// docker_image_inspect_command states. A document with no such line, two of them, or a value that is +// not an absolute path is Unreadable -- never the default. +fn docker_info_command() -> ArgvCommand { + argv_command(program: docker_binary_path, arguments: ["info"]) +} + +data docker_info_root_dir_label: String = "Docker Root Dir:" + +type DockerRootDirRead + = DockerRootDir { path: NonEmptyStr } + | DockerRootDirUnreadable { cause: String } + +fn docker_info_root_dir_from_stdout(stdout: String) -> DockerRootDirRead { + let values = flat_map(split(s: stdout, delimiter: "\n"), l => { + let t = trim(s: l) + if starts_with(s: t, prefix: docker_info_root_dir_label) { + [trim(s: substring(s: t, start: docker_info_root_dir_label.length(), end: t.length()))] + } else { + [] as List + } + }) + if count(values) != 1 { + DockerRootDirUnreadable { cause: join(["docker info names the Docker Root Dir ", to_string(count(values)), " times"], "") } + } else { + match first(values) { + Absent => DockerRootDirUnreadable { cause: "docker info names no Docker Root Dir" } + Present { value: v } => + if starts_with(s: v, prefix: "/") { + DockerRootDir { path: v as NonEmptyStr } + } else { + DockerRootDirUnreadable { cause: join(["docker info's Docker Root Dir is not an absolute path: ", v], "") } + } + } + } +} + // THE CONTAINER'S TWO IDENTITY FIELDS, AND NOTHING ELSE OFF THE INSPECT OBJECT. `docker container // inspect` returns the whole object extdeps.docker.container_inspect ContainerInspect models, // including Config.Env, whose values can carry secrets. This reader takes exactly .Id -- the full diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index 6d8b3f3fcba..ee8aba9cdf3 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -87,6 +87,7 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_image_save_command"), decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_image_load_command"), decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_image_inspect_command"), + decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_info_command"), decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_run_ephemeral_command"), decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_remove_force_command"), decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_rename_command"), diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 2f2cf2c398a..71a865e25d9 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -8,6 +8,7 @@ import gunbc.spark.pair_serving_apply { spark_pair_apply_receipt_path } import gunbc.spark.native_serving_apply { spark_native_apply_receipt_path } import gunbc.spark.v41_row_store_encode_run { v41_encode_run_receipt_path } import gunbc.spark.v41_row_store_readback_run { v41_readback_run_receipt_path } +import gunbc.spark.v41_runtime_image_converge { v41_distribution_receipt_path } import gunbc.ledger_row_coherence { heal_repair_declaration_artifact_path } import extdeps.cloud.gcp.secret_ref { SecretRef, secret_ref_access_url, secret_ref_version_resource } import gunbc.auth.ci_app_key_rotation { @@ -1019,6 +1020,14 @@ data gunbc_ci_spark_v41_runtime_image_build_target: GunbcRunStepTarget = GunbcRu function: "v41_runtime_image_produce_ci_wet", } +// THE DISTRIBUTION OF THAT PRODUCED IMAGE TO ANOTHER GROUP A SPARK: save on the host the production +// receipt names, scp through the executor, load on the selected Spark, and the configuration digest +// read back there. The same prelude, because docker's socket on both ends is root-owned. +data gunbc_ci_spark_v41_runtime_image_distribute_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/spark/v41_runtime_image_converge.dag", + function: "v41_runtime_image_distribute_ci_wet", +} + // THE PUBLISHED V4.1 CHECKPOINT ON ONE GROUP A SPARK (gunbc.spark.v41_checkpoint_materialize): the same // administrator credential as the runtime-image probe, because every leg runs privileged on the Spark. data gunbc_ci_spark_v41_checkpoint_materialize_target: GunbcRunStepTarget = GunbcRunStepTarget { @@ -2584,6 +2593,17 @@ fn gunbc_ci_spark_runtime_image_probe_invoke() -> String { // The production shares the probe's prelude, for the same reason and with the same credential: one // fleet-key agent from the job and one per-target administrator credential materialized to one file. +fn gunbc_ci_spark_v41_runtime_image_distribute_invoke() -> String { + gunbc_run_step_script_with_prelude( + prelude: gunbc_ci_spark_grant_install_credential_prelude(), + source_roots: witness_layer_roots, + entry: gunbc_ci_spark_v41_runtime_image_distribute_target.entry, + function: gunbc_ci_spark_v41_runtime_image_distribute_target.function, + claim_run: false, + receipt_rel: v41_distribution_receipt_path + ) +} + fn gunbc_ci_spark_v41_runtime_image_build_invoke() -> String { gunbc_run_step_script_with_prelude( prelude: gunbc_ci_spark_grant_install_credential_prelude(), diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 71198a026c0..a72c767d42c 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -63,6 +63,7 @@ import gunbc.fleet_workflow_steps { gunbc_ci_runner_host_file_step_timeout_minutes, gunbc_ci_spark_serving_step_timeout_minutes, gunbc_ci_spark_runtime_image_build_step_timeout_minutes, + gunbc_ci_spark_runtime_image_distribute_step_timeout_minutes, gunbc_ci_artifact_transfer_step_timeout_minutes, gunbc_ci_fleet_job_backstop_timeout_minutes, gunbc_ci_release_build_step_timeout_minutes, @@ -135,6 +136,7 @@ import gunbc.ci_spec { gunbc_ci_spark_v41_row_store_readback_invoke, gunbc_ci_spark_v41_index_selection_invoke, gunbc_ci_spark_v41_runtime_image_build_invoke, + gunbc_ci_spark_v41_runtime_image_distribute_invoke, gunbc_fleet_converge_plan_artifact_name, gunbc_ci_fleet_converge_plan_artifact_hash_env_name, gunbc_ci_live_deploy_transaction_invoke, @@ -241,6 +243,7 @@ type FleetConvergeWorkflowMode | SparkV41RowStoreEncode | SparkV41RowStoreReadback | SparkV41RuntimeImageBuild + | SparkV41RuntimeImageDistribute | DashboardDeploy | RlmLaunchDeploymentReceipt | HostResetReturn @@ -291,6 +294,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String SparkV41RowStoreEncode => "spark_v41_row_store_encode" SparkV41RowStoreReadback => "spark_v41_row_store_readback" SparkV41RuntimeImageBuild => "spark_v41_runtime_image_build" + SparkV41RuntimeImageDistribute => "spark_v41_runtime_image_distribute" DashboardDeploy => "dashboard_deploy" RlmLaunchDeploymentReceipt => "rlm_launch_deployment_receipt" HostResetReturn => "host_reset_return" @@ -340,7 +344,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41RuntimeImageBuild, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // // EVERY MODE IS NAMED, AND THE WILDCARD IS DELIBERATELY ABSENT. A `_ => none` would read the same @@ -385,6 +389,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc SparkV41RowStoreEncode => none SparkV41RowStoreReadback => none SparkV41RuntimeImageBuild => none + SparkV41RuntimeImageDistribute => none DashboardDeploy => Present { value: LiveDeploy } ApprovalBrokerDarkInstall => none MicrovmControllerInstall => none @@ -509,6 +514,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> SparkV41RowStoreEncode => FleetSshKeyConsumed SparkV41RowStoreReadback => FleetSshKeyConsumed SparkV41RuntimeImageBuild => FleetSshKeyConsumed + SparkV41RuntimeImageDistribute => FleetSshKeyConsumed DashboardDeploy => FleetSshKeyConsumed ApprovalBrokerDarkInstall => FleetSshKeyConsumed MicrovmControllerInstall => FleetSshKeyConsumed @@ -1061,6 +1067,7 @@ fn fleet_converge_mode_mutation_domain(mode: FleetConvergeWorkflowMode) -> Fleet SparkV41RowStoreEncode => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } SparkV41RowStoreReadback => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } SparkV41RuntimeImageBuild => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } + SparkV41RuntimeImageDistribute => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } DashboardDeploy => ExecutorDomain RlmLaunchDeploymentReceipt => ExecutorDomain HostResetReturn => ExecutorDomain @@ -1117,6 +1124,7 @@ data fleet_converge_spark_v41_row_store_encode_step_if: String = fleet_converge_ data fleet_converge_spark_v41_row_store_readback_step_if: String = fleet_converge_mode_step_if(mode: SparkV41RowStoreReadback) data fleet_converge_spark_v41_runtime_image_build_step_if: String = fleet_converge_mode_step_if(mode: SparkV41RuntimeImageBuild) +data fleet_converge_spark_v41_runtime_image_distribute_step_if: String = fleet_converge_mode_step_if(mode: SparkV41RuntimeImageDistribute) data fleet_converge_dashboard_deploy_step_if: String = fleet_converge_mode_step_if(mode: DashboardDeploy) @@ -2834,6 +2842,29 @@ fn fleet_converge_spark_v41_runtime_image_build_step() -> Step { } } +// THE PRODUCED V4.1 IMAGE ONTO THE SELECTED GROUP A SPARK (gunbc.spark.v41_runtime_image_converge +// v41_runtime_image_distribute_ci_wet): the source is the host the production receipt names, the target +// is this dispatch's `target`, and the image is named by the receipt's configuration digest. Group A's +// arm domain serializes it with the build, D0 and the pair apply. +fn fleet_converge_spark_v41_runtime_image_distribute_step() -> Step { + RunStep { + name: Present { value: "Spark V4.1 runtime image distribute (move the produced image, named by its configuration digest, from the host its production receipt names to the selected Group A Spark, and read the digest back there)" }, + id: Present { value: "spark_v41_runtime_image_distribute" }, + run: gunbc_ci_spark_v41_runtime_image_distribute_invoke(), + shell: none, + env: Present { + value: [ + kv(key: "SPARK_CONVERGE_TARGET", value: yaml_string(s: "${{ github.event.inputs.target }}")), + kv(key: "WIF_ACCESS_TOKEN", value: yaml_string(s: "${{ steps.wif_auth.outputs.access_token }}")), + ] + }, + working_directory: none, + if_condition: Present { value: fleet_converge_spark_v41_runtime_image_distribute_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_spark_runtime_image_distribute_step_timeout_minutes() } + } +} + // THE PUBLISHED V4.1 CHECKPOINT ON ONE GROUP A SPARK (gunbc.spark.v41_checkpoint_materialize): every // admitted file fetched at the pinned revision, digested against its manifest row before it is // published, then the storage-backed Engram spans read from the verified shards. Its domain is Group @@ -3162,6 +3193,7 @@ fn fleet_converge_job() -> Job { fleet_converge_spark_v41_row_store_encode_step(), fleet_converge_spark_v41_row_store_readback_step(), fleet_converge_spark_v41_runtime_image_build_step(), + fleet_converge_spark_v41_runtime_image_distribute_step(), fleet_converge_runner_host_file_observe_step(), fleet_converge_runner_host_file_converge_step(), fleet_converge_runner_password_session_tool_converge_step(), @@ -3381,7 +3413,7 @@ data fleet_converge_dispatch_inputs: List = [ }, DispatchInput { name: "mode", - description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction" }, + description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction" }, required: true, default: none, type: InputChoice { options: fleet_converge_mode_options }, diff --git a/dag/gunbc/fleet/fleet_workflow_steps.dag b/dag/gunbc/fleet/fleet_workflow_steps.dag index 274514d5b0c..403c4aa5e73 100644 --- a/dag/gunbc/fleet/fleet_workflow_steps.dag +++ b/dag/gunbc/fleet/fleet_workflow_steps.dag @@ -48,6 +48,7 @@ import gunbc.ci_spec { } import gunbc.fleet_host_key_enrollment { probe_identity_secret_access_requirement } import gunbc.spark.vllm_runtime_image_build { vllm_build_host_effect_term } +import gunbc.spark.v41_runtime_image_converge { v41_image_distribution_host_effect_term } import std.types { List, String, Int } import gunbc.roadmap_execution_contract { @@ -480,6 +481,13 @@ fn gunbc_ci_spark_runtime_image_build_step_timeout_minutes() -> Int { second_count(s: vllm_build_host_effect_term) / 60 } +// THE SAME DERIVATION FOR THE PRODUCED IMAGE'S DISTRIBUTION: its step budget is the term both of +// its host-effect claims are admitted for (gunbc.spark.v41_runtime_image_converge +// v41_image_distribution_host_effect_term). +fn gunbc_ci_spark_runtime_image_distribute_step_timeout_minutes() -> Int { + second_count(s: v41_image_distribution_host_effect_term) / 60 +} + data gunbc_ci_spark_serving_step_timeout_note: String = "Spark serving converge/observe step budget, raised from the shared aux 5m (operator authorization 2026-08-19). STATED AS AN ENVELOPE, NOT A MEASUREMENT, because the neighbouring gates in this module earn their numbers and this one cannot yet. What IS modeled: the ollama runtime archive at 1542416321 bytes, cited on extdeps.ollama.binary_release. What is NOT: the model blob closure has no modeled byte size at all — the desired model is identified by its manifest digest rather than a size — and throughput from the executor to a spark target has never been measured. So a cold install downloads a ~1.5GB runtime plus a model of unmodeled size over a link of unmeasured bandwidth, and no honest derivation produces a minute count from that. 5m was not a considered budget for this step, it was the shared aux default, and it cannot cover a cold install under any plausible bandwidth. WHAT IS OBSERVED, and why this is not a hypothetical envelope: run 32213195686 (2026-08-19T03:49:56Z, read-only spark_observe) reports state=absent for BOTH spark targets — on srv6 the user unit is-active exits 4, the api/version and api/tags probes exit 7 against a refused connection, and the runtime receipt file does not exist. So the first canary is a guaranteed cold install of the full closure rather than a converged no-op, and the step budget has to cover acquisition of an unmodeled model closure on a host observed to hold none of it. The 5m default would have killed that install mid-download every time, and because publication is split from acquisition it would have done so silently-safely — dying in the attempt workspace, publishing nothing, and looking like a timeout rather than a budget defect. diff --git a/dag/gunbc/spark/v41_runtime_image_converge.dag b/dag/gunbc/spark/v41_runtime_image_converge.dag index c12cf2dad34..59542708729 100644 --- a/dag/gunbc/spark/v41_runtime_image_converge.dag +++ b/dag/gunbc/spark/v41_runtime_image_converge.dag @@ -4,7 +4,11 @@ import std.types { String, Bool, List, NonEmptyStr, FilePath } import std.process { ProcessExit, ExitSuccess, exit_failure } import v2.std.optional { Present, Absent } import extdeps.filesystem.filesystem_io { Filesystem } -import extdeps.container.oci.digest { OciContentDigest, parse_oci_content_digest_wire, render_oci_content_digest_wire } +import extdeps.container.oci.digest { OciContentDigest, OciSha256Digest, parse_oci_content_digest_wire, render_oci_content_digest_wire } +import std.content_hash { Sha256Digest, Sha256DigestHex } +import std.measure { Second, second } +import product.placement_supply { HostIdentity, host_identity_eq } +import gunbc.spark.host_effect_quiescence { HostEffectResidueSpec } import extdeps.exec.command { argv_words } import extdeps.flashinfer.flashinfer { FlashInferWheel, flashinfer_0_6_18_post1_wheel_url } import extdeps.posix.sh_invocation { posix_sh_script_command } @@ -34,10 +38,14 @@ import gunbc.spark.vllm_runtime_image_build { SparkBuildHostSession, SparkBuildHostRan, SparkBuildHostRefused, SparkBuildHostWork, with_spark_build_host, vllm_build_root, vllm_build_host_effect_term, vllm_build_host_purpose, vllm_build_host_residue, vllm_image_receipt_line, + VllmImageDistributionSubject, distribute_vllm_image, vllm_image_distribution_wire, vllm_image_distribution_succeeded, + vllm_image_transfer_tar_path, } +import gunbc.spark.vllm_image_identity { vllm_image_tag_for_key } import gunbc.spark.v41_source_patch_converge { V41PatchedImageBuild, V41PatchedImageBuilt, V41PatchedImageRefused, converge_vllm_runtime_image_with_patches, + V41ObservedSourceIdentity, v41_observed_source_identity_of, v41_patched_image_build_key, } import gunbc.spark.v41_runtime_candidate { V41PatchesObserved, V41PatchesAbsentByObservation, V41PatchesUnestablished, @@ -474,3 +482,150 @@ fn v41_runtime_image_produce_ci_wet() -> ProcessExit } } } + +// ── DISTRIBUTION: THE PRODUCED IMAGE ON ANOTHER GROUP A HOST ────────────────────────────────── +// +// WHY A SECOND HOST NEEDS THESE BYTES AND NOT A SECOND BUILD. The produced image is one content +// address; a rebuild on another host is a new build whose digest nobody has admitted, and V4.1 +// serving and the Engram differential both have to run in THE image the production admitted. The +// row stores the differential reads live on srv5, the image was produced on srv8, and serving +// later needs it on all four Group A hosts -- so the image moves, by the one transport the fleet +// models for images (gunbc.spark.vllm_runtime_image_build distribute_vllm_image: save, scp through +// the executor, load, readback), and this entry supplies what that fold needs and nothing else. +// +// THE SUBJECT IS THE PRODUCTION RECEIPT, NOT A TAG. What the production run established is a +// configuration digest on a host, and that is the whole identity this row carries: the observed +// source tree the build read (from which the canonical tag is DERIVED here, through the same key +// fold the build used, rather than transcribed -- so a row whose tree does not re-derive the tag +// the build printed cannot exist silently), the digest the daemon answered, and the host that +// answered it. Transcribed from fleet-converge run 36207135528 (spark_v41_runtime_image_build on +// srv8, 2026-09-26), whose receipt reads: produced runtime tag=gunbc-vllm-dsv41-gb10:d73e307009a5d716 +// config_digest=sha256:ea39410ed01d396caad86339d877c1adb181a983baadffd22c722069109ea664 +// source_head=d2d649e674c75425d2d6975c87eb89fd4d55fff8 +// worktree_diff_sha256=79ee43471662502d1ae0ddff9723b5d17599b036725d17b45fc5909679b7ece7, and +// "probe identity JOINED to the produced image" at the same digest. +// +// A HELD RECEIPT IS NOT A PRODUCED STANDING. gunbc.spark.v41_runtime_realized V41ProducedRuntimeImage +// is sole_constructor so only a real build leg mints one, and this row does not pretend to: it +// names bytes to move, and the source's readback -- not this row -- is what establishes the bytes +// are still there under that digest when the move runs. +type V41ProducedImageHeld { + observed: V41ObservedSourceIdentity + config: OciContentDigest + held_on: HostIdentity + receipt_run: NonEmptyStr +} + +data v41_arm_a_produced_image: V41ProducedImageHeld = V41ProducedImageHeld { + observed: v41_observed_source_identity_of( + revision_head: "d2d649e674c75425d2d6975c87eb89fd4d55fff8" as NonEmptyStr, + worktree_diff_sha256: Sha256Digest { hex: "79ee43471662502d1ae0ddff9723b5d17599b036725d17b45fc5909679b7ece7" as Sha256DigestHex }, + ), + config: OciSha256Digest(Sha256Digest { hex: "ea39410ed01d396caad86339d877c1adb181a983baadffd22c722069109ea664" as Sha256DigestHex }), + held_on: "srv8" as HostIdentity, + receipt_run: "36207135528" as NonEmptyStr, +} + +fn v41_produced_image_subject(p: V41ProducedImageHeld) -> VllmImageDistributionSubject { + VllmImageDistributionSubject { + tag: vllm_image_tag_for_key(r: v41_candidate_recipe(), key: v41_patched_image_build_key(r: v41_candidate_recipe(), observed: p.observed)), + config: p.config, + } +} + +// WHAT THIS MODE DOES TO A HOST, handed to the host-effect claim on both ends: the source writes a +// tar of the image's size into /tmp and the target writes one there and then loads it. Two hours +// bounds a ~30 GB save, two scp legs and a load on the management LAN; an unreleased claim reads +// OVERDUE after it and still fences until the host is observed quiet. The residue a crashed run +// leaves is the digest-named tar, which every leg carries on its command line. +data v41_image_distribution_host_purpose: NonEmptyStr = "a runtime image distribution, which saves or loads the produced image through a digest-named staging tar" as NonEmptyStr +data v41_image_distribution_host_effect_term: Second = second(count: 7200) + +fn v41_image_distribution_host_residue(p: V41ProducedImageHeld) -> HostEffectResidueSpec { + HostEffectResidueSpec { process_pattern: vllm_image_transfer_tar_path(config: p.config), container: none } +} + +fn v41_with_host_effect_claim(host: HostIdentity, p: V41ProducedImageHeld, run: fn() -> ProcessExit) -> ProcessExit { + match claim_host_effect_live(host: host, term: v41_image_distribution_host_effect_term, purpose: v41_image_distribution_host_purpose, residue: v41_image_distribution_host_residue(p: p)) { + HostEffectClaimNotLanded { cause: why } => exit_failure(reason: why as String) + HostEffectClaimLanded { handle: handle } => release_host_effect_live(handle: Present { value: handle }, exit: run()) + } +} + +data v41_distribution_attempt_raw: String = "v41-runtime-image-distribute" +data v41_distribution_receipt_path: String = "target/v41-runtime-image-distribute-receipt.txt" + +fn v41_distribute_to(target_host: HostIdentity, p: V41ProducedImageHeld) -> ProcessExit { + let now = clock_now_probed_at() + let image = v41_produced_image_subject(p: p) + match with_spark_build_host( + host: p.held_on, + attempt_raw: v41_distribution_attempt_raw, + identity_file_name: "gunbc-v41-runtime-image-distribute-admin-password" as NonEmptyStr, + use: fn(target, context, ctx) { + let dist = distribute_vllm_image(source_ctx: ctx, source_endpoint: target.endpoint, image: image, targets: [target_host]) + SparkBuildHostWork { + receipt: vllm_image_distribution_wire(dist: dist), + converged: vllm_image_distribution_succeeded(dist: dist), + } + }, + ) { + SparkBuildHostRefused { cause: c } => exit_failure(reason: c) + SparkBuildHostRan { work: work, endpoint: endpoint, executor: executor, trust_receipt: trust_receipt } => { + let body = join([ + join(["v41 runtime image distribute probed_at=", probed_at_word(reading: now) as String, " source=", p.held_on as String, " endpoint=", endpoint as String, " target=", target_host as String, " executor=", executor, "\n"], ""), + trust_receipt, + join(["image tag=", image.tag as String, " config_digest=", render_oci_content_digest_wire(d: image.config), " receipt_run=", p.receipt_run as String, "\n"], ""), + work.receipt, + ], "") + let w = Filesystem.Write(path: v41_distribution_receipt_path, content: body) + if !w.success { + exit_failure(reason: join(["v41 runtime image distribute receipt write failed: ", w.error], "")) + } else if work.converged { + ExitSuccess + } else { + exit_failure(reason: join(["the V4.1 runtime image was not verified on the target:\n", body], "")) + } + } + } +} + +// ── THE MODE ENTRY ──────────────────────────────────────────────────────────────────────────── +// +// SPARK_CONVERGE_TARGET NAMES THE TARGET; THE SOURCE IS THE RECEIPT'S HOST. Where the bytes are is +// a fact the production run established, not an operator's choice, so it is not an input; an +// unset target refuses rather than fanning out, because this mode writes tens of gigabytes to the +// host it delivers to. +// +// BOTH HOSTS ARE ADMITTED BY THE RELATION THE PRODUCTION USES -- the Group A pair subject HOLDS +// them (gunbc.spark.host_commitment admit_host_held_by_subject) -- and both are CLAIMED, because +// both carry the effect: a save and a staging tar on one, a staging tar and a load on the other. A +// target that is the source holds the image already; the fold reads it as the Noop, and the one host +// is claimed once. Occupancy (gunbc.spark.host_occupancy_admission, #12351) answers a GPU and memory +// need, which a save and a load do not have; the need this effect does have is disk, and the fold +// states and checks it on every filesystem a copy lands on before a byte moves. +fn v41_runtime_image_distribute_ci_wet() -> ProcessExit +{ + let p = v41_arm_a_produced_image + match spark_converge_target_selection(execute_apply: true) { + SparkConvergeTargetRefused { reason: why } => exit_failure(reason: why) + SparkConvergeTargetAllObserveOnly => + exit_failure(reason: "SPARK_CONVERGE_TARGET is unset: distributing this image writes tens of gigabytes to the host it names, so it refuses rather than choosing one") + SparkConvergeTargetSingle { host: target_host } => + match admit_host_held_by_subject(raw: p.held_on, subject: PairServingUnitOn { group: FabricGroupA }) { + Present { value: why } => exit_failure(reason: why) + Absent => + match admit_host_held_by_subject(raw: target_host, subject: PairServingUnitOn { group: FabricGroupA }) { + Present { value: why } => exit_failure(reason: why) + Absent => + v41_with_host_effect_claim(host: p.held_on, p: p, run: fn() { + if host_identity_eq(a: target_host, b: p.held_on) { + v41_distribute_to(target_host: target_host, p: p) + } else { + v41_with_host_effect_claim(host: target_host, p: p, run: fn() { v41_distribute_to(target_host: target_host, p: p) }) + } + }) + } + } + } +} diff --git a/dag/gunbc/spark/vllm_runtime_image_build.dag b/dag/gunbc/spark/vllm_runtime_image_build.dag index 8a55d8a3533..94e1ecb534e 100644 --- a/dag/gunbc/spark/vllm_runtime_image_build.dag +++ b/dag/gunbc/spark/vllm_runtime_image_build.dag @@ -4,7 +4,7 @@ import std.types { String, Bool, List, NonEmptyStr, FilePath, Secret } import std.process { ProcessExit, ExitSuccess, exit_failure } import std.algebra { trim } import std.nat { Nat } -import std.measure { HardwareThreadCount, hardware_thread_count, hardware_thread_count_value, Second, second } +import std.measure { HardwareThreadCount, hardware_thread_count, hardware_thread_count_value, Second, second, ByteSize, byte_size, byte_size_count } import std.content_hash { ContentHash, content_hash_atom, serialize_content_hash } import std.artifact_store { ArtifactRealizationInput, artifact_realization_digest } import product.placement_supply { HostIdentity, host_identity_eq } @@ -18,7 +18,14 @@ import extdeps.docker.cli { DockerBuildArg, DockerBuildSpec, docker_build_spec_command, docker_image_digest_command, docker_image_save_command, docker_image_load_command, -} + docker_image_inspect_command, docker_image_inspect_names_no_such_image, + docker_image_inspect_id_from_stdout, DockerImageInspectId, DockerImageInspectIdUnreadable, + docker_image_inspect_size_from_stdout, DockerImageInspectSize, DockerImageInspectSizeUnreadable, + docker_info_command, docker_info_root_dir_from_stdout, DockerRootDir, DockerRootDirUnreadable, +} +import extdeps.container.oci.digest { OciContentDigest, render_oci_content_digest_wire, oci_content_digest_encoded_hex, parse_oci_content_digest_wire } +import std.checked_arithmetic { CheckedNatReady, CheckedNatOverflow, checked_int_magnitude } +import gunbc.host_disk_observation { HostDiskObservation, DiskObserved, DiskObservationRefused, DiskUnobserved, host_disk_free_space_argv, host_disk_observation_of_df } import extdeps.ssh.openssh_client_commands { scp_download_command, scp_upload_command } import extdeps.ssh.client_options { serialize_ssh_client_options } import extdeps.tools.chmod { chmod_set_mode_command, chmod_path_resolved_program } @@ -1209,46 +1216,171 @@ fn vllm_source_acquire_wet(host: String) -> ProcessExit } } -// ── DISTRIBUTION: THE BUILT IMAGE LEAVES ITS BUILD HOST ───────────────────────────────────── +// ── DISTRIBUTION: A BUILT IMAGE LEAVES ITS BUILD HOST, NAMED BY ITS CONFIGURATION DIGEST ────── // -// THE FLEET RUNS NO REGISTRY, so a lane-built image reaches the arm's other hosts as BYTES: -// docker save on the build host, scp through the executor (the only side holding the fleet -// private key), docker load on each target, and a digest readback that must equal the source's -- -// docker load preserves the config digest, so equality with the source's own readback IS the -// delivery proof. Rebuilding per host was the alternative and it re-pays a two-hour compile for a -// fact one host already holds; that is DESIGN §2's redundant work, refused. Modeled 2026-09-17 -// when the GLM derived image (33GB) had to leave srv10 for srv9/11/12. +// THE FLEET RUNS NO REGISTRY, so a lane-built image reaches another host as BYTES: docker save on +// the source, scp through the executor (the only side holding the fleet private key), docker load +// on each target, and a readback. Rebuilding per host was the alternative and it re-pays a +// multi-hour compile for a fact one host already holds; that is DESIGN §2's redundant work, +// refused. Modeled 2026-09-17 when the GLM derived image (33GB) had to leave srv10 for srv9/11/12, +// and made subject-generic 2026-09-26 when the V4.1 produced image had to leave srv8 for srv5. // -// THE TAR IS A STAGING ARTIFACT WITH A CONTENT-NAMED PATH: the build key names it on every host -// it passes through, so a stale tar from an earlier image can never wear this one's name, and -// every leg removes it once the readback has answered. +// THE IMAGE IS NAMED BY ITS CONFIGURATION DIGEST, NOT BY ITS TAG. A tag is a mutable pointer, and +// two hosts can hold two different images under one tag -- a build key names the recipe, and a +// rebuild of one recipe is not guaranteed to reproduce a digest. So the subject carries the digest +// the producing run read back, and every leg answers against THAT: the source must hold it under +// the tag before a byte moves, a target holding it under the tag is a Noop, a target holding +// something ELSE under the tag refuses rather than having the tag silently moved off it, and a +// load whose readback is not the digest refuses. `docker load` preserves the configuration digest +// (it is the hash of the config blob the tar carries), which is what makes equality the delivery +// proof. The tag travels inside the tar, so the target wears the build's canonical tag and no +// other; nothing is re-tagged and no image content changes. // -// THE scp LEGS RUN ON THE EXECUTOR under the same anchored trust as every other fleet leg: the -// trust words are the materialized anchor's own options, and authentication is the agent already -// verified at entry. The remote path half travels inside scp's principal@endpoint:path operand, -// whose discipline the builder's comment states. +// THE BYTES ARE STATED BEFORE THEY MOVE. The image's Size (extdeps.docker.cli +// docker_image_inspect_size_from_stdout) is the requirement at every place a copy lands: the +// source's staging directory (the saved tar), the executor's (the relayed tar), and on each target +// both the staging directory and the daemon's data root as the daemon itself reports it (the tar, +// then the loaded layers, which coexist until the tar is removed). The two target paths' +// filesystem identity is not read, and on the Sparks both sit on the root filesystem, so each is +// required to hold BOTH copies -- an upper bound, exact where they share a filesystem, which is the +// case measured. A filesystem that cannot hold its share refuses before the save, with the +// figures; a data root the daemon does not name refuses too, rather than reading a default path. +// Every figure is a ByteSize from the inspect reader onward, so a requirement and a free-space +// reading are one quantity compared as one type. data vllm_image_transfer_tar_stem: NonEmptyStr = "/tmp/gunbc-vllm-image" as NonEmptyStr +data vllm_image_transfer_staging_directory: NonEmptyStr = "/tmp" as NonEmptyStr -fn vllm_image_transfer_tar_path(build_key: ContentHash) -> NonEmptyStr { - join([vllm_image_transfer_tar_stem as String, "-", serialize_content_hash(hash: build_key) as String, ".tar"], "") as NonEmptyStr +type VllmImageDistributionSubject { + tag: NonEmptyStr + config: OciContentDigest +} + +// THE TAR IS NAMED BY THE DIGEST OF WHAT IT CARRIES, so a stale tar from another image can never +// wear this one's name on any host it passes through, and every leg removes it once the readback +// has answered. +fn vllm_image_transfer_tar_path(config: OciContentDigest) -> NonEmptyStr { + join([vllm_image_transfer_tar_stem as String, "-", oci_content_digest_encoded_hex(d: config), ".tar"], "") as NonEmptyStr } // THE LOCAL LEGS' ABSOLUTE PROGRAM PATHS ON THE EXECUTOR are filesystem observations, not -// spellings to assume (extdeps.exec.command program_spelling_is_an_observation_claim): both read -// 2026-09-17 on srv2. The ArgvCommand builders keep their PATH-resolved names; the runner is +// spellings to assume (extdeps.exec.command program_spelling_is_an_observation_claim): scp and rm +// read 2026-09-17 on srv2. The ArgvCommand builders keep their PATH-resolved names; the runner is // handed the observed absolute path beside the command's arguments. data vllm_image_transfer_scp_path: NonEmptyStr = "/usr/bin/scp" as NonEmptyStr data vllm_image_transfer_rm_path: NonEmptyStr = "/usr/bin/rm" as NonEmptyStr +data vllm_image_transfer_df_path: NonEmptyStr = "/usr/bin/df" as NonEmptyStr -// THE TARGETS ARE THE ARM'S ROSTER MINUS THE SOURCE: the image is already on the host that built -// it, and delivering there would move 33GB to no end. +// THE GLM ARM'S TARGETS ARE ITS ROSTER MINUS THE SOURCE: the image is already on the host that +// built it, and delivering there would move 33GB to no end. fn vllm_image_distribution_targets(source: HostIdentity) -> List { filter(glm_native_ranked_hosts(), h => !host_identity_eq(a: h, b: source)) } -type VllmImageHostDelivery - = VllmImageHostDelivered { host: HostIdentity, digest: NonEmptyStr, already_present: Bool } - | VllmImageHostDeliveryRefused { host: HostIdentity, step: NonEmptyStr, cause: String } +// WHAT ONE DAEMON HOLDS UNDER THE TAG, READ BY `docker image inspect`. Absence is the daemon's own +// "No such image", which is a fact; any other non-zero exit, and an Id or Size that does not read, +// is Unread -- never absence, because reading an unreachable daemon as "not held" is what would +// move 30GB onto a host that already had it, or report a host empty that was never asked. +type VllmImageHeld + = VllmImageHeldAs { id: NonEmptyStr, size: ByteSize } + | VllmImageNotHeld + | VllmImageHeldUnread { step: NonEmptyStr, cause: String } + +fn vllm_image_held_of_inspect(step: NonEmptyStr, stdout: String) -> VllmImageHeld { + match docker_image_inspect_id_from_stdout(stdout: stdout) { + DockerImageInspectIdUnreadable { cause: why } => VllmImageHeldUnread { step: step, cause: why } + DockerImageInspectId { id: id } => + match docker_image_inspect_size_from_stdout(stdout: stdout) { + DockerImageInspectSizeUnreadable { cause: why } => VllmImageHeldUnread { step: step, cause: why } + DockerImageInspectSize { bytes: n } => + match checked_int_magnitude(a: n) { + CheckedNatOverflow { cause: _ } => VllmImageHeldUnread { step: step, cause: "the image Size does not fit a Nat" } + CheckedNatReady { value: v } => VllmImageHeldAs { id: id, size: byte_size(count: v) } + } + } + } +} + +fn vllm_image_read_held(ctx: SparkMaterializeContext, step: NonEmptyStr, tag: NonEmptyStr) -> VllmImageHeld { + match spark_remote_run(ctx: ctx, argv: argv_words(command: docker_image_inspect_command(image: tag))) { + RemoteUnreachable { cause: why } => VllmImageHeldUnread { step: step, cause: why } + RemoteRan { exit_code: code, stdout: out, stderr: err } => + if code == 0 { + vllm_image_held_of_inspect(step: step, stdout: out) + } else if docker_image_inspect_names_no_such_image(stderr: err) { + VllmImageNotHeld + } else { + VllmImageHeldUnread { step: step, cause: join(["exit=", to_string(code), " stderr=", trim(s: err)], "") } + } + } +} + +// ONE FILESYSTEM'S SHARE OF THE REQUIREMENT. None admits; otherwise the refusal, with both figures. +// A COPY'S SHARE OF A FILESYSTEM THAT MUST HOLD `copies` OF THE IMAGE. +fn vllm_image_copies(size: ByteSize, copies: Nat) -> ByteSize { + byte_size(count: byte_size_count(b: size) * copies) +} + +fn vllm_image_disk_refusal(step: NonEmptyStr, place: String, required: ByteSize, reading: HostDiskObservation) -> String? { + match reading { + DiskObservationRefused { detail: d } => Present { value: join([step as String, ": the free-space read at ", place, " refused: ", d as String], "") } + DiskUnobserved { reason: r } => Present { value: join([step as String, ": the free-space read at ", place, " did not run: ", r], "") } + DiskObserved { free_bytes: free, observed_on: _ } => + if byte_size_count(b: free) < byte_size_count(b: required) { + Present { value: join([step as String, ": ", place, " has ", to_string(byte_size_count(b: free)), " bytes available and the image needs ", to_string(byte_size_count(b: required)), " there, so nothing was moved"], "") } + } else { + none + } + } +} + +fn vllm_image_remote_disk_refusal(ctx: SparkMaterializeContext, step: NonEmptyStr, path: NonEmptyStr, required: ByteSize) -> String? { + let place = join([ctx.host as String, ":", path as String], "") + match spark_remote_run(ctx: ctx, argv: host_disk_free_space_argv(path: path as String)) { + RemoteUnreachable { cause: why } => Present { value: join([step as String, ": the free-space read at ", place, " did not run: ", why], "") } + RemoteRan { exit_code: code, stdout: out, stderr: err } => + vllm_image_disk_refusal(step: step, place: place, required: required, + reading: host_disk_observation_of_df(exit_code: code, stdout: out, stderr: err, observed_on: ctx.host as NonEmptyStr)) + } +} + +fn vllm_image_local_disk_refusal(step: NonEmptyStr, path: NonEmptyStr, required: ByteSize) -> String? { + let place = join(["executor:", path as String], "") + let argv = host_disk_free_space_argv(path: path as String) + let r = gunbc.WitnessBin.Run( + workdir: "." as FilePath, + bin_path: vllm_image_transfer_df_path as FilePath, + args: argv.skip(n: 1), + expect: OutcomeIsData, + ) + vllm_image_disk_refusal(step: step, place: place, required: required, + reading: host_disk_observation_of_df(exit_code: r.exit_code, stdout: r.stdout, stderr: r.stderr, observed_on: "executor" as NonEmptyStr)) +} + +// THE TARGET'S TWO SHARES: the staging directory, and the data root the daemon names. The root is +// read from `docker info` on the target itself, so the free-space read is of the filesystem the +// load writes to, and a daemon that names no root refuses. +fn vllm_image_target_disk_refusal(ctx: SparkMaterializeContext, size: ByteSize) -> String? { + let both = vllm_image_copies(size: size, copies: 2) + match vllm_docker_leg(ctx: ctx, step: "deliver-data-root" as NonEmptyStr, command: docker_info_command()) { + ImageLegFailed { step: st, cause: why } => Present { value: join([st as String, ": docker info did not run, so the data root is unknown: ", why], "") } + ImageLegRan { stdout: out } => + match docker_info_root_dir_from_stdout(stdout: out) { + DockerRootDirUnreadable { cause: why } => Present { value: join(["deliver-data-root: ", why], "") } + DockerRootDir { path: root } => + vllm_image_first_refusal(refusals: [ + vllm_image_remote_disk_refusal(ctx: ctx, step: "deliver-disk-staging" as NonEmptyStr, path: vllm_image_transfer_staging_directory, required: both), + vllm_image_remote_disk_refusal(ctx: ctx, step: "deliver-disk-data-root" as NonEmptyStr, path: root, required: both), + ]) + } + } +} + +fn vllm_image_first_refusal(refusals: List) -> String? { + fold(refusals, init: none, f: (acc, r) => match acc { + Present { value: _ } => acc + Absent => r + }) +} // THE WORKDIR IS THE RUN'S OWN CWD, NOT /tmp: the trust words carry the anchor's path as the // materialization wrote it -- target/fleet-ssh-attempt-...-known-hosts-, CWD-RELATIVE, exactly @@ -1274,103 +1406,90 @@ fn vllm_image_local_scp_leg(step: NonEmptyStr, command: ArgvCommand) -> ImageLeg vllm_image_local_leg(step: step, bin_path: vllm_image_transfer_scp_path, command: command) } -// ONE HOST'S DELIVERY, IN ORDER: if the target's readback already equals the source digest the -// host is done without a byte moved; otherwise upload, load, read back, and only then clean the -// tar. A digest that reads back DIFFERENT after a successful load is a refusal, not a retry -- -// the bytes that landed are not the bytes the source holds, and that is a located fact. -fn vllm_image_deliver_to_host( - host: HostIdentity, - endpoint: NonEmptyStr, - ctx: SparkMaterializeContext, - trust_words: List, - local_tar: NonEmptyStr, - remote_tar: NonEmptyStr, - tag: NonEmptyStr, - expected_digest: NonEmptyStr, -) -> VllmImageHostDelivery -{ - match vllm_docker_leg(ctx: ctx, step: "deliver-digest-pre" as NonEmptyStr, command: docker_image_digest_command(image: tag)) { - ImageLegFailed { step: st, cause: why } => - VllmImageHostDeliveryRefused { host: host, step: st, cause: join(["the target's image query did not run, so nothing is known about whether the tag exists: ", why], "") } - ImageLegRan { stdout: prior } => - if prior == (expected_digest as String) { - VllmImageHostDelivered { host: host, digest: expected_digest, already_present: true } +type VllmImageHostDelivery + = VllmImageHostDelivered { host: HostIdentity, digest: NonEmptyStr, already_present: Bool } + | VllmImageHostDeliveryRefused { host: HostIdentity, step: NonEmptyStr, cause: String } + +// WHAT A TARGET'S PRE-READ DECIDES, BEFORE ANY BYTE IS SAVED. Holding the digest under the tag is +// the Noop; holding nothing under the tag is the one state a delivery may start from; holding a +// different image under the tag, or an unread answer, refuses -- the first because a load would +// move the canonical tag off an image some consumer may be running, the second because nothing is +// known. +type VllmImageTargetState + = VllmImageTargetHolds + | VllmImageTargetNeeds + | VllmImageTargetRefused { step: NonEmptyStr, cause: String } + +fn vllm_image_target_state(held: VllmImageHeld, image: VllmImageDistributionSubject) -> VllmImageTargetState { + let want = render_oci_content_digest_wire(d: image.config) + match held { + VllmImageNotHeld => VllmImageTargetNeeds + VllmImageHeldUnread { step: st, cause: why } => VllmImageTargetRefused { step: st, cause: why } + VllmImageHeldAs { id: id, size: _ } => + if (id as String) == want { + VllmImageTargetHolds } else { - match vllm_image_local_scp_leg(step: "deliver-upload" as NonEmptyStr, command: scp_upload_command(trust_words: trust_words, principal: spark_managed_executor_login, endpoint: endpoint, local_path: local_tar, remote_path: remote_tar)) { - ImageLegFailed { step: st, cause: why } => VllmImageHostDeliveryRefused { host: host, step: st, cause: why } - ImageLegRan { stdout: _ } => - match vllm_docker_leg(ctx: ctx, step: "deliver-load" as NonEmptyStr, command: docker_image_load_command(input_path: remote_tar)) { - ImageLegFailed { step: st, cause: why } => VllmImageHostDeliveryRefused { host: host, step: st, cause: why } - ImageLegRan { stdout: _ } => - match vllm_docker_leg(ctx: ctx, step: "deliver-readback" as NonEmptyStr, command: docker_image_digest_command(image: tag)) { - ImageLegFailed { step: st, cause: why } => - VllmImageHostDeliveryRefused { host: host, step: st, cause: join(["the load reported success and the readback did not run: ", why], "") } - ImageLegRan { stdout: readback } => - if readback != (expected_digest as String) { - VllmImageHostDeliveryRefused { host: host, step: "deliver-readback" as NonEmptyStr, cause: "the load reported success and the tag's digest is not the source's" } - } else { - match vllm_docker_leg(ctx: ctx, step: "deliver-cleanup" as NonEmptyStr, command: rm_force_command(path: remote_tar as String)) { - ImageLegFailed { step: st, cause: why } => VllmImageHostDeliveryRefused { host: host, step: st, cause: why } - ImageLegRan { stdout: _ } => - VllmImageHostDelivered { host: host, digest: expected_digest, already_present: false } - } - } - } - } + VllmImageTargetRefused { + step: "deliver-digest-pre" as NonEmptyStr, + cause: join(["the target holds ", id as String, " under ", image.tag as String, " and the subject is ", want, "; loading would move the tag off an image this lane did not produce, so nothing was moved"], ""), } } } } -// THE FOLD'S SHARED STATE, carried in the accumulator because a named effectful step function -// cannot close over the entry's bindings. -type VllmImageDistributionAcc { - deliveries: List - trust: FleetOpenSshTrustPolicy - trust_words: List - path: SparkReachPath - admin_credential: Secret - admin_credential_file: NonEmptyStr - local_tar: NonEmptyStr - remote_tar: NonEmptyStr - tag: NonEmptyStr - expected_digest: NonEmptyStr +fn vllm_image_target_ctx(source_ctx: SparkMaterializeContext, host: HostIdentity) -> SparkMaterializeContext { + SparkMaterializeContext { + host: host, trust: source_ctx.trust, path: source_ctx.path, + admin_credential: source_ctx.admin_credential, admin_credential_file: source_ctx.admin_credential_file, + executor: HostCli, + } } -fn vllm_image_deliver_fold_step(acc: VllmImageDistributionAcc, host: HostIdentity) -> VllmImageDistributionAcc +// ONE HOST'S DELIVERY, IN ORDER, from a host the pre-read found needing the image: the disk share, +// upload, load, the readback, and only then the tar's removal. A readback that is not the subject +// after a successful load is a refusal, not a retry -- the bytes that landed are not the bytes the +// source holds, and that is a located fact. +fn vllm_image_deliver_to_host( + host: HostIdentity, + source_ctx: SparkMaterializeContext, + trust_words: List, + tar: NonEmptyStr, + image: VllmImageDistributionSubject, + size: ByteSize, +) -> VllmImageHostDelivery { - match spark_reach_endpoint(host: host, path: acc.path) { - Absent => VllmImageDistributionAcc { - deliveries: concat(acc.deliveries, [VllmImageHostDeliveryRefused { - host: host, step: "deliver-reach" as NonEmptyStr, - cause: join(["no ", spark_reach_path_wire(p: acc.path), " endpoint for ", host as String], ""), - }]), - trust: acc.trust, trust_words: acc.trust_words, path: acc.path, - admin_credential: acc.admin_credential, admin_credential_file: acc.admin_credential_file, - local_tar: acc.local_tar, remote_tar: acc.remote_tar, tag: acc.tag, expected_digest: acc.expected_digest, - } - Present { value: endpoint } => { - let delivery = vllm_image_deliver_to_host( - host: host, - endpoint: endpoint as NonEmptyStr, - ctx: SparkMaterializeContext { - host: host, trust: acc.trust, path: acc.path, - admin_credential: acc.admin_credential, admin_credential_file: acc.admin_credential_file, - executor: HostCli, - }, - trust_words: acc.trust_words, - local_tar: acc.local_tar, - remote_tar: acc.remote_tar, - tag: acc.tag, - expected_digest: acc.expected_digest, - ) - VllmImageDistributionAcc { - deliveries: concat(acc.deliveries, [delivery]), - trust: acc.trust, trust_words: acc.trust_words, path: acc.path, - admin_credential: acc.admin_credential, admin_credential_file: acc.admin_credential_file, - local_tar: acc.local_tar, remote_tar: acc.remote_tar, tag: acc.tag, expected_digest: acc.expected_digest, + let ctx = vllm_image_target_ctx(source_ctx: source_ctx, host: host) + let want = render_oci_content_digest_wire(d: image.config) + match spark_reach_endpoint(host: host, path: ReachManagementLan) { + Absent => VllmImageHostDeliveryRefused { host: host, step: "deliver-reach" as NonEmptyStr, cause: join(["no ", spark_reach_path_wire(p: ReachManagementLan), " endpoint for ", host as String], "") } + Present { value: endpoint } => + match vllm_image_target_disk_refusal(ctx: ctx, size: size) { + Present { value: why } => VllmImageHostDeliveryRefused { host: host, step: "deliver-disk" as NonEmptyStr, cause: why } + Absent => + match vllm_image_local_scp_leg(step: "deliver-upload" as NonEmptyStr, command: scp_upload_command(trust_words: trust_words, principal: spark_managed_executor_login, endpoint: endpoint as NonEmptyStr, local_path: tar, remote_path: tar)) { + ImageLegFailed { step: st, cause: why } => VllmImageHostDeliveryRefused { host: host, step: st, cause: why } + ImageLegRan { stdout: _ } => + match vllm_docker_leg(ctx: ctx, step: "deliver-load" as NonEmptyStr, command: docker_image_load_command(input_path: tar)) { + ImageLegFailed { step: st, cause: why } => VllmImageHostDeliveryRefused { host: host, step: st, cause: why } + ImageLegRan { stdout: _ } => + match vllm_image_read_held(ctx: ctx, step: "deliver-readback" as NonEmptyStr, tag: image.tag) { + VllmImageHeldUnread { step: st, cause: why } => + VllmImageHostDeliveryRefused { host: host, step: st, cause: join(["the load reported success and the readback did not read: ", why], "") } + VllmImageNotHeld => + VllmImageHostDeliveryRefused { host: host, step: "deliver-readback" as NonEmptyStr, cause: "the load reported success and the daemon names no image under the tag" } + VllmImageHeldAs { id: id, size: _ } => + if (id as String) != want { + VllmImageHostDeliveryRefused { host: host, step: "deliver-readback" as NonEmptyStr, cause: join(["the load reported success and the tag names ", id as String, ", not ", want], "") } + } else { + match vllm_docker_leg(ctx: ctx, step: "deliver-cleanup" as NonEmptyStr, command: rm_force_command(path: tar as String)) { + ImageLegFailed { step: st, cause: why } => VllmImageHostDeliveryRefused { host: host, step: st, cause: why } + ImageLegRan { stdout: _ } => VllmImageHostDelivered { host: host, digest: want as NonEmptyStr, already_present: false } + } + } + } + } + } } - } } } @@ -1395,71 +1514,103 @@ fn vllm_image_deliveries_wire(ds: List) -> String { join(map(ds, d => vllm_image_delivery_wire(d: d)), "") } +fn vllm_image_distribution_wire(dist: VllmImageDistribution) -> String { + match dist { + VllmImageDistributionRefused { step: st, cause: why } => + join(["distribution REFUSED step=", st as String, " cause=", why, "\n"], "") + VllmImageDistributed { deliveries: ds } => vllm_image_deliveries_wire(ds: ds) + } +} + // AN EMPTY DELIVERY SET IS NOT A SUCCESS. The fold's unit is true, so a roster that produced no // target -- a one-host arm, or a source that matched every ranked host -- would report the // distribution succeeded having moved nothing. Delivering to nobody is the state this reader exists // to catch, so it is false here and the caller's refusal arm names it. -fn vllm_image_distribution_succeeded(deliveries: List) -> Bool { - count(deliveries) > 0 - && fold(deliveries, init: true, f: (ok, d) => ok && match d { - VllmImageHostDelivered { host: _, digest: _, already_present: _ } => true - VllmImageHostDeliveryRefused { host: _, step: _, cause: _ } => false - }) +fn vllm_image_distribution_succeeded(dist: VllmImageDistribution) -> Bool { + match dist { + VllmImageDistributionRefused { step: _, cause: _ } => false + VllmImageDistributed { deliveries: deliveries } => + count(deliveries) > 0 + && fold(deliveries, init: true, f: (ok, d) => ok && match d { + VllmImageHostDelivered { host: _, digest: _, already_present: _ } => true + VllmImageHostDeliveryRefused { host: _, step: _, cause: _ } => false + }) + } } -// THE DISTRIBUTION ITSELF: save on the source (as root through the privileged leg, then chmod so -// the executor principal can read it back out), read the source's digest as the expectation, -// pull the tar to the executor, deliver to every target, and remove the staging tars. The source -// digest is READ, not carried from the build receipt -- the receipt's number and the host's fact -// agree by construction only if the read says so. -fn distribute_vllm_image( - source: HostIdentity, - source_endpoint: NonEmptyStr, +type VllmImageTargetRead { + host: HostIdentity + state: VllmImageTargetState +} + +fn vllm_image_target_read(source_ctx: SparkMaterializeContext, host: HostIdentity, image: VllmImageDistributionSubject) -> VllmImageTargetRead { + VllmImageTargetRead { + host: host, + state: vllm_image_target_state(held: vllm_image_read_held(ctx: vllm_image_target_ctx(source_ctx: source_ctx, host: host), step: "deliver-digest-pre" as NonEmptyStr, tag: image.tag), image: image), + } +} + +fn vllm_image_target_needs(r: VllmImageTargetRead) -> Bool { + match r.state { + VllmImageTargetNeeds => true + VllmImageTargetHolds => false + VllmImageTargetRefused { step: _, cause: _ } => false + } +} + +// A TARGET THE PRE-READ SETTLED, WITHOUT A BYTE MOVED: the Noop, or the pre-read's refusal. +fn vllm_image_settled_delivery(r: VllmImageTargetRead, image: VllmImageDistributionSubject) -> VllmImageHostDelivery? { + match r.state { + VllmImageTargetNeeds => none + VllmImageTargetHolds => Present { value: VllmImageHostDelivered { host: r.host, digest: render_oci_content_digest_wire(d: image.config) as NonEmptyStr, already_present: true } } + VllmImageTargetRefused { step: st, cause: why } => Present { value: VllmImageHostDeliveryRefused { host: r.host, step: st, cause: why } } + } +} + +fn vllm_image_settled_deliveries(reads: List, image: VllmImageDistributionSubject) -> List { + flat_map(reads, r => match vllm_image_settled_delivery(r: r, image: image) { + Absent => [] as List + Present { value: d } => [d] + }) +} + +// THE MOVE ITSELF, ONCE THE SOURCE IS KNOWN TO HOLD THE SUBJECT AND AT LEAST ONE TARGET NEEDS IT: +// the source's and the executor's disk shares, save (as root through the privileged leg, then chmod +// so the executor principal can read it back out), pull the tar to the executor, deliver to each +// target that needs it, and remove the staging tars on the executor and the source. +fn vllm_image_move( source_ctx: SparkMaterializeContext, - trust: FleetOpenSshTrustPolicy, - path: SparkReachPath, - admin_credential: Secret, - admin_credential_file: NonEmptyStr, - tag: NonEmptyStr, - build_key: ContentHash, + source_endpoint: NonEmptyStr, + image: VllmImageDistributionSubject, + size: ByteSize, + needing: List, ) -> VllmImageDistribution { - let tar = vllm_image_transfer_tar_path(build_key: build_key) - let trust_words = serialize_ssh_client_options(options: fleet_openssh_trust_options(policy: trust)) - match vllm_docker_leg(ctx: source_ctx, step: "distribute-save" as NonEmptyStr, command: docker_image_save_command(image: tag, output_path: tar)) { - ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } - ImageLegRan { stdout: _ } => - match vllm_docker_leg(ctx: source_ctx, step: "distribute-save-chmod" as NonEmptyStr, command: chmod_set_mode_command(chmod_program: chmod_path_resolved_program, mode: "644" as NonEmptyStr, path: tar as String)) { + let tar = vllm_image_transfer_tar_path(config: image.config) + let trust_words = serialize_ssh_client_options(options: fleet_openssh_trust_options(policy: source_ctx.trust)) + match vllm_image_first_refusal(refusals: [ + vllm_image_remote_disk_refusal(ctx: source_ctx, step: "distribute-disk-source" as NonEmptyStr, path: vllm_image_transfer_staging_directory, required: vllm_image_copies(size: size, copies: 1)), + vllm_image_local_disk_refusal(step: "distribute-disk-executor" as NonEmptyStr, path: vllm_image_transfer_staging_directory, required: vllm_image_copies(size: size, copies: 1)), + ]) { + Present { value: why } => VllmImageDistributionRefused { step: "distribute-disk" as NonEmptyStr, cause: why } + Absent => + match vllm_docker_leg(ctx: source_ctx, step: "distribute-save" as NonEmptyStr, command: docker_image_save_command(image: image.tag, output_path: tar)) { ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } ImageLegRan { stdout: _ } => - match vllm_docker_leg(ctx: source_ctx, step: "distribute-source-digest" as NonEmptyStr, command: docker_image_digest_command(image: tag)) { + match vllm_docker_leg(ctx: source_ctx, step: "distribute-save-chmod" as NonEmptyStr, command: chmod_set_mode_command(chmod_program: chmod_path_resolved_program, mode: "644" as NonEmptyStr, path: tar as String)) { ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } - ImageLegRan { stdout: source_digest } => - if source_digest == "" { - VllmImageDistributionRefused { step: "distribute-source-digest" as NonEmptyStr, cause: "the source host answers no digest for the tag it just saved" } - } else { - match vllm_image_local_scp_leg(step: "distribute-download" as NonEmptyStr, command: scp_download_command(trust_words: trust_words, principal: spark_managed_executor_login, endpoint: source_endpoint, remote_path: tar, local_path: tar)) { - ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } - ImageLegRan { stdout: _ } => { - let acc = fold( - vllm_image_distribution_targets(source: source), - init: VllmImageDistributionAcc { - deliveries: [], - trust: trust, trust_words: trust_words, path: path, - admin_credential: admin_credential, admin_credential_file: admin_credential_file, - local_tar: tar, remote_tar: tar, - tag: tag, expected_digest: source_digest as NonEmptyStr, - }, - f: fn(a, h) { vllm_image_deliver_fold_step(acc: a, host: h) }, - ) - match vllm_image_local_leg(step: "distribute-local-cleanup" as NonEmptyStr, bin_path: vllm_image_transfer_rm_path, command: rm_force_command(path: tar as String)) { - ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } - ImageLegRan { stdout: _ } => - match vllm_docker_leg(ctx: source_ctx, step: "distribute-source-cleanup" as NonEmptyStr, command: rm_force_command(path: tar as String)) { - ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } - ImageLegRan { stdout: _ } => VllmImageDistributed { deliveries: acc.deliveries } - } - } + ImageLegRan { stdout: _ } => + match vllm_image_local_scp_leg(step: "distribute-download" as NonEmptyStr, command: scp_download_command(trust_words: trust_words, principal: spark_managed_executor_login, endpoint: source_endpoint, remote_path: tar, local_path: tar)) { + ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } + ImageLegRan { stdout: _ } => { + let delivered = map(needing, h => vllm_image_deliver_to_host(host: h, source_ctx: source_ctx, trust_words: trust_words, tar: tar, image: image, size: size)) + match vllm_image_local_leg(step: "distribute-local-cleanup" as NonEmptyStr, bin_path: vllm_image_transfer_rm_path, command: rm_force_command(path: tar as String)) { + ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } + ImageLegRan { stdout: _ } => + match vllm_docker_leg(ctx: source_ctx, step: "distribute-source-cleanup" as NonEmptyStr, command: rm_force_command(path: tar as String)) { + ImageLegFailed { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } + ImageLegRan { stdout: _ } => VllmImageDistributed { deliveries: delivered } + } } } } @@ -1468,109 +1619,113 @@ fn distribute_vllm_image( } } +// THE DISTRIBUTION: the source must hold the SUBJECT'S DIGEST under the tag -- read, not carried, +// and a different image under the tag refuses rather than distributing whatever the tag points at +// now. Then every target is pre-read; if none needs the image the distribution is complete with no +// byte moved, and otherwise the move runs for exactly the targets that need it. Admission of the +// hosts is the CALLER's: which subject holds them and which claims fence them differ by arm. +fn distribute_vllm_image( + source_ctx: SparkMaterializeContext, + source_endpoint: NonEmptyStr, + image: VllmImageDistributionSubject, + targets: List, +) -> VllmImageDistribution +{ + let want = render_oci_content_digest_wire(d: image.config) + match vllm_image_read_held(ctx: source_ctx, step: "distribute-source-digest" as NonEmptyStr, tag: image.tag) { + VllmImageHeldUnread { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } + VllmImageNotHeld => + VllmImageDistributionRefused { step: "distribute-source-digest" as NonEmptyStr, cause: join(["the source ", source_ctx.host as String, " holds no image under ", image.tag as String], "") } + VllmImageHeldAs { id: id, size: size } => + if (id as String) != want { + VllmImageDistributionRefused { + step: "distribute-source-digest" as NonEmptyStr, + cause: join(["the source ", source_ctx.host as String, " holds ", id as String, " under ", image.tag as String, " and the subject is ", want, "; the tag is not the identity, so nothing was moved"], ""), + } + } else if count(targets) == 0 { + VllmImageDistributionRefused { step: "distribute-targets" as NonEmptyStr, cause: "no target was named, so there is nothing to deliver to" } + } else { + let reads = map(targets, h => vllm_image_target_read(source_ctx: source_ctx, host: h, image: image)) + let settled = vllm_image_settled_deliveries(reads: reads, image: image) + let needing = map(filter(reads, r => vllm_image_target_needs(r: r)), r => r.host) + if count(needing) == 0 { + VllmImageDistributed { deliveries: settled } + } else { + match vllm_image_move(source_ctx: source_ctx, source_endpoint: source_endpoint, image: image, size: size, needing: needing) { + VllmImageDistributionRefused { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } + VllmImageDistributed { deliveries: moved } => VllmImageDistributed { deliveries: concat(settled, moved) } + } + } + } + } +} + data vllm_image_distribute_attempt_raw: String = "vllm-image-distribute" data vllm_image_distribute_receipt_path: String = "target/vllm-image-distribute-receipt.txt" -// THE GLM DERIVED IMAGE'S DISTRIBUTION ENTRY. SPARK_CONVERGE_TARGET names the SOURCE host (the -// host the build lane built on); the targets are the arm's roster minus it. The image identity -// is derived from the recipe and derived-stage keys -- the same fold the build used -- so the tag -// this distributes is the tag the build's receipt printed, not a re-typed string. +// THE GLM DERIVED IMAGE'S DISTRIBUTION ENTRY. SPARK_CONVERGE_TARGET names the SOURCE host (the host +// the build lane built on); the targets are the arm's roster minus it. The tag is derived from the +// recipe and derived-stage keys -- the same fold the build used. +// +// ITS DIGEST IS THE SOURCE'S READBACK, AND THAT IS WEAKER THAN THE V4.1 ENTRY'S RECEIPT. No GLM +// build receipt carrying a configuration digest is transcribed in this corpus, so this entry names +// the subject by what the source holds under the derived tag at the moment of the run -- which is +// what it did before the fold took a digest, now stated rather than implicit. Every target is then +// held to that digest exactly as the V4.1 arm's are. DECLARED FRONTIER: a GLM produced-image receipt +// row replaces the readback here, as gunbc.spark.v41_runtime_image_converge v41_arm_a_produced_image +// does for V4.1. fn glm53_image_distribute_wet() -> ProcessExit { let now = clock_now_probed_at() let base_key = vllm_image_build_key(r: vllm_8d09804_sm121_recipe) let identity_tag = glm53_derived_tag(base_key: base_key) - let identity_key = glm53_derived_key(base_key: base_key) match spark_converge_target_selection(execute_apply: true) { SparkConvergeTargetRefused { reason: why } => exit_failure(reason: why) SparkConvergeTargetAllObserveOnly => exit_failure(reason: "SPARK_CONVERGE_TARGET is unset: distribution reads bytes FROM one named host, so it refuses rather than choosing one") SparkConvergeTargetSingle { host: source } => - match observe_executor_reach() { - ExecutorReachUnknown { cause: c } => exit_failure(reason: join(["glm53_image_distribute: ", c], "")) - ExecutorReachKnown { short_hostname: executor, path: path } => - match read_bootstrap_credential() { - BootstrapCredentialNotMaterialized { cause: c } => exit_failure(reason: c as String) - BootstrapCredentialReady { secret: administrator } => - match fleet_ssh_attempt_identity(raw: vllm_image_distribute_attempt_raw) { - Absent => exit_failure(reason: "attempt identity is not a safe path segment") - Present { value: attempt } => - match probe_agent_credential_verification(attempt: attempt) { - FleetSshCredentialIdentityMismatch { expected_line: expected, observed_line: observed } => - exit_failure(reason: fleet_ssh_credential_verification_refusal_reason(verification: FleetSshCredentialIdentityMismatch { expected_line: expected, observed_line: observed })) - FleetSshCredentialDerivationRefused { cause: cause } => - exit_failure(reason: fleet_ssh_credential_verification_refusal_reason(verification: FleetSshCredentialDerivationRefused { cause: cause })) - FleetSshCredentialVerified { binding: binding } => { - let materialization = materialize_fleet_known_hosts_anchor(attempt: attempt) - match fleet_ssh_execution_context_of(outcome: materialization, credential: binding) { - Absent => exit_failure(reason: join(["trust could not be established, so no leg ran: ", known_hosts_materialization_receipt(outcome: materialization)], "")) - Present { value: context } => - match fleet_openssh_trust_policy_of(outcome: materialization) { - Absent => exit_failure(reason: "the trust policy could not be sealed for the privileged legs") - Present { value: trust } => - match spark_reach_endpoint(host: source, path: ReachManagementLan) { - Absent => exit_failure(reason: join(["no ", spark_reach_path_wire(p: ReachManagementLan), " endpoint for ", source as String], "")) - Present { value: endpoint } => - match with_materialized_ssh_key_file( - identity_file_name: "gunbc-vllm-image-distribute-admin-password" as NonEmptyStr, - secret_bytes: administrator as Secret, - use: fn(file) { - distribute_vllm_image( - source: source, - source_endpoint: endpoint as NonEmptyStr, - source_ctx: SparkMaterializeContext { - host: source, trust: trust, path: path, - admin_credential: administrator as Secret, - admin_credential_file: file, - executor: HostCli, - }, - trust: trust, - path: path, - admin_credential: administrator as Secret, - admin_credential_file: file, - tag: identity_tag, - build_key: identity_key, - ) - }, - ) { - MaterializedSshKeyFileBracketRefused { cause: c } => exit_failure(reason: c as String) - MaterializedSshKeyFileBracketCompleted { value: dist } => { - let dist_lines = match dist { - VllmImageDistributionRefused { step: st, cause: why } => - join(["distribution REFUSED step=", st as String, " cause=", why, "\n"], "") - VllmImageDistributed { deliveries: ds } => - vllm_image_deliveries_wire(ds: ds) - } - let body = join([ - join(["vllm image distribute probed_at=", probed_at_word(reading: now) as String, " source=", source as String, " endpoint=", endpoint, " executor=", executor, "\n"], ""), - known_hosts_materialization_receipt(outcome: materialization), - join(["image tag=", identity_tag as String, " key=", serialize_content_hash(hash: identity_key) as String, "\n"], ""), - dist_lines, - ], "") - let w = Filesystem.Write(path: vllm_image_distribute_receipt_path, content: body) - if !w.success { - exit_failure(reason: join(["vllm image distribute receipt write failed: ", w.error], "")) - } else { - match dist { - VllmImageDistributionRefused { step: _, cause: _ } => - exit_failure(reason: join(["vllm image distribute did not converge:\n", body], "")) - VllmImageDistributed { deliveries: ds } => - if vllm_image_distribution_succeeded(deliveries: ds) { - ExitSuccess - } else { - exit_failure(reason: join(["vllm image distribute did not converge:\n", body], "")) - } - } - } - } - } - } - } - } - } - } + match with_spark_build_host( + host: source, + attempt_raw: vllm_image_distribute_attempt_raw, + identity_file_name: "gunbc-vllm-image-distribute-admin-password" as NonEmptyStr, + use: fn(target, context, ctx) { + let dist = match vllm_image_read_held(ctx: ctx, step: "distribute-source-digest" as NonEmptyStr, tag: identity_tag) { + VllmImageHeldUnread { step: st, cause: why } => VllmImageDistributionRefused { step: st, cause: why } + VllmImageNotHeld => VllmImageDistributionRefused { step: "distribute-source-digest" as NonEmptyStr, cause: "the source holds no image under the derived tag" } + VllmImageHeldAs { id: id, size: _ } => + match parse_oci_content_digest_wire(raw: id as String) { + Absent => VllmImageDistributionRefused { step: "distribute-source-digest" as NonEmptyStr, cause: join(["the source answered ", id as String, ", which is not an OCI content digest"], "") } + Present { value: config } => + distribute_vllm_image( + source_ctx: ctx, + source_endpoint: target.endpoint, + image: VllmImageDistributionSubject { tag: identity_tag, config: config }, + targets: vllm_image_distribution_targets(source: source), + ) } } + SparkBuildHostWork { + receipt: join([join(["image tag=", identity_tag as String, "\n"], ""), vllm_image_distribution_wire(dist: dist)], ""), + converged: vllm_image_distribution_succeeded(dist: dist), + } + }, + ) { + SparkBuildHostRefused { cause: c } => exit_failure(reason: c) + SparkBuildHostRan { work: work, endpoint: endpoint, executor: executor, trust_receipt: trust_receipt } => { + let body = join([ + join(["vllm image distribute probed_at=", probed_at_word(reading: now) as String, " source=", source as String, " endpoint=", endpoint as String, " executor=", executor, "\n"], ""), + trust_receipt, + work.receipt, + ], "") + let w = Filesystem.Write(path: vllm_image_distribute_receipt_path, content: body) + if !w.success { + exit_failure(reason: join(["vllm image distribute receipt write failed: ", w.error], "")) + } else if work.converged { + ExitSuccess + } else { + exit_failure(reason: join(["vllm image distribute did not converge:\n", body], "")) + } + } } } } diff --git a/dag/test/claim/docker_image_digest_command_witness_test.dag b/dag/test/claim/docker_image_digest_command_witness_test.dag index 7335b2edbf7..341f61bc361 100644 --- a/dag/test/claim/docker_image_digest_command_witness_test.dag +++ b/dag/test/claim/docker_image_digest_command_witness_test.dag @@ -10,6 +10,8 @@ import extdeps.docker.cli { docker_image_digest_command, docker_image_inspect_command, docker_image_inspect_id_from_stdout, DockerImageInspectId, DockerImageInspectIdUnreadable, docker_image_inspect_names_no_such_image, + docker_image_inspect_size_from_stdout, DockerImageInspectSize, DockerImageInspectSizeUnreadable, + docker_info_root_dir_from_stdout, DockerRootDir, DockerRootDirUnreadable, docker_container_inspect_env_from_stdout, DockerContainerInspectEnv, DockerContainerInspectEnvUnreadable, } import gunbc.typed_argv_exec { typed_argv_shell_safe_refusal_argv } @@ -71,6 +73,49 @@ test fn inspect_stdout_json_yields_the_unique_id() -> Bool { } } +// THE Size IS READ OFF THE SAME DOCUMENT: an integer reads through as the byte requirement a +// transport states; a string, a negative and an absent member are each Unreadable, never zero. +fn inspect_size_json_document(size: String) -> String { + join(["[{\"Id\":\"", inspect_id_fixture as String, "\",\"Size\":", size, "}]"], "") +} + +test fn inspect_stdout_json_yields_the_image_size() -> Bool { + match docker_image_inspect_size_from_stdout(stdout: inspect_size_json_document(size: "31457280000")) { + DockerImageInspectSize { bytes: n } => n == 31457280000 + DockerImageInspectSizeUnreadable { cause: _ } => false + } + && match docker_image_inspect_size_from_stdout(stdout: inspect_size_json_document(size: "\"31457280000\"")) { + DockerImageInspectSizeUnreadable { cause: why } => string_contains(s: why, pattern: "not a number") + DockerImageInspectSize { bytes: _ } => false + } + && match docker_image_inspect_size_from_stdout(stdout: inspect_size_json_document(size: "-1")) { + DockerImageInspectSizeUnreadable { cause: why } => string_contains(s: why, pattern: "negative") + DockerImageInspectSize { bytes: _ } => false + } + && match docker_image_inspect_size_from_stdout(stdout: inspect_id_json_document()) { + DockerImageInspectSizeUnreadable { cause: why } => string_contains(s: why, pattern: "names no Size") + DockerImageInspectSize { bytes: _ } => false + } +} + +// THE DATA ROOT IS THE DAEMON'S ANSWER: the one ` Docker Root Dir:` line of untemplated `docker info` +// reads through, including a relocated root; a document naming none, or naming it twice, is +// Unreadable -- never the /var/lib/docker default. +test fn docker_info_names_the_data_root_or_is_unreadable() -> Bool { + match docker_info_root_dir_from_stdout(stdout: "Client:\n Version: 29.2.1\nServer:\n Storage Driver: overlay2\n Docker Root Dir: /data/docker\n Debug Mode: false\n") { + DockerRootDir { path: p } => (p as String) == "/data/docker" + DockerRootDirUnreadable { cause: _ } => false + } + && match docker_info_root_dir_from_stdout(stdout: "Server:\n Storage Driver: overlay2\n") { + DockerRootDirUnreadable { cause: why } => string_contains(s: why, pattern: "0 times") + DockerRootDir { path: _ } => false + } + && match docker_info_root_dir_from_stdout(stdout: " Docker Root Dir: /a\n Docker Root Dir: /b\n") { + DockerRootDirUnreadable { cause: why } => string_contains(s: why, pattern: "2 times") + DockerRootDir { path: _ } => false + } +} + test fn inspect_stdout_that_is_not_one_object_is_unreadable() -> Bool { match docker_image_inspect_id_from_stdout(stdout: "[]") { DockerImageInspectIdUnreadable { cause: why } => string_contains(s: why, pattern: "one-element") diff --git a/dag/test/claim/spark/glm53_runtime_image_build_witness_test.dag b/dag/test/claim/spark/glm53_runtime_image_build_witness_test.dag index 9738e4df511..c3143a07c84 100644 --- a/dag/test/claim/spark/glm53_runtime_image_build_witness_test.dag +++ b/dag/test/claim/spark/glm53_runtime_image_build_witness_test.dag @@ -1,7 +1,8 @@ module test.claim.spark.glm53_runtime_image_build_witness import std.types { Bool, List, NonEmptyStr, String } -import std.content_hash { serialize_content_hash } +import std.content_hash { serialize_content_hash, Sha256Digest, Sha256DigestHex } +import extdeps.container.oci.digest { OciSha256Digest } import gunbc.spark.vllm_runtime_image_build { vllm_recipe_arch_is_buildable, glm53_derived_dockerfile, glm53_derived_prepare_script, glm53_derived_build_spec, glm53_derived_context_dir, glm53_derived_prepare_steps, glm53_derived_prepare_script_path, glm53_derived_dockerfile_path, vllm_image_transfer_tar_path, vllm_image_distribution_targets, } @@ -108,13 +109,16 @@ test fn the_derived_build_spec_selects_the_named_stage() -> Bool { // ── DISTRIBUTION ───────────────────────────────────────────────────────────────────────────── -// THE TRANSFER TAR IS CONTENT-NAMED: the derived image's build key is in the path, so a stale tar -// from another build cannot wear this one's name on any host it passes through. +// THE TRANSFER TAR IS CONTENT-NAMED: the digest of the image it carries is in the path, so a stale +// tar from another image cannot wear this one's name on any host it passes through, and two images +// never share a path. test fn the_transfer_tar_path_is_content_named() -> Bool { - let key = glm53_derived_key(base_key: vllm_image_build_key(r: vllm_8d09804_sm121_recipe)) - let p = vllm_image_transfer_tar_path(build_key: key) - string_contains(s: p as String, pattern: serialize_content_hash(hash: key) as String) + let a = OciSha256Digest(Sha256Digest { hex: "ea39410ed01d396caad86339d877c1adb181a983baadffd22c722069109ea664" as Sha256DigestHex }) + let b = OciSha256Digest(Sha256Digest { hex: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" as Sha256DigestHex }) + let p = vllm_image_transfer_tar_path(config: a) + string_contains(s: p as String, pattern: "ea39410ed01d396caad86339d877c1adb181a983baadffd22c722069109ea664") && string_contains(s: p as String, pattern: ".tar") + && (p as String) != (vllm_image_transfer_tar_path(config: b) as String) } // THE SAVE AND LOAD ARGV ARE UPSTREAM'S OWN VERBS with the paths where upstream reads them. diff --git a/dag/test/claim/spark/vllm_image_distribution_witness_test.dag b/dag/test/claim/spark/vllm_image_distribution_witness_test.dag new file mode 100644 index 00000000000..3798bf170b8 --- /dev/null +++ b/dag/test/claim/spark/vllm_image_distribution_witness_test.dag @@ -0,0 +1,127 @@ +module test.claim.spark.vllm_image_distribution_witness + +import std.types { Bool, List, NonEmptyStr, String } +import product.placement_supply { HostIdentity } +import std.measure { byte_size, byte_size_count } +import std.content_hash { Sha256Digest, Sha256DigestHex } +import extdeps.container.oci.digest { OciSha256Digest, render_oci_content_digest_wire } +import gunbc.host_disk_observation { DiskObserved, DiskUnobserved } +import gunbc.spark.vllm_runtime_image_build { + VllmImageDistributionSubject, + VllmImageHeldAs, VllmImageNotHeld, VllmImageHeldUnread, vllm_image_held_of_inspect, + VllmImageTargetHolds, VllmImageTargetNeeds, VllmImageTargetRefused, vllm_image_target_state, + vllm_image_disk_refusal, + VllmImageDistributed, VllmImageDistributionRefused, VllmImageHostDelivered, VllmImageHostDeliveryRefused, + vllm_image_distribution_succeeded, +} +import gunbc.spark.v41_runtime_image_converge { v41_arm_a_produced_image, v41_produced_image_subject } + +// ── THE DISTRIBUTION'S DECISIONS, OVER SUPPLIED DAEMON ANSWERS ──────────────────────────────── +// +// Every claim here supplies what a daemon or df answered and asks the decision the fold makes of +// it. The real path -- save, scp, load, readback against a live daemon -- is the fleet-converge +// mode spark_v41_runtime_image_distribute, whose receipt is the inhabitance evidence for these +// shapes (DESIGN §3, a witness discriminates at one interface). + +data subject_hex: String = "ea39410ed01d396caad86339d877c1adb181a983baadffd22c722069109ea664" +data other_hex: String = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +fn subject() -> VllmImageDistributionSubject { + VllmImageDistributionSubject { + tag: "gunbc-vllm-dsv41-gb10:fixture" as NonEmptyStr, + config: OciSha256Digest(Sha256Digest { hex: subject_hex as Sha256DigestHex }), + } +} + +fn id_of(hex: String) -> NonEmptyStr { + join(["sha256:", hex], "") as NonEmptyStr +} + +// THE NOOP: a target already holding the subject's digest under the tag moves nothing. +test fn a_target_holding_the_digest_is_the_noop() -> Bool { + match vllm_image_target_state(held: VllmImageHeldAs { id: id_of(hex: subject_hex), size: byte_size(count: 10) }, image: subject()) { + VllmImageTargetHolds => true + VllmImageTargetNeeds => false + VllmImageTargetRefused { step: _, cause: _ } => false + } +} + +test fn a_target_holding_nothing_under_the_tag_needs_the_image() -> Bool { + match vllm_image_target_state(held: VllmImageNotHeld, image: subject()) { + VllmImageTargetNeeds => true + VllmImageTargetHolds => false + VllmImageTargetRefused { step: _, cause: _ } => false + } +} + +// THE RED THE TAG CANNOT CATCH: a different image under the same tag refuses rather than having the +// tag moved off it by a load -- two images under one tag is exactly why the tag is not the identity. +test fn a_different_image_under_the_tag_refuses() -> Bool { + match vllm_image_target_state(held: VllmImageHeldAs { id: id_of(hex: other_hex), size: byte_size(count: 10) }, image: subject()) { + VllmImageTargetRefused { step: _, cause: why } => string_contains(s: why, pattern: other_hex) && string_contains(s: why, pattern: subject_hex) + VllmImageTargetHolds => false + VllmImageTargetNeeds => false + } +} + +// AN UNREAD DAEMON IS NOT AN EMPTY ONE: reading it as "needs" would move 30GB onto a host nobody asked. +test fn an_unread_target_refuses_rather_than_reading_as_empty() -> Bool { + match vllm_image_target_state(held: VllmImageHeldUnread { step: "deliver-digest-pre" as NonEmptyStr, cause: "exit=1 stderr=permission denied" }, image: subject()) { + VllmImageTargetRefused { step: _, cause: why } => string_contains(s: why, pattern: "permission denied") + VllmImageTargetHolds => false + VllmImageTargetNeeds => false + } +} + +test fn an_inspect_document_reads_as_id_and_size() -> Bool { + let doc = join(["[{\"Id\":\"sha256:", subject_hex, "\",\"Size\":31457280000}]"], "") + match vllm_image_held_of_inspect(step: "s" as NonEmptyStr, stdout: doc) { + VllmImageHeldAs { id: id, size: n } => (id as String) == join(["sha256:", subject_hex], "") && byte_size_count(b: n) == 31457280000 + VllmImageNotHeld => false + VllmImageHeldUnread { step: _, cause: _ } => false + } + && match vllm_image_held_of_inspect(step: "s" as NonEmptyStr, stdout: join(["[{\"Id\":\"sha256:", subject_hex, "\"}]"], "")) { + VllmImageHeldUnread { step: _, cause: why } => string_contains(s: why, pattern: "Size") + VllmImageHeldAs { id: _, size: _ } => false + VllmImageNotHeld => false + } +} + +// THE DISK SHARE REFUSES SHORT OF ITS REQUIREMENT WITH BOTH FIGURES, ADMITS AT IT, AND AN +// UNOBSERVED FILESYSTEM IS A REFUSAL -- never room assumed. +test fn a_filesystem_short_of_its_share_refuses_with_the_figures() -> Bool { + match vllm_image_disk_refusal(step: "deliver-disk-staging" as NonEmptyStr, place: "srv5:/tmp", required: byte_size(count: 200), reading: DiskObserved { free_bytes: byte_size(count: 199), observed_on: "srv5" as NonEmptyStr }) { + Present { value: why } => string_contains(s: why, pattern: "199") && string_contains(s: why, pattern: "200") + Absent => false + } + && match vllm_image_disk_refusal(step: "deliver-disk-staging" as NonEmptyStr, place: "srv5:/tmp", required: byte_size(count: 200), reading: DiskObserved { free_bytes: byte_size(count: 200), observed_on: "srv5" as NonEmptyStr }) { + Present { value: _ } => false + Absent => true + } + && match vllm_image_disk_refusal(step: "deliver-disk-staging" as NonEmptyStr, place: "srv5:/tmp", required: byte_size(count: 200), reading: DiskUnobserved { reason: "df did not run" }) { + Present { value: why } => string_contains(s: why, pattern: "df did not run") + Absent => false + } +} + +// DELIVERING TO NOBODY IS NOT A SUCCESS, AND ONE REFUSED HOST FAILS THE WHOLE. +test fn distribution_succeeds_only_over_a_nonempty_all_delivered_set() -> Bool { + let ok = VllmImageHostDelivered { host: "srv5" as HostIdentity, digest: id_of(hex: subject_hex), already_present: false } + let noop = VllmImageHostDelivered { host: "srv6" as HostIdentity, digest: id_of(hex: subject_hex), already_present: true } + let bad = VllmImageHostDeliveryRefused { host: "srv7" as HostIdentity, step: "deliver-disk" as NonEmptyStr, cause: "short" } + vllm_image_distribution_succeeded(dist: VllmImageDistributed { deliveries: [ok, noop] }) + && !vllm_image_distribution_succeeded(dist: VllmImageDistributed { deliveries: [] }) + && !vllm_image_distribution_succeeded(dist: VllmImageDistributed { deliveries: [ok, bad] }) + && !vllm_image_distribution_succeeded(dist: VllmImageDistributionRefused { step: "distribute-save" as NonEmptyStr, cause: "x" }) +} + +// THE ARM A RECEIPT ROW RE-DERIVES THE TAG ITS PRODUCTION PRINTED. fleet-converge run 36207135528 +// printed tag=gunbc-vllm-dsv41-gb10:d73e307009a5d716 for this source tree; the row carries only the +// tree, and the tag comes out of the candidate's recipe through the build's own key fold. A row +// whose tree, or a recipe that has moved since, no longer derives that tag goes red here. +test fn the_arm_a_receipt_derives_its_printed_tag_and_carries_its_digest() -> Bool { + let s = v41_produced_image_subject(p: v41_arm_a_produced_image) + (s.tag as String) == "gunbc-vllm-dsv41-gb10:d73e307009a5d716" + && render_oci_content_digest_wire(d: s.config) == join(["sha256:", subject_hex], "") + && (v41_arm_a_produced_image.held_on as String) == "srv8" +}