V4.1 runtime image distribution by config digest (spark_v41_runtime_image_distribute)

dag/extdeps/docker/cli.dag

@@ -472,6 +472,95 @@ fn docker_image_inspect_id_from_stdout(stdout: String) -> DockerImageInspectIdRe
   }
 }
 
+// THE IMAGE'S SIZE, READ OFF THE SAME INSPECT DOCUMENT AS ITS Id. `Size` is the image's total
+// uncompressed layer bytes (docs.docker.com/reference/api/engine/version/v1.47/#tag/Image/operation/ImageInspect:
+// "Total size of the image including all layers it is composed of"), a JSON integer. It is what a
+// `docker save` tar of the image carries, give or take the tar's own headers and the config and
+// manifest members, and what a `docker load` on a host holding none of its layers writes -- so it is
+// the byte requirement a byte transport between daemons states before moving anything. A member that
+// is absent, duplicated, not an integer or negative is Unreadable, never a zero.
+type DockerImageInspectSizeRead
+  = DockerImageInspectSize { bytes: Int }
+  | DockerImageInspectSizeUnreadable { cause: String }
+
+fn docker_image_inspect_size_from_stdout(stdout: String) -> DockerImageInspectSizeRead {
+  match parse_json_document(s: trim(s: stdout)) {
+    JsonDocumentUnreadable { gap: g } =>
+      DockerImageInspectSizeUnreadable { cause: json_document_gap_text(gap: g) }
+    JsonDocumentParsed { value: v } =>
+      match docker_inspect_json_first_of_one(v: v) {
+        Absent => DockerImageInspectSizeUnreadable { cause: "docker image inspect JSON is not a one-element array" }
+        Present { value: obj } =>
+          match json_object_unique_member(v: obj, key: "Size") {
+            JsonMemberAbsent => DockerImageInspectSizeUnreadable { cause: "docker image inspect JSON names no Size" }
+            JsonMemberNotAnObject => DockerImageInspectSizeUnreadable { cause: "docker image inspect JSON element is not an object" }
+            JsonMemberDuplicated { count: n } =>
+              DockerImageInspectSizeUnreadable { cause: join(["docker image inspect JSON names Size ", to_string(n), " times"], "") }
+            JsonMemberFound { value: sv } =>
+              match sv {
+                JsonNumber { lexeme: l } =>
+                  match parse_int(s: l) {
+                    Absent => DockerImageInspectSizeUnreadable { cause: join(["docker image inspect Size is not an integer: ", l], "") }
+                    Present { value: n } =>
+                      if n < 0 {
+                        DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is negative" }
+                      } else {
+                        DockerImageInspectSize { bytes: n }
+                      }
+                  }
+                JsonNull => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" }
+                JsonBool { value: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" }
+                JsonString { value: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" }
+                JsonArray { elements: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" }
+                JsonObject { members: _ } => DockerImageInspectSizeUnreadable { cause: "docker image inspect Size is not a number" }
+              }
+          }
+      }
+  }
+}
+
+// THE DAEMON'S DATA ROOT, WHERE `docker load` WRITES LAYERS, READ FROM THE DAEMON. dockerd's
+// --data-root defaults to /var/lib/docker (docs.docker.com/reference/cli/dockerd/) and daemon.json can
+// move it, so the default is not a reading: a free-space read of the default path on a host whose
+// root moved answers for the wrong filesystem. `docker info` with no template prints the server's
+// state as text, one ` Docker Root Dir: <path>` line among them
+// (docs.docker.com/reference/cli/docker/system/info/); no --format, for the portable-word reason
+// docker_image_inspect_command states. A document with no such line, two of them, or a value that is
+// not an absolute path is Unreadable -- never the default.
+fn docker_info_command() -> ArgvCommand {
+  argv_command(program: docker_binary_path, arguments: ["info"])
+}
+
+data docker_info_root_dir_label: String = "Docker Root Dir:"
+
+type DockerRootDirRead
+  = DockerRootDir { path: NonEmptyStr }
+  | DockerRootDirUnreadable { cause: String }
+
+fn docker_info_root_dir_from_stdout(stdout: String) -> DockerRootDirRead {
+  let values = flat_map(split(s: stdout, delimiter: "\n"), l => {
+    let t = trim(s: l)
+    if starts_with(s: t, prefix: docker_info_root_dir_label) {
+      [trim(s: substring(s: t, start: docker_info_root_dir_label.length(), end: t.length()))]
+    } else {
+      [] as List<String>
+    }
+  })
+  if count(values) != 1 {
+    DockerRootDirUnreadable { cause: join(["docker info names the Docker Root Dir ", to_string(count(values)), " times"], "") }
+  } else {
+    match first(values) {
+      Absent => DockerRootDirUnreadable { cause: "docker info names no Docker Root Dir" }
+      Present { value: v } =>
+        if starts_with(s: v, prefix: "/") {
+          DockerRootDir { path: v as NonEmptyStr }
+        } else {
+          DockerRootDirUnreadable { cause: join(["docker info's Docker Root Dir is not an absolute path: ", v], "") }
+        }
+    }
+  }
+}
+
 // THE CONTAINER'S TWO IDENTITY FIELDS, AND NOTHING ELSE OFF THE INSPECT OBJECT. `docker container
 // inspect` returns the whole object extdeps.docker.container_inspect ContainerInspect models,
 // including Config.Env, whose values can carry secrets. This reader takes exactly .Id -- the full

dag/extdeps/exec/command.dag

@@ -87,6 +87,7 @@ fn argv_command(program: NonEmptyStr, arguments: List<String>) -> ArgvCommand
     decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_image_save_command"),
     decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_image_load_command"),
     decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_image_inspect_command"),
+    decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_info_command"),
     decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_run_ephemeral_command"),
     decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_remove_force_command"),
     decl_ref(module_path: "extdeps.docker.cli", decl_name: "docker_rename_command"),

dag/gunbc/ci/ci_spec.dag

@@ -8,6 +8,7 @@ import gunbc.spark.pair_serving_apply { spark_pair_apply_receipt_path }
 import gunbc.spark.native_serving_apply { spark_native_apply_receipt_path }
 import gunbc.spark.v41_row_store_encode_run { v41_encode_run_receipt_path }
 import gunbc.spark.v41_row_store_readback_run { v41_readback_run_receipt_path }
+import gunbc.spark.v41_runtime_image_converge { v41_distribution_receipt_path }
 import gunbc.ledger_row_coherence { heal_repair_declaration_artifact_path }
 import extdeps.cloud.gcp.secret_ref { SecretRef, secret_ref_access_url, secret_ref_version_resource }
 import gunbc.auth.ci_app_key_rotation {
@@ -1019,6 +1020,14 @@ data gunbc_ci_spark_v41_runtime_image_build_target: GunbcRunStepTarget = GunbcRu
   function: "v41_runtime_image_produce_ci_wet",
 }
 
+// THE DISTRIBUTION OF THAT PRODUCED IMAGE TO ANOTHER GROUP A SPARK: save on the host the production
+// receipt names, scp through the executor, load on the selected Spark, and the configuration digest
+// read back there. The same prelude, because docker's socket on both ends is root-owned.
+data gunbc_ci_spark_v41_runtime_image_distribute_target: GunbcRunStepTarget = GunbcRunStepTarget {
+  entry: "dag/gunbc/spark/v41_runtime_image_converge.dag",
+  function: "v41_runtime_image_distribute_ci_wet",
+}
+
 // THE PUBLISHED V4.1 CHECKPOINT ON ONE GROUP A SPARK (gunbc.spark.v41_checkpoint_materialize): the same
 // administrator credential as the runtime-image probe, because every leg runs privileged on the Spark.
 data gunbc_ci_spark_v41_checkpoint_materialize_target: GunbcRunStepTarget = GunbcRunStepTarget {
@@ -2584,6 +2593,17 @@ fn gunbc_ci_spark_runtime_image_probe_invoke() -> String {
 
 // The production shares the probe's prelude, for the same reason and with the same credential: one
 // fleet-key agent from the job and one per-target administrator credential materialized to one file.
+fn gunbc_ci_spark_v41_runtime_image_distribute_invoke() -> String {
+  gunbc_run_step_script_with_prelude(
+    prelude: gunbc_ci_spark_grant_install_credential_prelude(),
+    source_roots: witness_layer_roots,
+    entry: gunbc_ci_spark_v41_runtime_image_distribute_target.entry,
+    function: gunbc_ci_spark_v41_runtime_image_distribute_target.function,
+    claim_run: false,
+    receipt_rel: v41_distribution_receipt_path
+  )
+}
+
 fn gunbc_ci_spark_v41_runtime_image_build_invoke() -> String {
   gunbc_run_step_script_with_prelude(
     prelude: gunbc_ci_spark_grant_install_credential_prelude(),