feat(computer)!: the ubuntu-24.04 disk form and Linux disk composition

.github/dependabot.yml

@@ -56,6 +56,9 @@ updates:
       - "/src/agent_env/agents/agents/pydantic-ai"
       - "/src/agent_env/examples/mail-mcp/envs/mail"
       - "/src/agent_env/computers/artifacts/ubuntu-24.04"
+      - "/src/agent_env/computers/disk/linux/boot"
+      - "/src/agent_env/computers/disk/linux/tools"
+      - "/src/agent_env/computers/disk/linux/launcher"
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 5

.github/scripts/check_clean_install.py

@@ -85,10 +85,8 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]:
         found.append(f"agent-env found the config file {config}; the gate runs with none")
     for plugin in plugins():
         found.append(f"{plugin} is installed; the gate tests agent-env without plugins")
-    installed_src = Path(agent_env.__file__).parent.parent
     for image in PUBLISHED_IMAGES:
-        checkout = PublishedImage(image.name, CHECKOUT_SRC / image.dockerfile.relative_to(installed_src))
-        if (tag := image.tag()) != (expected := checkout.tag()):
+        if (tag := image.tag()) != (expected := _checkout(image).tag()):
             found.append(f"{image.name}: the installed sources tag it {tag}, the checkout's {expected}; an install "
                          "wouldn't find the image published for its sources")
     bundle = find_bundle(name)
@@ -121,6 +119,13 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]:
     return found
 
 
+def _checkout(image: PublishedImage) -> PublishedImage:
+    """``image`` as the checkout's sources make it, with the images it's built from."""
+    installed_src = Path(agent_env.__file__).parent.parent
+    return PublishedImage(image.name, CHECKOUT_SRC / image.dockerfile.relative_to(installed_src),
+                          tuple((arg, _checkout(built)) for arg, built in image.args))
+
+
 def plugins() -> list[str]:
     """Entry points in an ``agent_env.*`` group from any distribution other than agent-env's own."""
     return sorted(f"{dist.name}'s {ep.group} entry point {ep.name}"

.github/scripts/infra_build.py

@@ -4,8 +4,11 @@
 images agent-env ships before it's built, pushed or recorded.
 
 ``check ENTRY`` fails unless ENTRY, a JSON object as ``infra_images.py plan`` lists an image, names an image agent-env
-ships, with its Dockerfile, context, repository and a source tag. ``exists REF`` exits 0 when the registry has REF, 1
-when it says it doesn't, and 2 when it can't tell. ``smoke NAME IMAGE`` checks that IMAGE, a local build of NAME, runs.
+ships, with its Dockerfile, context, repository and a source tag, and for a disk form, the build args naming its base
+computer's image and the disk tools by their repositories and source tags. ``exists REF`` exits 0 when the registry has
+REF, 1 when it says it doesn't, and 2 when it can't tell. ``config IMAGE`` prints the config object of IMAGE's OCI
+config, base64'd, as a disk form's build takes its base's. ``smoke NAME IMAGE`` checks that IMAGE, a local build of
+NAME, runs: a disk form boots to its desktop in the QEMU launcher built here as ``agentenv-qemu-launcher``.
 ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the registry serves
 under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve
 yet the agentenv-framework-protocol an agent image's build installs.
@@ -14,7 +17,9 @@
 from __future__ import annotations
 
 import argparse
+import base64
 import json
+import platform
 import posixpath
 import re
 import subprocess
@@ -43,12 +48,20 @@
     "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile",
     "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile",
     "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile",
+    "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile",
+    "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile",
+    "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile",
 }
+# Each disk form's build args, by the images agent-env ships that they name.
+DISK_ARGS = {"agentenv-ubuntu-24.04-disk": {"ROOTFS": "agentenv-ubuntu-24.04", "TOOLS": "agentenv-disk-tools"}}
 _TAG = re.compile(r"^src-[0-9a-f]{64}$")
 _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
 _HEALTHY_WITHIN_SECONDS = 120
 _HEALTH_POLL_SECONDS = 2
 _LOG_TAIL_LINES = 30  # of a container that never turned healthy, in the error
+# How long a disk form may take to boot to its desktop: seconds with KVM, minutes under TCG.
+_BOOTED_WITHIN_SECONDS = 900
+_BOOTED = re.compile(r"success: (xvfb|dbus|desktop) entered RUNNING")
 _PRINTED_CHARS = 200  # of what docker printed in place of a manifest, in the error
 # What a registry answers when it rate-limits a request: ECR Public's 429, and the toomanyrequests error code GHCR and
 # Docker Hub send.
@@ -94,6 +107,11 @@
                               " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);"
                               " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;"
                               " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"],
+    "agentenv-disk-tools": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
+                            "command -v disk image-env mkfs.ext4 mkfs.vfat mcopy sfdisk qemu-img"],
+    "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c",
+                               "qemu-system-x86_64 --version && qemu-system-aarch64 --version"
+                               " && test -f /usr/share/OVMF/OVMF_CODE_4M.fd && test -f /usr/share/AAVMF/AAVMF_CODE.fd"],
 }
 
 
@@ -102,6 +120,7 @@ def main() -> int:
     commands = parser.add_subparsers(dest="command", required=True)
     commands.add_parser("check").add_argument("entry")
     commands.add_parser("exists").add_argument("ref")
+    commands.add_parser("config").add_argument("image")
     smoke = commands.add_parser("smoke")
     smoke.add_argument("name", choices=sorted(IMAGES))
     smoke.add_argument("image")
@@ -112,6 +131,9 @@ def main() -> int:
         return _tried_again(lambda: _streamed(args.argv)).returncode
     if args.command == "exists":
         return _exists(args.ref)
+    if args.command == "config":
+        print(_config(args.image))
+        return 0
     if args.command == "smoke":
         return _smoke(args.name, args.image)
     problems = _check(args.entry) if args.command == "check" else _verify_record(args.path)
@@ -134,11 +156,26 @@ def _check(entry: str) -> list[str]:
                 for key, value in expected.items() if image.get(key) != value]
     if not _TAG.match(str(image.get("tag"))):
         problems.append(f"{name}: {image.get('tag')!r} isn't a source tag")
-    if extra := sorted(set(image) - {*expected, "tag"}):
+    if name in DISK_ARGS:
+        problems += _check_args(name, image.get("args"))
+    if extra := sorted(set(image) - {*expected, "tag", *(("args",) if name in DISK_ARGS else ())}):
         problems.append(f"{name}: unexpected {', '.join(extra)}")
     return problems
 
 
+def _check_args(name: str, args: object) -> list[str]:
+    """The problems with a disk form's build args: each must name the image agent-env ships it's for, by a source tag."""
+    wanted = DISK_ARGS[name]
+    if not isinstance(args, dict) or set(args) != set(wanted):
+        return [f"{name}: args is {args!r}, not {', '.join(sorted(wanted))}"]
+    problems = []
+    for arg, image in wanted.items():
+        repository, _, tag = str(args[arg]).rpartition(":")
+        if repository != f"{REGISTRY}/{image}" or not _TAG.match(tag):
+            problems.append(f"{name}: {arg} is {args[arg]!r}, not {REGISTRY}/{image} under a source tag")
+    return problems
+
+
 def _tried_again(attempt: Callable[[], subprocess.CompletedProcess]) -> subprocess.CompletedProcess:
     """``attempt()``, and again after each of ``_RETRY_WAITS_SECONDS`` while what it printed says a later try can work
     (``_why_again``)."""
@@ -232,16 +269,54 @@ def _verify_record(path: Path) -> list[str]:
     return problems
 
 
+def _config(image: str) -> str:
+    """The config object of ``image``'s OCI config, a local image's, as base64'd JSON."""
+    inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", image],
+                               capture_output=True, text=True, check=True)
+    return base64.b64encode(inspected.stdout.strip().encode()).decode()
+
+
 def _smoke(name: str, image: str) -> int:
     if name == "agentenv-service-db":
         return _healthy(image)
+    if name in DISK_ARGS:
+        return _boots(image)
     command = [part.replace("{image}", image) for part in _SMOKES[name]]
     result = subprocess.run(command, capture_output=True, text=True)
     if result.returncode != 0:
         print(f"::error::{name}: `{' '.join(command)}` exited {result.returncode}:\n{result.stdout}{result.stderr}")
     return result.returncode
 
 
+def _boots(image: str) -> int:
+    """Boot ``image``, a disk form, in the launcher built here, with KVM where it can, and wait for its desktop's
+    programs to run, which its console shows."""
+    arch = subprocess.run(["docker", "image", "inspect", "--format", "{{.Architecture}}", image], capture_output=True,
+                          text=True, check=True).stdout.strip()
+    machine = f"{image}-machine"
+    subprocess.run(["docker", "build", "-q", "-t", machine, "-"], text=True, check=True, capture_output=True,
+                   input=f"FROM --platform=linux/{arch} {image} AS disk\nFROM agentenv-qemu-launcher\n"
+                         "COPY --from=disk /disk/ /disk/\n")
+    kvm = ["--device", "/dev/kvm"] if Path("/dev/kvm").exists() else []
+    container = subprocess.run(["docker", "run", "-d", *kvm, "-e", f"VM_ARCH={arch}", machine], capture_output=True,
+                               text=True, check=True).stdout.strip()
+    try:
+        deadline, logs = time.monotonic() + _BOOTED_WITHIN_SECONDS, ""
+        while time.monotonic() < deadline:
+            logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True, errors="replace")
+            logs = logs.stdout + logs.stderr
+            if len(set(_BOOTED.findall(logs))) == 3:
+                native = {"x86_64": "amd64", "aarch64": "arm64"}.get(platform.machine()) == arch
+                print(f"{image} booted to its desktop {'with KVM' if kvm and native else 'under TCG'}")
+                return 0
+            time.sleep(_HEALTH_POLL_SECONDS)
+        tail = "\n".join(logs.splitlines()[-_LOG_TAIL_LINES:])
+        print(f"::error::{image} didn't boot to its desktop within {_BOOTED_WITHIN_SECONDS}s:\n{tail}")
+        return 1
+    finally:
+        subprocess.run(["docker", "rm", "-f", container], capture_output=True)
+
+
 def _healthy(image: str) -> int:
     """Start ``image`` and wait for its health check to pass, as compose does before starting what depends on it."""
     container = subprocess.run(["docker", "run", "-d", image], capture_output=True, text=True, check=True).stdout.strip()

.github/scripts/infra_images.py

@@ -2,11 +2,13 @@
 the release workflow's jobs that only read: this imports agent-env, so it runs where agent-env and its dependencies are
 installed, and no job that holds a token to push images or publish to PyPI runs it (those run ``infra_build.py``).
 
-``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, ``--missing`` only those the
+``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, and for an image built over other
+published images, the build args naming each by its repository and tag, ``--missing`` only those the
 registry hasn't published: it has no image under the tag, or refuses to show one, as GHCR does for a package that
-doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, and ``--only agents``
+doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, ``--only agents``
 only the images that install the protocol, the agents' and the website browser's, which a release builds after it
-publishes the protocol, as their protocol floor may name it. ``write PATH`` records the digest the registry serves under every
+publishes the protocol, as their protocol floor may name it, and ``--only disks`` the base computers' disk forms, built
+over their published images. ``write PATH`` records the digest the registry serves under every
 image's tag as a release wheel carries them, and fails if any is missing. ``public`` warns about each image a user, who
 pulls anonymously, can't. The registry is read with ``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously
 otherwise, and always anonymously by ``public``.
@@ -21,7 +23,7 @@
 from pathlib import Path
 
 from agent_env.env.bootstrap import PLAYWRIGHT_MCP_DOCKERFILE, PUBLISHED_IMAGES
-from agent_env.env.published_images import BUNDLE_IMAGES, REGISTRY, PublishedImage, write_release_digests
+from agent_env.env.published_images import BUNDLE_IMAGES, DISK_IMAGES, REGISTRY, PublishedImage, write_release_digests
 from agent_env.store.image_store.oci_registry_credentials import RegistryAuth
 from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError, pin_digest
 
@@ -30,7 +32,8 @@
 # example env's, and the Playwright MCP browser's, whose recorder installs it, matched by its Dockerfile.
 _AFTER_PROTOCOL = BUNDLE_IMAGES + tuple(image for image in PUBLISHED_IMAGES
                                        if image.dockerfile == PLAYWRIGHT_MCP_DOCKERFILE)
-_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL), "agents": _AFTER_PROTOCOL}
+_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL + DISK_IMAGES),
+           "agents": _AFTER_PROTOCOL, "disks": DISK_IMAGES}
 
 
 def main() -> int:
@@ -59,9 +62,13 @@ def main() -> int:
     return 0
 
 
-def _described(image: PublishedImage) -> dict[str, str]:
-    return {"name": image.name, "repository": image.repository, "tag": image.tag(),
-            "dockerfile": str(image.dockerfile.relative_to(REPO)), "context": str(image.dockerfile.parent.relative_to(REPO))}
+def _described(image: PublishedImage) -> dict:
+    described = {"name": image.name, "repository": image.repository, "tag": image.tag(),
+                 "dockerfile": str(image.dockerfile.relative_to(REPO)),
+                 "context": str(image.dockerfile.parent.relative_to(REPO))}
+    if image.args:
+        described["args"] = {arg: built.ref() for arg, built in image.args}
+    return described
 
 
 def _digest(image: PublishedImage) -> str | None:

.github/workflows/local-backends.yml

@@ -138,6 +138,9 @@ jobs:
         run: |
           docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2
           timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done'
+      # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below.
+      - name: Let the local provider boot disks with KVM
+        run: sudo chmod 666 /dev/kvm
       - name: Slow integration tier on the local defaults
         env:
           AGENTENV_TEST_SHARD: ${{ matrix.shard }}/8