diff --git a/.github/workflows/build-vm-driver.yml b/.github/workflows/build-vm-driver.yml
index 9df710cb82..5ca5dd99f0 100644
--- a/.github/workflows/build-vm-driver.yml
+++ b/.github/workflows/build-vm-driver.yml
@@ -25,8 +25,88 @@ permissions:
contents: read
jobs:
+ host-supervisor-macos:
+ name: native host supervisor (aarch64-apple-darwin)
+ permissions:
+ contents: read
+ uses: ./.github/workflows/build-binaries.yml
+ with:
+ package: openshell-supervisor
+ binary: openshell-supervisor
+ triple: aarch64-apple-darwin
+ runner: macos-15-xlarge
+ dev-shell: .#devShells.aarch64-darwin.default
+ cargo-version: ${{ inputs.cargo-version }}
+ image-tag: ${{ inputs.image-tag }}
+ checkout-ref: ${{ inputs.checkout-ref }}
+ secrets: inherit
+
+ helper-runtime:
+ name: helper runtime (${{ matrix.arch }})
+ strategy:
+ matrix:
+ include:
+ - arch: x86_64
+ docker_arch: amd64
+ runner: linux-amd64-cpu8
+ - arch: aarch64
+ docker_arch: arm64
+ runner: linux-arm64-cpu8
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: ${{ inputs['checkout-ref'] || github.sha }}
+
+ - name: Download openshell-sandbox
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: openshell-sandbox-${{ matrix.arch }}-unknown-linux-musl
+ path: sandbox
+
+ - name: Download openshell-supervisor
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: openshell-supervisor-${{ matrix.arch }}-unknown-linux-musl
+ path: supervisor
+
+ - name: Assemble trusted helper runtime
+ shell: bash
+ run: |
+ binary_dir="deploy/docker/.build/prebuilt-binaries/${{ matrix.docker_arch }}"
+ install -d "$binary_dir" artifacts
+ install -m 0555 sandbox/openshell-sandbox "$binary_dir/openshell-sandbox"
+ install -m 0555 supervisor/openshell-supervisor "$binary_dir/openshell-supervisor"
+
+ image="openshell-vm-helper-runtime:${{ matrix.docker_arch }}-${GITHUB_RUN_ID}"
+ container=""
+ cleanup() {
+ if [ -n "$container" ]; then docker rm -f "$container" >/dev/null 2>&1 || true; fi
+ docker image rm "$image" >/dev/null 2>&1 || true
+ }
+ trap cleanup EXIT
+
+ docker build \
+ --build-arg "TARGETARCH=${{ matrix.docker_arch }}" \
+ --file deploy/docker/Dockerfile.supervisor \
+ --tag "$image" \
+ .
+ container="$(docker create "$image")"
+ docker cp "$container:/openshell-runtime" - \
+ | zstd -19 -T0 -o artifacts/openshell-runtime.tar.zst
+ test -s artifacts/openshell-runtime.tar.zst
+
+ - name: Upload trusted helper runtime
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: openshell-vm-helper-runtime-${{ matrix.arch }}
+ path: artifacts/openshell-runtime.tar.zst
+ if-no-files-found: error
+
build:
name: openshell-driver-vm (${{ matrix.triple }})
+ needs: [helper-runtime, host-supervisor-macos]
strategy:
matrix:
include:
@@ -64,15 +144,40 @@ jobs:
name: openshell-sandbox-${{ matrix.arch }}-unknown-linux-musl
path: sandbox
+ - name: Download openshell-supervisor
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: openshell-supervisor-${{ matrix.arch }}-unknown-linux-musl
+ path: supervisor
+
+ - name: Download native macOS host supervisor
+ if: endsWith(matrix.triple, '-apple-darwin')
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: openshell-supervisor-aarch64-apple-darwin
+ path: host-supervisor
+
+ - name: Download trusted helper runtime
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: openshell-vm-helper-runtime-${{ matrix.arch }}
+ path: helper-runtime
+
- name: Build VM runtime
run: nix build .#vm-runtime
- name: Assemble compressed VM runtime
run: |
compressed_dir="${RUNNER_TEMP}/vm-runtime-compressed"
+ host_supervisor="supervisor/openshell-supervisor"
+ if [[ "${{ matrix.triple }}" == *-apple-darwin ]]; then
+ host_supervisor="host-supervisor/openshell-supervisor"
+ fi
install -d "$compressed_dir"
cp result/compressed/*.zst "$compressed_dir/"
zstd -19 -T1 sandbox/openshell-sandbox -o "$compressed_dir/openshell-sandbox.zst"
+ zstd -19 -T1 "$host_supervisor" -o "$compressed_dir/openshell-supervisor.zst"
+ cp helper-runtime/openshell-runtime.tar.zst "$compressed_dir/"
- name: Build openshell-driver-vm
uses: ./.github/actions/build-rust-binary
diff --git a/.github/workflows/release-vm-kernel.yml b/.github/workflows/release-vm-kernel.yml
index 76f00cb784..4ec5ec2159 100644
--- a/.github/workflows/release-vm-kernel.yml
+++ b/.github/workflows/release-vm-kernel.yml
@@ -1,6 +1,6 @@
name: Release VM Kernel
-# Build custom libkrunfw (kernel firmware) + libkrun (VMM) + gvproxy for all
+# Build custom libkrunfw (kernel firmware) + libkrun (VMM) for all
# supported openshell-driver-vm platforms. Artifacts are uploaded to the
# rolling "vm-runtime" GitHub Release and consumed by normal dev/tag release
# workflows when building the openshell-driver-vm binary.
@@ -16,6 +16,12 @@ name: Release VM Kernel
on:
workflow_dispatch:
+ inputs:
+ release-tag:
+ description: Rolling prerelease tag to create or update
+ required: false
+ default: vm-runtime
+ type: string
permissions:
contents: write
@@ -23,7 +29,7 @@ permissions:
# Serialize runtime release updates.
concurrency:
- group: vm-runtime-release
+ group: vm-runtime-release-${{ inputs.release-tag || 'vm-runtime' }}
cancel-in-progress: false
defaults:
@@ -194,23 +200,28 @@ jobs:
release/vm-runtime-darwin-aarch64.tar.zst
- name: Ensure vm-runtime tag exists
+ env:
+ RELEASE_TAG: ${{ inputs.release-tag || 'vm-runtime' }}
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
- git tag -fa vm-runtime -m "VM Runtime Development Build" "${GITHUB_SHA}"
- git push --force origin vm-runtime
+ git tag -fa "$RELEASE_TAG" -m "VM Runtime Development Build" "${GITHUB_SHA}"
+ git push --force origin "$RELEASE_TAG"
- name: Prune stale runtime assets from vm-runtime release
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
+ env:
+ RELEASE_TAG: ${{ inputs.release-tag || 'vm-runtime' }}
with:
script: |
const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/');
+ const tag = process.env.RELEASE_TAG;
let release;
try {
- release = await github.rest.repos.getReleaseByTag({ owner, repo, tag: 'vm-runtime' });
+ release = await github.rest.repos.getReleaseByTag({ owner, repo, tag });
} catch (err) {
if (err.status === 404) {
- core.info('No existing vm-runtime release; will create fresh.');
+ core.info(`No existing ${tag} release; will create fresh.`);
return;
}
throw err;
@@ -228,7 +239,7 @@ jobs:
with:
name: OpenShell VM Runtime
prerelease: true
- tag_name: vm-runtime
+ tag_name: ${{ inputs.release-tag || 'vm-runtime' }}
target_commitish: ${{ github.sha }}
body: |
Build of the OpenShell VM runtime artifacts used by `openshell-driver-vm`.
@@ -237,7 +248,7 @@ jobs:
### Kernel Runtime Artifacts
- Pre-built kernel runtime (libkrunfw + libkrun + gvproxy + umoci) for embedding
+ Pre-built kernel runtime (libkrunfw + libkrun + umoci) for embedding
into the `openshell-driver-vm` binary. These are rebuilt on demand when the
kernel config or pinned dependency versions change.
@@ -250,7 +261,7 @@ jobs:
### Verify
```bash
- gh release download vm-runtime -R NVIDIA/OpenShell -p vm-runtime-linux-x86_64.tar.zst
+ gh release download ${{ inputs.release-tag || 'vm-runtime' }} -R NVIDIA/OpenShell -p vm-runtime-linux-x86_64.tar.zst
gh attestation verify vm-runtime-linux-x86_64.tar.zst -R NVIDIA/OpenShell
```
diff --git a/Cargo.lock b/Cargo.lock
index 83bea726ef..0322d2e96c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4071,6 +4071,7 @@ dependencies = [
name = "openshell-driver-vm"
version = "0.0.0"
dependencies = [
+ "base64 0.22.1",
"bollard",
"clap",
"flate2",
@@ -4083,6 +4084,7 @@ dependencies = [
"oci-client",
"openshell-core",
"openshell-driver-podman",
+ "openshell-isolation-interface",
"openshell-otel",
"openshell-otel-test-support",
"openshell-policy",
@@ -4092,6 +4094,7 @@ dependencies = [
"polling",
"prost",
"prost-types",
+ "rand 0.9.4",
"rustix 1.1.4",
"serde",
"serde_json",
diff --git a/architecture/sandbox.md b/architecture/sandbox.md
index 22edd7c515..90fe34cdc7 100644
--- a/architecture/sandbox.md
+++ b/architecture/sandbox.md
@@ -320,11 +320,12 @@ file and builds the `Proxy-Authorization: Basic` header; a credential that is
empty, contains control characters, or is not in `user:pass` form is fatal on
both sides.
-The VM driver runs `openshell-supervisor` on the host. Corporate-proxy
-credentials, private CA keys, policy, and gateway credentials never enter the
-guest. The NIC-less guest reaches the host supervisor only through the
-authenticated vsock channel; the host supervisor performs DNS and upstream
-connections.
+The VM driver starts `openshell-supervisor` on the host and
+`openshell-sandbox` as capability-free guest PID 1. Corporate proxy arguments,
+credentials, private CA keys, policy, and gateway credentials stay host-side.
+Both libkrun and QEMU guests are NIC-less; intercepted workload connections
+cross the authenticated vsock channel. A gateway-host proxy is addressed as
+`host.openshell.internal`, which the host supervisor normalizes to `127.0.0.1`.
The Docker driver runs `openshell-supervisor` in a separate companion container.
Its private named volume contains supervisor bootstrap and channel material.
diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs
index e26ea53f7a..39dfe8a871 100644
--- a/crates/openshell-core/src/container_paths.rs
+++ b/crates/openshell-core/src/container_paths.rs
@@ -66,31 +66,6 @@ pub const VM_GUEST_SANDBOX_TOKEN_PATH: &str = "/opt/openshell/auth/sandbox.jwt";
pub const VM_GUEST_INIT_DROPIN_DIR: &str = "/opt/openshell/init.d";
pub const VM_GUEST_INIT_DROPIN_MANIFEST: &str = "/opt/openshell/init.d.manifest";
-/// Guest path for the corporate upstream-proxy credential in VM sandboxes.
-///
-/// The VM driver stages the `user:pass` credential here (mode `0600`,
-/// root-only) inside the per-sandbox overlay upperdir, and passes only this
-/// path on the supervisor's argv. A microVM has no bind mounts or container
-/// secrets, so this is the same delivery the per-sandbox JWT already uses.
-pub const VM_GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = "/opt/openshell/auth/upstream-proxy";
-
-/// Guest path for the corporate proxy CA bundle in VM sandboxes.
-///
-/// A CA certificate is not secret, so unlike the credential this is staged
-/// world-readable. The supervisor trusts it for the handshake with an
-/// `https://` proxy and for server certificates re-signed by a
-/// TLS-intercepting proxy.
-pub const VM_GUEST_PROXY_CA_PATH: &str = "/opt/openshell/tls/proxy-ca.pem";
-
-/// Guest path for the driver-authored supervisor argument list in VM sandboxes.
-///
-/// Podman and Kubernetes build the supervisor's command line directly; the VM
-/// guest init script execs a fixed argv, so driver-owned arguments travel
-/// through this file instead. The driver writes it into the overlay upperdir
-/// on every launch — empty when it has no arguments to pass — so a sandbox
-/// image can neither forge entries nor shadow the driver's copy, and the
-/// guest appends exactly what it finds there and nothing else.
-pub const VM_GUEST_SUPERVISOR_ARGS_PATH: &str = "/opt/openshell/supervisor-args";
pub const VM_UMOCI_PATH: &str = "/opt/openshell/bin/umoci";
pub const VM_SANDBOX_OWNER_NORMALIZED_MARKER: &str = "/opt/openshell/.sandbox-owner-normalized";
@@ -129,9 +104,6 @@ mod tests {
VM_GUEST_SANDBOX_TOKEN_PATH,
VM_GUEST_INIT_DROPIN_DIR,
VM_GUEST_INIT_DROPIN_MANIFEST,
- VM_GUEST_UPSTREAM_PROXY_AUTH_PATH,
- VM_GUEST_PROXY_CA_PATH,
- VM_GUEST_SUPERVISOR_ARGS_PATH,
VM_UMOCI_PATH,
VM_SANDBOX_OWNER_NORMALIZED_MARKER,
];
diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml
index 0d250b6e71..b46d4b9049 100644
--- a/crates/openshell-driver-vm/Cargo.toml
+++ b/crates/openshell-driver-vm/Cargo.toml
@@ -17,45 +17,83 @@ path = "src/lib.rs"
[[bin]]
name = "openshell-driver-vm"
path = "src/main.rs"
+required-features = ["compute-driver"]
[dependencies]
openshell-core = { path = "../openshell-core", default-features = false }
-openshell-otel = { path = "../openshell-otel" }
-openshell-policy = { path = "../openshell-policy" }
-openshell-driver-podman = { path = "../openshell-driver-podman" }
-openshell-vfio = { path = "../openshell-vfio" }
+openshell-isolation-interface = { path = "../openshell-isolation-interface" }
+openshell-otel = { path = "../openshell-otel", optional = true }
+openshell-policy = { path = "../openshell-policy", optional = true }
+openshell-driver-podman = { path = "../openshell-driver-podman", optional = true }
+openshell-vfio = { path = "../openshell-vfio", optional = true }
-bollard = { version = "0.20", features = ["ssh"] }
+base64 = { workspace = true, optional = true }
+bollard = { version = "0.20", features = ["ssh"], optional = true }
tokio = { workspace = true }
-tonic = { workspace = true, features = ["transport"] }
-tower-http = { workspace = true }
-http = { workspace = true }
-prost = { workspace = true }
-prost-types = { workspace = true }
-futures = { workspace = true }
-tokio-stream = { workspace = true, features = ["net"] }
-nix = { workspace = true }
-clap = { workspace = true }
-tracing = { workspace = true }
-tracing-subscriber = { workspace = true }
-opentelemetry = { workspace = true }
-opentelemetry_sdk = { workspace = true }
-tracing-opentelemetry = { workspace = true }
-miette = { workspace = true }
-url = { workspace = true }
+tonic = { workspace = true, features = ["transport"], optional = true }
+tower-http = { workspace = true, optional = true }
+http = { workspace = true, optional = true }
+prost = { workspace = true, optional = true }
+prost-types = { workspace = true, optional = true }
+futures = { workspace = true, optional = true }
+tokio-stream = { workspace = true, features = ["net"], optional = true }
+nix = { workspace = true, optional = true }
+clap = { workspace = true, optional = true }
+tracing = { workspace = true, optional = true }
+tracing-subscriber = { workspace = true, optional = true }
+opentelemetry = { workspace = true, optional = true }
+opentelemetry_sdk = { workspace = true, optional = true }
+tracing-opentelemetry = { workspace = true, optional = true }
+miette = { workspace = true, optional = true }
+rand = { workspace = true, optional = true }
+url = { workspace = true, optional = true }
serde = { workspace = true }
serde_json = { workspace = true }
-oci-client = "0.16"
+oci-client = { version = "0.16", optional = true }
libc = "0.2"
-rustix = { workspace = true }
-libloading = "0.8"
-tar = "0.4"
-flate2 = "1"
-sha2 = "0.10"
-zstd = "0.13"
+rustix = { workspace = true, optional = true }
+libloading = { version = "0.8", optional = true }
+tar = { version = "0.4", optional = true }
+flate2 = { version = "1", optional = true }
+sha2 = { version = "0.10", optional = true }
+zstd = { version = "0.13", optional = true }
[features]
-default = ["telemetry"]
+default = ["compute-driver", "telemetry"]
+## Build the standalone compute driver and its host runtime implementation.
+compute-driver = [
+ "dep:base64",
+ "dep:bollard",
+ "dep:clap",
+ "dep:flate2",
+ "dep:futures",
+ "dep:http",
+ "dep:libloading",
+ "dep:miette",
+ "dep:nix",
+ "dep:oci-client",
+ "dep:openshell-otel",
+ "dep:openshell-policy",
+ "dep:openshell-driver-podman",
+ "dep:openshell-vfio",
+ "dep:opentelemetry",
+ "dep:opentelemetry_sdk",
+ "dep:polling",
+ "dep:prost",
+ "dep:prost-types",
+ "dep:rand",
+ "dep:rustix",
+ "dep:sha2",
+ "dep:tar",
+ "dep:tokio-stream",
+ "dep:tonic",
+ "dep:tower-http",
+ "dep:tracing",
+ "dep:tracing-opentelemetry",
+ "dep:tracing-subscriber",
+ "dep:url",
+ "dep:zstd",
+]
## Compile in telemetry support (forwards to openshell-core/telemetry). On by
## default; build with `--no-default-features` for a telemetry-free VM driver
## that reports telemetry disabled to the sandboxes it launches.
@@ -68,7 +106,7 @@ telemetry = ["openshell-core/telemetry"]
## enabling it alongside `telemetry` is a compile error rather than a silent
## telemetry-on build. Kept in sync with `default` by
## `rust:verify:defaults-without-telemetry`.
-defaults-without-telemetry = []
+defaults-without-telemetry = ["compute-driver"]
[dev-dependencies]
openshell-otel-test-support = { path = "../openshell-otel-test-support" }
@@ -82,7 +120,7 @@ opentelemetry_sdk = { workspace = true, features = ["testing"] }
# nix::sys::prctl::set_pdeathsig there keeps the Linux path a single
# syscall with no helper thread.
[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly"))'.dependencies]
-polling = "3.11"
+polling = { version = "3.11", optional = true }
[lints]
workspace = true
diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md
index 5c61ae1823..30365aaff9 100644
--- a/crates/openshell-driver-vm/README.md
+++ b/crates/openshell-driver-vm/README.md
@@ -1,33 +1,40 @@
# openshell-driver-vm
-> Status: Experimental. The VM compute driver is under active development and the interface still has VM-specific plumbing that will be generalized.
+> Status: Experimental. The VM compute driver is under active development.
-Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy), guest OCI unpacker, and sandbox supervisor are embedded directly in the binary; each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir.
+Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess and talks to it over the `openshell.compute.v1.ComputeDriver` Unix-socket surface. `openshell-supervisor` runs as a native host process, while `openshell-sandbox` runs as capability-free PID 1 inside each microVM and applies guest-local isolation over virtio-vsock.
+
+The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest sandbox, and the custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir.
## How it fits together
```mermaid
flowchart LR
- subgraph host["Host process"]
+ subgraph host["Host"]
gateway["openshell-gateway
(vm::spawn)"]
- driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"]
+ driver["openshell-driver-vm
libkrun"]
+ supervisor["openshell-supervisor
host policy supervisor"]
gateway <-->|"gRPC over UDS
compute-driver.sock"| driver
+ supervisor <-->|"authenticated gRPC
policy + relay"| gateway
end
subgraph guest["Per-sandbox microVM"]
init["/srv/openshell-vm-
sandbox-init.sh"]
- supervisor["/opt/openshell/bin/
openshell-sandbox
(PID 1)"]
- init --> supervisor
+ sandbox["openshell-sandbox
capability-free guest PID 1"]
+ workload["sandbox workload"]
+ init --> sandbox --> workload
end
driver -->|"CreateSandbox
boots via libkrun"| guest
- supervisor -.->|"gRPC callback
--grpc-endpoint"| gateway
+ supervisor <-->|"mutual TLS RFC 0012
over virtio-vsock"| sandbox
- client["openshell-cli"] -->|"SSH proxy
127.0.0.1:<port>"| supervisor
+ client["openshell-cli"] -->|"connect / exec / forward"| gateway
client -->|"CreateSandbox / Watch"| gateway
```
-Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside the VM. gvproxy exposes a single inbound SSH port (`host:` → `guest:2222`) and provides virtio-net egress.
+The supervisor owns gateway credentials, admitted policy, provider resolution, middleware, the network proxy, and relay registration. The sandbox receives no gateway JWT. Each VM generation receives distinct sandbox and supervisor channel keys; the guest consumes and unlinks its private bootstrap files before launching the workload.
+
+VM-specific RFC 0012 code under `src/isolation/` only chooses the vsock transport and binds immutable VM generation and image claims into the protected guest config and host descriptor. Lifecycle, authentication, process control, binary identity, forwarding, and streaming come from `openshell-isolation-interface` and `openshell-sandbox`.
## Quick start (recommended)
@@ -35,7 +42,7 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th
mise run gateway:vm
```
-First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached.
+First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/umoci and `mise run vm:supervisor` builds the portable Linux guest sandbox plus its trusted helper runtime. The development task also builds the native host supervisor. Subsequent runs are cached.
By default `mise run gateway:vm`:
@@ -96,13 +103,13 @@ rm -rf "${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/vm-dev"
If you want to drive the launch yourself instead of using `mise run gateway:vm` (i.e. `tasks/scripts/gateway-vm.sh`):
```shell
-# 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/
+# 1. Stage runtime artifacts + guest sandbox into target/vm-runtime-compressed/
mise run vm:setup
-mise run vm:supervisor # if openshell-sandbox.zst is not already present
+mise run vm:supervisor # builds the Linux guest sandbox and trusted helper runtime
-# 2. Build both binaries with the staged artifacts embedded
+# 2. Build gateway, native host supervisor, and driver
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \
- cargo build -p openshell-gateway -p openshell-driver-vm
+ cargo build -p openshell-gateway -p openshell-supervisor -p openshell-driver-vm
# 3. macOS only: codesign the driver for Hypervisor.framework
codesign \
@@ -121,7 +128,7 @@ disable_tls = true
[openshell.drivers.vm]
default_image = ""
-grpc_endpoint = "http://host.containers.internal:18081"
+grpc_endpoint = "http://127.0.0.1:18081"
driver_dir = "$PWD/target/debug"
state_dir = "/tmp/openshell-vm-driver-$USER-vm-dev"
EOF
@@ -142,8 +149,8 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr
| Configuration key | Default | Purpose |
|---|---|---|
-| `grpc_endpoint` | empty | Required. URL the sandbox guest dials to reach the gateway. Use `http://host.containers.internal:` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. |
-| `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. |
+| `grpc_endpoint` | empty | Required. URL the native host supervisor uses to reach the gateway. Host loopback such as `http://127.0.0.1:` is valid. Legacy guest aliases are normalized to host loopback. This endpoint is never sent into the VM. |
+| `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. Relative paths are resolved to absolute paths at driver startup. |
| `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. |
| `default_image` | OpenShell base image | Sandbox image used when a create request omits one. |
| `bootstrap_image` | unset | VM runtime image used as the immutable bootstrap root disk. Defaults to the sandbox image when unset. |
@@ -151,17 +158,17 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr
| `mem_mib` | `2048` | Memory per sandbox, in MiB. |
| `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. |
| `krun_log_level` | `1` | libkrun verbosity (0-5). |
-| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. |
-| `guest_tls_cert` | unset | Guest client certificate. |
-| `guest_tls_key` | unset | Guest client private key. |
-| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend (GPU sandboxes) has no such NAT and its nftables rules expose only the gateway port to the guest, so a gateway-host proxy URL is rejected at launch there; use an address routable from the guest's masqueraded egress. |
+| `guest_tls_ca` | unset | Historical key name for the host supervisor's gateway CA certificate. Required when `grpc_endpoint` uses `https://`; never copied into the guest. |
+| `guest_tls_cert` | unset | Historical key name for the host supervisor's client certificate; never copied into the guest. |
+| `guest_tls_key` | unset | Historical key name for the host supervisor's client private key; never copied into the guest. |
+| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) that host control chains policy-approved TLS CONNECT egress through. Host-loopback proxy URLs work because control runs on the gateway host. |
| `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. |
| `proxy_auth_file` | unset | Gateway-host path to a `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox. |
| `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. |
| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP targets. |
| `proxy_ca_bundle` | unset | Gateway-host path to a PEM CA bundle trusted for an `https://` proxy and for certificates a TLS-intercepting proxy re-signs. |
-The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial.
+The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and the driver passes them only to native host control. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial.
See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface.
@@ -219,9 +226,17 @@ marked sandboxes without launching compute. Start removes the marker and uses
the normal persisted restore path with the existing overlay. Delete removes the
entire sandbox state directory, including a stop marker and overlay.
-The driver records a terminal tombstone when the canonical main process exits.
-Driver startup reports that sandbox as terminal instead of relaunching the VM,
-even when the process exited successfully.
+The host control writes and syncs a terminal tombstone when the canonical main
+process exits, before it reports completion and while it retains the boundary
+for exec and forwarding. Driver startup reports that sandbox as terminal
+instead of relaunching the VM, even when the process exited successfully.
+
+The driver embeds a platform-native host supervisor and extracts it into
+`/host-runtime`. It accepts a cached binary only when its SHA-256
+content matches the embedded supervisor and it remains an executable regular
+file. Replacement is written and synced under a temporary name, then atomically
+renamed into place. `OPENSHELL_VM_SUPERVISOR_BIN` remains an explicit
+development override.
## Logs and debugging
@@ -233,34 +248,23 @@ RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \
```
The VM guest's serial console is appended to `//console.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions and removes same-owner stale sockets. On clean shutdown, the gateway sends the managed driver `SIGTERM`, waits up to five seconds for it to flush telemetry and exit, then force-kills it if necessary and removes the socket. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development.
+The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development.
-## Host-side nftables rules
-
-The VM driver creates a per-VM nftables table on the host (`openshell_vm_vmtap_`) with three chains. These rules serve two purposes: NAT infrastructure (required for VM connectivity) and defense-in-depth host isolation. Primary security enforcement — proxy-only egress and bypass detection — is handled by the sandbox supervisor's own nftables rules inside the VM guest.
-
-**`postrouting` (NAT):** Masquerades outbound VM traffic so it can be routed from the VM's private subnet to the external network. This chain handles forwarded traffic (VM → internet), not traffic destined for the host.
-
-**`forward` (defense-in-depth):** Accepts all outbound traffic from the VM (security enforcement happens guest-side) and accepts established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces.
-
-**`input` (defense-in-depth):** Accepts traffic from the VM to the gateway port on the host. Drops all other traffic from the VM destined for the host itself. This limits what a compromised guest can reach on the host to the gateway service only.
-
-The `input` and `postrouting` chains handle different traffic paths: `input` covers packets addressed to the host (VM → host), while `postrouting` covers packets the host is forwarding on behalf of the VM (VM → internet). A packet from the VM goes through one path or the other, never both.
-
-All chains use `policy accept`, so non-TAP traffic is unaffected. Because nftables evaluates multiple base chains on the same hook independently, host firewalls interact with these rules as follows:
-
-- **Open host (no other firewall):** Our chains are the only filter. The defense-in-depth drop rules block unsolicited inbound and non-gateway host access. Non-TAP traffic passes through.
-- **Restrictive host firewall (e.g. firewalld):** The host firewall's chains may additionally drop TAP traffic that our chains accept. A `drop` verdict from any chain is final — our `accept` cannot override it. If VM connectivity fails, verify that the host firewall allows forwarding and input for `vmtap-*` interfaces.
+## Network isolation
-Each table is created atomically via `nft -f` on VM start and torn down atomically via `nft delete table` when the VM is destroyed.
+VM sandboxes boot without a virtual NIC. The guest exposes only the protected
+vsock channel used by `openshell-sandbox`; `openshell-supervisor` performs DNS,
+policy evaluation, and external networking on the host. The driver does not
+create TAP devices or install nftables/iptables rules.
## Prerequisites
- macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM
- Rust toolchain
- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection
-- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch):
- - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest)
- - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary.
+- Guest-sandbox cross-compile toolchain (needed on macOS, and on Linux when host arch differs from the guest):
+ - Matching static guest target: `rustup target add aarch64-unknown-linux-musl` (or `x86_64-unknown-linux-musl` for an amd64 guest)
+ - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` cross-compiles the Linux guest `openshell-sandbox` and its matching `openshell-supervisor`.
- [mise](https://mise.jdx.dev/) task runner
- Docker or Podman socket on the local CLI/gateway host when using
`openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI
@@ -291,11 +295,11 @@ The RPM gateway package is configured for the Podman driver.
On Apple Silicon macOS, `install.sh` stages the generated `openshell.rb`
formula from the selected release in the `nvidia/openshell` Homebrew tap.
-Homebrew installs `openshell`, `openshell-gateway`, and
-`openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement
-in `post_install`, and owns the `brew services` gateway lifecycle. The service
-also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless
-the user explicitly overrides it.
+Homebrew installs `openshell`, `openshell-gateway`, and the self-contained
+`openshell-driver-vm` with its embedded native supervisor. It ad-hoc signs the
+driver with the Hypervisor entitlement in `post_install` and owns the `brew
+services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` unset
+so driver choice remains automatic unless the user explicitly overrides it.
## TODOs
diff --git a/crates/openshell-driver-vm/build.rs b/crates/openshell-driver-vm/build.rs
index 92532ed7b2..04ade9944c 100644
--- a/crates/openshell-driver-vm/build.rs
+++ b/crates/openshell-driver-vm/build.rs
@@ -3,13 +3,17 @@
//! Build script for openshell-driver-vm.
//!
-//! This crate embeds the sandbox supervisor plus the minimal libkrun runtime
+//! This crate embeds the sandbox, host supervisor, and minimal libkrun runtime
//! artifacts it needs to boot VMs without a separate VM runtime binary.
use std::path::{Path, PathBuf};
use std::{env, fs};
fn main() {
+ if env::var_os("CARGO_FEATURE_COMPUTE_DRIVER").is_none() {
+ return;
+ }
+
println!("cargo:rerun-if-env-changed=OPENSHELL_VM_RUNTIME_COMPRESSED_DIR");
if let Ok(dir) = env::var("OPENSHELL_VM_RUNTIME_COMPRESSED_DIR") {
@@ -19,8 +23,8 @@ fn main() {
"libkrunfw.so.5.zst",
"libkrun.dylib.zst",
"libkrunfw.5.dylib.zst",
- "gvproxy.zst",
"openshell-sandbox.zst",
+ "openshell-supervisor.zst",
"umoci.zst",
] {
println!("cargo:rerun-if-changed={dir}/{name}");
@@ -38,7 +42,14 @@ fn main() {
println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}");
generate_stub_resources(
&out_dir,
- &["libkrun", "libkrunfw", "openshell-sandbox.zst", "umoci.zst"],
+ &[
+ "libkrun",
+ "libkrunfw",
+ "openshell-sandbox.zst",
+ "openshell-supervisor.zst",
+ "openshell-runtime.tar.zst",
+ "umoci.zst",
+ ],
);
return;
}
@@ -54,8 +65,9 @@ fn main() {
&[
&format!("{libkrun_name}.zst"),
&format!("{libkrunfw_name}.zst"),
- "gvproxy.zst",
"openshell-sandbox.zst",
+ "openshell-supervisor.zst",
+ "openshell-runtime.tar.zst",
"umoci.zst",
],
);
@@ -74,11 +86,18 @@ fn main() {
format!("{libkrunfw_name}.zst"),
format!("{libkrunfw_name}.zst"),
),
- ("gvproxy.zst".to_string(), "gvproxy.zst".to_string()),
(
"openshell-sandbox.zst".to_string(),
"openshell-sandbox.zst".to_string(),
),
+ (
+ "openshell-supervisor.zst".to_string(),
+ "openshell-supervisor.zst".to_string(),
+ ),
+ (
+ "openshell-runtime.tar.zst".to_string(),
+ "openshell-runtime.tar.zst".to_string(),
+ ),
("umoci.zst".to_string(), "umoci.zst".to_string()),
];
diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md
index b686874ba2..5e299aa7e1 100644
--- a/crates/openshell-driver-vm/runtime/README.md
+++ b/crates/openshell-driver-vm/runtime/README.md
@@ -11,15 +11,18 @@ runtime/
openshell.kconfig
```
-`openshell-driver-vm` embeds libkrun, libkrunfw, gvproxy, umoci for guest-side
-OCI image unpacking, and the bundled `openshell-sandbox` supervisor.
+`openshell-driver-vm` embeds libkrun, libkrunfw, umoci for guest-side OCI image
+unpacking, and the portable capability-free `openshell-sandbox` role.
+VMs do not attach a guest NIC. The boundary carries control, mediated network,
+and DNS streams over the authenticated vsock channel.
## Why
-The stock `libkrunfw` kernel does not include the bridge, netfilter,
-conntrack, cgroup, seccomp, and Landlock features the sandbox supervisor needs
-inside each microVM. `kernel/openshell.kconfig` extends the libkrunfw kernel so
-VM sandboxes can run the same supervisor enforcement path as other backends.
+The stock `libkrunfw` kernel does not include every cgroup, seccomp, and
+Landlock feature the sandbox needs inside each microVM.
+`kernel/openshell.kconfig` extends the libkrunfw kernel so VM sandboxes retain
+guest-local process, network-syscall, and filesystem enforcement while the
+supervisor runs on the host.
## Build Scripts
@@ -27,7 +30,7 @@ VM sandboxes can run the same supervisor enforcement path as other backends.
|---|---|---|
| `tasks/scripts/vm/build-libkrun.sh` | Linux | Builds libkrunfw and libkrun from source with the custom kernel config |
| `tasks/scripts/vm/build-libkrun-macos.sh` | macOS | Builds portable libkrunfw and libkrun from a prebuilt `kernel.c` |
-| `tasks/scripts/vm/package-vm-runtime.sh` | Any | Packages `vm-runtime-.tar.zst` with libraries, gvproxy, umoci, and provenance |
+| `tasks/scripts/vm/package-vm-runtime.sh` | Any | Packages `vm-runtime-.tar.zst` with libraries, umoci, and provenance |
| `tasks/scripts/vm/download-kernel-runtime.sh` | Any | Downloads runtime tarballs from the `vm-runtime` release and stages compressed files |
## Local Flow
@@ -36,12 +39,12 @@ VM sandboxes can run the same supervisor enforcement path as other backends.
# Download the current pre-built runtime and stage compressed artifacts
mise run vm:setup
-# Build the bundled guest supervisor
+# Build the portable Linux guest sandbox and trusted helper runtime (requires Docker Buildx)
mise run vm:supervisor
-# Build the gateway and VM driver with embedded runtime artifacts
+# Build the gateway, native host supervisor, and VM driver
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \
- cargo build -p openshell-gateway -p openshell-driver-vm
+ cargo build -p openshell-gateway -p openshell-supervisor -p openshell-driver-vm
```
Use `FROM_SOURCE=1 mise run vm:setup` to build the runtime from source instead
@@ -62,7 +65,7 @@ publish the driver binary next to `openshell-gateway`.
## Provenance
`package-vm-runtime.sh` writes `provenance.json` into each runtime tarball with
-the platform, libkrunfw commit, kernel version, gvproxy and umoci versions,
+the platform, libkrunfw commit, kernel version, and umoci version,
GitHub SHA, and build time. The driver logs this metadata when it extracts and
loads a runtime bundle.
diff --git a/crates/openshell-driver-vm/runtime/kernel/openshell.kconfig b/crates/openshell-driver-vm/runtime/kernel/openshell.kconfig
index e8d826c53e..4249e71121 100644
--- a/crates/openshell-driver-vm/runtime/kernel/openshell.kconfig
+++ b/crates/openshell-driver-vm/runtime/kernel/openshell.kconfig
@@ -1,136 +1,37 @@
# Custom kernel config fragment for libkrunfw (OpenShell VM)
#
-# This fragment is applied on top of libkrunfw's base kernel config
-# to enable bridge CNI, netfilter/iptables, and conntrack support
-# required for Kubernetes pod networking in the VM.
+# This fragment is applied on top of libkrunfw's base kernel config. VM
+# sandboxes have no virtual NIC; the base loopback, TCP/UDP, Unix-socket,
+# route-netlink, and vsock support is enough for the capability-free sandbox.
#
# Apply with: scripts/merge-kconfig.sh
#
# See also: check-vm-capabilities.sh for runtime verification.
-# ── Root disk transport and filesystem ─────────────────────────────────
+# Root disk transport and filesystem.
CONFIG_BLOCK=y
CONFIG_BLK_DEV=y
CONFIG_VIRTIO_BLK=y
CONFIG_EXT4_FS=y
CONFIG_EXT4_USE_FOR_EXT2=y
-# ── Network Namespaces (required for pod isolation) ─────────────────────
-CONFIG_NET_NS=y
-CONFIG_NAMESPACES=y
-
-# ── Virtual Ethernet (veth pairs for pod networking) ────────────────────
-CONFIG_VETH=y
-
-# ── Linux Bridge (required for bridge CNI plugin) ──────────────────────
-CONFIG_BRIDGE=y
-CONFIG_BRIDGE_NETFILTER=y
-CONFIG_BRIDGE_IGMP_SNOOPING=y
-
-# ── Netfilter framework ────────────────────────────────────────────────
-CONFIG_NETFILTER=y
-CONFIG_NETFILTER_ADVANCED=y
-CONFIG_NETFILTER_INGRESS=y
-CONFIG_NETFILTER_NETLINK=y
-CONFIG_NETFILTER_NETLINK_QUEUE=y
-CONFIG_NETFILTER_NETLINK_LOG=y
-
-# ── Connection tracking (required for NAT and kube-proxy) ──────────────
-CONFIG_NF_CONNTRACK=y
-CONFIG_NF_CT_NETLINK=y
-CONFIG_NF_CONNTRACK_EVENTS=y
-CONFIG_NF_CONNTRACK_TIMEOUT=y
-CONFIG_NF_CONNTRACK_TIMESTAMP=y
-
-# ── Netfilter xtables match modules (required by kube-proxy & kubelet) ─
-# kube-proxy uses xt_conntrack for stateful rules and xt_comment for
-# labeling chains. Without these, iptables fails with:
-# "Couldn't load match 'conntrack': No such file or directory"
-CONFIG_NETFILTER_XTABLES=y
-CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
-CONFIG_NETFILTER_XT_MATCH_COMMENT=y
-CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
-CONFIG_NETFILTER_XT_MATCH_MARK=y
-CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
-CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y
-CONFIG_NETFILTER_XT_MATCH_RECENT=y
-CONFIG_NETFILTER_XT_MATCH_LIMIT=y
-CONFIG_NETFILTER_XT_TARGET_LOG=y
-CONFIG_NETFILTER_XT_TARGET_MARK=y
-CONFIG_NETFILTER_XT_TARGET_CONNMARK=y
-CONFIG_NETFILTER_XT_MATCH_CONNMARK=y
-
-# ── NAT (required for service VIP / DNAT / SNAT) ──────────────────────
-CONFIG_NF_NAT=y
-CONFIG_NF_NAT_MASQUERADE_IPV4=y
-
-# ── iptables (CNI bridge masquerade + compat) ──────────────────────────
-CONFIG_IP_NF_IPTABLES=y
-CONFIG_IP_NF_FILTER=y
-CONFIG_IP_NF_NAT=y
-CONFIG_IP_NF_MANGLE=y
-CONFIG_IP_NF_TARGET_MASQUERADE=y
-CONFIG_IP_NF_TARGET_REJECT=y
-
-# ── nftables (kube-proxy nftables mode — primary proxy backend) ─────────
-# kube-proxy nftables proxier requires: numgen (random LB), fib (local
-# address detection), counter, ct, nat, masq, reject, limit, redir.
-CONFIG_NF_TABLES=y
-CONFIG_NF_TABLES_INET=y
-CONFIG_NFT_CT=y
-CONFIG_NFT_NAT=y
-CONFIG_NFT_MASQ=y
-CONFIG_NFT_REJECT=y
-CONFIG_NFT_COMPAT=y
-CONFIG_NFT_NUMGEN=y
-CONFIG_NFT_FIB_IPV4=y
-CONFIG_NFT_FIB_IPV6=y
-CONFIG_NFT_LIMIT=y
-CONFIG_NFT_LOG=y
-CONFIG_NFT_REDIR=y
-CONFIG_NFT_TPROXY=y
-
-# ── IP forwarding and routing (required for pod-to-pod) ────────────────
-CONFIG_IP_ADVANCED_ROUTER=y
-CONFIG_IP_MULTIPLE_TABLES=y
-CONFIG_IP_ROUTE_MULTIPATH=y
-CONFIG_NET_IP_TUNNEL=y
-
-# ── IPVS (optional: kube-proxy IPVS mode) ─────────────────────────────
-CONFIG_IP_VS=y
-CONFIG_IP_VS_PROTO_TCP=y
-CONFIG_IP_VS_PROTO_UDP=y
-CONFIG_IP_VS_RR=y
-CONFIG_IP_VS_WRR=y
-CONFIG_IP_VS_SH=y
-CONFIG_IP_VS_NFCT=y
-
-# ── Misc networking required by Kubernetes ─────────────────────────────
-CONFIG_NET_SCH_HTB=y
-CONFIG_NET_CLS_CGROUP=y
-CONFIG_CGROUP_NET_PRIO=y
-CONFIG_CGROUP_NET_CLASSID=y
-
-# ── Dummy interface (fallback networking) ──────────────────────────────
-CONFIG_DUMMY=y
-
-# ── TUN/TAP (used by some CNI plugins) ────────────────────────────────
-CONFIG_TUN=y
-
-# ── Cgroups (already in base, ensure v2 is available) ──────────────────
+# Cgroups used for process supervision and resource limits.
CONFIG_CGROUPS=y
CONFIG_CGROUP_DEVICE=y
CONFIG_CGROUP_CPUACCT=y
CONFIG_CGROUP_PIDS=y
CONFIG_MEMCG=y
-# ── Disable kernel headers archive (avoids cpio issues in CI) ──────────
+# Disable the kernel headers archive to avoid cpio issues in CI.
# CONFIG_IKHEADERS is not set
-# ── POSIX message queues (required by runc to mount /dev/mqueue in containers) ─
+# POSIX message queues used by OCI workloads.
CONFIG_POSIX_MQUEUE=y
CONFIG_POSIX_MQUEUE_SYSCTL=y
-# ── Security features required by the sandbox runtime ───────────────────
+# Capability-free sandbox enforcement.
+CONFIG_SECURITY=y
CONFIG_SECURITY_LANDLOCK=y
+CONFIG_LSM="landlock,lockdown,yama,loadpin,safesetid,ipe,bpf"
+CONFIG_SECCOMP=y
CONFIG_SECCOMP_FILTER=y
diff --git a/crates/openshell-driver-vm/runtime/pins.env b/crates/openshell-driver-vm/runtime/pins.env
index 34a9f0bf33..9977eddcc1 100644
--- a/crates/openshell-driver-vm/runtime/pins.env
+++ b/crates/openshell-driver-vm/runtime/pins.env
@@ -29,10 +29,6 @@ COMMUNITY_SANDBOX_IMAGE="${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-com
# during local development. Override via IMAGE_REPO_BASE and IMAGE_TAG
# environment variables (defaults: openshell/gateway:dev).
-# ── gvproxy (networking proxy) ──────────────────────────────────────────
-# Repo: https://github.com/containers/gvisor-tap-vsock
-GVPROXY_VERSION="${GVPROXY_VERSION:-v0.8.9}"
-
# ── umoci (guest OCI unpacker) ──────────────────────────────────────────
# Repo: https://github.com/opencontainers/umoci
UMOCI_VERSION="${UMOCI_VERSION:-v0.6.0}"
diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh
index 32d6ed1dff..980ee81368 100644
--- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh
+++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh
@@ -3,9 +3,9 @@
# SPDX-License-Identifier: Apache-2.0
# Minimal init for sandbox VMs. Runs as PID 1 inside the guest, mounts the
-# essential filesystems, configures networking (gvproxy DHCP or TAP static),
-# optionally loads NVIDIA GPU drivers, then execs the OpenShell sandbox
-# supervisor.
+# essential filesystems, optionally loads NVIDIA GPU drivers, then execs the
+# portable VM sandbox. Workload networking crosses the authenticated
+# boundary channel; the VM does not receive a network interface.
set -euo pipefail
@@ -14,27 +14,9 @@ set -euo pipefail
unset KRUN_INIT_PID1
BOOT_START=$(date +%s%3N 2>/dev/null || date +%s)
-# gvisor-tap-vsock subnet layout:
-# 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT
-# proxy arbitrary host ports.
-# 192.168.127.254 — host-loopback: NAT-rewritten to host's 127.0.0.1 by
-# gvproxy's TCP/UDP/ICMP forwarder. Use this address
-# (or any of the host.* hostnames below) to reach a
-# service the host is listening on.
-# The host.openshell.internal / host.containers.internal /
-# host.docker.internal DNS records served by gvproxy's embedded resolver
-# point at 192.168.127.254. We mirror that in /etc/hosts so the supervisor
-# can reach the gateway even when gvproxy's DNS is not in resolv.conf
-# (e.g. DHCP failed and we fell back to 8.8.8.8).
-GVPROXY_GATEWAY_IP="192.168.127.1"
-GVPROXY_HOST_LOOPBACK_IP="192.168.127.254"
-GATEWAY_IP="$GVPROXY_GATEWAY_IP"
SANDBOX_OWNER_NORMALIZED_MARKER="/opt/openshell/.sandbox-owner-normalized"
GPU_ENABLED="${GPU_ENABLED:-false}"
-VM_NET_IP="${VM_NET_IP:-}"
-VM_NET_GW="${VM_NET_GW:-}"
-VM_NET_DNS="${VM_NET_DNS:-}"
ts() {
local now
@@ -117,6 +99,10 @@ ensure_target_runtime() {
cp /opt/openshell/bin/openshell-sandbox "$image_root/opt/openshell/bin/openshell-sandbox"
chmod 0755 "$image_root/opt/openshell/bin/openshell-sandbox"
fi
+ if [ -d /opt/openshell/bin/openshell-runtime ]; then
+ rm -rf "$image_root/opt/openshell/bin/openshell-runtime"
+ cp -a /opt/openshell/bin/openshell-runtime "$image_root/opt/openshell/bin/openshell-runtime"
+ fi
touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow"
if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then
@@ -192,67 +178,6 @@ prepare_guest_image_rootfs() {
rm -rf "$payload_dir"
}
-# Driver-owned arguments appended to the supervisor's command line.
-#
-# The VM driver cannot build the supervisor's argv the way the container
-# drivers do, so it writes the arguments it chose into the overlay upperdir
-# and this script appends them verbatim. Populated by
-# read_supervisor_extra_args; empty until then.
-SUPERVISOR_EXTRA_ARGS=()
-
-# Upper bound on driver-supplied supervisor arguments.
-#
-# The corporate proxy settings are the only producer today and top out at ten
-# entries. The cap exists so a corrupt or oversized file cannot expand into an
-# unbounded command line.
-SUPERVISOR_EXTRA_ARGS_MAX=32
-
-read_supervisor_extra_args() {
- # Read the driver-authored supervisor argument list, one argument per
- # line, verbatim -- no word splitting, globbing, or expansion, so values
- # containing spaces (e.g. a NO_PROXY list) survive intact.
- #
- # Security: this is the operator-owned egress boundary. The driver writes
- # this file into the overlay upperdir on every launch, including an empty
- # file when it has no arguments to pass, so the upperdir copy always
- # shadows the read-only image layer. A sandbox image can therefore neither
- # supply its own supervisor arguments by baking a file at this path nor
- # disable the operator's by omitting one. A missing file means the driver
- # passed nothing; a file it cannot read means the overlay is broken, and
- # we fail closed rather than start a supervisor with a silently truncated
- # egress configuration.
- local args_file
- args_file="$(root_path /opt/openshell/supervisor-args)"
-
- SUPERVISOR_EXTRA_ARGS=()
- if [ ! -f "$args_file" ]; then
- return 0
- fi
- if [ ! -r "$args_file" ]; then
- ts "FATAL: supervisor argument list ${args_file} is not readable"
- exit 1
- fi
-
- local arg
- while IFS= read -r arg; do
- # render_guest_supervisor_args never emits a blank line, so one means
- # the file was truncated or tampered with after the driver wrote it.
- if [ -z "$arg" ]; then
- ts "FATAL: empty entry in supervisor argument list"
- exit 1
- fi
- if [ "${#SUPERVISOR_EXTRA_ARGS[@]}" -ge "$SUPERVISOR_EXTRA_ARGS_MAX" ]; then
- ts "FATAL: supervisor argument list exceeds ${SUPERVISOR_EXTRA_ARGS_MAX} entries"
- exit 1
- fi
- SUPERVISOR_EXTRA_ARGS+=("$arg")
- done < "$args_file"
-
- if [ "${#SUPERVISOR_EXTRA_ARGS[@]}" -gt 0 ]; then
- ts "supervisor arguments from driver: ${#SUPERVISOR_EXTRA_ARGS[@]} entries"
- fi
-}
-
exec_supervisor_in_newroot() {
local chroot_bin
local bootstrap="/.openshell-bootstrap"
@@ -275,16 +200,14 @@ exec_supervisor_in_newroot() {
"${bootstrap}/lib64/ld-linux-aarch64.so.1"; do
if [ -x "/newroot${loader}" ]; then
lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu"
- exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" \
- "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}"
+ exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" "$@"
fi
done
- exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}"
+ exec "$chroot_bin" /newroot "$supervisor" "$@"
fi
if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then
- exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \
- --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}"
+ exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox "$@"
fi
done
@@ -355,170 +278,6 @@ setup_overlay_root() {
run_post_overlay_setup
}
-parse_endpoint() {
- local endpoint="$1"
- local scheme rest authority path host port
-
- case "$endpoint" in
- *://*)
- scheme="${endpoint%%://*}"
- rest="${endpoint#*://}"
- ;;
- *)
- return 1
- ;;
- esac
-
- authority="${rest%%/*}"
- path="${rest#"$authority"}"
- if [ "$path" = "$rest" ]; then
- path=""
- fi
-
- if [[ "$authority" =~ ^\[([^]]+)\]:(.+)$ ]]; then
- host="${BASH_REMATCH[1]}"
- port="${BASH_REMATCH[2]}"
- elif [[ "$authority" =~ ^\[([^]]+)\]$ ]]; then
- host="${BASH_REMATCH[1]}"
- port=""
- elif [[ "$authority" == *:* ]]; then
- host="${authority%%:*}"
- port="${authority##*:}"
- else
- host="$authority"
- port=""
- fi
-
- if [ -z "$port" ]; then
- case "$scheme" in
- https) port="443" ;;
- *) port="80" ;;
- esac
- fi
-
- printf '%s\n%s\n%s\n%s\n' "$scheme" "$host" "$port" "$path"
-}
-
-tcp_probe() {
- local host="$1"
- local port="$2"
-
- if command -v timeout >/dev/null 2>&1; then
- timeout 2 bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1
- else
- bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1
- fi
-}
-
-ensure_host_gateway_aliases() {
- # Seed /etc/hosts with the well-known gvproxy hostnames so the supervisor
- # can reach the OpenShell server even when gvproxy's built-in DNS is not
- # in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8).
- #
- # Critical distinction: host.* aliases point at the gvproxy *host-loopback*
- # IP (192.168.127.254), not the gateway IP (192.168.127.1). Only the
- # host-loopback IP carries NAT rewriting to the host's 127.0.0.1 — the
- # gateway IP only listens on gvproxy's own service ports (DNS:53, DHCP,
- # HTTP API:80). Pinning host.containers.internal to the gateway IP
- # silently breaks guest→host port reachability for arbitrary ports.
- local host_aliases="host.openshell.internal host.containers.internal host.docker.internal"
- local gateway_aliases="gateway.containers.internal"
- local filter='(^|[[:space:]])(host\.openshell\.internal|host\.containers\.internal|host\.docker\.internal|gateway\.containers\.internal)([[:space:]]|$)'
-
- write_host_gateway_aliases "$(root_path /etc/hosts)" "$(root_path "/tmp/openshell-hosts.$$.tmp")" || true
- if [ -n "${ROOT_PREFIX:-}" ]; then
- write_host_gateway_aliases "/etc/hosts" "/tmp/openshell-hosts.$$.tmp" || true
- fi
-}
-
-write_host_gateway_aliases() {
- local hosts_path="$1"
- local hosts_tmp="$2"
- mkdir -p "$(dirname "$hosts_path")" 2>/dev/null || true
- mkdir -p "$(dirname "$hosts_tmp")" 2>/dev/null || true
- if [ -f "$hosts_path" ]; then
- grep -vE "$filter" "$hosts_path" > "$hosts_tmp" || true
- else
- : > "$hosts_tmp"
- fi
-
- # In TAP/GPU mode, GATEWAY_IP is overridden to VM_NET_GW (the host-side
- # of the TAP), and the gateway is reachable directly there. In gvproxy
- # mode, host.openshell.internal etc. need GVPROXY_HOST_LOOPBACK_IP
- # (192.168.127.254) which is gvproxy's host-NAT entry, while
- # gateway.containers.internal points at the gvproxy gateway itself.
- if [ "${GATEWAY_IP}" = "${GVPROXY_GATEWAY_IP}" ]; then
- printf '%s %s\n' "$GVPROXY_HOST_LOOPBACK_IP" "$host_aliases" >> "$hosts_tmp"
- printf '%s %s\n' "$GVPROXY_GATEWAY_IP" "$gateway_aliases" >> "$hosts_tmp"
- else
- # TAP networking: gateway and host are both reachable at GATEWAY_IP.
- printf '%s %s %s\n' "$GATEWAY_IP" "$host_aliases" "$gateway_aliases" >> "$hosts_tmp"
- fi
- if ! cat "$hosts_tmp" > "$hosts_path" 2>/dev/null; then
- rm -f "$hosts_tmp"
- ts "WARNING: could not update ${hosts_path}"
- return 1
- fi
- rm -f "$hosts_tmp"
-}
-
-rewrite_openshell_endpoint_if_needed() {
- local endpoint="${OPENSHELL_ENDPOINT:-}"
- [ -n "$endpoint" ] || return 0
-
- local parsed
- if ! parsed="$(parse_endpoint "$endpoint")"; then
- ts "WARNING: could not parse OPENSHELL_ENDPOINT=$endpoint"
- return 0
- fi
-
- local scheme host port path
- scheme="$(printf '%s\n' "$parsed" | sed -n '1p')"
- host="$(printf '%s\n' "$parsed" | sed -n '2p')"
- port="$(printf '%s\n' "$parsed" | sed -n '3p')"
- path="$(printf '%s\n' "$parsed" | sed -n '4p')"
-
- if tcp_probe "$host" "$port"; then
- return 0
- fi
-
- # Probe candidates in preference order. Hostnames first for informative
- # log output, then a bare IP as a final safety net. In gvproxy mode the
- # bare IP is the host-loopback (192.168.127.254). In TAP/GPU mode it's
- # the TAP host gateway.
- local fallback_ip="$GVPROXY_HOST_LOOPBACK_IP"
- if [ "${GATEWAY_IP}" != "${GVPROXY_GATEWAY_IP}" ]; then
- fallback_ip="$GATEWAY_IP"
- fi
- local candidates="host.openshell.internal host.containers.internal host.docker.internal"
- if [ "$scheme" != "https" ]; then
- candidates="${candidates} ${fallback_ip}"
- fi
-
- for candidate in $candidates; do
- if [ "$candidate" = "$host" ]; then
- continue
- fi
- if tcp_probe "$candidate" "$port"; then
- local authority="$candidate"
- if ! { [ "$scheme" = "http" ] && [ "$port" = "80" ]; } \
- && ! { [ "$scheme" = "https" ] && [ "$port" = "443" ]; }; then
- authority="${authority}:${port}"
- fi
- export OPENSHELL_ENDPOINT="${scheme}://${authority}${path}"
- ts "rewrote OPENSHELL_ENDPOINT to ${OPENSHELL_ENDPOINT}"
- return 0
- fi
- done
-
- if [ "$scheme" = "https" ]; then
- ts "WARNING: could not preflight HTTPS OpenShell endpoint ${host}:${port}; preserving hostname for TLS verification"
- return 0
- fi
-
- ts "WARNING: could not reach OpenShell endpoint ${host}:${port}"
-}
-
create_gpu_device_nodes_mknod() {
# Mode 666 is intentional: single-tenant microVM with the VM itself as the
# isolation boundary. The sandbox user is the only non-root user.
@@ -740,116 +499,27 @@ run_post_overlay_setup() {
mount -t cgroup2 cgroup2 "$(root_path /sys/fs/cgroup)" 2>/dev/null &
wait
- # Allow nftables LOG rules to work in non-init network namespaces.
- # Without this, the kernel's nf_log_syslog silently suppresses output
- # from the sandbox's network namespace.
- if [ -f /proc/sys/net/netfilter/nf_log_all_netns ]; then
- echo 1 > /proc/sys/net/netfilter/nf_log_all_netns 2>/dev/null || true
- fi
-
setup_sandbox_workdir
configure_hostname
- ip link set lo up 2>/dev/null || true
-
-# Networking: use TAP static config if VM_NET_IP is set (QEMU path),
-# otherwise fall back to gvproxy DHCP on eth0 (libkrun path).
-if [ -n "${VM_NET_IP}" ] && [ -n "${VM_NET_GW}" ]; then
- ts "configuring TAP networking (static ${VM_NET_IP} gw ${VM_NET_GW})"
- GATEWAY_IP="${VM_NET_GW}"
-
- TAP_NIC=""
- NIC_WAIT=0
- while [ -z "$TAP_NIC" ] && [ "$NIC_WAIT" -lt 10 ]; do
- for candidate in eth0 ens3 enp0s2; do
- if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then
- TAP_NIC="$candidate"
- break
- fi
- done
- if [ -z "$TAP_NIC" ]; then
- for sys_nic in /sys/class/net/*; do
- [ -e "$sys_nic" ] || continue
- candidate="${sys_nic##*/}"
- if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then
- TAP_NIC="$candidate"
- break
- fi
- done
- fi
- if [ -z "$TAP_NIC" ]; then
- sleep 1
- NIC_WAIT=$((NIC_WAIT + 1))
- fi
- done
-
- if [ -n "$TAP_NIC" ]; then
- ts "using NIC ${TAP_NIC} for TAP networking"
- ip link set "$TAP_NIC" up 2>/dev/null || true
- ip addr add "${VM_NET_IP}/30" dev "$TAP_NIC" 2>/dev/null || true
- ip route add default via "${VM_NET_GW}" 2>/dev/null || true
- else
- ts "WARNING: no network interface found for TAP networking"
- fi
-
- if [ -n "${VM_NET_DNS}" ]; then
- echo "nameserver ${VM_NET_DNS}" > "$(root_path /etc/resolv.conf)"
- elif [ ! -s "$(root_path /etc/resolv.conf)" ]; then
- echo "nameserver 8.8.8.8" > "$(root_path /etc/resolv.conf)"
- echo "nameserver 8.8.4.4" >> "$(root_path /etc/resolv.conf)"
- fi
-
- ensure_host_gateway_aliases
-elif ip link show eth0 >/dev/null 2>&1; then
- ts "detected eth0 (gvproxy networking)"
- ip link set eth0 up 2>/dev/null || true
-
- if command -v udhcpc >/dev/null 2>&1; then
- UDHCPC_SCRIPT="$(root_path /run/openshell-udhcpc.script)"
- mkdir -p "$(dirname "$UDHCPC_SCRIPT")"
- cat > "$UDHCPC_SCRIPT" <<'DHCP_SCRIPT'
-#!/bin/sh
-case "$1" in
- bound|renew)
- ip addr flush dev "$interface"
- ip addr add "$ip/$mask" dev "$interface"
- if [ -n "$router" ]; then
- ip route add default via "$router" dev "$interface"
- fi
- if [ -n "$dns" ]; then
- resolv_conf="${OPENSHELL_RESOLV_CONF:-/etc/resolv.conf}"
- mkdir -p "$(dirname "$resolv_conf")" 2>/dev/null || true
- : > "$resolv_conf" 2>/dev/null || true
- for d in $dns; do
- echo "nameserver $d" >> "$resolv_conf" 2>/dev/null || true
- done
- fi
- ;;
-esac
-DHCP_SCRIPT
- chmod +x "$UDHCPC_SCRIPT"
-
- if ! OPENSHELL_RESOLV_CONF="$(root_path /etc/resolv.conf)" \
- udhcpc -i eth0 -f -q -n -T 1 -t 3 -A 1 -s "$UDHCPC_SCRIPT" 2>&1; then
- ts "WARNING: DHCP failed, falling back to static config"
- ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true
- ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true
- fi
- else
- ts "no DHCP client, using static config"
- ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true
- ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true
+ if ! ip link set lo up; then
+ ts "FATAL: failed to bring up the loopback interface"
+ exit 1
fi
- if [ ! -s "$(root_path /etc/resolv.conf)" ]; then
- echo "nameserver 8.8.8.8" > "$(root_path /etc/resolv.conf)"
- echo "nameserver 8.8.4.4" >> "$(root_path /etc/resolv.conf)"
+ # The capability-free sandbox owns a loopback-only DNS relay. Guest init
+ # grants the low port before handing control to the zero-capability UID.
+ if ! echo 0 > /proc/sys/net/ipv4/ip_unprivileged_port_start; then
+ ts "FATAL: failed to permit the unprivileged DNS relay to bind port 53"
+ exit 1
fi
+ cat >"$(root_path /etc/resolv.conf)" <<'EOF'
+nameserver 127.0.0.53
+options timeout:2 attempts:2
+EOF
- ensure_host_gateway_aliases
-else
- ts "WARNING: no network interface found; supervisor will start without guest egress"
-fi
+# The boundary transport mediates network and DNS requests. Only loopback is
+# configured in the guest; no public resolver or guest NIC is needed.
export HOME=/sandbox
export USER=sandbox
@@ -876,33 +546,42 @@ fi
run_openshell_init_dropins
-rewrite_openshell_endpoint_if_needed
-
-# Log supervisor connectivity state for debugging stuck-in-Provisioning issues
-if [ -n "${OPENSHELL_ENDPOINT:-}" ]; then
- _ep_parsed="$(parse_endpoint "$OPENSHELL_ENDPOINT" 2>/dev/null || true)"
- if [ -n "$_ep_parsed" ]; then
- _ep_host="$(printf '%s\n' "$_ep_parsed" | sed -n '2p')"
- _ep_port="$(printf '%s\n' "$_ep_parsed" | sed -n '3p')"
- if tcp_probe "$_ep_host" "$_ep_port"; then
- ts "gateway reachable at ${_ep_host}:${_ep_port}"
- else
- ts "WARNING: gateway NOT reachable at ${_ep_host}:${_ep_port} — supervisor may fail to connect"
- fi
- fi
- ts "OPENSHELL_ENDPOINT=${OPENSHELL_ENDPOINT}"
-fi
if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then
ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}"
fi
-read_supervisor_extra_args
-
-ts "starting openshell-sandbox supervisor"
+ts "starting OpenShell VM sandbox"
+_sandbox_owner="$(sandbox_owner)"
+_sandbox_uid="${_sandbox_owner%%:*}"
+_sandbox_gid="${_sandbox_owner##*:}"
+_sandbox_bootstrap_guest="${OPENSHELL_VM_SANDBOX_BOOTSTRAP:-/.openshell/state/bootstrap.json}"
+_sandbox_bootstrap="$(root_path "$_sandbox_bootstrap_guest")"
+_sandbox_state_dir="${_sandbox_bootstrap%/*}"
+if [ ! -f "$_sandbox_bootstrap" ]; then
+ ts "FATAL: capability-free sandbox bootstrap is missing"
+ exit 1
+fi
+chown "${_sandbox_uid}:${_sandbox_gid}" "$_sandbox_state_dir"
+chmod 0700 "$_sandbox_state_dir"
+for _sandbox_private_file in "$_sandbox_state_dir"/*; do
+ [ -f "$_sandbox_private_file" ] || continue
+ chown "${_sandbox_uid}:${_sandbox_gid}" "$_sandbox_private_file"
+ chmod 0600 "$_sandbox_private_file"
+done
+if [ "${OPENSHELL_VM_INIT_MODE:-sandbox}" = "capability-probe" ]; then
+ ts "starting capability-free VM qualification as ${_sandbox_uid}:${_sandbox_gid}"
+ if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then
+ exec_supervisor_in_newroot capability-probe-launch "$_sandbox_uid" "$_sandbox_gid"
+ fi
+ exec /opt/openshell/bin/openshell-sandbox \
+ capability-probe-launch "$_sandbox_uid" "$_sandbox_gid"
+fi
if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then
- exec_supervisor_in_newroot
+ exec_supervisor_in_newroot \
+ launch-capability-free "$_sandbox_uid" "$_sandbox_gid" "$_sandbox_bootstrap_guest"
fi
-exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}"
+exec /opt/openshell/bin/openshell-sandbox \
+ launch-capability-free "$_sandbox_uid" "$_sandbox_gid" "$_sandbox_bootstrap_guest"
}
if [ "${1:-}" != "--post-overlay" ]; then
diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs
index 8e051afffd..8eda9158a8 100644
--- a/crates/openshell-driver-vm/src/driver.rs
+++ b/crates/openshell-driver-vm/src/driver.rs
@@ -1,17 +1,19 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-use crate::gpu::{
- GpuInventory, SubnetAllocator, allocate_vsock_cid, mac_from_sandbox_id, tap_device_name,
-};
+#![allow(unsafe_code)]
+
+use crate::gpu::{GpuInventory, allocate_vsock_cid};
+
use crate::lifecycle::{
BackendFeature, GuestInitDropin, LaunchAbortReason, LaunchPlan, LifecycleExtensionRegistry,
RestoreContext, extension_state_dir,
};
use crate::rootfs::{
clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir,
- extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path,
- set_rootfs_image_file_mode, write_rootfs_image_file,
+ extract_host_supervisor, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root,
+ recover_rootfs_image, sandbox_guest_init_path, sandbox_guest_runtime_identity,
+ set_rootfs_image_file_mode, validate_host_supervisor, write_rootfs_image_file,
};
use crate::runtime::VmBackend;
use bollard::Docker;
@@ -54,15 +56,21 @@ use openshell_core::proto::compute::v1::{
use openshell_core::proto_struct::{
deserialize_optional_non_empty_string_list, struct_to_json_value,
};
+use openshell_isolation_interface::boundary_protocol::{
+ BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryMutualTlsMaterial,
+ BoundaryServerTls, BoundaryTopology, BoundaryTransport, generate_boundary_mutual_tls_material,
+};
use openshell_vfio::SysfsRoot;
use opentelemetry::trace::TraceContextExt as _;
use prost::Message;
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
+use std::fmt::Write as _;
use std::fs;
use std::future::Future;
-use std::io::Read;
-use std::net::{IpAddr, Ipv4Addr};
+
+use crate::isolation::VmBoundarySpec;
+use std::io::{Read, Seek, SeekFrom};
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::{Component, Path, PathBuf};
@@ -91,6 +99,7 @@ const MAX_REGISTRY_LAYER_DOWNLOAD_CONCURRENCY: usize = 16;
const REGISTRY_REQUEST_MAX_ATTEMPTS: usize = 4;
const REGISTRY_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(250);
const REGISTRY_RETRY_MAX_DELAY: Duration = Duration::from_secs(1);
+const VM_CONSOLE_DIAGNOSTIC_BYTES: u64 = 8 * 1024;
#[derive(Debug, Clone, Default, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
@@ -125,39 +134,34 @@ impl VmSandboxDriverConfig {
}
}
-/// gvproxy host-loopback IP — gvproxy's TCP/UDP/ICMP forwarder NAT-rewrites
-/// this destination to the host's `127.0.0.1` and dials out from the host
-/// process. This is the only address that transparently reaches host-bound
-/// services without explicit `expose` rules.
-///
-/// See gvisor-tap-vsock `cmd/gvproxy/config.go` (default NAT entry
-/// `HostIP -> 127.0.0.1`) and `pkg/services/forwarder/tcp.go` (NAT lookup
-/// before `net.Dial`).
-///
-/// Code paths route via `GVPROXY_HOST_LOOPBACK_ALIAS` (DNS / /etc/hosts)
-/// instead so logs stay readable; this constant is kept for documentation
-/// and parity with the guest init script.
-#[allow(dead_code)] // Documentation/parity anchor; all routing goes via the alias.
-const GVPROXY_HOST_LOOPBACK_IP: &str = "192.168.127.254";
const OPENSHELL_HOST_GATEWAY_ALIAS: &str = "host.openshell.internal";
-/// Hostname gvproxy resolves (via its embedded DNS) to the host-loopback IP.
-///
-/// We rewrite loopback URLs to this hostname rather than the bare IP because:
-/// * the guest init script seeds /etc/hosts with the same mapping, so it
-/// resolves even when gvproxy's DNS is not in resolv.conf;
-/// * keeping a recognisable hostname makes log messages clearer than a bare
-/// 192.168.127.254 reference;
-/// * package-managed gateway certificates include this SAN for guest mTLS.
-///
-/// Both names ultimately route through the gvproxy NAT path on
-/// `GVPROXY_HOST_LOOPBACK_IP` — they do **not** go through the gateway IP.
-const GVPROXY_HOST_LOOPBACK_ALIAS: &str = OPENSHELL_HOST_GATEWAY_ALIAS;
+const HOST_LOOPBACK_ALIASES: &[&str] = &[
+ OPENSHELL_HOST_GATEWAY_ALIAS,
+ "host.containers.internal",
+ "host.docker.internal",
+];
+#[allow(dead_code)]
const GUEST_SSH_SOCKET_PATH: &str = openshell_core::container_paths::SSH_SOCKET_PATH;
+#[allow(dead_code)]
const GUEST_TLS_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CA_PATH;
+#[allow(dead_code)]
const GUEST_TLS_CERT_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CERT_PATH;
+#[allow(dead_code)]
const GUEST_TLS_KEY_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_KEY_PATH;
+#[allow(dead_code)]
const GUEST_SANDBOX_TOKEN_PATH: &str = openshell_core::container_paths::VM_GUEST_SANDBOX_TOKEN_PATH;
const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_DIR;
+const GUEST_BOUNDARY_CONFIG_DIR: &str = "/.openshell/state";
+const GUEST_BOUNDARY_CONFIG_ENV: &str = "OPENSHELL_VM_SANDBOX_BOOTSTRAP";
+const HOST_SANDBOX_TOKEN_FILE: &str = "sandbox.jwt";
+const HOST_TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload";
+/// The backend this driver's deployment admits, delivered to the supervisor on
+/// a channel separate from the topology descriptor so descriptor verification
+/// is not self-referential.
+const DRIVER_ADMITTED_BACKEND: &str = "vm";
+const HOST_SUPERVISOR_BINARY: &str = "host-runtime/openshell-supervisor";
+const VM_CONTROL_SOCKET: &str = "control.sock";
+const VM_CONTROL_PORT: u32 = 5500;
/// Guest path of the driver-authored manifest enumerating which
/// `init.d` drop-ins the guest init script is allowed to execute.
///
@@ -167,19 +171,6 @@ const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_IN
/// upperdir on every launch, so the image cannot forge or shadow it.
const GUEST_INIT_DROPIN_MANIFEST: &str =
openshell_core::container_paths::VM_GUEST_INIT_DROPIN_MANIFEST;
-/// Guest path of the root-only corporate proxy credential staged by the driver.
-const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str =
- openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH;
-/// Guest path of the corporate proxy CA bundle staged by the driver.
-const GUEST_PROXY_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_PROXY_CA_PATH;
-/// Guest path of the driver-authored supervisor argument list.
-///
-/// The counterpart of [`GUEST_INIT_DROPIN_MANIFEST`] for the supervisor's own
-/// command line: written into the overlay upperdir on every launch (empty
-/// when there is nothing to pass) so the guest appends exactly the arguments
-/// the driver chose and a sandbox image cannot forge or shadow them.
-const GUEST_SUPERVISOR_ARGS_PATH: &str =
- openshell_core::container_paths::VM_GUEST_SUPERVISOR_ARGS_PATH;
const IMAGE_CACHE_ROOT_DIR: &str = "images";
const IMAGE_CACHE_ROOTFS_IMAGE: &str = "rootfs.ext4";
const OVERLAY_TEMPLATE_CACHE_DIR: &str = "overlay-templates";
@@ -194,7 +185,7 @@ const GUEST_IMAGE_CONFIG_DIR: &str = "openshell-image";
const GUEST_IMAGE_OCI_LAYOUT_DIR: &str = "oci";
const GUEST_IMAGE_OCI_REF: &str = "openshell";
const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar";
-const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v3";
+const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v4";
const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v3";
const IMAGE_IDENTITY_FILE: &str = "image-identity";
const IMAGE_REFERENCE_FILE: &str = "image-reference";
@@ -263,15 +254,14 @@ pub struct VmDriverConfig {
pub sandbox_gid: Option,
/// Corporate forward proxy URL (`http://host:port` or `https://host:port`)
- /// passed to the in-guest supervisor.
+ /// passed to the host supervisor.
///
/// The supervisor chains policy-approved TLS tunnels through this proxy
/// with HTTP CONNECT instead of dialing destinations directly. This is an
/// operator-owned egress boundary: it travels on the supervisor's argv,
/// which sandbox spec/template environment and image `ENV` cannot
- /// influence. A proxy on the gateway host's loopback is reachable from the
- /// guest only through the gvproxy host alias
- /// (`host.openshell.internal`).
+ /// influence. `host.openshell.internal` resolves to host loopback for the
+ /// host supervisor.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub https_proxy: Option,
@@ -432,8 +422,8 @@ impl VmDriverConfig {
/// Validate the operator's corporate upstream-proxy settings, fail-closed.
///
/// Delegates to the validator shared with the Podman and Kubernetes
- /// drivers and with the in-guest supervisor, so a value accepted here is
- /// never rejected inside the guest — and no misconfiguration can silently
+ /// drivers and with the host supervisor, so a value accepted here is
+ /// never rejected by the host supervisor — and no misconfiguration can silently
/// degrade to a direct dial.
///
/// # Errors
@@ -465,7 +455,7 @@ impl VmDriverConfig {
if provided.iter().all(Option::is_none) {
return if self.requires_tls_materials() {
Err(
- "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so sandbox VMs can authenticate to the gateway"
+ "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so the host supervisor can authenticate to the gateway"
.to_string(),
)
} else {
@@ -524,9 +514,27 @@ fn validate_openshell_endpoint(endpoint: &str) -> Result<(), String> {
Ok(())
}
+fn host_control_openshell_endpoint(endpoint: &str) -> Result<(String, Option), String> {
+ let mut url = Url::parse(endpoint)
+ .map_err(|err| format!("invalid openshell endpoint '{endpoint}': {err}"))?;
+ let Some(host) = url.host_str().map(str::to_string) else {
+ return Ok((endpoint.to_string(), None));
+ };
+ if !HOST_LOOPBACK_ALIASES.contains(&host.as_str()) {
+ return Ok((endpoint.to_string(), None));
+ }
+
+ // The supervisor runs on the host, so guest aliases dial loopback while
+ // retaining the configured hostname for TLS certificate verification.
+ url.set_host(Some("127.0.0.1"))
+ .map_err(|error| format!("failed to rewrite host endpoint '{endpoint}': {error}"))?;
+ Ok((url.into(), Some(host)))
+}
+
#[derive(Debug)]
struct VmProcess {
child: Child,
+ supervisor: Child,
deleting: bool,
}
@@ -536,7 +544,6 @@ struct SandboxRecord {
process: Option>>,
provisioning_task: Option>,
gpu_bdf: Option,
- qemu_network_allocated: bool,
deleting: bool,
}
@@ -575,7 +582,6 @@ pub struct VmDriver {
image_cache_lock: Arc>,
events: broadcast::Sender,
gpu_inventory: Option>>,
- subnet_allocator: Arc>,
lifecycle_extensions: Arc,
}
@@ -585,7 +591,7 @@ impl VmDriver {
}
pub async fn new_with_extensions(
- config: VmDriverConfig,
+ mut config: VmDriverConfig,
lifecycle_extensions: LifecycleExtensionRegistry,
) -> Result {
lifecycle_extensions
@@ -598,13 +604,11 @@ impl VmDriver {
}
validate_openshell_endpoint(&config.openshell_endpoint)?;
let _ = config.tls_paths()?;
+ config.state_dir = absolute_state_dir(&config.state_dir)?;
#[cfg(target_os = "linux")]
if config.gpu_enabled {
check_gpu_privileges()?;
- tokio::task::spawn_blocking(crate::cleanup_stale_tap_interfaces)
- .await
- .map_err(|e| format!("cleanup stale TAP interfaces panicked: {e}"))?;
}
let state_root = sandboxes_root_dir(&config.state_dir);
@@ -643,11 +647,6 @@ impl VmDriver {
None
};
- let subnet_allocator = Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- )));
-
let (events, _) = broadcast::channel(WATCH_BUFFER);
let driver = Self {
config,
@@ -656,13 +655,181 @@ impl VmDriver {
image_cache_lock: Arc::new(Mutex::new(())),
events,
gpu_inventory,
- subnet_allocator,
lifecycle_extensions: Arc::new(lifecycle_extensions),
};
driver.restore_persisted_sandboxes().await;
Ok(driver)
}
+ async fn host_supervisor_binary(&self) -> Result {
+ if let Some(configured) = std::env::var_os("OPENSHELL_VM_SUPERVISOR_BIN") {
+ let configured = PathBuf::from(configured);
+ if configured.is_file() {
+ return Ok(configured);
+ }
+ return Err(Status::failed_precondition(format!(
+ "configured host supervisor does not exist: {}",
+ configured.display()
+ )));
+ }
+
+ let destination = self.config.state_dir.join(HOST_SUPERVISOR_BINARY);
+ if validate_host_supervisor(&destination).is_ok() {
+ return Ok(destination);
+ }
+ let _cache_guard = self.image_cache_lock.lock().await;
+ if validate_host_supervisor(&destination).is_ok() {
+ return Ok(destination);
+ }
+ let destination_for_extract = destination.clone();
+ tokio::task::spawn_blocking(move || extract_host_supervisor(&destination_for_extract))
+ .await
+ .map_err(|error| {
+ Status::internal(format!("host supervisor extraction panicked: {error}"))
+ })?
+ .map_err(Status::failed_precondition)?;
+ validate_host_supervisor(&destination).map_err(Status::failed_precondition)?;
+ Ok(destination)
+ }
+
+ async fn spawn_host_supervisor(
+ &self,
+ sandbox: &Sandbox,
+ state_dir: &Path,
+ tls_paths: Option<&VmDriverTlsPaths>,
+ topology: &BoundaryTopology,
+ ) -> Result {
+ let supervisor_binary = self.host_supervisor_binary().await?;
+ let (openshell_endpoint, gateway_tls_server_name) =
+ host_control_openshell_endpoint(&self.config.openshell_endpoint)
+ .map_err(Status::failed_precondition)?;
+ let token = sandbox
+ .spec
+ .as_ref()
+ .map(|spec| spec.sandbox_token.as_str())
+ .filter(|token| !token.is_empty())
+ .ok_or_else(|| Status::failed_precondition("VM sandbox gateway token is required"))?;
+ let token_path = state_dir.join(HOST_SANDBOX_TOKEN_FILE);
+ tokio::fs::write(&token_path, format!("{token}\n"))
+ .await
+ .map_err(|error| Status::internal(format!("write host sandbox token: {error}")))?;
+ #[cfg(unix)]
+ tokio::fs::set_permissions(&token_path, fs::Permissions::from_mode(0o600))
+ .await
+ .map_err(|error| Status::internal(format!("restrict host sandbox token: {error}")))?;
+
+ let descriptor = topology
+ .descriptor(DRIVER_ADMITTED_BACKEND)
+ .map_err(|error| Status::internal(error.to_string()))?;
+ // The payload carries the boundary bootstrap token, so it must not
+ // appear in the world-readable process cmdline; deliver it through a
+ // driver-owned 0600 file like the gateway token.
+ let payload_path = state_dir.join(HOST_TOPOLOGY_PAYLOAD_FILE);
+ tokio::fs::write(&payload_path, &descriptor.payload)
+ .await
+ .map_err(|error| Status::internal(format!("write host topology payload: {error}")))?;
+ #[cfg(unix)]
+ tokio::fs::set_permissions(&payload_path, fs::Permissions::from_mode(0o600))
+ .await
+ .map_err(|error| {
+ Status::internal(format!("restrict host topology payload: {error}"))
+ })?;
+ let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(
+ sandbox.spec.as_ref(),
+ )
+ .map_err(|error| Status::internal(format!("encode main process spec: {error}")))?;
+ let sandbox_user_id = self.config.resolve_sandbox_uid();
+ let primary_group_id = self.config.resolve_sandbox_gid(sandbox_user_id);
+ let upstream_proxy_args = upstream_proxy_cli_args(&self.config)
+ .map_err(|error| Status::invalid_argument(format!("render upstream proxy: {error}")))?;
+ let mut command = Command::new(&supervisor_binary);
+ isolate_host_control_environment(&mut command);
+ command
+ .kill_on_drop(true)
+ .stdin(Stdio::null())
+ .stdout(Stdio::from(
+ fs::File::create(state_dir.join("supervisor.log"))
+ .map_err(|error| Status::internal(format!("create supervisor log: {error}")))?,
+ ))
+ .stderr(Stdio::from(
+ fs::File::create(state_dir.join("supervisor.err.log")).map_err(|error| {
+ Status::internal(format!("create supervisor error log: {error}"))
+ })?,
+ ))
+ .arg(format!(
+ "--topology-backend-name={}",
+ descriptor.backend_name
+ ))
+ .arg("--topology-payload-file")
+ .arg(&payload_path)
+ .arg("--workdir")
+ .arg("/sandbox")
+ .args(upstream_proxy_args)
+ .env(
+ openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND,
+ DRIVER_ADMITTED_BACKEND,
+ )
+ .env(
+ openshell_core::sandbox_env::MAIN_PROCESS_SPEC,
+ main_process_spec,
+ )
+ .env(openshell_core::sandbox_env::ENDPOINT, openshell_endpoint)
+ .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id)
+ .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name)
+ .env(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, &token_path)
+ .env(
+ openshell_core::sandbox_env::SSH_SOCKET_PATH,
+ state_dir.join("ssh.sock"),
+ )
+ .env(
+ openshell_core::sandbox_env::PROXY_TLS_DIR,
+ state_dir.join("proxy-tls"),
+ )
+ .env(
+ openshell_core::sandbox_env::SANDBOX_UID,
+ sandbox_user_id.to_string(),
+ )
+ .env(
+ openshell_core::sandbox_env::SANDBOX_GID,
+ primary_group_id.to_string(),
+ )
+ .env(openshell_core::sandbox_env::OCI_IMAGE_USER, "")
+ .env(
+ openshell_core::sandbox_env::LOG_LEVEL,
+ openshell_core::driver_utils::sandbox_log_level(sandbox, &self.config.log_level),
+ )
+ .env(
+ openshell_core::sandbox_env::TELEMETRY_ENABLED,
+ openshell_core::telemetry::enabled_env_value(),
+ );
+ if let Some(server_name) = gateway_tls_server_name {
+ command.env(
+ openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME,
+ server_name,
+ );
+ }
+ configure_main_exit_marker(&mut command, state_dir);
+ if let Some(tls) = tls_paths {
+ command
+ .env(openshell_core::sandbox_env::TLS_CA, &tls.ca)
+ .env(openshell_core::sandbox_env::TLS_CERT, &tls.cert)
+ .env(openshell_core::sandbox_env::TLS_KEY, &tls.key);
+ }
+ #[cfg(target_os = "linux")]
+ unsafe {
+ command.pre_exec(|| {
+ nix::sys::prctl::set_pdeathsig(Signal::SIGKILL)
+ .map_err(|error| std::io::Error::other(error.to_string()))
+ });
+ }
+ command.spawn().map_err(|error| {
+ Status::internal(format!(
+ "start host supervisor '{}': {error}",
+ supervisor_binary.display()
+ ))
+ })
+ }
+
#[must_use]
pub fn capabilities(&self) -> GetCapabilitiesResponse {
GetCapabilitiesResponse {
@@ -726,7 +893,6 @@ impl VmDriver {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
},
);
@@ -893,6 +1059,8 @@ impl VmDriver {
let root_disk = image_plan.root_disk;
let image_disk = image_plan.image_disk;
let overlay_disk = disk_paths.overlay_disk;
+ let bootstrap_token = random_boundary_token();
+ let boundary_generation = random_boundary_token();
self.publish_platform_event(
sandbox.id.clone(),
@@ -904,16 +1072,7 @@ impl VmDriver {
),
);
if let Err(err) = self
- .prepare_runtime_overlay(
- &overlay_disk,
- tls_paths.as_ref(),
- sandbox
- .spec
- .as_ref()
- .map(|spec| spec.sandbox_token.as_str())
- .filter(|token| !token.is_empty()),
- overlay_preparation,
- )
+ .prepare_runtime_overlay(&overlay_disk, overlay_preparation)
.await
{
return Err(Status::internal(format!(
@@ -946,24 +1105,11 @@ impl VmDriver {
match self.build_vm_launch_plan(&sandbox.id, needs_qemu, is_gpu, gpu_bdf.clone()) {
Ok(plan) => plan,
Err(err) => {
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
return Err(err);
}
};
- // `build_vm_launch_plan` already allocated the QEMU subnet, so record
- // it as allocated now — before the cancellable `configure_launch` /
- // `before_launch` hooks run. If a delete aborts provisioning while
- // one of those hooks is awaiting, the aborted future never runs its
- // own release path, and the delete cleanup is gated on this flag; if
- // the flag were still unset the subnet would leak.
- if plan.backend == VmBackend::Qemu
- && let Err(err) = self.mark_qemu_network_allocated(&sandbox.id).await
- {
- self.release_gpu_and_subnet(&sandbox.id);
- return Err(err);
- }
-
if let Err(err) = self
.lifecycle_extensions
.configure_launch(&sandbox, &state_dir, &mut plan)
@@ -976,7 +1122,7 @@ impl VmDriver {
LaunchAbortReason::BeforeLaunchHookFailed,
)
.await;
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
let message = format!(
"vm lifecycle extension rejected sandbox launch plan: {}",
err.message()
@@ -990,8 +1136,8 @@ impl VmDriver {
// Resolve and validate the backend from the requirements that
// `configure_launch` extensions contributed. After this point the
- // plan's backend, sizing, and host allocations (subnet, tap, vsock)
- // are final; the `before_launch` hook below may still mutate
+ // plan's backend, sizing, and host allocations are final; the
+ // `before_launch` hook below may still mutate
// `plan.env` and `plan.guest_init_dropins` and may abort the launch,
// but it MUST NOT change `plan.backend`, `plan.required_backends`,
// or `plan.required_backend_features` -- those are enforced as a
@@ -1006,7 +1152,7 @@ impl VmDriver {
LaunchAbortReason::BeforeLaunchHookFailed,
)
.await;
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
return Err(err);
}
@@ -1018,7 +1164,7 @@ impl VmDriver {
LaunchAbortReason::BeforeLaunchHookFailed,
)
.await;
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
return Err(err);
}
@@ -1034,7 +1180,7 @@ impl VmDriver {
LaunchAbortReason::BeforeLaunchHookFailed,
)
.await;
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
let message = format!(
"vm lifecycle extension rejected sandbox launch: {}",
err.message()
@@ -1050,29 +1196,60 @@ impl VmDriver {
self.lifecycle_extensions
.after_launch_failed(&sandbox, &state_dir, LaunchAbortReason::GuestPrepareFailed)
.await;
- self.release_gpu_and_subnet(&sandbox.id);
- return Err(err);
- }
-
- // Staged on every launch, including a restart onto a preserved
- // overlay, so the driver's copy always shadows the image layer.
- if let Err(err) = inject_guest_upstream_proxy(&overlay_disk, &self.config).await {
- self.lifecycle_extensions
- .after_launch_failed(&sandbox, &state_dir, LaunchAbortReason::GuestPrepareFailed)
- .await;
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
return Err(err);
}
- let endpoint_override = if plan.backend == VmBackend::Qemu {
- plan.host_ip.as_deref().map(|host_ip| {
- guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip)
- })
+ let console_output = state_dir.join("rootfs-console.log");
+ let control_socket = state_dir.join(VM_CONTROL_SOCKET);
+ let channel_tls = generate_boundary_mutual_tls_material()
+ .map_err(|error| Status::internal(error.to_string()))?;
+ let supervisor_tls = BoundaryClientTls {
+ server_name: channel_tls.server_name.clone(),
+ ca_certificate_pem: channel_tls.ca_certificate_pem.clone(),
+ certificate_chain_pem: channel_tls.supervisor_certificate_pem.clone(),
+ private_key_pem: channel_tls.supervisor_private_key_pem.clone(),
+ };
+ let transport = if plan.backend == VmBackend::Qemu {
+ BoundaryTransport::Vsock {
+ guest_cid: plan.vsock_cid.ok_or_else(|| {
+ Status::internal("QEMU launch plan is missing a guest vsock CID")
+ })?,
+ control_port: VM_CONTROL_PORT,
+ tls: supervisor_tls,
+ }
} else {
- None
+ BoundaryTransport::Unix {
+ socket_path: control_socket.clone(),
+ tls: supervisor_tls,
+ }
};
-
- let console_output = state_dir.join("rootfs-console.log");
+ let sandbox_user_id = self.config.resolve_sandbox_uid();
+ let provisioning = VmBoundarySpec {
+ boundary_id: sandbox.id.clone(),
+ bootstrap_token,
+ generation: boundary_generation.clone(),
+ session_epoch: random_boundary_token(),
+ image_identity,
+ transport,
+ sandbox_tls: guest_boundary_tls_paths(&boundary_generation),
+ control_port: VM_CONTROL_PORT,
+ agent_uid: sandbox_user_id,
+ agent_gid: self.config.resolve_sandbox_gid(sandbox_user_id),
+ child_env: merged_environment(&sandbox),
+ }
+ .provision()
+ .map_err(|error| Status::failed_precondition(error.to_string()))?;
+ let guest_boundary_config_path =
+ guest_boundary_config_path(&provisioning.boundary_config.generation);
+ inject_guest_boundary_bundle(
+ &overlay_disk,
+ &guest_boundary_config_path,
+ &provisioning.boundary_config,
+ &channel_tls,
+ )
+ .map_err(|error| Status::internal(format!("inject VM boundary configuration: {error}")))?;
+ let topology = provisioning.topology;
let mut command = Command::new(&self.launcher_bin);
command.kill_on_drop(true);
command.stdin(Stdio::null());
@@ -1098,24 +1275,16 @@ impl VmDriver {
if let Some(bdf) = plan.gpu_bdf.as_deref() {
command.arg("--vm-gpu-bdf").arg(bdf);
}
- if let Some(tap) = plan.tap_device.as_deref() {
- command.arg("--vm-tap-device").arg(tap);
- }
- if let Some(guest_ip) = plan.guest_ip.as_deref() {
- command.arg("--vm-guest-ip").arg(guest_ip);
- }
- if let Some(host_ip) = plan.host_ip.as_deref() {
- command.arg("--vm-host-ip").arg(host_ip);
- }
if let Some(vsock_cid) = plan.vsock_cid {
command.arg("--vm-vsock-cid").arg(vsock_cid.to_string());
}
- if let Some(guest_mac) = plan.guest_mac.as_deref() {
- command.arg("--vm-guest-mac").arg(guest_mac);
- }
- if let Some(port) = plan.gateway_port {
- command.arg("--vm-gateway-port").arg(port.to_string());
- }
+ } else {
+ let _ = tokio::fs::remove_file(&control_socket).await;
+ command
+ .arg("--vm-vsock-control-port")
+ .arg(VM_CONTROL_PORT.to_string())
+ .arg("--vm-vsock-control-socket")
+ .arg(&control_socket);
}
self.ensure_provisioning_active(&sandbox.id).await?;
@@ -1124,9 +1293,12 @@ impl VmDriver {
.arg("--vm-krun-log-level")
.arg(self.config.krun_log_level.to_string());
- for env in build_guest_environment(&sandbox, &self.config, endpoint_override.as_deref()) {
+ for env in build_guest_environment(&sandbox, &self.config) {
command.arg("--vm-env").arg(env);
}
+ command.arg("--vm-env").arg(format!(
+ "{GUEST_BOUNDARY_CONFIG_ENV}={guest_boundary_config_path}"
+ ));
for env in &plan.env {
command.arg("--vm-env").arg(env);
}
@@ -1137,7 +1309,7 @@ impl VmDriver {
console_output = %console_output.display(),
"vm driver: spawning VM launcher"
);
- let child = match spawn_vm_launcher(&mut command, &sandbox.id, &plan.backend) {
+ let mut child = match command.spawn() {
Ok(child) => child,
Err(err) => {
warn!(
@@ -1152,7 +1324,7 @@ impl VmDriver {
LaunchAbortReason::LauncherSpawnFailed,
)
.await;
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
return Err(Status::internal(format!(
"failed to launch vm helper '{}': {err}",
self.launcher_bin.display()
@@ -1164,8 +1336,27 @@ impl VmDriver {
launcher_pid = child.id().unwrap_or(0),
"vm driver: launcher spawned"
);
+ let supervisor = match self
+ .spawn_host_supervisor(&sandbox, &state_dir, tls_paths.as_ref(), &topology)
+ .await
+ {
+ Ok(supervisor) => supervisor,
+ Err(error) => {
+ let _ = terminate_vm_process(&mut child).await;
+ self.lifecycle_extensions
+ .after_launch_failed(
+ &sandbox,
+ &state_dir,
+ LaunchAbortReason::LauncherSpawnFailed,
+ )
+ .await;
+ self.release_gpu(&sandbox.id);
+ return Err(error);
+ }
+ };
let process = Arc::new(Mutex::new(VmProcess {
child,
+ supervisor,
deleting: false,
}));
@@ -1177,7 +1368,6 @@ impl VmDriver {
Some(record) if !record.deleting => {
record.process = Some(process.clone());
record.gpu_bdf.clone_from(&gpu_bdf);
- record.qemu_network_allocated = plan.backend == VmBackend::Qemu;
snapshot_to_publish = Some(record.snapshot.clone());
}
_ => {
@@ -1190,11 +1380,11 @@ impl VmDriver {
{
let mut process = process.lock().await;
process.deleting = true;
- terminate_vm_process(&mut process.child)
+ terminate_sandbox_processes(&mut process)
.await
- .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?;
+ .map_err(|err| Status::internal(format!("failed to stop sandbox: {err}")))?;
}
- self.release_gpu_and_subnet(&sandbox.id);
+ self.release_gpu(&sandbox.id);
return Err(Status::cancelled("sandbox provisioning cancelled"));
}
@@ -1255,7 +1445,7 @@ impl VmDriver {
.await
.map_err(|err| Status::internal(format!("persist stop marker failed: {err}")))?;
- let (process, provisioning_task, has_gpu, has_qemu_network, snapshot) = {
+ let (process, provisioning_task, has_gpu, snapshot) = {
let mut registry = self.registry.lock().await;
let record = registry
.get_mut(&record_id)
@@ -1264,7 +1454,6 @@ impl VmDriver {
record.process.take(),
record.provisioning_task.take(),
record.gpu_bdf.take().is_some(),
- std::mem::take(&mut record.qemu_network_allocated),
record.snapshot.clone(),
)
};
@@ -1275,14 +1464,16 @@ impl VmDriver {
if let Some(process) = process {
let mut process = process.lock().await;
process.deleting = true;
- terminate_vm_process(&mut process.child)
+ terminate_sandbox_processes(&mut process)
.await
- .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?;
+ .map_err(|err| Status::internal(format!("failed to stop sandbox: {err}")))?;
}
self.lifecycle_extensions
.after_launch_failed(&snapshot, &state_dir, LaunchAbortReason::Stopped)
.await;
- self.release_allocations(&record_id, has_gpu, has_qemu_network);
+ if has_gpu {
+ self.release_gpu(&record_id);
+ }
if let Some(snapshot) = self
.set_snapshot_condition(&record_id, stopped_condition(), false)
@@ -1382,14 +1573,7 @@ impl VmDriver {
return span_status.finish(Ok(DeleteSandboxResponse { deleted: false }));
};
- let (
- state_dir,
- process,
- gpu_bdf,
- qemu_network_allocated,
- provisioning_task,
- sandbox_snapshot,
- ) = {
+ let (state_dir, process, gpu_bdf, provisioning_task, sandbox_snapshot) = {
let mut registry = self.registry.lock().await;
let Some(record) = registry.get_mut(&record_id) else {
return span_status.finish(Ok(DeleteSandboxResponse { deleted: false }));
@@ -1399,7 +1583,6 @@ impl VmDriver {
record.state_dir.clone(),
record.process.clone(),
record.gpu_bdf.clone(),
- record.qemu_network_allocated,
record.provisioning_task.take(),
record.snapshot.clone(),
)
@@ -1419,16 +1602,18 @@ impl VmDriver {
if let Some(process) = process {
let mut process = process.lock().await;
process.deleting = true;
- terminate_vm_process(&mut process.child)
+ terminate_sandbox_processes(&mut process)
.await
- .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?;
+ .map_err(|err| Status::internal(format!("failed to stop sandbox: {err}")))?;
}
self.lifecycle_extensions
.after_delete(&sandbox_snapshot, &state_dir)
.await;
- self.release_allocations(&record_id, gpu_bdf.is_some(), qemu_network_allocated);
+ if gpu_bdf.is_some() {
+ self.release_gpu(&record_id);
+ }
remove_sandbox_state_dir(&self.config.state_dir, &state_dir).await?;
@@ -1564,7 +1749,6 @@ impl VmDriver {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
});
drop(registry);
@@ -1592,7 +1776,6 @@ impl VmDriver {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
});
drop(registry);
@@ -1681,7 +1864,6 @@ impl VmDriver {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
},
);
@@ -1765,26 +1947,6 @@ impl VmDriver {
}
}
- fn release_subnet(&self, sandbox_id: &str) {
- if let Ok(mut alloc) = self.subnet_allocator.lock() {
- alloc.release(sandbox_id);
- }
- }
-
- fn release_allocations(&self, sandbox_id: &str, has_gpu: bool, has_qemu_network: bool) {
- if has_gpu {
- self.release_gpu(sandbox_id);
- }
- if has_qemu_network {
- self.release_subnet(sandbox_id);
- }
- }
-
- fn release_gpu_and_subnet(&self, sandbox_id: &str) {
- self.release_gpu(sandbox_id);
- self.release_subnet(sandbox_id);
- }
-
async fn ensure_extension_state_dirs(&self, state_dir: &Path) -> Result<(), Status> {
for extension_name in self.lifecycle_extensions.names() {
let extension_dir = extension_state_dir(state_dir, &extension_name).map_err(|err| {
@@ -1834,10 +1996,12 @@ impl VmDriver {
Ok(())
}
- #[allow(clippy::result_large_err)]
+ // Keep the fallible shape used by launch-plan resolution: driver-local
+ // backends may add allocation failures here without changing callers.
+ #[allow(clippy::result_large_err, clippy::unnecessary_wraps)]
fn configure_qemu_launch_plan(
&self,
- sandbox_id: &str,
+ _sandbox_id: &str,
is_gpu: bool,
gpu_bdf: Option,
plan: &mut LaunchPlan,
@@ -1850,44 +2014,10 @@ impl VmDriver {
if plan.gpu_bdf.is_none() {
plan.gpu_bdf = gpu_bdf;
}
- if !has_complete_qemu_network(plan) {
- let subnet = self
- .subnet_allocator
- .lock()
- .map_err(|e| Status::internal(format!("subnet allocator lock poisoned: {e}")))?
- .allocate(sandbox_id)
- .map_err(Status::failed_precondition)?;
- let mac = mac_from_sandbox_id(sandbox_id);
- plan.tap_device = Some(tap_device_name(sandbox_id));
- plan.guest_ip = Some(subnet.guest_ip.to_string());
- plan.host_ip = Some(subnet.host_ip.to_string());
- plan.vsock_cid = Some(allocate_vsock_cid());
- plan.guest_mac = Some(format!(
- "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
- mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]
- ));
- plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint);
- }
-
- // The corporate-proxy host-loopback recipe is a libkrun/gvproxy
- // property and has no QEMU/TAP equivalent (see
- // `proxy_url_targets_gateway_host`). Run it here, after the subnet
- // allocation above has settled `plan.host_ip`, because the address to
- // compare against is this sandbox's own TAP host address. Fail the
- // create with the reason rather than boot a sandbox whose
- // policy-approved CONNECTs all time out against an unreachable proxy.
- if let Some(url) = self.config.https_proxy.as_deref()
- && proxy_url_targets_gateway_host(url, plan.host_ip.as_deref())
- {
- let tap_host = plan.host_ip.as_deref().unwrap_or("the TAP host address");
- return Err(Status::failed_precondition(format!(
- "https_proxy '{url}' addresses the gateway host, which a QEMU/TAP sandbox \
- (GPU sandboxes) cannot reach: host.openshell.internal resolves to this \
- sandbox's TAP host address {tap_host} and the driver's nftables rules allow \
- only the gateway port from the guest. Configure a proxy address routable \
- from the guest's masqueraded egress, or run this sandbox without a GPU"
- )));
+ if plan.vsock_cid.is_some() {
+ return Ok(());
}
+ plan.vsock_cid = Some(allocate_vsock_cid());
Ok(())
}
@@ -1966,21 +2096,12 @@ impl VmDriver {
Ok(())
}
- async fn mark_qemu_network_allocated(&self, sandbox_id: &str) -> Result<(), Status> {
- let mut registry = self.registry.lock().await;
- match registry.get_mut(sandbox_id) {
- Some(record) if !record.deleting => {
- record.qemu_network_allocated = true;
- Ok(())
- }
- _ => Err(Status::cancelled("sandbox provisioning cancelled")),
- }
- }
-
- #[allow(clippy::result_large_err)]
+ // Keep the fallible shape used by provisioning and lifecycle tests even
+ // though NIC/subnet allocation no longer introduces a failure today.
+ #[allow(clippy::result_large_err, clippy::unnecessary_wraps)]
fn build_vm_launch_plan(
&self,
- sandbox_id: &str,
+ _sandbox_id: &str,
needs_qemu: bool,
is_gpu: bool,
gpu_bdf: Option,
@@ -1995,32 +2116,13 @@ impl VmDriver {
kernel_profile: None,
kernel_image: None,
gpu_bdf: None,
- tap_device: None,
- guest_ip: None,
- host_ip: None,
vsock_cid: None,
- guest_mac: None,
- gateway_port: None,
guest_init_dropins: Vec::new(),
env: Vec::new(),
});
}
- let subnet = self
- .subnet_allocator
- .lock()
- .map_err(|e| Status::internal(format!("subnet allocator lock poisoned: {e}")))?
- .allocate(sandbox_id)
- .map_err(Status::failed_precondition)?;
let vsock_cid = allocate_vsock_cid();
- let mac = mac_from_sandbox_id(sandbox_id);
- let mac_str = format!(
- "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}",
- mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]
- );
- let tap = tap_device_name(sandbox_id);
- let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint);
-
let (vcpus, mem_mib) = if is_gpu {
(self.config.gpu_vcpus, self.config.gpu_mem_mib)
} else {
@@ -2036,12 +2138,7 @@ impl VmDriver {
kernel_profile: None,
kernel_image: None,
gpu_bdf,
- tap_device: Some(tap),
- guest_ip: Some(subnet.guest_ip.to_string()),
- host_ip: Some(subnet.host_ip.to_string()),
vsock_cid: Some(vsock_cid),
- guest_mac: Some(mac_str),
- gateway_port,
guest_init_dropins: Vec::new(),
env: Vec::new(),
})
@@ -2113,7 +2210,7 @@ impl VmDriver {
message: &str,
remove_state: bool,
) {
- self.release_gpu_and_subnet(sandbox_id);
+ self.release_gpu(sandbox_id);
let snapshot = {
let mut registry = self.registry.lock().await;
let Some(record) = registry.get_mut(sandbox_id) else {
@@ -2124,7 +2221,6 @@ impl VmDriver {
}
record.process = None;
record.gpu_bdf = None;
- record.qemu_network_allocated = false;
record.snapshot.status = Some(status_with_condition(
&record.snapshot,
error_condition(reason, message),
@@ -2217,16 +2313,9 @@ impl VmDriver {
async fn prepare_runtime_overlay(
&self,
overlay_disk: &Path,
- tls_paths: Option<&VmDriverTlsPaths>,
- sandbox_token: Option<&str>,
preparation: OverlayPreparation,
) -> Result<(), String> {
let span_status = openshell_otel::ErrorStatusGuard::current();
- let tls_materials = match tls_paths {
- Some(paths) => Some(read_guest_tls_materials(paths).await?),
- None => None,
- };
- let sandbox_token = sandbox_token.map(str::to_string);
let overlay_disk = overlay_disk.to_path_buf();
let overlay_size_bytes = self
.config
@@ -2240,6 +2329,10 @@ impl VmDriver {
})?;
let template_path = overlay_template_image(&self.config.state_dir, overlay_size_bytes);
+ let recover_preserved_overlay = preparation == OverlayPreparation::PreserveExisting
+ && tokio::fs::metadata(&overlay_disk)
+ .await
+ .is_ok_and(|metadata| metadata.is_file());
if !overlay_template_image_ready(&template_path, overlay_size_bytes).await? {
let _cache_guard = self.image_cache_lock.lock().await;
let template_path = template_path.clone();
@@ -2250,19 +2343,24 @@ impl VmDriver {
.map_err(|err| format!("overlay template preparation panicked: {err}"))??;
}
+ let overlay_to_recover = overlay_disk.clone();
let result = tokio::task::spawn_blocking(move || {
prepare_sandbox_overlay_image(
&template_path,
&overlay_disk,
- tls_materials.as_ref(),
- sandbox_token.as_deref(),
preparation,
overlay_size_bytes,
)
})
.await
.map_err(|err| format!("overlay image preparation panicked: {err}"))?;
- span_status.finish(result)
+ result?;
+ if recover_preserved_overlay {
+ tokio::task::spawn_blocking(move || recover_rootfs_image(&overlay_to_recover))
+ .await
+ .map_err(|error| format!("overlay recovery panicked: {error}"))??;
+ }
+ span_status.finish(Ok(()))
}
fn resolved_sandbox_image(&self, sandbox: &Sandbox) -> Option {
@@ -3347,62 +3445,89 @@ impl VmDriver {
process.clone()
};
- let exit_status = {
+ let poll_result = {
let mut process = process.lock().await;
if process.deleting {
return;
}
match process.child.try_wait() {
- Ok(status) => status,
- Err(err) => {
- if let Some(snapshot) = self
- .set_snapshot_condition(
- &sandbox_id,
- error_condition("ProcessPollFailed", &err.to_string()),
- false,
- )
- .await
- {
- self.publish_snapshot(snapshot);
- }
- self.publish_platform_event(
- sandbox_id.clone(),
- platform_event(
- "vm",
- "Warning",
- "ProcessPollFailed",
- format!("Failed to poll VM helper process: {err}"),
- ),
- );
- return;
+ Ok(Some(status)) => Ok(Some(("VM", status))),
+ Ok(None) => process
+ .supervisor
+ .try_wait()
+ .map(|status| status.map(|status| ("host supervisor", status))),
+ Err(error) => Err(error),
+ }
+ };
+
+ let exit_status = match poll_result {
+ Ok(status) => status,
+ Err(err) => {
+ if let Some(snapshot) = self
+ .set_snapshot_condition(
+ &sandbox_id,
+ error_condition("ProcessPollFailed", &err.to_string()),
+ false,
+ )
+ .await
+ {
+ self.publish_snapshot(snapshot);
}
+ self.publish_platform_event(
+ sandbox_id.clone(),
+ platform_event(
+ "vm",
+ "Warning",
+ "ProcessPollFailed",
+ format!("Failed to poll VM sandbox process: {err}"),
+ ),
+ );
+ return;
}
};
- if let Some(status) = exit_status {
+ if let Some((component, status)) = exit_status {
let state_dir = {
let registry = self.registry.lock().await;
registry
.get(&sandbox_id)
.map(|record| record.state_dir.clone())
};
- if let Some(state_dir) = state_dir
- && let Err(error) = write_private_file(
- &state_dir.join(MAIN_PROCESS_EXITED_FILE),
- b"terminal\n".to_vec(),
- )
- .await
+ if let Some(ref state_dir) = state_dir {
+ let marker = state_dir.join(MAIN_PROCESS_EXITED_FILE);
+ if !tokio::fs::try_exists(&marker).await.unwrap_or(false)
+ && let Err(error) =
+ write_private_file(&marker, b"terminal\n".to_vec()).await
+ {
+ warn!(
+ sandbox_id = %sandbox_id,
+ %error,
+ "vm driver: failed to persist canonical-process exit tombstone"
+ );
+ }
+ }
{
- warn!(
- sandbox_id = %sandbox_id,
- %error,
- "vm driver: failed to persist canonical-process exit tombstone"
- );
+ let mut process = process.lock().await;
+ if component == "VM" {
+ let _ = terminate_vm_process(&mut process.supervisor).await;
+ } else {
+ let _ = terminate_vm_process(&mut process.child).await;
+ }
}
- let message = status.code().map_or_else(
- || "VM process exited".to_string(),
- |code| format!("VM process exited with status {code}"),
+ let mut message = status.code().map_or_else(
+ || format!("{component} process exited"),
+ |code| format!("{component} process exited with status {code}"),
);
+ if component == "VM"
+ && let Some(state_dir) = state_dir.as_deref()
+ && let Some(console) = read_vm_console_tail(
+ &state_dir.join("rootfs-console.log"),
+ VM_CONSOLE_DIAGNOSTIC_BYTES,
+ )
+ {
+ write!(message, "; guest console tail:\n{console}")
+ .expect("writing to String cannot fail");
+ }
if let Some(snapshot) = self
.set_snapshot_condition(
&sandbox_id,
@@ -3417,17 +3542,14 @@ impl VmDriver {
sandbox_id.clone(),
platform_event("vm", "Warning", "ProcessExited", message),
);
- let (has_gpu, has_qemu_network, cleanup_ctx) = {
+ let (has_gpu, cleanup_ctx) = {
let registry = self.registry.lock().await;
- registry
- .get(&sandbox_id)
- .map_or((false, false, None), |record| {
- (
- record.gpu_bdf.is_some(),
- record.qemu_network_allocated,
- Some((record.snapshot.clone(), record.state_dir.clone())),
- )
- })
+ registry.get(&sandbox_id).map_or((false, None), |record| {
+ (
+ record.gpu_bdf.is_some(),
+ Some((record.snapshot.clone(), record.state_dir.clone())),
+ )
+ })
};
// Give lifecycle extensions a chance to release host
// resources they allocated in `before_launch` (e.g. device
@@ -3441,7 +3563,9 @@ impl VmDriver {
.after_launch_failed(&sandbox, &state_dir, LaunchAbortReason::ProcessExited)
.await;
}
- self.release_allocations(&sandbox_id, has_gpu, has_qemu_network);
+ if has_gpu {
+ self.release_gpu(&sandbox_id);
+ }
return;
}
@@ -3504,6 +3628,27 @@ impl VmDriver {
}
}
+fn read_vm_console_tail(path: &Path, limit: u64) -> Option {
+ if limit == 0 {
+ return None;
+ }
+ let mut file = fs::File::open(path).ok()?;
+ let length = file.metadata().ok()?.len();
+ file.seek(SeekFrom::Start(length.saturating_sub(limit)))
+ .ok()?;
+ let mut bytes = Vec::with_capacity(usize::try_from(length.min(limit)).ok()?);
+ file.read_to_end(&mut bytes).ok()?;
+ let text = String::from_utf8_lossy(&bytes);
+ let text = text.trim_matches(['\0', '\n', '\r']);
+ (!text.is_empty()).then(|| text.to_string())
+}
+
+fn configure_main_exit_marker(command: &mut Command, state_dir: &Path) {
+ command
+ .arg("--main-exit-marker")
+ .arg(state_dir.join(MAIN_PROCESS_EXITED_FILE));
+}
+
#[tonic::async_trait]
impl ComputeDriver for VmDriver {
async fn authenticate_sandbox(
@@ -3699,8 +3844,8 @@ impl ComputeDriver for VmDriver {
fn check_gpu_privileges() -> Result<(), String> {
if !rustix::process::geteuid().is_root() {
return Err(
- "GPU support requires root privileges for VFIO bind/unbind and TAP networking. \
- Run with sudo or ensure CAP_SYS_ADMIN + CAP_NET_ADMIN capabilities are set."
+ "GPU support requires root privileges for VFIO bind/unbind. \
+ Run with sudo or grant the host device-management capabilities required by VFIO."
.to_string(),
);
}
@@ -4718,147 +4863,25 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap {
environment
}
-/// Rewrites loopback host references in a gateway URL to a hostname the guest
-/// can reach via gvproxy.
-///
-/// The driver receives the gateway endpoint from `--openshell-endpoint`, which
-/// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is
-/// useless inside the guest because the guest's loopback interface is its own,
-/// not the host's. Inside the guest we need a name that gvproxy will translate
-/// into the host's loopback address.
-///
-/// We rewrite to `host.openshell.internal`, which gvproxy's embedded DNS resolves
-/// to the host-loopback IP `192.168.127.254`. gvproxy installs a default NAT entry
-/// rewriting that destination to the host's `127.0.0.1` and dialing out from the
-/// host process, so any port the host is listening on becomes reachable. The
-/// gateway IP `192.168.127.1` does **not** do this — it only listens on gvproxy's
-/// own service ports (DNS, DHCP, HTTP API). The guest init script also seeds the
-/// hostname in `/etc/hosts` so resolution works even if gvproxy's DNS isn't in
-/// resolv.conf (e.g. when DHCP fails).
-///
-/// Non-loopback URLs are returned unchanged.
-fn guest_visible_openshell_endpoint(endpoint: &str) -> String {
- let Ok(mut url) = Url::parse(endpoint) else {
- return endpoint.to_string();
- };
-
- let should_rewrite = match url.host() {
- Some(Host::Ipv4(ip)) => ip.is_loopback(),
- Some(Host::Ipv6(ip)) => ip.is_loopback(),
- Some(Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"),
- None => false,
- };
-
- if should_rewrite && url.set_host(Some(GVPROXY_HOST_LOOPBACK_ALIAS)).is_ok() {
- return url.to_string();
- }
-
- endpoint.to_string()
-}
-
-/// Whether a corporate proxy URL points at the gateway host itself, as seen
-/// from a QEMU/TAP guest whose TAP host address is `tap_host_ip`.
-///
-/// On the libkrun backend gvproxy NATs the host-loopback alias
-/// `host.openshell.internal` (and any loopback URL, which the driver rewrites
-/// to that alias) to the gateway host's `127.0.0.1`, so a proxy bound to host
-/// loopback is reachable from the guest. The QEMU/TAP backend used for GPU
-/// sandboxes has no equivalent: `host.openshell.internal` resolves to the TAP
-/// host address, and the driver's own nftables `input` chain accepts only the
-/// gateway port from the guest and drops the rest, so no proxy on the gateway
-/// host is reachable regardless of the address it binds.
-///
-/// The gateway host is therefore reached from a QEMU guest under exactly three
-/// spellings: the guest's own loopback (never the host's, but a configuration
-/// that plainly means the host), the documented host aliases that
-/// `write_host_gateway_aliases` seeds to the TAP host address, and that TAP
-/// host address written literally. `tap_host_ip` is this sandbox's allocated
-/// address, so the comparison must be made after the launch plan's subnet
-/// allocation; `None` means the plan carries no TAP host and only the
-/// address-independent spellings are classified.
-///
-/// gvproxy's `GVPROXY_HOST_LOOPBACK_IP` is deliberately **not** matched here.
-/// It is special only to libkrun; on QEMU/TAP it is an ordinary address that
-/// may well be routable through the guest's masqueraded egress, and rejecting
-/// it would refuse a working configuration.
-///
-/// Used to reject an unreachable configuration up front on the QEMU path
-/// instead of letting every policy-approved CONNECT time out.
-fn proxy_url_targets_gateway_host(url: &str, tap_host_ip: Option<&str>) -> bool {
- let Ok(parsed) = Url::parse(url) else {
- // Unparseable URLs are rejected by shared validation before launch.
- return false;
- };
- let tap_host = tap_host_ip.and_then(|ip| ip.parse::().ok());
- match parsed.host() {
- Some(Host::Ipv4(ip)) => ip.is_loopback() || tap_host == Some(IpAddr::V4(ip)),
- Some(Host::Ipv6(ip)) => ip.is_loopback() || tap_host == Some(IpAddr::V6(ip)),
- Some(Host::Domain(host)) => {
- host.eq_ignore_ascii_case("localhost")
- || host.eq_ignore_ascii_case(OPENSHELL_HOST_GATEWAY_ALIAS)
- || host.eq_ignore_ascii_case("host.containers.internal")
- || host.eq_ignore_ascii_case("host.docker.internal")
- }
- None => false,
- }
-}
-
-fn gateway_port_from_endpoint(endpoint: &str) -> Option {
- Url::parse(endpoint).ok().and_then(|url| url.port())
-}
-
-fn has_complete_qemu_network(plan: &LaunchPlan) -> bool {
- plan.tap_device.is_some()
- && plan.guest_ip.is_some()
- && plan.host_ip.is_some()
- && plan.vsock_cid.is_some()
- && plan.guest_mac.is_some()
-}
-
-fn guest_visible_openshell_endpoint_for_tap(endpoint: &str, host_ip: &str) -> String {
- let Ok(mut url) = Url::parse(endpoint) else {
- return endpoint.to_string();
- };
- if url.set_host(Some(host_ip)).is_ok() {
- url.to_string()
- } else {
- endpoint.to_string()
+fn random_boundary_token() -> String {
+ let mut token = String::with_capacity(64);
+ for byte in rand::random::<[u8; 32]>() {
+ write!(&mut token, "{byte:02x}").expect("writing to String cannot fail");
}
+ token
}
-fn build_guest_environment(
- sandbox: &Sandbox,
- config: &VmDriverConfig,
- endpoint_override: Option<&str>,
-) -> Vec {
- let openshell_endpoint = endpoint_override.map_or_else(
- || guest_visible_openshell_endpoint(&config.openshell_endpoint),
- String::from,
- );
- // 1. User-supplied environment (lowest priority).
- let user_env = merged_environment(sandbox);
+fn build_guest_environment(sandbox: &Sandbox, config: &VmDriverConfig) -> Vec {
+ // The guest receives only driver-owned boot metadata. Gateway credentials,
+ // TLS material, and logical-supervisor configuration remain on the host;
+ // workload environment is carried in the authenticated BoundaryConfig.
let mut environment: HashMap = HashMap::new();
- environment.extend(user_env.clone());
- if !user_env.is_empty()
- && let Ok(json) = serde_json::to_string(&user_env)
- {
- environment.insert(
- openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(),
- json,
- );
- }
-
- // 2. Required driver vars (highest priority -- always overwrite).
environment.insert("HOME".to_string(), "/root".to_string());
environment.insert(
"PATH".to_string(),
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(),
);
environment.insert("TERM".to_string(), "xterm".to_string());
- environment.insert(
- openshell_core::sandbox_env::ENDPOINT.to_string(),
- openshell_endpoint,
- );
environment.insert(
openshell_core::sandbox_env::SANDBOX_ID.to_string(),
sandbox.id.clone(),
@@ -4867,68 +4890,14 @@ fn build_guest_environment(
openshell_core::sandbox_env::SANDBOX.to_string(),
sandbox.name.clone(),
);
- environment.insert(
- openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(),
- GUEST_SSH_SOCKET_PATH.to_string(),
- );
- // The libkrun guest environment path does not preserve spaces in values
- // before guest startup. Use a whitespace-free base64url envelope so
- // command arguments remain lossless.
- let main_process =
- openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec_base64url(
- sandbox.spec.as_ref(),
- )
- .expect("main process config serialization cannot fail");
- environment.insert(
- openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(),
- main_process,
- );
environment.insert(
openshell_core::sandbox_env::LOG_LEVEL.to_string(),
openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level),
);
- if config.requires_tls_materials() {
- environment.insert(
- openshell_core::sandbox_env::TLS_CA.to_string(),
- GUEST_TLS_CA_PATH.to_string(),
- );
- environment.insert(
- openshell_core::sandbox_env::TLS_CERT.to_string(),
- GUEST_TLS_CERT_PATH.to_string(),
- );
- environment.insert(
- openshell_core::sandbox_env::TLS_KEY.to_string(),
- GUEST_TLS_KEY_PATH.to_string(),
- );
- }
environment.insert(
openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(),
openshell_core::telemetry::enabled_env_value().to_string(),
);
- // Runtime capabilities are driver-owned. The VM driver does not yet
- // provide policy DNS and transparent TCP interception.
- environment.insert(
- openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(),
- String::new(),
- );
- environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN);
- environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE);
- // Prevent user-supplied environment from overriding the TLS server name
- // the supervisor verifies — a sandbox user who can redirect the gateway
- // hostname could otherwise present a certificate for a name they control
- // and intercept the sandbox JWT.
- environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME);
- if sandbox
- .spec
- .as_ref()
- .is_some_and(|spec| !spec.sandbox_token.is_empty())
- {
- environment.insert(
- openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(),
- GUEST_SANDBOX_TOKEN_PATH.to_string(),
- );
- }
-
let mut pairs = environment.into_iter().collect::>();
pairs.sort_by(|left, right| left.0.cmp(&right.0));
pairs
@@ -5136,8 +5105,9 @@ fn write_oci_layout_for_manifest(
fn bootstrap_image_cache_identity(image_identity: &str) -> String {
format!(
- "{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}",
- openshell_core::VERSION
+ "{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:guest-{}:{image_identity}",
+ openshell_core::VERSION,
+ sandbox_guest_runtime_identity()
)
}
@@ -5256,26 +5226,6 @@ fn validate_restored_sandbox_state(
Ok(())
}
-#[derive(Debug, Clone)]
-struct GuestTlsMaterials {
- ca: Vec,
- cert: Vec,
- key: Vec,
-}
-
-async fn read_guest_tls_materials(paths: &VmDriverTlsPaths) -> Result {
- let ca = tokio::fs::read(&paths.ca)
- .await
- .map_err(|err| format!("read {}: {err}", paths.ca.display()))?;
- let cert = tokio::fs::read(&paths.cert)
- .await
- .map_err(|err| format!("read {}: {err}", paths.cert.display()))?;
- let key = tokio::fs::read(&paths.key)
- .await
- .map_err(|err| format!("read {}: {err}", paths.key.display()))?;
- Ok(GuestTlsMaterials { ca, cert, key })
-}
-
async fn overlay_template_image_ready(path: &Path, size_bytes: u64) -> Result {
match tokio::fs::metadata(path).await {
Ok(metadata) => Ok(metadata.is_file() && metadata.len() == size_bytes),
@@ -5360,36 +5310,19 @@ fn create_empty_sandbox_overlay_image(overlay_disk: &Path, size_bytes: u64) -> R
fn create_sandbox_overlay_image_from_template(
template_path: &Path,
overlay_disk: &Path,
- tls_materials: Option<&GuestTlsMaterials>,
- sandbox_token: Option<&str>,
) -> Result<(), String> {
- clone_or_copy_sparse_file(template_path, overlay_disk)?;
- if let Some(tls) = tls_materials {
- inject_guest_tls_materials(overlay_disk, tls)?;
- }
- if let Some(token) = sandbox_token {
- inject_guest_sandbox_token(overlay_disk, token)?;
- }
- Ok(())
+ clone_or_copy_sparse_file(template_path, overlay_disk)
}
fn prepare_sandbox_overlay_image(
template_path: &Path,
overlay_disk: &Path,
- tls_materials: Option<&GuestTlsMaterials>,
- sandbox_token: Option<&str>,
preparation: OverlayPreparation,
expected_size_bytes: u64,
) -> Result<(), String> {
if preparation == OverlayPreparation::PreserveExisting {
match fs::metadata(overlay_disk) {
Ok(metadata) if metadata.is_file() && metadata.len() == expected_size_bytes => {
- if let Some(tls) = tls_materials {
- inject_guest_tls_materials(overlay_disk, tls)?;
- }
- if let Some(token) = sandbox_token {
- inject_guest_sandbox_token(overlay_disk, token)?;
- }
return Ok(());
}
Ok(metadata) if metadata.is_file() => {
@@ -5416,37 +5349,63 @@ fn prepare_sandbox_overlay_image(
}
}
- create_sandbox_overlay_image_from_template(
- template_path,
- overlay_disk,
- tls_materials,
- sandbox_token,
- )
+ create_sandbox_overlay_image_from_template(template_path, overlay_disk)
}
-fn inject_guest_tls_materials(
+fn inject_guest_boundary_bundle(
overlay_disk: &Path,
- materials: &GuestTlsMaterials,
+ guest_path: &str,
+ config: &BoundaryConfig,
+ material: &BoundaryMutualTlsMaterial,
) -> Result<(), String> {
- write_rootfs_image_file(
- overlay_disk,
- &overlay_upper_path(GUEST_TLS_CA_PATH),
- &materials.ca,
- )?;
- write_rootfs_image_file(
- overlay_disk,
- &overlay_upper_path(GUEST_TLS_CERT_PATH),
- &materials.cert,
- )?;
- let key_path = overlay_upper_path(GUEST_TLS_KEY_PATH);
- write_rootfs_image_file(overlay_disk, &key_path, &materials.key)?;
- set_rootfs_image_file_mode(overlay_disk, &key_path, 0o600)
-}
-
-fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), String> {
- let token_path = overlay_upper_path(GUEST_SANDBOX_TOKEN_PATH);
- write_rootfs_image_file(overlay_disk, &token_path, format!("{token}\n").as_bytes())?;
- set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600)
+ let tls = match &config.listener {
+ BoundaryListener::Unix { tls, .. }
+ | BoundaryListener::TlsTcp { tls, .. }
+ | BoundaryListener::Vsock { tls, .. } => tls.clone(),
+ };
+ let encoded_config = config
+ .encode()
+ .map_err(|error| format!("encode VM boundary configuration: {error}"))?;
+ let config_path = overlay_upper_path(guest_path);
+ write_rootfs_image_file(overlay_disk, &config_path, &encoded_config)?;
+ set_rootfs_image_file_mode(overlay_disk, &config_path, 0o600)?;
+ for (guest_path, contents) in [
+ (
+ tls.certificate_chain_path,
+ material.sandbox_certificate_pem.as_bytes(),
+ ),
+ (
+ tls.private_key_path,
+ material.sandbox_private_key_pem.as_bytes(),
+ ),
+ (
+ tls.client_ca_certificate_path,
+ material.ca_certificate_pem.as_bytes(),
+ ),
+ ] {
+ let path = overlay_upper_path(guest_path.to_string_lossy().as_ref());
+ write_rootfs_image_file(overlay_disk, &path, contents)?;
+ set_rootfs_image_file_mode(overlay_disk, &path, 0o600)?;
+ }
+ Ok(())
+}
+
+fn guest_boundary_config_path(generation: &str) -> String {
+ format!("{GUEST_BOUNDARY_CONFIG_DIR}/bootstrap-{generation}.json")
+}
+
+fn guest_boundary_tls_paths(generation: &str) -> BoundaryServerTls {
+ BoundaryServerTls {
+ certificate_chain_path: PathBuf::from(format!(
+ "{GUEST_BOUNDARY_CONFIG_DIR}/sandbox-{generation}.crt"
+ )),
+ private_key_path: PathBuf::from(format!(
+ "{GUEST_BOUNDARY_CONFIG_DIR}/sandbox-{generation}.key"
+ )),
+ client_ca_certificate_path: PathBuf::from(format!(
+ "{GUEST_BOUNDARY_CONFIG_DIR}/supervisor-ca-{generation}.crt"
+ )),
+ }
}
#[allow(clippy::result_large_err)]
@@ -5496,26 +5455,34 @@ fn inject_guest_init_dropins(
span_status.finish(Ok(()))
}
-/// Build the corporate upstream-proxy arguments passed to the guest supervisor.
+/// Build the corporate upstream-proxy arguments passed to host control.
///
/// This operator-owned egress boundary travels on the supervisor's argv,
/// which sandbox spec/template environment and image `ENV` cannot influence.
-/// Credentials are never on argv — only the root-only guest path is passed;
-/// the supervisor reads the credential from that file.
-fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec {
+/// Credentials are never on argv; the supervisor reads them from the
+/// operator-owned host file.
+fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Result, String> {
let mut args = Vec::new();
if let Some(url) = &config.https_proxy {
args.push("--upstream-proxy".to_string());
args.push(url.clone());
+ let proxy_url = Url::parse(url)
+ .map_err(|error| format!("invalid upstream proxy endpoint '{url}': {error}"))?;
+ if proxy_url
+ .host_str()
+ .is_some_and(|host| HOST_LOOPBACK_ALIASES.contains(&host))
+ {
+ args.push("--upstream-proxy-dial-ip".to_string());
+ args.push("127.0.0.1".to_string());
+ }
}
if let Some(list) = &config.no_proxy {
args.push("--upstream-no-proxy".to_string());
args.push(list.clone());
}
- if config.proxy_auth_file.is_some() {
+ if let Some(path) = &config.proxy_auth_file {
args.push("--upstream-proxy-auth-file".to_string());
- // The guest path, never the gateway-host path the operator configured.
- args.push(GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string());
+ args.push(path.clone());
}
// Config validation guarantees the acknowledgement is `true` whenever an
// auth file is configured against an http:// proxy; the supervisor
@@ -5528,148 +5495,11 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec {
if config.proxy_connect_by_hostname == Some(true) {
args.push("--upstream-proxy-connect-by-hostname".to_string());
}
- if config.proxy_ca_bundle.is_some() {
+ if let Some(path) = &config.proxy_ca_bundle {
args.push("--upstream-proxy-ca-bundle".to_string());
- args.push(GUEST_PROXY_CA_PATH.to_string());
- }
- args
-}
-
-/// Render the supervisor argument list as newline-separated arguments.
-///
-/// One argument per line, verbatim: the guest reads the lines into an array
-/// without word splitting or globbing, so values containing spaces survive
-/// intact. An empty list renders an empty file, which the guest reads as "no
-/// extra arguments".
-fn render_guest_supervisor_args(args: &[String]) -> Vec {
- let mut body = args.join("\n");
- if !body.is_empty() {
- body.push('\n');
- }
- body.into_bytes()
-}
-
-/// Reject argument values the newline-delimited guest file cannot represent.
-///
-/// Every value here is operator-supplied config, so this is a guard against
-/// misconfiguration rather than an attack: a stray newline would otherwise
-/// split one value into two arguments in the guest.
-fn validate_guest_supervisor_args(args: &[String]) -> Result<(), String> {
- for arg in args {
- if arg.contains('\n') || arg.contains('\r') || arg.contains('\0') {
- return Err(
- "corporate proxy settings must not contain newline or NUL characters".to_string(),
- );
- }
+ args.push(path.clone());
}
- Ok(())
-}
-
-/// Read and validate the corporate proxy credential from the gateway host.
-///
-/// Uses the validators shared with the supervisor, so a credential accepted
-/// here is never rejected inside the guest. The error never carries the file
-/// contents.
-async fn read_sandbox_proxy_credential(path: &str) -> Result {
- let path_owned = path.to_string();
- let raw = tokio::task::spawn_blocking(move || {
- openshell_core::driver_utils::read_upstream_proxy_credential_file(&path_owned)
- })
- .await
- .map_err(|err| Status::internal(format!("proxy_auth_file read task failed: {err}")))?
- .map_err(Status::invalid_argument)?;
- let credential = openshell_core::driver_utils::parse_upstream_proxy_credential(&raw)
- .map_err(|err| Status::invalid_argument(format!("proxy_auth_file '{path}': {err}")))?;
- Ok(credential.to_string())
-}
-
-/// Read and validate the corporate proxy CA bundle from the gateway host.
-///
-/// Uses the reader shared with the supervisor, so the bundle is bounded and
-/// non-regular files are rejected (an operator path such as `/dev/zero` can
-/// otherwise exhaust driver memory), and a bundle accepted here contributes at
-/// least one trust anchor rustls accepts rather than merely looking like PEM.
-/// Checked here rather than only in the guest so the operator gets an error
-/// attributable to `proxy_ca_bundle` instead of an opaque supervisor startup
-/// failure inside every sandbox. The error never carries the file contents.
-async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> {
- let path_owned = path.to_string();
- let pem = tokio::task::spawn_blocking(move || {
- openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file(
- &path_owned,
- "proxy_ca_bundle",
- )
- })
- .await
- .map_err(|err| Status::internal(format!("proxy_ca_bundle read task failed: {err}")))?
- .map_err(Status::invalid_argument)?;
- Ok(pem.into_bytes())
-}
-
-/// Stage the corporate upstream-proxy configuration into the guest overlay.
-///
-/// Writes three files into the overlay upperdir the driver owns:
-///
-/// * the credential at [`GUEST_UPSTREAM_PROXY_AUTH_PATH`], mode `0600`;
-/// * the CA bundle at [`GUEST_PROXY_CA_PATH`], mode `0644` (a CA certificate
-/// is not secret);
-/// * the supervisor argument list at [`GUEST_SUPERVISOR_ARGS_PATH`], mode
-/// `0644`.
-///
-/// All three are written on every launch, empty when the corresponding
-/// setting is absent. Writing rather than skipping is what makes the channel
-/// unforgeable: the upperdir copy always shadows the read-only image layer, so
-/// a sandbox image cannot supply its own arguments or credential by baking a
-/// file at these paths, and cannot disable the operator's by omitting one. It
-/// also clears material a previous launch staged into a preserved overlay
-/// after the operator removed the setting.
-///
-/// A microVM has no bind mounts or container secrets, so the credential lives
-/// at rest inside the per-sandbox overlay disk on the host — the same
-/// delivery the per-sandbox gateway JWT already uses. It is removed with the
-/// sandbox when the state directory is deleted.
-#[allow(clippy::result_large_err)]
-async fn inject_guest_upstream_proxy(
- overlay_disk: &Path,
- config: &VmDriverConfig,
-) -> Result<(), Status> {
- // Written whether or not they are configured. Writing empty files when
- // the operator removed a setting clears material a previous launch staged
- // into a preserved overlay, and shadows anything an image baked at these
- // paths, so a staged file is only ever the one this launch produced.
- let credential = match config.proxy_auth_file.as_deref() {
- Some(path) => format!("{}\n", read_sandbox_proxy_credential(path).await?).into_bytes(),
- None => Vec::new(),
- };
- let credential_path = overlay_upper_path(GUEST_UPSTREAM_PROXY_AUTH_PATH);
- write_rootfs_image_file(overlay_disk, &credential_path, &credential)
- .map_err(|err| Status::internal(format!("write VM guest proxy credential: {err}")))?;
- set_rootfs_image_file_mode(overlay_disk, &credential_path, 0o600)
- .map_err(|err| Status::internal(format!("set VM guest proxy credential mode: {err}")))?;
-
- let ca_bundle = match config.proxy_ca_bundle.as_deref() {
- Some(path) => read_sandbox_proxy_ca_bundle(path).await?,
- None => Vec::new(),
- };
- let ca_path = overlay_upper_path(GUEST_PROXY_CA_PATH);
- write_rootfs_image_file(overlay_disk, &ca_path, &ca_bundle)
- .map_err(|err| Status::internal(format!("write VM guest proxy CA bundle: {err}")))?;
- set_rootfs_image_file_mode(overlay_disk, &ca_path, 0o644)
- .map_err(|err| Status::internal(format!("set VM guest proxy CA bundle mode: {err}")))?;
-
- let args = upstream_proxy_cli_args(config);
- validate_guest_supervisor_args(&args).map_err(Status::failed_precondition)?;
- let guest_path = overlay_upper_path(GUEST_SUPERVISOR_ARGS_PATH);
- write_rootfs_image_file(
- overlay_disk,
- &guest_path,
- &render_guest_supervisor_args(&args),
- )
- .map_err(|err| Status::internal(format!("write VM guest supervisor arguments: {err}")))?;
- set_rootfs_image_file_mode(overlay_disk, &guest_path, 0o644).map_err(|err| {
- Status::internal(format!("set VM guest supervisor arguments mode: {err}"))
- })?;
- Ok(())
+ Ok(args)
}
/// Render the drop-in allow-list as newline-separated, ASCII-sorted,
@@ -5869,47 +5699,6 @@ fn dir_size_bytes(path: &Path) -> Result {
Ok(total)
}
-#[cfg(test)]
-fn stage_guest_tls_materials(
- staging_dir: &Path,
- materials: &GuestTlsMaterials,
-) -> Result<(), String> {
- let tls_dir = staging_dir
- .join("upper")
- .join(GUEST_TLS_CA_PATH.trim_start_matches('/'))
- .parent()
- .ok_or_else(|| "guest TLS CA path has no parent".to_string())?
- .to_path_buf();
- fs::create_dir_all(&tls_dir)
- .map_err(|err| format!("create guest TLS dir {}: {err}", tls_dir.display()))?;
-
- let ca_path = staging_dir
- .join("upper")
- .join(GUEST_TLS_CA_PATH.trim_start_matches('/'));
- let cert_path = staging_dir
- .join("upper")
- .join(GUEST_TLS_CERT_PATH.trim_start_matches('/'));
- let key_path = staging_dir
- .join("upper")
- .join(GUEST_TLS_KEY_PATH.trim_start_matches('/'));
- fs::write(&ca_path, &materials.ca)
- .map_err(|err| format!("write guest TLS CA {}: {err}", ca_path.display()))?;
- fs::write(&cert_path, &materials.cert)
- .map_err(|err| format!("write guest TLS cert {}: {err}", cert_path.display()))?;
- fs::write(&key_path, &materials.key)
- .map_err(|err| format!("write guest TLS key {}: {err}", key_path.display()))?;
-
- #[cfg(unix)]
- {
- use std::os::unix::fs::PermissionsExt as _;
-
- fs::set_permissions(&key_path, fs::Permissions::from_mode(0o600))
- .map_err(|err| format!("chmod guest TLS key {}: {err}", key_path.display()))?;
- }
-
- Ok(())
-}
-
fn overlay_staging_dir(overlay_disk: &Path) -> PathBuf {
let parent = overlay_disk.parent().unwrap_or_else(|| Path::new("."));
parent.join(format!(
@@ -5939,6 +5728,33 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
}
}
+async fn terminate_sandbox_processes(process: &mut VmProcess) -> Result<(), std::io::Error> {
+ let supervisor_error = terminate_vm_process(&mut process.supervisor).await.err();
+ let vm_error = terminate_vm_process(&mut process.child).await.err();
+
+ match (supervisor_error, vm_error) {
+ (None, None) => Ok(()),
+ (Some(error), None) => Err(std::io::Error::other(format!("stop supervisor: {error}"))),
+ (None, Some(error)) => Err(std::io::Error::other(format!("stop vm: {error}"))),
+ (Some(supervisor), Some(vm)) => Err(std::io::Error::other(format!(
+ "stop supervisor: {supervisor}; stop vm: {vm}"
+ ))),
+ }
+}
+
+fn absolute_state_dir(state_dir: &Path) -> Result {
+ if state_dir.is_absolute() {
+ return Ok(state_dir.to_path_buf());
+ }
+ std::env::current_dir()
+ .map(|working_dir| working_dir.join(state_dir))
+ .map_err(|err| format!("failed to resolve VM driver state directory: {err}"))
+}
+
+fn isolate_host_control_environment(command: &mut Command) {
+ command.env_clear();
+}
+
#[tracing::instrument(
name = "vm.launch",
skip(command),
@@ -5949,6 +5765,7 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> {
vm.backend = ?backend,
)
)]
+#[allow(dead_code)]
fn spawn_vm_launcher(
command: &mut Command,
sandbox_id: &str,
@@ -6133,7 +5950,7 @@ fn pulling_layer_detail(metadata: &HashMap) -> Option {
#[cfg(test)]
mod tests {
use super::*;
- use crate::gpu::{SubnetAllocator, allocate_vsock_cid, mac_from_sandbox_id, tap_device_name};
+ use crate::gpu::allocate_vsock_cid;
use openshell_core::progress::{
PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY,
PROGRESS_COMPLETE_STEP_KEY,
@@ -6152,6 +5969,23 @@ mod tests {
static ENV_LOCK: std::sync::LazyLock> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(()));
+ #[test]
+ fn vm_console_diagnostic_is_bounded_to_the_tail() {
+ let directory = tempfile::tempdir().unwrap();
+ let console = directory.path().join("rootfs-console.log");
+ fs::write(&console, b"discard-this\nFATAL: sandbox startup failed\n").unwrap();
+
+ assert_eq!(
+ read_vm_console_tail(&console, 30).as_deref(),
+ Some("FATAL: sandbox startup failed")
+ );
+ assert_eq!(read_vm_console_tail(&console, 0), None);
+ assert_eq!(
+ read_vm_console_tail(&directory.path().join("missing"), 30),
+ None
+ );
+ }
+
#[test]
fn registry_throttling_errors_are_retryable() {
let error = OciDistributionError::RegistryError {
@@ -6655,7 +6489,7 @@ mod tests {
let parent = tracing::info_span!("vm.provision");
let result = driver
- .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh)
+ .prepare_runtime_overlay(Path::new("/unused"), OverlayPreparation::Fresh)
.instrument(parent)
.await;
assert!(result.is_err(), "overflow should stop before disk I/O");
@@ -7344,7 +7178,6 @@ mod tests {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
},
);
@@ -7387,8 +7220,6 @@ mod tests {
prepare_sandbox_overlay_image(
&template,
&overlay,
- None,
- None,
OverlayPreparation::PreserveExisting,
"saved-overlay".len() as u64,
)
@@ -7410,8 +7241,6 @@ mod tests {
prepare_sandbox_overlay_image(
&template,
&overlay,
- None,
- None,
OverlayPreparation::PreserveExisting,
"fresh-overlay".len() as u64,
)
@@ -7425,8 +7254,8 @@ mod tests {
#[test]
fn overlay_upper_path_targets_overlay_upperdir() {
assert_eq!(
- overlay_upper_path(GUEST_TLS_KEY_PATH),
- "/upper/opt/openshell/tls/tls.key"
+ overlay_upper_path(&guest_boundary_config_path("generation-123")),
+ "/upper/.openshell/state/bootstrap-generation-123.json"
);
}
@@ -7442,16 +7271,30 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
assert_eq!(driver.capabilities().default_image, "openshell/sandbox:dev");
}
+ #[test]
+ fn host_control_receives_driver_owned_completion_marker() {
+ let mut command = Command::new("openshell-sandbox");
+ configure_main_exit_marker(&mut command, Path::new("/private/sandboxes/sb-1"));
+ let args = command
+ .as_std()
+ .get_args()
+ .map(|arg| arg.to_string_lossy().into_owned())
+ .collect::>();
+ assert_eq!(
+ args,
+ [
+ "--main-exit-marker".to_string(),
+ "/private/sandboxes/sb-1/main-process-exited".to_string(),
+ ]
+ );
+ }
+
#[test]
fn resolved_sandbox_image_prefers_template_image() {
let driver = VmDriver {
@@ -7464,10 +7307,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
let sandbox = Sandbox {
@@ -7499,10 +7338,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
let sandbox = Sandbox {
@@ -7528,10 +7363,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
let sandbox = Sandbox {
@@ -7558,10 +7389,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
@@ -7583,10 +7410,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
@@ -7605,10 +7428,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
@@ -7640,7 +7459,7 @@ mod tests {
}
#[test]
- fn build_guest_environment_sets_supervisor_defaults() {
+ fn build_guest_environment_sets_sandbox_boot_metadata() {
let config = VmDriverConfig {
openshell_endpoint: "http://127.0.0.1:8080".to_string(),
..Default::default()
@@ -7652,16 +7471,18 @@ mod tests {
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
+ let env = build_guest_environment(&sandbox, &config);
assert!(env.contains(&"HOME=/root".to_string()));
- assert!(env.contains(&format!(
- "OPENSHELL_ENDPOINT=http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/"
- )));
assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string()));
assert!(env.contains(&"OPENSHELL_SANDBOX=breezy-rhinoceros".to_string()));
- assert!(env.contains(&format!(
- "OPENSHELL_SSH_SOCKET_PATH={GUEST_SSH_SOCKET_PATH}"
- )));
+ assert!(
+ !env.iter()
+ .any(|entry| entry.starts_with("OPENSHELL_ENDPOINT="))
+ );
+ assert!(
+ !env.iter()
+ .any(|entry| entry.starts_with("OPENSHELL_SSH_SOCKET_PATH="))
+ );
}
#[test]
@@ -7675,78 +7496,45 @@ mod tests {
}
#[test]
- fn persisted_legacy_sandbox_without_command_uses_scratch_main() {
+ fn build_guest_environment_keeps_user_values_in_child_channel() {
let config = VmDriverConfig {
openshell_endpoint: "http://127.0.0.1:8080".to_string(),
..Default::default()
};
- // Requests persisted before the canonical-main contract have a
- // present DriverSandboxSpec but no command or tty fields.
let sandbox = Sandbox {
- id: "legacy-sandbox".to_string(),
- name: "legacy-sandbox".to_string(),
- spec: Some(SandboxSpec::default()),
- ..Default::default()
- };
-
- let env = build_guest_environment(&sandbox, &config, None);
- let encoded = env
- .iter()
- .find_map(|entry| {
- entry.strip_prefix(&format!(
- "{}=",
- openshell_core::sandbox_env::MAIN_PROCESS_SPEC
- ))
- })
- .expect("main process environment");
- let main = openshell_core::sandbox_env::MainProcessConfig::decode(encoded)
- .expect("legacy persisted request should produce a valid main config");
-
- assert_eq!(
- main,
- openshell_core::sandbox_env::MainProcessConfig::scratch()
- );
- }
-
- #[test]
- fn build_guest_environment_preserves_main_command_spaces() {
- let config = VmDriverConfig {
- openshell_endpoint: "https://127.0.0.1:8080".to_string(),
- ..Default::default()
- };
- let command = vec![
- "sh".to_string(),
- "-lc".to_string(),
- "echo ready; while true; do sleep 1; done".to_string(),
- ];
- let sandbox = Sandbox {
- id: "space-command".to_string(),
- name: "space-command".to_string(),
+ id: "sandbox-123".to_string(),
+ name: "sandbox-123".to_string(),
spec: Some(SandboxSpec {
- command: command.clone(),
+ environment: HashMap::from([
+ ("LD_PRELOAD".to_string(), "/workload/evil.so".to_string()),
+ ("BAD;touch /root/pwned".to_string(), "value".to_string()),
+ ]),
..Default::default()
}),
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
- let encoded = env
- .iter()
- .find_map(|entry| {
- entry.strip_prefix(&format!(
- "{}=",
- openshell_core::sandbox_env::MAIN_PROCESS_SPEC
- ))
- })
- .expect("main process environment");
+ let env = build_guest_environment(&sandbox, &config);
- assert!(!encoded.contains(char::is_whitespace));
- let main = openshell_core::sandbox_env::MainProcessConfig::decode(encoded).unwrap();
- assert_eq!(main.command, command);
+ assert!(!env.iter().any(|entry| entry.starts_with("LD_PRELOAD=")));
+ assert!(!env.iter().any(|entry| entry.starts_with("BAD;")));
+ assert!(
+ !env.iter()
+ .any(|entry| { entry.starts_with(openshell_core::sandbox_env::USER_ENVIRONMENT) })
+ );
+ let child_env = merged_environment(&sandbox);
+ assert_eq!(
+ child_env.get("LD_PRELOAD"),
+ Some(&"/workload/evil.so".to_string())
+ );
+ assert_eq!(
+ child_env.get("BAD;touch /root/pwned"),
+ Some(&"value".to_string())
+ );
}
#[test]
- fn build_guest_environment_uses_token_file_without_raw_token_env() {
+ fn build_guest_environment_excludes_all_gateway_credentials() {
let config = VmDriverConfig {
openshell_endpoint: "http://127.0.0.1:8080".to_string(),
..Default::default()
@@ -7765,16 +7553,16 @@ mod tests {
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
+ let env = build_guest_environment(&sandbox, &config);
assert!(!env.iter().any(|v| v.starts_with(&format!(
"{}=",
openshell_core::sandbox_env::SANDBOX_TOKEN
))));
- assert!(env.contains(&format!(
- "{}={GUEST_SANDBOX_TOKEN_PATH}",
+ assert!(!env.iter().any(|v| v.starts_with(&format!(
+ "{}=",
openshell_core::sandbox_env::SANDBOX_TOKEN_FILE
- )));
+ ))));
}
#[test]
@@ -7796,7 +7584,7 @@ mod tests {
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
+ let env = build_guest_environment(&sandbox, &config);
assert!(
!env.iter().any(|v| v.starts_with(&format!(
@@ -7833,7 +7621,7 @@ mod tests {
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
+ let env = build_guest_environment(&sandbox, &config);
let telemetry_entries = env
.iter()
.filter(|entry| {
@@ -7853,102 +7641,6 @@ mod tests {
);
}
- #[test]
- fn build_guest_environment_clears_unsupported_network_capabilities() {
- let config = VmDriverConfig {
- openshell_endpoint: "http://127.0.0.1:8080".to_string(),
- ..Default::default()
- };
- let sandbox = Sandbox {
- id: "sandbox-123".to_string(),
- name: "sandbox-123".to_string(),
- spec: Some(SandboxSpec {
- environment: HashMap::from([(
- openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(),
- openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(),
- )]),
- ..Default::default()
- }),
- ..Default::default()
- };
- let env = build_guest_environment(&sandbox, &config, None);
- assert!(env.contains(&format!(
- "{}=",
- openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES
- )));
- assert!(!env.contains(&format!(
- "{}={}",
- openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES,
- openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY
- )));
- }
-
- #[test]
- fn build_guest_environment_uses_endpoint_override_for_tap() {
- let config = VmDriverConfig {
- openshell_endpoint: "http://127.0.0.1:8080".to_string(),
- ..Default::default()
- };
- let sandbox = Sandbox {
- id: "sandbox-123".to_string(),
- name: "sandbox-123".to_string(),
- spec: Some(SandboxSpec::default()),
- ..Default::default()
- };
-
- let env = build_guest_environment(&sandbox, &config, Some("http://10.0.128.1:8080"));
- assert!(
- env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()),
- "TAP endpoint override must replace the default"
- );
- let endpoint_count = env
- .iter()
- .filter(|e| e.starts_with("OPENSHELL_ENDPOINT="))
- .count();
- assert_eq!(
- endpoint_count, 1,
- "must have exactly one OPENSHELL_ENDPOINT"
- );
- }
-
- #[test]
- fn guest_visible_openshell_endpoint_rewrites_loopback_hosts_to_gvproxy_host_alias() {
- assert_eq!(
- guest_visible_openshell_endpoint("http://127.0.0.1:8080"),
- format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/")
- );
- assert_eq!(
- guest_visible_openshell_endpoint("http://localhost:8080"),
- format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/")
- );
- assert_eq!(
- guest_visible_openshell_endpoint("https://[::1]:8443"),
- format!("https://{GVPROXY_HOST_LOOPBACK_ALIAS}:8443/")
- );
- }
-
- #[test]
- fn guest_visible_openshell_endpoint_preserves_non_loopback_hosts() {
- assert_eq!(
- guest_visible_openshell_endpoint(&format!(
- "http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080"
- )),
- format!("http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080")
- );
- assert_eq!(
- guest_visible_openshell_endpoint(&format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080")),
- format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080")
- );
- assert_eq!(
- guest_visible_openshell_endpoint("http://192.168.127.1:8080"),
- "http://192.168.127.1:8080"
- );
- assert_eq!(
- guest_visible_openshell_endpoint("https://gateway.internal:8443"),
- "https://gateway.internal:8443"
- );
- }
-
#[test]
fn image_reference_registry_host_defaults_to_docker_hub() {
assert_eq!(image_reference_registry_host("ubuntu:24.04"), "docker.io");
@@ -8086,7 +7778,7 @@ mod tests {
}
#[test]
- fn build_guest_environment_includes_tls_paths_for_https_endpoint() {
+ fn build_guest_environment_keeps_tls_paths_host_side() {
let config = VmDriverConfig {
openshell_endpoint: "https://127.0.0.1:8443".to_string(),
guest_tls_ca: Some(PathBuf::from("/host/ca.crt")),
@@ -8101,10 +7793,8 @@ mod tests {
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
- assert!(env.contains(&format!("OPENSHELL_TLS_CA={GUEST_TLS_CA_PATH}")));
- assert!(env.contains(&format!("OPENSHELL_TLS_CERT={GUEST_TLS_CERT_PATH}")));
- assert!(env.contains(&format!("OPENSHELL_TLS_KEY={GUEST_TLS_KEY_PATH}")));
+ let env = build_guest_environment(&sandbox, &config);
+ assert!(!env.iter().any(|entry| entry.starts_with("OPENSHELL_TLS_")));
}
#[test]
@@ -8134,10 +7824,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
@@ -8169,6 +7855,7 @@ mod tests {
record.state_dir = retry_state_dir;
record.process = Some(Arc::new(Mutex::new(VmProcess {
child: spawn_exited_child(),
+ supervisor: spawn_exited_child(),
deleting: false,
})));
}
@@ -8198,10 +7885,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
@@ -8221,7 +7904,6 @@ mod tests {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
},
);
@@ -8254,10 +7936,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(LifecycleExtensionRegistry::new()),
};
@@ -8278,7 +7956,6 @@ mod tests {
process: None,
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
},
);
@@ -8364,6 +8041,61 @@ mod tests {
.expect("dns endpoint should be accepted");
}
+ #[test]
+ fn host_control_endpoint_rewrites_guest_host_aliases() {
+ for host in HOST_LOOPBACK_ALIASES {
+ assert_eq!(
+ host_control_openshell_endpoint(&format!("https://{host}:8443/control"))
+ .expect("guest alias should be rewritten"),
+ (
+ "https://127.0.0.1:8443/control".to_string(),
+ Some((*host).to_string()),
+ ),
+ "host alias {host}"
+ );
+ }
+ }
+
+ #[test]
+ fn host_control_endpoint_preserves_remote_gateway() {
+ assert_eq!(
+ host_control_openshell_endpoint("https://gateway.internal:8443")
+ .expect("remote gateway should be preserved"),
+ ("https://gateway.internal:8443".to_string(), None)
+ );
+ }
+
+ #[test]
+ fn relative_state_dir_is_resolved_from_the_working_directory() {
+ let working_dir = std::env::current_dir().expect("working directory");
+ assert_eq!(
+ absolute_state_dir(Path::new("target/driver-state")).expect("resolve state dir"),
+ working_dir.join("target/driver-state")
+ );
+
+ let absolute = working_dir.join("existing-absolute-state");
+ assert_eq!(
+ absolute_state_dir(&absolute).expect("preserve absolute state dir"),
+ absolute
+ );
+ }
+
+ #[test]
+ fn host_control_environment_contains_only_explicit_values() {
+ let mut command = Command::new("openshell-sandbox");
+ command.env("UNTRUSTED_PARENT_VALUE", "must-not-leak");
+ isolate_host_control_environment(&mut command);
+ command.env("DRIVER_OWNED_VALUE", "kept");
+
+ let environment = command.as_std().get_envs().collect::>();
+ assert_eq!(environment.len(), 1);
+ assert_eq!(environment[0].0, "DRIVER_OWNED_VALUE");
+ assert_eq!(
+ environment[0].1.and_then(std::ffi::OsStr::to_str),
+ Some("kept")
+ );
+ }
+
#[test]
fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() {
assert_eq!(
@@ -8376,14 +8108,14 @@ mod tests {
}
#[test]
- fn bootstrap_image_cache_identity_includes_rootfs_layout_and_openshell_version() {
- assert_eq!(
- bootstrap_image_cache_identity("sha256:bootstrap-image"),
- format!(
- "sandbox-bootstrap-rootfs-ext4-v3:openshell-{}:sha256:bootstrap-image",
- openshell_core::VERSION
- )
- );
+ fn bootstrap_image_cache_identity_includes_rootfs_layout_version_and_guest_runtime() {
+ let identity = bootstrap_image_cache_identity("sha256:bootstrap-image");
+ assert!(identity.starts_with(&format!(
+ "sandbox-bootstrap-rootfs-ext4-v4:openshell-{}:guest-",
+ openshell_core::VERSION
+ )));
+ assert!(identity.ends_with(":sha256:bootstrap-image"));
+ assert!(identity.contains(&sandbox_guest_runtime_identity()));
}
#[test]
@@ -8483,96 +8215,6 @@ mod tests {
);
}
- #[tokio::test]
- async fn read_guest_tls_materials_reports_missing_input() {
- let base = unique_temp_dir();
- let source_dir = base.join("missing-source");
-
- let err = read_guest_tls_materials(&VmDriverTlsPaths {
- ca: source_dir.join("ca.crt"),
- cert: source_dir.join("tls.crt"),
- key: source_dir.join("tls.key"),
- })
- .await
- .expect_err("missing TLS materials should fail before image injection");
-
- assert!(err.contains("ca.crt"));
-
- let _ = std::fs::remove_dir_all(base);
- }
-
- #[cfg(unix)]
- #[test]
- fn stage_guest_tls_materials_places_files_in_overlay_upper_with_private_key_mode() {
- use std::os::unix::fs::PermissionsExt as _;
-
- let base = unique_temp_dir();
- let materials = GuestTlsMaterials {
- ca: b"ca".to_vec(),
- cert: b"cert".to_vec(),
- key: b"key".to_vec(),
- };
-
- stage_guest_tls_materials(&base, &materials).expect("stage TLS materials");
-
- assert_eq!(
- fs::read(
- base.join("upper")
- .join(GUEST_TLS_CA_PATH.trim_start_matches('/'))
- )
- .unwrap(),
- b"ca"
- );
- assert_eq!(
- fs::read(
- base.join("upper")
- .join(GUEST_TLS_CERT_PATH.trim_start_matches('/'))
- )
- .unwrap(),
- b"cert"
- );
- let key_path = base
- .join("upper")
- .join(GUEST_TLS_KEY_PATH.trim_start_matches('/'));
- assert_eq!(fs::read(&key_path).unwrap(), b"key");
- assert_eq!(
- fs::metadata(&key_path).unwrap().permissions().mode() & 0o777,
- 0o600
- );
-
- let _ = std::fs::remove_dir_all(base);
- }
-
- #[test]
- fn subnet_allocator_assigns_and_releases() {
- let mut alloc = SubnetAllocator::new(Ipv4Addr::new(10, 0, 128, 0), 17);
- let s1 = alloc.allocate("sandbox-1").unwrap();
- assert_eq!(s1.host_ip, Ipv4Addr::new(10, 0, 128, 1));
- assert_eq!(s1.guest_ip, Ipv4Addr::new(10, 0, 128, 2));
- assert_eq!(s1.prefix_len, 30);
-
- let s2 = alloc.allocate("sandbox-2").unwrap();
- assert_ne!(s1.host_ip, s2.host_ip);
-
- alloc.release("sandbox-1");
- let s3 = alloc.allocate("sandbox-3").unwrap();
- assert!(s3.host_ip != s2.host_ip);
- }
-
- #[test]
- fn tap_device_name_fits_ifnamsiz() {
- let name = tap_device_name("sandbox-abc-def-ghi");
- assert!(name.len() <= 15);
- assert!(name.starts_with("vmtap-"));
- }
-
- #[test]
- fn mac_address_is_locally_administered() {
- let mac = mac_from_sandbox_id("test-sandbox");
- assert_eq!(mac[0] & 0x02, 0x02);
- assert_eq!(mac[0] & 0x01, 0x00);
- }
-
#[test]
fn vsock_cid_monotonically_increases() {
let cid1 = allocate_vsock_cid();
@@ -8617,6 +8259,7 @@ mod tests {
};
let process = Arc::new(Mutex::new(VmProcess {
child,
+ supervisor: spawn_exited_child(),
deleting: false,
}));
@@ -8629,7 +8272,6 @@ mod tests {
process: Some(process),
provisioning_task: None,
gpu_bdf: None,
- qemu_network_allocated: false,
deleting: false,
},
);
@@ -8657,10 +8299,6 @@ mod tests {
image_cache_lock: Arc::new(Mutex::new(())),
events,
gpu_inventory: None,
- subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
- Ipv4Addr::new(10, 0, 128, 0),
- 17,
- ))),
lifecycle_extensions: Arc::new(extensions),
}
}
@@ -8739,11 +8377,7 @@ mod tests {
assert_eq!(plan.backend, VmBackend::Libkrun);
assert_eq!(plan.vcpus, 2);
assert_eq!(plan.mem_mib, 2048);
- assert!(plan.tap_device.is_none());
- assert!(plan.guest_ip.is_none());
- assert!(plan.host_ip.is_none());
assert!(plan.vsock_cid.is_none());
- assert!(plan.guest_mac.is_none());
assert!(plan.gpu_bdf.is_none());
assert!(plan.env.is_empty());
}
@@ -8766,11 +8400,7 @@ mod tests {
assert_eq!(plan.vcpus, 8);
assert_eq!(plan.mem_mib, 16384);
assert_eq!(plan.gpu_bdf.as_deref(), Some("0000:01:00.0"));
- assert!(plan.tap_device.is_some());
- assert!(plan.guest_ip.is_some());
- assert!(plan.host_ip.is_some());
assert!(plan.vsock_cid.is_some());
- assert!(plan.guest_mac.is_some());
}
#[test]
@@ -8784,12 +8414,7 @@ mod tests {
kernel_profile: None,
kernel_image: Some(PathBuf::from("/tmp/openshell-test-kernel")),
gpu_bdf: None,
- tap_device: None,
- guest_ip: None,
- host_ip: None,
vsock_cid: None,
- guest_mac: None,
- gateway_port: None,
guest_init_dropins: Vec::new(),
env: Vec::new(),
};
@@ -8822,13 +8447,7 @@ mod tests {
.expect("backend feature should resolve");
assert_eq!(plan.backend, VmBackend::Qemu);
- assert!(plan.tap_device.is_some());
- assert!(plan.guest_ip.is_some());
- assert!(plan.host_ip.is_some());
assert!(plan.vsock_cid.is_some());
- assert!(plan.guest_mac.is_some());
-
- driver.release_subnet("sandbox-vfio");
}
#[test]
@@ -8844,11 +8463,7 @@ mod tests {
.expect("backend requirement should resolve");
assert_eq!(plan.backend, VmBackend::Qemu);
- assert!(plan.tap_device.is_some());
- assert!(plan.guest_ip.is_some());
- assert!(plan.host_ip.is_some());
-
- driver.release_subnet("sandbox-qemu");
+ assert!(plan.vsock_cid.is_some());
}
#[test]
@@ -8864,7 +8479,6 @@ mod tests {
.expect("guest init feature should resolve");
assert_eq!(plan.backend, VmBackend::Libkrun);
- assert!(plan.tap_device.is_none());
}
#[test]
@@ -8927,12 +8541,7 @@ mod tests {
kernel_profile: None,
kernel_image: None,
gpu_bdf: None,
- tap_device: None,
- guest_ip: None,
- host_ip: None,
vsock_cid: None,
- guest_mac: None,
- gateway_port: None,
guest_init_dropins: Vec::new(),
env: Vec::new(),
};
@@ -8951,12 +8560,7 @@ mod tests {
kernel_profile: None,
kernel_image: None,
gpu_bdf: None,
- tap_device: Some("vmtap-x".to_string()),
- guest_ip: Some("10.0.0.2".to_string()),
- host_ip: Some("10.0.0.1".to_string()),
vsock_cid: Some(7),
- guest_mac: Some("02:00:00:00:00:01".to_string()),
- gateway_port: Some(8080),
guest_init_dropins: Vec::new(),
env: Vec::new(),
};
@@ -8984,12 +8588,7 @@ mod tests {
kernel_profile: None,
kernel_image: None,
gpu_bdf: None,
- tap_device: Some("vmtap-x".to_string()),
- guest_ip: Some("10.0.0.2".to_string()),
- host_ip: Some("10.0.0.1".to_string()),
vsock_cid: Some(7),
- guest_mac: Some("02:00:00:00:00:01".to_string()),
- gateway_port: Some(8080),
guest_init_dropins: Vec::new(),
env: Vec::new(),
};
@@ -9048,72 +8647,43 @@ mod tests {
);
}
- #[test]
- fn proxy_material_is_staged_inside_the_per_sandbox_overlay() {
- // Everything the driver stages lands in the overlay upperdir, which
- // lives in the sandbox's own state directory. That is what makes the
- // credential removable with the sandbox (remove_sandbox_state_dir
- // deletes the whole directory) and unforgeable by the guest image
- // (the upperdir shadows the read-only image layer).
- for guest_path in [
- GUEST_UPSTREAM_PROXY_AUTH_PATH,
- GUEST_PROXY_CA_PATH,
- GUEST_SUPERVISOR_ARGS_PATH,
- ] {
- assert!(
- guest_path.starts_with("/opt/openshell/"),
- "{guest_path} must be under the reserved guest control root"
- );
- assert_eq!(
- overlay_upper_path(guest_path),
- format!("/upper{guest_path}"),
- "{guest_path} must be staged into the overlay upperdir"
- );
- }
- }
-
#[test]
fn upstream_proxy_args_are_empty_without_a_configured_proxy() {
- assert!(upstream_proxy_cli_args(&VmDriverConfig::default()).is_empty());
- // The file is still written, empty, so the guest cannot fall back to
- // an image-baked argument list.
- assert!(render_guest_supervisor_args(&[]).is_empty());
+ assert!(
+ upstream_proxy_cli_args(&VmDriverConfig::default())
+ .unwrap()
+ .is_empty()
+ );
}
#[test]
- fn upstream_proxy_args_pass_guest_paths_not_host_paths() {
+ fn upstream_proxy_args_pass_host_paths_to_host_control() {
let config = proxy_config(
Some("http://proxy.corp.test:3128"),
Some("/etc/openshell/secrets/proxy-auth"),
Some("/etc/openshell/tls/corp-ca.pem"),
);
- let args = upstream_proxy_cli_args(&config);
+ let args = upstream_proxy_cli_args(&config).unwrap();
- // The credential and CA live at fixed guest paths; the gateway-host
- // paths the operator configured must never reach the guest argv.
+ // Control runs on the gateway host and receives the operator-owned
+ // paths directly; neither path is copied into the guest.
let auth = args
.iter()
.position(|arg| arg == "--upstream-proxy-auth-file")
.map(|i| args[i + 1].as_str());
- assert_eq!(auth, Some(GUEST_UPSTREAM_PROXY_AUTH_PATH));
+ assert_eq!(auth, Some("/etc/openshell/secrets/proxy-auth"));
let ca = args
.iter()
.position(|arg| arg == "--upstream-proxy-ca-bundle")
.map(|i| args[i + 1].as_str());
- assert_eq!(ca, Some(GUEST_PROXY_CA_PATH));
- assert!(
- !args
- .iter()
- .any(|arg| arg.contains("/etc/openshell/secrets") || arg.contains("corp-ca.pem")),
- "host paths leaked into the guest argv: {args:?}"
- );
+ assert_eq!(ca, Some("/etc/openshell/tls/corp-ca.pem"));
}
#[test]
fn upstream_proxy_args_pass_only_explicit_opt_ins() {
let mut config = proxy_config(Some("https://proxy.corp.test:3130"), None, None);
config.no_proxy = Some("10.0.0.0/8,.svc.cluster.local".to_string());
- let args = upstream_proxy_cli_args(&config);
+ let args = upstream_proxy_cli_args(&config).unwrap();
assert_eq!(
args,
vec![
@@ -9129,44 +8699,35 @@ mod tests {
config.proxy_connect_by_hostname = Some(false);
assert!(
!upstream_proxy_cli_args(&config)
+ .unwrap()
.iter()
.any(|arg| arg == "--upstream-proxy-connect-by-hostname")
);
config.proxy_connect_by_hostname = Some(true);
assert!(
upstream_proxy_cli_args(&config)
+ .unwrap()
.iter()
.any(|arg| arg == "--upstream-proxy-connect-by-hostname")
);
}
#[test]
- fn guest_supervisor_args_render_one_argument_per_line() {
- let args = vec![
- "--upstream-proxy".to_string(),
- "http://proxy.corp.test:3128".to_string(),
- "--upstream-no-proxy".to_string(),
- "a.example, b.example".to_string(),
- ];
- // A value containing a space stays one line, so the guest reads it
- // back as a single argument rather than word-splitting it.
- assert_eq!(
- String::from_utf8(render_guest_supervisor_args(&args)).unwrap(),
- "--upstream-proxy\nhttp://proxy.corp.test:3128\n--upstream-no-proxy\na.example, b.example\n"
- );
- }
-
- #[test]
- fn guest_supervisor_args_reject_line_breaking_values() {
- // A newline would split one operator value into two guest arguments.
- for bad in ["a\nb", "a\rb", "a\0b"] {
- assert!(
- validate_guest_supervisor_args(&[bad.to_string()]).is_err(),
- "{bad:?} must be rejected"
+ fn upstream_proxy_args_route_vm_host_aliases_to_host_loopback() {
+ for alias in HOST_LOOPBACK_ALIASES {
+ let config = proxy_config(Some(&format!("http://{alias}:3128")), None, None);
+ let args = upstream_proxy_cli_args(&config).unwrap();
+ assert_eq!(
+ args,
+ vec![
+ "--upstream-proxy".to_string(),
+ format!("http://{alias}:3128"),
+ "--upstream-proxy-dial-ip".to_string(),
+ "127.0.0.1".to_string(),
+ ],
+ "host alias {alias} must dial host loopback without changing its TLS identity"
);
}
- validate_guest_supervisor_args(&["--upstream-proxy".to_string()])
- .expect("ordinary arguments are accepted");
}
#[test]
@@ -9200,192 +8761,16 @@ mod tests {
assert!(err.contains("proxy_auth_allow_insecure"), "{err}");
}
- #[tokio::test]
- async fn proxy_ca_bundle_without_a_certificate_fails_the_sandbox() {
- let dir = std::env::temp_dir().join(format!("openshell-vm-ca-{}", std::process::id()));
- std::fs::create_dir_all(&dir).unwrap();
- let path = dir.join("not-a-ca.pem");
- std::fs::write(&path, b"this is not a certificate\n").unwrap();
-
- let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap())
- .await
- .expect_err("a certificate-free bundle must fail closed");
- assert_eq!(err.code(), Code::InvalidArgument);
- assert!(err.message().contains("no PEM certificate"), "{err}");
-
- std::fs::write(&path, b"").unwrap();
- let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap())
- .await
- .expect_err("an empty bundle must fail closed");
- assert!(err.message().contains("no PEM certificate"), "{err}");
-
- // PEM framing that base64-decodes but is not X.509 DER: accepted by
- // `rustls_pemfile` alone, contributes zero trust anchors at runtime,
- // and so would make every guest supervisor fail after boot.
- std::fs::write(
- &path,
- b"-----BEGIN CERTIFICATE-----\nAQID\n-----END CERTIFICATE-----\n",
- )
- .unwrap();
- let err = read_sandbox_proxy_ca_bundle(path.to_str().unwrap())
- .await
- .expect_err("a bundle with invalid DER must fail closed");
- assert_eq!(err.code(), Code::InvalidArgument);
- assert!(err.message().contains("no usable trust anchors"), "{err}");
-
- let err = read_sandbox_proxy_ca_bundle(dir.join("missing.pem").to_str().unwrap())
- .await
- .expect_err("an unreadable bundle must fail closed");
- assert!(err.message().contains("could not be read"), "{err}");
-
- // A special file must be rejected on its type, not read: an
- // unbounded read of /dev/zero would exhaust driver memory.
- #[cfg(unix)]
- {
- let err = read_sandbox_proxy_ca_bundle("/dev/zero")
- .await
- .expect_err("a non-regular bundle path must fail closed");
- assert_eq!(err.code(), Code::InvalidArgument);
- assert!(err.message().contains("not a regular file"), "{err}");
- }
-
- // Oversized regular file: rejected on the stat'd length, again
- // without reading it whole.
- let oversized = dir.join("oversized.pem");
- let bound = openshell_core::driver_utils::MAX_UPSTREAM_PROXY_CA_BUNDLE_BYTES;
- std::fs::write(&oversized, vec![b'x'; usize::try_from(bound).unwrap() + 1]).unwrap();
- let err = read_sandbox_proxy_ca_bundle(oversized.to_str().unwrap())
- .await
- .expect_err("an oversized bundle must fail closed");
- assert_eq!(err.code(), Code::InvalidArgument);
- assert!(err.message().contains("exceeds"), "{err}");
-
- std::fs::remove_dir_all(&dir).unwrap();
- }
-
- #[test]
- fn qemu_backend_rejects_a_gateway_host_proxy() {
- // gvproxy's host-loopback NAT has no QEMU/TAP equivalent, so a proxy
- // on the gateway host is unreachable from a GPU sandbox and must be
- // rejected rather than time out on every CONNECT. The address that
- // reaches the gateway host from a QEMU guest is this sandbox's own
- // TAP host address, so the classifier is parameterized by it.
- let tap_host = Some("10.0.128.1");
- for url in [
- "http://host.openshell.internal:8080",
- "http://host.containers.internal:8080",
- "http://host.docker.internal:8080",
- "http://127.0.0.1:8080",
- "http://localhost:8080",
- "https://[::1]:8080",
- // The address the aliases above resolve to inside the guest.
- "http://10.0.128.1:8080",
- ] {
- assert!(proxy_url_targets_gateway_host(url, tap_host), "{url}");
- }
- for url in [
- "http://proxy.corp.example:8080",
- "https://10.1.2.3:3128",
- // Special only to libkrun/gvproxy. On QEMU/TAP it is an ordinary
- // address that may be routable through the guest's masqueraded
- // egress, so rejecting it would refuse a working configuration.
- "http://192.168.127.254:8080",
- // Another sandbox's TAP host, not this one's.
- "http://10.0.128.5:8080",
- "not a url",
- ] {
- assert!(!proxy_url_targets_gateway_host(url, tap_host), "{url}");
- }
-
- // Without an allocated TAP host only the address-independent
- // spellings classify; the loopback and alias guards still hold.
- assert!(proxy_url_targets_gateway_host(
- "http://127.0.0.1:8080",
- None
- ));
- assert!(proxy_url_targets_gateway_host(
- "http://host.openshell.internal:8080",
- None
- ));
- assert!(!proxy_url_targets_gateway_host(
- "http://10.0.128.1:8080",
- None
- ));
- }
-
- #[test]
- fn qemu_launch_plan_rejects_a_proxy_at_the_allocated_tap_host() {
- // The preflight has to run against the address this sandbox actually
- // got, which only exists once the launch plan's subnet is allocated.
- // A proxy there is what `host.openshell.internal` resolves to in the
- // guest, and the driver's own nftables input chain drops the port.
- let probe = test_driver_with_extensions(LifecycleExtensionRegistry::new());
- let tap_host = probe
- .build_vm_launch_plan("sandbox-proxy-tap", true, true, None)
- .expect("gpu plan should build")
- .host_ip
- .expect("a QEMU plan carries a TAP host address");
- probe.release_subnet("sandbox-proxy-tap");
-
- let driver = test_driver_with_proxy(&format!("http://{tap_host}:8080"));
- let mut plan = driver
- .build_vm_launch_plan("sandbox-proxy-tap", true, true, None)
- .expect("gpu plan should build");
- assert_eq!(plan.host_ip.as_deref(), Some(tap_host.as_str()));
-
- let err = driver
- .resolve_launch_plan_backend("sandbox-proxy-tap", true, None, &mut plan)
- .expect_err("a proxy at the TAP host address is unreachable from the guest");
- assert_eq!(err.code(), Code::FailedPrecondition);
- assert!(err.message().contains(&tap_host), "{err}");
-
- driver.release_subnet("sandbox-proxy-tap");
- }
-
#[test]
- fn qemu_launch_plan_allows_a_proxy_at_the_gvproxy_host_loopback_address() {
- // 192.168.127.254 carries no meaning on QEMU/TAP, so a launch must
- // proceed rather than be refused for a libkrun-only reason.
- let driver = test_driver_with_proxy(&format!("http://{GVPROXY_HOST_LOOPBACK_IP}:8080"));
+ fn qemu_launch_plan_uses_vsock_only_with_host_proxy() {
+ let driver = test_driver_with_proxy("http://127.0.0.1:8080");
let mut plan = driver
- .build_vm_launch_plan("sandbox-proxy-gvproxy", true, true, None)
+ .build_vm_launch_plan("sandbox-proxy-vsock", true, true, None)
.expect("gpu plan should build");
- assert_ne!(plan.host_ip.as_deref(), Some(GVPROXY_HOST_LOOPBACK_IP));
-
driver
- .resolve_launch_plan_backend("sandbox-proxy-gvproxy", true, None, &mut plan)
- .expect("a routable proxy address must not block a GPU launch");
- assert_eq!(plan.backend, VmBackend::Qemu);
-
- driver.release_subnet("sandbox-proxy-gvproxy");
- }
-
- #[tokio::test]
- async fn proxy_credential_is_validated_against_the_supervisor_rules() {
- let dir = std::env::temp_dir().join(format!("openshell-vm-cred-{}", std::process::id()));
- std::fs::create_dir_all(&dir).unwrap();
- let path = dir.join("proxy-auth");
-
- std::fs::write(&path, "proxyuser:proxypass\n").unwrap();
- assert_eq!(
- read_sandbox_proxy_credential(path.to_str().unwrap())
- .await
- .expect("a well-formed credential is accepted"),
- "proxyuser:proxypass"
- );
-
- // Rejected here rather than inside every sandbox's supervisor.
- std::fs::write(&path, "no-separator\n").unwrap();
- let err = read_sandbox_proxy_credential(path.to_str().unwrap())
- .await
- .expect_err("a malformed credential must fail closed");
- assert_eq!(err.code(), Code::InvalidArgument);
- assert!(
- !err.message().contains("no-separator"),
- "the error must not echo credential file contents: {err}"
- );
-
- std::fs::remove_dir_all(&dir).unwrap();
+ .resolve_launch_plan_backend("sandbox-proxy-vsock", true, None, &mut plan)
+ .expect("host control can reach a host-loopback proxy");
+ assert!(plan.vsock_cid.is_some());
}
#[test]
@@ -9416,7 +8801,7 @@ mod tests {
..Default::default()
};
- let env = build_guest_environment(&sandbox, &config, None);
+ let env = build_guest_environment(&sandbox, &config);
assert!(
!env.iter().any(|entry| entry.starts_with("--upstream")),
"driver environment must never carry supervisor arguments: {env:?}"
diff --git a/crates/openshell-driver-vm/src/embedded_runtime.rs b/crates/openshell-driver-vm/src/embedded_runtime.rs
index 70626edd98..4b75bd4ba8 100644
--- a/crates/openshell-driver-vm/src/embedded_runtime.rs
+++ b/crates/openshell-driver-vm/src/embedded_runtime.rs
@@ -10,7 +10,6 @@ use std::path::{Path, PathBuf};
mod resources {
pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.dylib.zst"));
pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.5.dylib.zst"));
- pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst"));
pub const LIBKRUN_NAME: &str = "libkrun.dylib";
pub const LIBKRUNFW_NAME: &str = "libkrunfw.5.dylib";
}
@@ -19,7 +18,6 @@ mod resources {
mod resources {
pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.so.zst"));
pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.so.5.zst"));
- pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst"));
pub const LIBKRUN_NAME: &str = "libkrun.so";
pub const LIBKRUNFW_NAME: &str = "libkrunfw.so.5";
}
@@ -28,7 +26,6 @@ mod resources {
mod resources {
pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.so.zst"));
pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.so.5.zst"));
- pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst"));
pub const LIBKRUN_NAME: &str = "libkrun.so";
pub const LIBKRUNFW_NAME: &str = "libkrunfw.so.5";
}
@@ -41,7 +38,6 @@ mod resources {
mod resources {
pub const LIBKRUN: &[u8] = &[];
pub const LIBKRUNFW: &[u8] = &[];
- pub const GVPROXY: &[u8] = &[];
pub const LIBKRUN_NAME: &str = "libkrun";
pub const LIBKRUNFW_NAME: &str = "libkrunfw";
}
@@ -82,7 +78,6 @@ pub fn ensure_runtime_extracted() -> Result {
resources::LIBKRUNFW,
&cache_dir.join(resources::LIBKRUNFW_NAME),
)?;
- extract_resource(resources::GVPROXY, &cache_dir.join("gvproxy"))?;
#[cfg(target_os = "macos")]
{
@@ -96,22 +91,13 @@ pub fn ensure_runtime_extracted() -> Result {
fs::write(&version_marker, cache_key)
.map_err(|e| format!("write runtime marker {}: {e}", version_marker.display()))?;
- #[cfg(unix)]
- {
- use std::os::unix::fs::PermissionsExt as _;
- fs::set_permissions(cache_dir.join("gvproxy"), fs::Permissions::from_mode(0o755))
- .map_err(|e| format!("chmod gvproxy: {e}"))?;
- }
-
Ok(cache_dir)
}
pub fn validate_runtime_dir(dir: &Path) -> Result<(), String> {
let libkrun = dir.join(resources::LIBKRUN_NAME);
let libkrunfw = dir.join(resources::LIBKRUNFW_NAME);
- let gvproxy = dir.join("gvproxy");
-
- for path in [&libkrun, &libkrunfw, &gvproxy] {
+ for path in [&libkrun, &libkrunfw] {
if !path.is_file() {
return Err(format!("missing runtime file: {}", path.display()));
}
@@ -128,7 +114,6 @@ fn runtime_cache_key() -> String {
let mut fp: u64 = 0;
for (index, chunk) in [resources::LIBKRUN, resources::LIBKRUNFW]
.into_iter()
- .chain(std::iter::once(resources::GVPROXY))
.enumerate()
{
let sample = &chunk[..chunk.len().min(64)];
diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs
index 423ad6f05b..f84ea35743 100644
--- a/crates/openshell-driver-vm/src/ffi.rs
+++ b/crates/openshell-driver-vm/src/ffi.rs
@@ -52,23 +52,8 @@ type KrunSetConsoleOutput = unsafe extern "C" fn(ctx_id: u32, filepath: *const c
type KrunStartEnter = unsafe extern "C" fn(ctx_id: u32) -> i32;
type KrunDisableImplicitVsock = unsafe extern "C" fn(ctx_id: u32) -> i32;
type KrunAddVsock = unsafe extern "C" fn(ctx_id: u32, tsi_features: u32) -> i32;
-#[cfg(target_os = "macos")]
-type KrunAddNetUnixgram = unsafe extern "C" fn(
- ctx_id: u32,
- c_path: *const c_char,
- fd: i32,
- c_mac: *const u8,
- features: u32,
- flags: u32,
-) -> i32;
-type KrunAddNetUnixstream = unsafe extern "C" fn(
- ctx_id: u32,
- c_path: *const c_char,
- fd: i32,
- c_mac: *const u8,
- features: u32,
- flags: u32,
-) -> i32;
+type KrunAddVsockPort2 =
+ unsafe extern "C" fn(ctx_id: u32, port: u32, filepath: *const c_char, listen: bool) -> i32;
// Field names mirror the libkrun C API symbol names (`krun_*`); preserving
// the prefix keeps the FFI binding 1:1 with the upstream library.
@@ -86,10 +71,7 @@ pub struct LibKrun {
pub krun_start_enter: KrunStartEnter,
pub krun_disable_implicit_vsock: KrunDisableImplicitVsock,
pub krun_add_vsock: KrunAddVsock,
- #[cfg(target_os = "macos")]
- pub krun_add_net_unixgram: KrunAddNetUnixgram,
- #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode.
- pub krun_add_net_unixstream: KrunAddNetUnixstream,
+ pub krun_add_vsock_port2: KrunAddVsockPort2,
}
static LIBKRUN: OnceLock = OnceLock::new();
@@ -151,13 +133,7 @@ impl LibKrun {
&libkrun_path,
)?,
krun_add_vsock: load_symbol(library, b"krun_add_vsock\0", &libkrun_path)?,
- #[cfg(target_os = "macos")]
- krun_add_net_unixgram: load_symbol(library, b"krun_add_net_unixgram\0", &libkrun_path)?,
- krun_add_net_unixstream: load_symbol(
- library,
- b"krun_add_net_unixstream\0",
- &libkrun_path,
- )?,
+ krun_add_vsock_port2: load_symbol(library, b"krun_add_vsock_port2\0", &libkrun_path)?,
})
}
}
diff --git a/crates/openshell-driver-vm/src/gpu.rs b/crates/openshell-driver-vm/src/gpu.rs
index dc5883b5ba..e7b0233fdf 100644
--- a/crates/openshell-driver-vm/src/gpu.rs
+++ b/crates/openshell-driver-vm/src/gpu.rs
@@ -5,8 +5,6 @@ use openshell_vfio::{
GpuBindGuard, GpuBindState, GpuBinding, GpuInfo, SysfsRoot, prepare_gpu_for_passthrough,
probe_host_nvidia_vfio_readiness, reconcile_stale_bindings, validate_bdf,
};
-use std::collections::HashMap;
-use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU32, Ordering};
@@ -156,164 +154,16 @@ pub struct GpuAssignment {
pub iommu_group: u32,
}
-// ---------------------------------------------------------------------------
-// Subnet allocation for per-sandbox TAP networking
-// ---------------------------------------------------------------------------
-
-/// Allocates /30 subnets from a pool for per-sandbox TAP networking.
-pub struct SubnetAllocator {
- base: Ipv4Addr,
- prefix_len: u8,
- next_offset: u32,
- allocated: HashMap,
-}
-
-pub struct SubnetAllocation {
- pub host_ip: Ipv4Addr,
- pub guest_ip: Ipv4Addr,
- pub prefix_len: u8,
- pub offset: u32,
-}
-
static NEXT_VSOCK_CID: AtomicU32 = AtomicU32::new(3);
-impl SubnetAllocator {
- pub fn new(base: Ipv4Addr, prefix_len: u8) -> Self {
- Self {
- base,
- prefix_len,
- next_offset: 0,
- allocated: HashMap::new(),
- }
- }
-
- pub fn allocate(&mut self, sandbox_id: &str) -> Result {
- let pool_size = 1u32 << (32 - self.prefix_len);
- let max_subnets = pool_size / 4;
-
- if u32::try_from(self.allocated.len()).unwrap_or(u32::MAX) >= max_subnets {
- return Err("subnet pool exhausted".to_string());
- }
-
- while self
- .allocated
- .values()
- .any(|a| a.offset == self.next_offset)
- {
- self.next_offset = (self.next_offset + 1) % max_subnets;
- }
-
- let base_u32 = u32::from(self.base);
- let subnet_base = base_u32 + (self.next_offset * 4);
- let host_ip = Ipv4Addr::from(subnet_base + 1);
- let guest_ip = Ipv4Addr::from(subnet_base + 2);
-
- let allocation = SubnetAllocation {
- host_ip,
- guest_ip,
- prefix_len: 30,
- offset: self.next_offset,
- };
-
- self.allocated.insert(sandbox_id.to_string(), allocation);
- self.next_offset = (self.next_offset + 1) % max_subnets;
-
- let alloc = &self.allocated[sandbox_id];
- Ok(SubnetAllocation {
- host_ip: alloc.host_ip,
- guest_ip: alloc.guest_ip,
- prefix_len: alloc.prefix_len,
- offset: alloc.offset,
- })
- }
-
- pub fn release(&mut self, sandbox_id: &str) {
- self.allocated.remove(sandbox_id);
- }
-}
-
pub fn allocate_vsock_cid() -> u32 {
NEXT_VSOCK_CID.fetch_add(1, Ordering::Relaxed)
}
-/// Generate a locally-administered MAC from sandbox ID using FNV-1a.
-pub fn mac_from_sandbox_id(sandbox_id: &str) -> [u8; 6] {
- let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
- for byte in sandbox_id.as_bytes() {
- hash ^= u64::from(*byte);
- hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
- }
- let bytes = hash.to_le_bytes();
- let mut mac = [bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5]];
- mac[0] = (mac[0] & 0xFE) | 0x02;
- mac
-}
-
-/// TAP device name from sandbox ID (fits `IFNAMSIZ=16`).
-pub fn tap_device_name(sandbox_id: &str) -> String {
- let mut end = sandbox_id.len().min(8);
- // Walk back to a UTF-8 char boundary (str::floor_char_boundary requires
- // Rust 1.91 — we still build on older toolchains).
- while end > 0 && !sandbox_id.is_char_boundary(end) {
- end -= 1;
- }
- let prefix = &sandbox_id[..end];
- format!("vmtap-{prefix}")
-}
-
#[cfg(test)]
mod tests {
use super::*;
- #[test]
- fn subnet_allocator_assigns_sequential_blocks() {
- let mut alloc = SubnetAllocator::new(Ipv4Addr::new(10, 0, 128, 0), 17);
-
- let s1 = alloc.allocate("sandbox-1").unwrap();
- assert_eq!(s1.host_ip, Ipv4Addr::new(10, 0, 128, 1));
- assert_eq!(s1.guest_ip, Ipv4Addr::new(10, 0, 128, 2));
- assert_eq!(s1.prefix_len, 30);
-
- let s2 = alloc.allocate("sandbox-2").unwrap();
- assert_eq!(s2.host_ip, Ipv4Addr::new(10, 0, 128, 5));
- assert_eq!(s2.guest_ip, Ipv4Addr::new(10, 0, 128, 6));
- }
-
- #[test]
- fn subnet_allocator_recycles_after_release() {
- let mut alloc = SubnetAllocator::new(Ipv4Addr::new(10, 0, 128, 0), 17);
-
- let _s1 = alloc.allocate("sandbox-1").unwrap();
- let _s2 = alloc.allocate("sandbox-2").unwrap();
- alloc.release("sandbox-1");
-
- let s3 = alloc.allocate("sandbox-3").unwrap();
- assert_eq!(s3.host_ip, Ipv4Addr::new(10, 0, 128, 9));
- }
-
- #[test]
- fn tap_device_name_truncates_long_ids() {
- assert_eq!(tap_device_name("abc"), "vmtap-abc");
- assert_eq!(tap_device_name("abcdefghijklmnop"), "vmtap-abcdefgh");
- }
-
- #[test]
- fn mac_from_sandbox_id_sets_locally_administered_bit() {
- let mac = mac_from_sandbox_id("sandbox-123");
- assert_eq!(mac[0] & 0x02, 0x02, "locally-administered bit must be set");
- assert_eq!(mac[0] & 0x01, 0x00, "multicast bit must be clear");
- }
-
- #[test]
- fn mac_from_sandbox_id_deterministic() {
- let mac1 = mac_from_sandbox_id("sandbox-x");
- let mac2 = mac_from_sandbox_id("sandbox-x");
- assert_eq!(mac1, mac2);
-
- let mac3 = mac_from_sandbox_id("sandbox-y");
- assert_ne!(mac1, mac3);
- }
-
#[test]
fn vsock_cid_increments() {
let cid1 = allocate_vsock_cid();
diff --git a/crates/openshell-driver-vm/src/isolation/mod.rs b/crates/openshell-driver-vm/src/isolation/mod.rs
new file mode 100644
index 0000000000..398d71cbe4
--- /dev/null
+++ b/crates/openshell-driver-vm/src/isolation/mod.rs
@@ -0,0 +1,156 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! VM provisioning for the shared authenticated boundary protocol.
+//!
+//! This module deliberately contains no lifecycle, process, network, or wire
+//! implementation. The driver chooses the host transport and binds immutable
+//! VM claims; `openshell-isolation-interface` and `openshell-sandbox` provide
+//! the common control and boundary behavior.
+
+use openshell_isolation_interface::boundary_protocol::{
+ BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, BoundaryTransport,
+};
+use openshell_isolation_interface::contract::{
+ BackendError, DriverFenceEvidence, ResolvedWorkloadIdentity,
+};
+use std::collections::{BTreeMap, HashMap};
+
+/// Driver-owned inputs that bind one VM generation to one supervisor boundary.
+pub struct VmBoundarySpec {
+ pub boundary_id: String,
+ pub bootstrap_token: String,
+ pub generation: String,
+ pub session_epoch: String,
+ pub image_identity: String,
+ pub transport: BoundaryTransport,
+ pub sandbox_tls: BoundaryServerTls,
+ pub control_port: u32,
+ pub agent_uid: u32,
+ pub agent_gid: u32,
+ pub child_env: HashMap,
+}
+
+/// The protected guest config and matching host descriptor for one VM.
+pub struct VmBoundaryProvisioning {
+ pub boundary_config: BoundaryConfig,
+ pub topology: BoundaryTopology,
+}
+
+impl VmBoundarySpec {
+ /// Produce both sides of the common protocol from one set of immutable
+ /// driver inputs so their identity claims cannot drift.
+ pub fn provision(self) -> Result {
+ let workload_identity = ResolvedWorkloadIdentity::new(
+ self.agent_uid,
+ self.agent_gid,
+ Vec::new(),
+ "vm-config".to_string(),
+ self.image_identity.clone(),
+ )?;
+ let resource_claims = BTreeMap::from([
+ ("vm.generation".to_string(), self.generation.clone()),
+ ("vm.image_identity".to_string(), self.image_identity),
+ ]);
+ let driver_fence = DriverFenceEvidence::Vm {
+ generation: self.generation.clone(),
+ network_device_count: 0,
+ };
+ Ok(VmBoundaryProvisioning {
+ boundary_config: BoundaryConfig {
+ boundary_id: self.boundary_id.clone(),
+ generation: self.generation.clone(),
+ session_epoch: self.session_epoch.clone(),
+ bootstrap_token: self.bootstrap_token.clone(),
+ listener: BoundaryListener::Vsock {
+ control_port: self.control_port,
+ tls: self.sandbox_tls,
+ },
+ resource_claims: resource_claims.clone(),
+ resource_claim_files: BTreeMap::new(),
+ workload_identity: workload_identity.clone(),
+ driver_fence: driver_fence.clone(),
+ child_env: self.child_env,
+ },
+ topology: BoundaryTopology {
+ boundary_id: self.boundary_id,
+ generation: self.generation,
+ session_epoch: self.session_epoch,
+ workload_identity,
+ transport: self.transport,
+ // The host-side control process is the network broker, so
+ // reserved host aliases terminate at its loopback address
+ // after crossing the authenticated boundary channel.
+ host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)),
+ resource_claims,
+ driver_fence,
+ bootstrap_token: self.bootstrap_token,
+ },
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use openshell_isolation_interface::boundary_protocol::{
+ BoundaryClientTls, generate_boundary_mutual_tls_material,
+ };
+
+ #[test]
+ fn provisioning_binds_identical_resource_claims() {
+ let material = generate_boundary_mutual_tls_material().unwrap();
+ let provisioned = VmBoundarySpec {
+ boundary_id: "sandbox-1".to_string(),
+ bootstrap_token: "a".repeat(64),
+ generation: "generation-1".to_string(),
+ session_epoch: "epoch-1".to_string(),
+ image_identity: "sha256:image".to_string(),
+ transport: BoundaryTransport::Vsock {
+ guest_cid: 42,
+ control_port: 5500,
+ tls: BoundaryClientTls {
+ server_name: material.server_name,
+ ca_certificate_pem: material.ca_certificate_pem,
+ certificate_chain_pem: material.supervisor_certificate_pem,
+ private_key_pem: material.supervisor_private_key_pem,
+ },
+ },
+ sandbox_tls: BoundaryServerTls {
+ certificate_chain_path: "/.openshell/state/sandbox.crt".into(),
+ private_key_path: "/.openshell/state/sandbox.key".into(),
+ client_ca_certificate_path: "/.openshell/state/client-ca.crt".into(),
+ },
+ control_port: 5500,
+ agent_uid: 1000,
+ agent_gid: 1000,
+ child_env: HashMap::new(),
+ }
+ .provision()
+ .unwrap();
+
+ assert_eq!(
+ provisioned.boundary_config.resource_claims,
+ provisioned.topology.resource_claims
+ );
+ assert_eq!(
+ provisioned.topology.resource_claims["vm.generation"],
+ "generation-1"
+ );
+ assert_eq!(
+ provisioned.topology.host_gateway_ip,
+ Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST))
+ );
+ assert_eq!(
+ provisioned.boundary_config.driver_fence,
+ provisioned.topology.driver_fence
+ );
+ assert!(
+ provisioned
+ .topology
+ .driver_fence
+ .validate_for_backend("vm")
+ .is_ok()
+ );
+ }
+}
diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs
index f34c7dda8d..8e3f41a15d 100644
--- a/crates/openshell-driver-vm/src/lib.rs
+++ b/crates/openshell-driver-vm/src/lib.rs
@@ -11,24 +11,36 @@ compile_error!(
build a telemetry-free VM driver with `--no-default-features --features defaults-without-telemetry`"
);
+#[cfg(feature = "compute-driver")]
pub mod driver;
+#[cfg(feature = "compute-driver")]
mod embedded_runtime;
+#[cfg(feature = "compute-driver")]
mod ffi;
+#[cfg(feature = "compute-driver")]
pub mod gpu;
+#[cfg(feature = "compute-driver")]
+mod isolation;
+#[cfg(feature = "compute-driver")]
pub mod lifecycle;
-mod nft_ruleset;
+#[cfg(feature = "compute-driver")]
pub mod otel_tracing;
+#[cfg(feature = "compute-driver")]
pub mod procguard;
+#[cfg(feature = "compute-driver")]
mod rootfs;
+#[cfg(feature = "compute-driver")]
mod runtime;
+#[cfg(feature = "compute-driver")]
pub use driver::{VmDriver, VmDriverConfig};
+#[cfg(feature = "compute-driver")]
pub use lifecycle::{
BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason,
LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult,
RestoreContext,
};
+#[cfg(feature = "compute-driver")]
pub use runtime::{
- VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces,
- configured_runtime_dir, run_vm,
+ VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, VsockPortMap, configured_runtime_dir, run_vm,
};
diff --git a/crates/openshell-driver-vm/src/lifecycle.rs b/crates/openshell-driver-vm/src/lifecycle.rs
index 25ec91db67..51977ccef2 100644
--- a/crates/openshell-driver-vm/src/lifecycle.rs
+++ b/crates/openshell-driver-vm/src/lifecycle.rs
@@ -108,9 +108,6 @@ pub enum BackendFeature {
/// QEMU-only and currently rejected for non-GPU sandboxes pending the
/// non-GPU QEMU launch path landing.
PciPassthrough,
- /// Extension needs a host TAP device wired into the guest. Currently
- /// QEMU-only (libkrun does not expose a TAP transport).
- TapNetworking,
}
impl BackendFeature {
@@ -120,7 +117,6 @@ impl BackendFeature {
Self::ExternalKernelImage => "external-kernel-image",
Self::GuestInitDropins => "guest-init-dropins",
Self::PciPassthrough => "pci-passthrough",
- Self::TapNetworking => "tap-networking",
}
}
@@ -130,10 +126,7 @@ impl BackendFeature {
/// exists.
#[must_use]
pub fn requires_qemu(self) -> bool {
- matches!(
- self,
- Self::ExternalKernelImage | Self::PciPassthrough | Self::TapNetworking
- )
+ matches!(self, Self::ExternalKernelImage | Self::PciPassthrough)
}
}
@@ -226,12 +219,7 @@ pub struct LaunchPlan {
pub kernel_profile: Option,
pub kernel_image: Option,
pub gpu_bdf: Option,
- pub tap_device: Option,
- pub guest_ip: Option,
- pub host_ip: Option,
pub vsock_cid: Option,
- pub guest_mac: Option,
- pub gateway_port: Option,
pub guest_init_dropins: Vec,
pub env: Vec,
}
@@ -296,7 +284,7 @@ pub enum ExtensionActivation {
/// (kernel profile, guest init drop-ins, etc.). Called before the driver
/// has resolved the final backend.
/// 2. Driver resolves [`LaunchPlan::backend`] from declared requirements
-/// and allocates backend-specific host resources (subnet, tap, vsock).
+/// and allocates backend-specific host resources such as a vsock CID.
/// 3. [`before_launch`](Self::before_launch) — perform host-side
/// side effects with the resolved plan in hand, optionally append
/// additional guest env via [`LaunchPlan::env`].
@@ -353,7 +341,7 @@ pub trait LifecycleExtension: std::fmt::Debug + Send + Sync {
/// At this point [`LaunchPlan::backend`],
/// [`LaunchPlan::required_backends`], and
/// [`LaunchPlan::required_backend_features`] are finalized and any
- /// backend-specific host resources (subnet, tap, vsock) have been
+ /// backend-specific host resources have been
/// allocated. This hook is the right place to bind PCI devices, set
/// up filesystem state, or otherwise prepare the host.
///
@@ -953,12 +941,7 @@ mod tests {
kernel_profile: None,
kernel_image: None,
gpu_bdf: None,
- tap_device: None,
- guest_ip: None,
- host_ip: None,
vsock_cid: None,
- guest_mac: None,
- gateway_port: None,
guest_init_dropins: Vec::new(),
env: Vec::new(),
}
diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs
index b8788e4fc9..56f76bb4f7 100644
--- a/crates/openshell-driver-vm/src/main.rs
+++ b/crates/openshell-driver-vm/src/main.rs
@@ -8,7 +8,9 @@ use openshell_core::VERSION;
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
#[cfg(target_os = "macos")]
use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir};
-use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm};
+use openshell_driver_vm::{
+ VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, VsockPortMap, procguard, run_vm,
+};
use std::io;
use std::net::SocketAddr;
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
@@ -144,7 +146,7 @@ struct Args {
sandbox_gid: Option,
// Corporate forward proxy for sandbox egress. Operator-owned: these reach
- // the guest supervisor on its argv, which the sandbox image and the
+ // the host supervisor on its argv, which the sandbox image and the
// user-supplied environment cannot influence.
#[arg(long, env = "OPENSHELL_VM_HTTPS_PROXY")]
https_proxy: Option,
@@ -173,37 +175,22 @@ struct Args {
#[arg(long, hide = true)]
vm_gpu_bdf: Option,
- #[arg(long, hide = true)]
- vm_tap_device: Option,
-
- #[arg(long, hide = true)]
- vm_guest_ip: Option,
-
- #[arg(long, hide = true)]
- vm_host_ip: Option,
-
#[arg(long, hide = true)]
vm_vsock_cid: Option,
#[arg(long, hide = true)]
- vm_guest_mac: Option,
+ vm_vsock_control_port: Option,
#[arg(long, hide = true)]
- vm_gateway_port: Option,
+ vm_vsock_control_socket: Option,
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
if args.internal_run_vm {
- // We intentionally defer procguard arming until `run_vm()` so
- // that the only arm is the one that knows how to clean up
- // gvproxy. Racing two watchers against the same parent-death
- // event causes the bare arm's `exit(1)` to win, skipping the
- // gvproxy cleanup and leaking the helper. The risk window
- // before `run_vm` arms procguard is ~a few syscalls long
- // (`build_vm_launch_config`, `configured_runtime_dir`), which
- // is negligible next to the parent gRPC server's uptime.
+ // The VM launcher arms procguard after resolving its runtime so its
+ // libkrun worker cannot outlive the launcher.
maybe_reexec_internal_vm_with_runtime_env()?;
let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?;
run_vm(&config).map_err(|err| miette::miette!("{err}"))?;
@@ -226,7 +213,7 @@ async fn main() -> Result<()> {
// we also die. Without this the driver is reparented to init and
// keeps its per-sandbox VM launchers alive forever. Launchers have
// their own procguards (armed in `run_vm`) which cascade cleanup of
- // gvproxy and the libkrun worker the moment this driver exits.
+ // the libkrun worker the moment this driver exits.
if let Err(err) = procguard::die_with_parent() {
tracing::warn!(
error = %err,
@@ -562,12 +549,24 @@ fn build_vm_launch_config(args: &Args) -> std::result::Result Some(VsockPortMap {
+ guest_port,
+ host_socket,
+ host_initiated: true,
+ }),
+ (None, None) => None,
+ _ => {
+ return Err(
+ "--vm-vsock-control-port and --vm-vsock-control-socket must be set together"
+ .to_string(),
+ );
+ }
+ },
})
}
diff --git a/crates/openshell-driver-vm/src/nft_ruleset.rs b/crates/openshell-driver-vm/src/nft_ruleset.rs
deleted file mode 100644
index fe3e86c902..0000000000
--- a/crates/openshell-driver-vm/src/nft_ruleset.rs
+++ /dev/null
@@ -1,92 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-use std::fmt::Write;
-
-/// Sanitize a TAP device name for use as an nftables table name suffix.
-/// Assumes device names match `vmtap-[a-f0-9]+` (driver-controlled).
-fn sanitize_table_name(device: &str) -> String {
- device.replace('-', "_")
-}
-
-/// Return the nftables table name for a TAP device.
-pub fn teardown_table_name(device: &str) -> String {
- format!("openshell_vm_{}", sanitize_table_name(device))
-}
-
-/// Generate the nftables ruleset for VM TAP networking.
-pub fn generate_tap_ruleset(tap_device: &str, subnet: &str, gateway_port: u16) -> String {
- let table_name = teardown_table_name(tap_device);
- let mut ruleset = String::with_capacity(512);
-
- writeln!(ruleset, "table ip {table_name} {{").unwrap();
- writeln!(ruleset, " chain postrouting {{").unwrap();
- writeln!(
- ruleset,
- " type nat hook postrouting priority 100; policy accept;"
- )
- .unwrap();
- writeln!(ruleset, " ip saddr {subnet} masquerade").unwrap();
- writeln!(ruleset, " }}").unwrap();
- writeln!(ruleset, " chain forward {{").unwrap();
- writeln!(
- ruleset,
- " type filter hook forward priority 0; policy accept;"
- )
- .unwrap();
- writeln!(ruleset, " iifname \"{tap_device}\" accept").unwrap();
- writeln!(
- ruleset,
- " oifname \"{tap_device}\" ct state related,established accept"
- )
- .unwrap();
- writeln!(ruleset, " oifname \"{tap_device}\" drop").unwrap();
- writeln!(ruleset, " }}").unwrap();
- writeln!(ruleset, " chain input {{").unwrap();
- writeln!(
- ruleset,
- " type filter hook input priority 0; policy accept;"
- )
- .unwrap();
- writeln!(
- ruleset,
- " iifname \"{tap_device}\" tcp dport {gateway_port} accept"
- )
- .unwrap();
- writeln!(ruleset, " iifname \"{tap_device}\" drop").unwrap();
- writeln!(ruleset, " }}").unwrap();
- writeln!(ruleset, "}}").unwrap();
-
- ruleset
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn generates_tap_setup_ruleset() {
- let ruleset = generate_tap_ruleset("vmtap-abcd", "10.0.128.0/30", 8080);
- assert!(ruleset.contains("table ip openshell_vm_vmtap_abcd {"));
- assert!(ruleset.contains("type nat hook postrouting priority 100; policy accept;"));
- assert!(ruleset.contains("ip saddr 10.0.128.0/30 masquerade"));
- assert!(ruleset.contains("type filter hook forward priority 0; policy accept;"));
- assert!(ruleset.contains("iifname \"vmtap-abcd\" accept"));
- assert!(ruleset.contains("oifname \"vmtap-abcd\" ct state related,established accept"));
- assert!(ruleset.contains("oifname \"vmtap-abcd\" drop"));
- assert!(ruleset.contains("type filter hook input priority 0; policy accept;"));
- assert!(ruleset.contains("iifname \"vmtap-abcd\" tcp dport 8080 accept"));
- }
-
- #[test]
- fn table_name_sanitizes_device_name() {
- let ruleset = generate_tap_ruleset("vmtap-abc-123", "10.0.128.0/30", 8080);
- assert!(ruleset.contains("table ip openshell_vm_vmtap_abc_123 {"));
- }
-
- #[test]
- fn teardown_command_targets_correct_table() {
- let cmd = teardown_table_name("vmtap-abcd");
- assert_eq!(cmd, "openshell_vm_vmtap_abcd");
- }
-}
diff --git a/crates/openshell-driver-vm/src/procguard.rs b/crates/openshell-driver-vm/src/procguard.rs
index fd4d3c872c..5f89f0848c 100644
--- a/crates/openshell-driver-vm/src/procguard.rs
+++ b/crates/openshell-driver-vm/src/procguard.rs
@@ -4,10 +4,9 @@
//! Cross-platform "die when my parent dies" primitive.
//!
//! The VM driver spawns a chain of subprocesses (compute driver → `--internal-run-vm`
-//! launcher → gvproxy + libkrun fork). If any link in that chain is killed
+//! launcher → libkrun fork). If any link in that chain is killed
//! with SIGKILL — or simply crashes — the children are reparented to init
-//! and survive indefinitely, leaking libkrun workers and gvproxy
-//! instances.
+//! and survive indefinitely, leaking libkrun workers.
//!
//! This module exposes two functions:
//! * [`die_with_parent`] — configure the kernel (Linux) or a helper
@@ -17,7 +16,7 @@
//! the runtime.rs comment at the single call site).
//! * [`die_with_parent_cleanup`] — same as above, but on the BSD path a
//! best-effort cleanup callback runs *before* this process exits.
-//! This matters when we own a non-Rust child (e.g. gvproxy) that
+//! This matters when we own a non-Rust child that
//! cannot arm its own procguard; the callback lets us SIGTERM it
//! first.
//!
diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs
index 9046913c9d..67c9c76fb1 100644
--- a/crates/openshell-driver-vm/src/rootfs.rs
+++ b/crates/openshell-driver-vm/src/rootfs.rs
@@ -1,6 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
+use sha2::{Digest, Sha256};
use std::fs;
use std::fs::File;
#[cfg(test)]
@@ -8,9 +9,13 @@ use std::io::BufWriter;
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::process::Command;
+use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
-const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst"));
+const SANDBOX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst"));
+const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-supervisor.zst"));
+const SUPERVISOR_RUNTIME: &[u8] =
+ include_bytes!(concat!(env!("OUT_DIR"), "/openshell-runtime.tar.zst"));
const UMOCI: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/umoci.zst"));
const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant";
const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh";
@@ -18,6 +23,7 @@ const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_C
const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH;
const SANDBOX_OWNER_NORMALIZED_MARKER: &str =
openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER;
+const SANDBOX_SUPERVISOR_RUNTIME_PATH: &str = "/opt/openshell/bin/openshell-runtime";
const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024;
const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024;
const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024;
@@ -27,6 +33,141 @@ pub const fn sandbox_guest_init_path() -> &'static str {
SANDBOX_GUEST_INIT_PATH
}
+/// Identity of every embedded artifact materialized into a bootstrap rootfs.
+///
+/// Including this in the image-cache key makes local, uncommitted guest-sandbox
+/// changes invalidate the cache even when the `OpenShell` version is unchanged.
+pub fn sandbox_guest_runtime_identity() -> String {
+ let mut hasher = Sha256::new();
+ hasher.update(SANDBOX);
+ hasher.update(SUPERVISOR_RUNTIME);
+ hasher.update(UMOCI);
+ hasher.update(include_bytes!("../scripts/openshell-vm-sandbox-init.sh"));
+ format!("{:x}", hasher.finalize())
+}
+
+/// Materialize the supervisor embedded in the VM driver for host-side use.
+pub fn extract_host_supervisor(path: &Path) -> Result<(), String> {
+ if SANDBOX.is_empty() {
+ return Err(
+ "host supervisor is not embedded; run `mise run vm:supervisor` and rebuild openshell-driver-vm"
+ .to_string(),
+ );
+ }
+ let supervisor = embedded_host_supervisor()?;
+ install_host_supervisor_atomically(path, &supervisor)?;
+ validate_host_supervisor(path)
+}
+
+pub fn validate_host_supervisor(path: &Path) -> Result<(), String> {
+ validate_host_supervisor_digest(path, embedded_host_supervisor_digest()?)
+}
+
+fn validate_host_supervisor_digest(path: &Path, expected: [u8; 32]) -> Result<(), String> {
+ let metadata = fs::symlink_metadata(path)
+ .map_err(|error| format!("inspect cached host supervisor {}: {error}", path.display()))?;
+ if !metadata.file_type().is_file() {
+ return Err(format!(
+ "cached host supervisor is not a regular file: {}",
+ path.display()
+ ));
+ }
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt as _;
+ if metadata.permissions().mode() & 0o111 == 0 {
+ return Err(format!(
+ "cached host supervisor is not executable: {}",
+ path.display()
+ ));
+ }
+ }
+ let actual = sha256_reader(
+ File::open(path)
+ .map_err(|error| format!("open cached host supervisor {}: {error}", path.display()))?,
+ )
+ .map_err(|error| format!("hash cached host supervisor {}: {error}", path.display()))?;
+ if actual != expected {
+ return Err(format!(
+ "cached host supervisor content does not match embedded runtime: {}",
+ path.display()
+ ));
+ }
+ Ok(())
+}
+
+fn embedded_host_supervisor() -> Result, String> {
+ zstd::decode_all(Cursor::new(SUPERVISOR))
+ .map_err(|error| format!("decompress host supervisor: {error}"))
+}
+
+fn embedded_host_supervisor_digest() -> Result<[u8; 32], String> {
+ static DIGEST: OnceLock> = OnceLock::new();
+ DIGEST
+ .get_or_init(|| embedded_host_supervisor().map(|bytes| sha256_bytes(&bytes)))
+ .clone()
+}
+
+fn sha256_bytes(bytes: &[u8]) -> [u8; 32] {
+ Sha256::digest(bytes).into()
+}
+
+fn sha256_reader(mut reader: impl Read) -> std::io::Result<[u8; 32]> {
+ let mut hasher = Sha256::new();
+ let mut buffer = [0_u8; 16 * 1024];
+ loop {
+ let read = reader.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..read]);
+ }
+ Ok(hasher.finalize().into())
+}
+
+fn install_host_supervisor_atomically(path: &Path, bytes: &[u8]) -> Result<(), String> {
+ let parent = path
+ .parent()
+ .filter(|parent| !parent.as_os_str().is_empty())
+ .ok_or_else(|| format!("host supervisor path has no parent: {}", path.display()))?;
+ fs::create_dir_all(parent).map_err(|error| format!("create {}: {error}", parent.display()))?;
+ let temporary = parent.join(format!(
+ ".openshell-sandbox.tmp-{}-{}",
+ std::process::id(),
+ INJECTION_COUNTER.fetch_add(1, Ordering::Relaxed)
+ ));
+ let result = (|| {
+ let mut options = fs::OpenOptions::new();
+ options.write(true).create_new(true);
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::OpenOptionsExt as _;
+ options.mode(0o755);
+ }
+ let mut file = options
+ .open(&temporary)
+ .map_err(|error| format!("create {}: {error}", temporary.display()))?;
+ file.write_all(bytes)
+ .map_err(|error| format!("write {}: {error}", temporary.display()))?;
+ file.sync_all()
+ .map_err(|error| format!("sync {}: {error}", temporary.display()))?;
+ fs::rename(&temporary, path).map_err(|error| {
+ format!(
+ "commit cached host supervisor {} to {}: {error}",
+ temporary.display(),
+ path.display()
+ )
+ })?;
+ File::open(parent)
+ .and_then(|directory| directory.sync_all())
+ .map_err(|error| format!("sync host supervisor cache {}: {error}", parent.display()))
+ })();
+ if result.is_err() {
+ let _ = fs::remove_file(&temporary);
+ }
+ result
+}
+
#[allow(clippy::similar_names)]
pub fn prepare_sandbox_rootfs_from_image_root(
rootfs: &Path,
@@ -198,6 +339,41 @@ pub fn set_rootfs_image_file_mode(
)
}
+/// Replay the ext4 journal and repair automatically correctable filesystem
+/// state before the driver mutates a preserved guest disk offline.
+pub fn recover_rootfs_image(image_path: &Path) -> Result<(), String> {
+ let mut failures = Vec::new();
+ let mut unavailable = Vec::new();
+
+ for candidate in e2fs_tool_candidates("e2fsck") {
+ let label = candidate.display().to_string();
+ match Command::new(&candidate)
+ .arg("-p")
+ .arg("-f")
+ .arg(image_path)
+ .output()
+ {
+ Ok(output) if matches!(output.status.code(), Some(0..=2)) => return Ok(()),
+ Ok(output) => failures.push(format!(
+ "{label} failed with status {}\nstdout: {}\nstderr: {}",
+ output.status,
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ )),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
+ unavailable.push(format!("{label} not found"));
+ }
+ Err(error) => failures.push(format!("run {label}: {error}")),
+ }
+ }
+
+ Err(if failures.is_empty() {
+ unavailable.join("\n")
+ } else {
+ failures.join("\n")
+ })
+}
+
#[cfg(target_os = "macos")]
fn try_clone_file(source: &Path, dest: &Path) -> Result<(), String> {
let output = Command::new("cp")
@@ -376,6 +552,8 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) ->
}
ensure_supervisor_binary(rootfs)?;
+ ensure_supervisor_runtime(rootfs)?;
+ ensure_guest_init_ip(rootfs)?;
ensure_umoci_binary(rootfs)?;
let opt_dir = rootfs.join("opt/openshell");
@@ -392,9 +570,55 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) ->
Ok(())
}
+fn ensure_guest_init_ip(rootfs: &Path) -> Result<(), String> {
+ const IP_PATHS: [&str; 4] = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"];
+ if IP_PATHS.iter().any(|path| rootfs.join(path).is_file()) {
+ return Ok(());
+ }
+
+ // Guest init runs before the sandbox can enter its trusted helper
+ // runtime. Images such as stock Ubuntu do not ship iproute2, so install a
+ // driver-owned launcher that executes the embedded musl helper explicitly.
+ // The helper and loader are both materialized from the trusted runtime,
+ // never from the workload image.
+ let path = rootfs.join("usr/sbin/ip");
+ let parent = path
+ .parent()
+ .ok_or_else(|| format!("guest ip launcher path has no parent: {}", path.display()))?;
+ fs::create_dir_all(parent).map_err(|error| format!("create {}: {error}", parent.display()))?;
+ fs::write(
+ &path,
+ r#"#!/bin/sh
+set -eu
+runtime=/opt/openshell/bin/openshell-runtime
+for loader in "$runtime"/lib/ld-musl-*.so.1; do
+ if [ -x "$loader" ]; then
+ for helper in "$runtime"/sbin/ip "$runtime"/usr/sbin/ip "$runtime"/bin/ip "$runtime"/usr/bin/ip; do
+ if [ -x "$helper" ]; then
+ exec "$loader" --library-path "$runtime/lib:$runtime/usr/lib" "$helper" "$@"
+ fi
+ done
+ fi
+done
+echo "trusted OpenShell ip helper is unavailable" >&2
+exit 127
+"#,
+ )
+ .map_err(|error| format!("write {}: {error}", path.display()))?;
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt as _;
+
+ fs::set_permissions(&path, fs::Permissions::from_mode(0o755))
+ .map_err(|error| format!("chmod {}: {error}", path.display()))?;
+ }
+ Ok(())
+}
+
pub fn validate_sandbox_rootfs(rootfs: &Path) -> Result<(), String> {
require_rootfs_path(rootfs, SANDBOX_GUEST_INIT_PATH)?;
require_rootfs_path(rootfs, SANDBOX_SUPERVISOR_PATH)?;
+ validate_supervisor_runtime(rootfs)?;
require_rootfs_path(rootfs, SANDBOX_UMOCI_PATH)?;
require_any_rootfs_path(rootfs, &["/bin/bash"])?;
require_any_rootfs_path(rootfs, &["/bin/mount", "/usr/bin/mount"])?;
@@ -795,20 +1019,20 @@ fn ensure_sandbox_guest_user(
let etc_dir = rootfs.join("etc");
fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?;
- ensure_line_in_file(
+ replace_or_append_line(
&etc_dir.join("group"),
&format!("sandbox:x:{sandbox_gid}:"),
|line| line.starts_with("sandbox:"),
)?;
- ensure_line_in_file(&etc_dir.join("gshadow"), "sandbox:!::", |line| {
+ replace_or_append_line(&etc_dir.join("gshadow"), "sandbox:!::", |line| {
line.starts_with("sandbox:")
})?;
- ensure_line_in_file(
+ replace_or_append_line(
&etc_dir.join("passwd"),
&format!("sandbox:x:{sandbox_uid}:{sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/bash"),
|line| line.starts_with("sandbox:"),
)?;
- ensure_line_in_file(
+ replace_or_append_line(
&etc_dir.join("shadow"),
"sandbox:!:20123:0:99999:7:::",
|line| line.starts_with("sandbox:"),
@@ -817,28 +1041,36 @@ fn ensure_sandbox_guest_user(
Ok(())
}
-fn ensure_line_in_file(
+fn replace_or_append_line(
path: &Path,
line: &str,
- exists: impl Fn(&str) -> bool,
+ matches: impl Fn(&str) -> bool,
) -> Result<(), String> {
- let mut contents = if path.exists() {
+ let contents = if path.exists() {
fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))?
} else {
String::new()
};
-
- if contents.lines().any(exists) {
- return Ok(());
+ let mut output = String::with_capacity(contents.len().max(line.len() + 1));
+ let mut replaced = false;
+ for existing in contents.lines() {
+ if matches(existing) {
+ if replaced {
+ continue;
+ }
+ output.push_str(line);
+ replaced = true;
+ } else {
+ output.push_str(existing);
+ }
+ output.push('\n');
}
-
- if !contents.is_empty() && !contents.ends_with('\n') {
- contents.push('\n');
+ if !replaced {
+ output.push_str(line);
+ output.push('\n');
}
- contents.push_str(line);
- contents.push('\n');
- fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display()))
+ fs::write(path, output).map_err(|e| format!("write {}: {e}", path.display()))
}
fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> {
@@ -855,9 +1087,9 @@ fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> {
fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?;
}
- let supervisor = zstd::decode_all(Cursor::new(SUPERVISOR))
- .map_err(|e| format!("decompress supervisor: {e}"))?;
- fs::write(&path, supervisor).map_err(|e| format!("write {}: {e}", path.display()))?;
+ let sandbox = zstd::decode_all(Cursor::new(SANDBOX))
+ .map_err(|e| format!("decompress sandbox: {e}"))?;
+ fs::write(&path, sandbox).map_err(|e| format!("write {}: {e}", path.display()))?;
}
#[cfg(unix)]
@@ -871,6 +1103,81 @@ fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> {
Ok(())
}
+fn ensure_supervisor_runtime(rootfs: &Path) -> Result<(), String> {
+ if SUPERVISOR_RUNTIME.is_empty() {
+ return validate_supervisor_runtime(rootfs).map_err(|_| {
+ "trusted supervisor helper runtime not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set and run `mise run vm:supervisor` first"
+ .to_string()
+ });
+ }
+
+ install_supervisor_runtime_archive(rootfs, SUPERVISOR_RUNTIME)
+}
+
+fn install_supervisor_runtime_archive(rootfs: &Path, archive_bytes: &[u8]) -> Result<(), String> {
+ let destination = rootfs.join("opt/openshell/bin");
+ fs::create_dir_all(&destination)
+ .map_err(|e| format!("create {}: {e}", destination.display()))?;
+ let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/'));
+ match fs::symlink_metadata(&runtime) {
+ Ok(metadata) if metadata.file_type().is_dir() => fs::remove_dir_all(&runtime)
+ .map_err(|e| format!("remove untrusted runtime {}: {e}", runtime.display()))?,
+ Ok(_) => fs::remove_file(&runtime)
+ .map_err(|e| format!("remove untrusted runtime {}: {e}", runtime.display()))?,
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+ Err(error) => return Err(format!("inspect runtime {}: {error}", runtime.display())),
+ }
+ let decoder = zstd::Decoder::new(Cursor::new(archive_bytes))
+ .map_err(|e| format!("decompress supervisor runtime: {e}"))?;
+ let mut archive = tar::Archive::new(decoder);
+ for entry in archive
+ .entries()
+ .map_err(|e| format!("open supervisor runtime archive: {e}"))?
+ {
+ let mut entry = entry.map_err(|e| format!("read supervisor runtime archive: {e}"))?;
+ let kind = entry.header().entry_type();
+ if !kind.is_file() && !kind.is_dir() {
+ return Err(
+ "supervisor runtime archive contains a non-materialized link or special file"
+ .to_string(),
+ );
+ }
+ if !entry
+ .unpack_in(&destination)
+ .map_err(|e| format!("extract supervisor runtime archive: {e}"))?
+ {
+ return Err("supervisor runtime archive contains a path outside its root".to_string());
+ }
+ }
+ validate_supervisor_runtime(rootfs)
+}
+
+fn validate_supervisor_runtime(rootfs: &Path) -> Result<(), String> {
+ let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/'));
+ let has_ip = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"]
+ .iter()
+ .any(|path| runtime.join(path).is_file());
+ let has_loader = fs::read_dir(runtime.join("lib"))
+ .ok()
+ .into_iter()
+ .flatten()
+ .filter_map(Result::ok)
+ .any(|entry| {
+ entry
+ .file_name()
+ .to_str()
+ .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1"))
+ });
+ if has_ip && has_loader {
+ Ok(())
+ } else {
+ Err(format!(
+ "trusted supervisor helper runtime '{}' is incomplete",
+ runtime.display()
+ ))
+ }
+}
+
fn ensure_umoci_binary(rootfs: &Path) -> Result<(), String> {
let path = rootfs.join(SANDBOX_UMOCI_PATH.trim_start_matches('/'));
if UMOCI.is_empty() {
@@ -944,10 +1251,52 @@ fn remove_rootfs_path(rootfs: &Path, relative: &str) -> Result<(), String> {
#[cfg(test)]
mod tests {
use super::*;
+ #[cfg(unix)]
+ use std::os::unix::fs::PermissionsExt as _;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
+ #[cfg(target_os = "linux")]
+ #[test]
+ fn host_supervisor_cache_rejects_wrong_content_and_installs_atomically() {
+ let directory = tempfile::tempdir().expect("cache directory");
+ let destination = directory.path().join("openshell-sandbox");
+ fs::write(&destination, b"stale executable").expect("write stale cache");
+ fs::set_permissions(&destination, fs::Permissions::from_mode(0o755))
+ .expect("make stale cache executable");
+ let expected = sha256_bytes(b"trusted supervisor");
+
+ assert!(validate_host_supervisor_digest(&destination, expected).is_err());
+ install_host_supervisor_atomically(&destination, b"trusted supervisor")
+ .expect("atomically replace cache");
+ validate_host_supervisor_digest(&destination, expected).expect("validate installed cache");
+ assert_eq!(fs::read(&destination).unwrap(), b"trusted supervisor");
+ assert!(fs::read_dir(directory.path()).unwrap().all(|entry| {
+ !entry
+ .unwrap()
+ .file_name()
+ .to_string_lossy()
+ .contains(".tmp-")
+ }));
+ }
+
+ #[test]
+ fn guest_init_gets_driver_owned_ip_launcher_when_image_omits_iproute2() {
+ let rootfs = tempfile::tempdir().expect("create rootfs");
+ ensure_guest_init_ip(rootfs.path()).expect("install guest ip launcher");
+
+ let launcher = rootfs.path().join("usr/sbin/ip");
+ let contents = fs::read_to_string(&launcher).expect("read guest ip launcher");
+ assert!(contents.contains("/opt/openshell/bin/openshell-runtime"));
+ assert!(contents.contains("ld-musl-"));
+ #[cfg(unix)]
+ assert_eq!(
+ fs::metadata(launcher).unwrap().permissions().mode() & 0o777,
+ 0o755
+ );
+ }
+
#[test]
fn prepare_sandbox_rootfs_rewrites_guest_layout() {
let dir = unique_temp_dir();
@@ -959,10 +1308,10 @@ mod tests {
write_fake_runtime_binaries(&rootfs);
fs::write(
rootfs.join("etc/passwd"),
- "root:x:0:0:root:/root:/bin/bash\n",
+ "root:x:0:0:root:/root:/bin/bash\nsandbox:x:998:997:Sandbox:/sandbox:/bin/sh\n",
)
.expect("write passwd");
- fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group");
+ fs::write(rootfs.join("etc/group"), "root:x:0:\nsandbox:x:997:\n").expect("write group");
fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts");
fs::create_dir_all(rootfs.join("bin")).expect("create bin");
fs::create_dir_all(rootfs.join("sbin")).expect("create sbin");
@@ -979,6 +1328,24 @@ mod tests {
assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file());
assert!(rootfs.join("opt/openshell/bin/umoci").is_file());
+ validate_supervisor_runtime(&rootfs).expect("trusted helper runtime remains complete");
+ let init_script = fs::read_to_string(rootfs.join("srv/openshell-vm-sandbox-init.sh"))
+ .expect("read guest init");
+ assert!(
+ init_script.contains(
+ "launch-capability-free \"$_sandbox_uid\" \"$_sandbox_gid\" \"$_sandbox_bootstrap_guest\""
+ )
+ );
+ assert!(init_script.contains("OPENSHELL_VM_SANDBOX_BOOTSTRAP"));
+ assert!(
+ init_script
+ .contains("chown \"${_sandbox_uid}:${_sandbox_gid}\" \"$_sandbox_state_dir\"")
+ );
+ assert!(init_script.contains("chmod 0700 \"$_sandbox_state_dir\""));
+ assert!(!init_script.contains("--topology-backend-name=in-pod"));
+ assert!(!init_script.contains("@ISOLATION_INTERFACE_VERSION@"));
+ assert!(!init_script.contains("8.8.8.8"));
+ assert!(!init_script.contains("VM_NET_"));
assert!(rootfs.join("sandbox").is_dir());
assert!(rootfs.join("image-cache").is_dir());
assert!(rootfs.join("lower").is_dir());
@@ -990,18 +1357,14 @@ mod tests {
.next()
.is_none()
);
- assert!(
- fs::read_to_string(rootfs.join("etc/passwd"))
- .expect("read passwd")
- .contains(&format!(
- "sandbox:x:{uid}:{uid}:OpenShell Sandbox:/sandbox:/bin/bash"
- ))
- );
- assert!(
- fs::read_to_string(rootfs.join("etc/group"))
- .expect("read group")
- .contains(&format!("sandbox:x:{uid}:"))
- );
+ let passwd = fs::read_to_string(rootfs.join("etc/passwd")).expect("read passwd");
+ assert!(passwd.contains(&format!(
+ "sandbox:x:{uid}:{uid}:OpenShell Sandbox:/sandbox:/bin/bash"
+ )));
+ assert!(!passwd.contains("sandbox:x:998:997:"));
+ let group = fs::read_to_string(rootfs.join("etc/group")).expect("read group");
+ assert!(group.contains(&format!("sandbox:x:{uid}:")));
+ assert!(!group.contains("sandbox:x:997:"));
assert_eq!(
fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"),
"127.0.0.1 localhost\n"
@@ -1010,6 +1373,55 @@ mod tests {
let _ = fs::remove_dir_all(&dir);
}
+ #[test]
+ fn supervisor_runtime_archive_materializes_below_the_trusted_path() {
+ let dir = unique_temp_dir();
+ let rootfs = dir.join("rootfs");
+ let untrusted_runtime = rootfs.join("opt/openshell/bin/openshell-runtime");
+ fs::create_dir_all(untrusted_runtime.join("usr/sbin")).expect("create untrusted runtime");
+ fs::write(untrusted_runtime.join("usr/sbin/ip"), b"untrusted")
+ .expect("write untrusted helper");
+ fs::write(untrusted_runtime.join("untrusted-extra"), b"untrusted")
+ .expect("write untrusted extra file");
+ let mut tar_bytes = Vec::new();
+ {
+ let mut archive = tar::Builder::new(&mut tar_bytes);
+ for (path, bytes, mode) in [
+ ("openshell-runtime/usr/sbin/ip", b"ip".as_slice(), 0o755),
+ (
+ "openshell-runtime/lib/ld-musl-test.so.1",
+ b"loader".as_slice(),
+ 0o755,
+ ),
+ ] {
+ let mut header = tar::Header::new_gnu();
+ header.set_size(bytes.len() as u64);
+ header.set_mode(mode);
+ header.set_entry_type(tar::EntryType::Regular);
+ header.set_cksum();
+ archive
+ .append_data(&mut header, path, bytes)
+ .expect("append runtime entry");
+ }
+ archive.finish().expect("finish runtime archive");
+ }
+ let compressed = zstd::encode_all(Cursor::new(tar_bytes), 1).expect("compress runtime");
+
+ install_supervisor_runtime_archive(&rootfs, &compressed).expect("install runtime");
+ validate_supervisor_runtime(&rootfs).expect("validate runtime");
+ assert_eq!(
+ fs::read(rootfs.join("opt/openshell/bin/openshell-runtime/usr/sbin/ip"))
+ .expect("read installed helper"),
+ b"ip"
+ );
+ assert!(
+ !rootfs
+ .join("opt/openshell/bin/openshell-runtime/untrusted-extra")
+ .exists(),
+ "embedded runtime replacement must discard bootstrap-image helpers"
+ );
+ }
+
#[test]
fn prepare_sandbox_rootfs_preserves_image_workdir_contents_in_rootfs() {
let dir = unique_temp_dir();
@@ -1100,6 +1512,28 @@ mod tests {
let _ = fs::remove_dir_all(&dir);
}
+ #[test]
+ fn recover_rootfs_image_accepts_clean_ext4_image() {
+ if !e2fs_tool_candidates("e2fsck")
+ .iter()
+ .any(|candidate| Command::new(candidate).arg("-V").output().is_ok())
+ {
+ return;
+ }
+
+ let dir = unique_temp_dir();
+ let source = dir.join("source");
+ let image = dir.join("overlay.ext4");
+ fs::create_dir_all(source.join("upper")).expect("create source upperdir");
+ fs::create_dir_all(source.join("work")).expect("create source workdir");
+ create_ext4_image_from_dir_with_size(&source, &image, 64 * 1024 * 1024)
+ .expect("create ext4 image");
+
+ recover_rootfs_image(&image).expect("recover clean ext4 image");
+
+ let _ = fs::remove_dir_all(&dir);
+ }
+
#[test]
fn sandbox_guest_user_ids_reads_existing_sandbox_user() {
let dir = unique_temp_dir();
@@ -1230,6 +1664,12 @@ mod tests {
}
fn write_fake_runtime_binaries(rootfs: &Path) {
+ let helper_runtime = rootfs.join("opt/openshell/bin/openshell-runtime");
+ fs::create_dir_all(helper_runtime.join("usr/sbin")).expect("create helper bin directory");
+ fs::create_dir_all(helper_runtime.join("lib")).expect("create helper lib directory");
+ fs::write(helper_runtime.join("usr/sbin/ip"), b"ip").expect("write ip helper");
+ fs::write(helper_runtime.join("lib/ld-musl-test.so.1"), b"loader")
+ .expect("write helper loader");
fs::write(
rootfs.join("opt/openshell/bin/openshell-sandbox"),
b"sandbox",
diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs
index f6020af829..4d0680bffe 100644
--- a/crates/openshell-driver-vm/src/runtime.rs
+++ b/crates/openshell-driver-vm/src/runtime.rs
@@ -5,12 +5,12 @@
use std::ffi::CString;
use std::path::{Path, PathBuf};
-use std::process::{Child as StdChild, Command as StdCommand, Stdio};
+use std::process::{Command as StdCommand, Stdio};
use std::ptr;
use std::sync::atomic::{AtomicI32, Ordering};
-use std::time::{Duration, Instant};
+use std::time::Duration;
-use crate::{embedded_runtime, ffi, nft_ruleset, procguard, rootfs};
+use crate::{embedded_runtime, ffi, procguard, rootfs};
pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR";
const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1";
@@ -19,31 +19,18 @@ const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1";
/// Used by the SIGTERM/SIGINT handler to forward signals to the VM.
static CHILD_PID: AtomicI32 = AtomicI32::new(0);
-/// PID of the helper process (gvproxy for libkrun; zero for QEMU).
-/// Zero when not running. Used by the SIGTERM/SIGINT handler and
-/// procguard cleanup callback to ensure the helper doesn't outlive the
-/// launcher (especially on macOS where `PR_SET_PDEATHSIG` is absent).
-static GVPROXY_PID: AtomicI32 = AtomicI32::new(0);
-
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VmBackend {
Libkrun,
Qemu,
}
-// virtio-net feature bits (see Linux `include/uapi/linux/virtio_net.h`).
-const NET_FEATURE_CSUM: u32 = 1 << 0;
-const NET_FEATURE_GUEST_CSUM: u32 = 1 << 1;
-const NET_FEATURE_GUEST_TSO4: u32 = 1 << 7;
-const NET_FEATURE_GUEST_UFO: u32 = 1 << 10;
-const NET_FEATURE_HOST_TSO4: u32 = 1 << 11;
-const NET_FEATURE_HOST_UFO: u32 = 1 << 14;
-const COMPAT_NET_FEATURES: u32 = NET_FEATURE_CSUM
- | NET_FEATURE_GUEST_CSUM
- | NET_FEATURE_GUEST_TSO4
- | NET_FEATURE_GUEST_UFO
- | NET_FEATURE_HOST_TSO4
- | NET_FEATURE_HOST_UFO;
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct VsockPortMap {
+ pub guest_port: u32,
+ pub host_socket: PathBuf,
+ pub host_initiated: bool,
+}
pub struct VmLaunchConfig {
pub root_disk: PathBuf,
@@ -60,12 +47,8 @@ pub struct VmLaunchConfig {
pub console_output: PathBuf,
pub backend: VmBackend,
pub gpu_bdf: Option,
- pub tap_device: Option,
- pub guest_ip: Option,
- pub host_ip: Option,
pub vsock_cid: Option,
- pub guest_mac: Option,
- pub gateway_port: Option,
+ pub vsock_port_map: Option,
}
pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> {
@@ -80,25 +63,9 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> {
.gpu_bdf
.as_deref()
.ok_or("gpu_bdf is required for QEMU backend")?;
- let tap_device = config
- .tap_device
- .as_deref()
- .ok_or("tap_device is required for QEMU backend")?;
- let guest_mac = config
- .guest_mac
- .as_deref()
- .ok_or("guest_mac is required for QEMU backend")?;
let vsock_cid = config
.vsock_cid
.ok_or("vsock_cid is required for QEMU backend")?;
- let _guest_ip = config
- .guest_ip
- .as_deref()
- .ok_or("guest_ip is required for QEMU backend")?;
- let host_ip = config
- .host_ip
- .as_deref()
- .ok_or("host_ip is required for QEMU backend")?;
if !config.root_disk.is_file() {
return Err(format!(
@@ -125,13 +92,9 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> {
#[cfg(target_os = "linux")]
check_kvm_access()?;
- let guest_env = qemu_guest_env_vars(config, host_dns_server());
+ let guest_env = qemu_guest_env_vars(config);
write_guest_env_file(&config.overlay_disk, &guest_env)?;
- let gw_port = config.gateway_port.unwrap_or(0);
- setup_tap_networking(tap_device, host_ip, gw_port)?;
- let mut tap_guard = TapGuard::new(tap_device.to_string(), host_ip.to_string(), gw_port);
-
let vmlinux = if let Some(kernel_image) = &config.kernel_image {
kernel_image.clone()
} else {
@@ -155,21 +118,12 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> {
.arg(format!("{}M", config.mem_mib))
.arg("-nographic")
.arg("-no-reboot")
+ .args(qemu_network_args())
.arg("-kernel")
.arg(&vmlinux)
.arg("-append")
.arg(&kernel_cmdline)
.args(qemu_disk_args(config))
- .arg("-netdev")
- .arg(format!(
- "tap,id=net0,ifname={tap_device},script=no,downscript=no"
- ))
- .arg("-device")
- .arg("pcie-root-port,id=net_root,slot=3")
- .arg("-device")
- .arg(format!(
- "virtio-net-pci-non-transitional,netdev=net0,mac={guest_mac},bus=net_root"
- ))
.arg("-device")
.arg("pcie-root-port,id=vsock_root,slot=1")
.arg("-device")
@@ -211,8 +165,6 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> {
.map_err(|e| format!("failed to wait for QEMU: {e}"))?;
CHILD_PID.store(0, Ordering::Relaxed);
- teardown_tap_networking(tap_device, host_ip, gw_port);
- tap_guard.disarm();
if status.success() {
Ok(())
@@ -221,6 +173,10 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> {
}
}
+fn qemu_network_args() -> [&'static str; 2] {
+ ["-nic", "none"]
+}
+
fn qemu_disk_args(config: &VmLaunchConfig) -> Vec {
let mut args = vec![
"-drive".to_string(),
@@ -271,20 +227,8 @@ fn write_guest_env_file(overlay_disk: &Path, env_vars: &[String]) -> Result<(),
)
}
-fn qemu_guest_env_vars(config: &VmLaunchConfig, dns_server: Option) -> Vec {
+fn qemu_guest_env_vars(config: &VmLaunchConfig) -> Vec {
let mut env_vars = config.env.clone();
-
- if let Some(ip) = &config.guest_ip
- && let Some(host_ip) = &config.host_ip
- {
- env_vars.push(format!("VM_NET_IP={ip}"));
- env_vars.push(format!("VM_NET_GW={host_ip}"));
- }
-
- if let Some(dns) = dns_server {
- env_vars.push(format!("VM_NET_DNS={dns}"));
- }
-
if config.gpu_bdf.is_some() {
env_vars.push("GPU_ENABLED=true".to_string());
}
@@ -312,12 +256,6 @@ fn build_kernel_cmdline(config: &VmLaunchConfig) -> String {
format!("init={}", config.exec_path),
];
- if let Some(ip) = &config.guest_ip
- && let Some(host_ip) = &config.host_ip
- {
- parts.push(format!("ip={ip}::{host_ip}:255.255.255.252:sandbox::off"));
- }
-
if config.gpu_bdf.is_some() {
parts.push("firmware_class.path=/lib/firmware".to_string());
}
@@ -325,326 +263,16 @@ fn build_kernel_cmdline(config: &VmLaunchConfig) -> String {
parts.join(" ")
}
-fn host_dns_server() -> Option {
- // Prefer systemd-resolved upstream config (skips the 127.0.0.53
- // stub listener which is unreachable from inside QEMU/TAP guests).
- for path in &["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"] {
- let Ok(resolv) = std::fs::read_to_string(path) else {
- continue;
- };
- for line in resolv.lines() {
- let line = line.trim();
- if let Some(server) = line.strip_prefix("nameserver") {
- let server = server.trim();
- if server == "127.0.0.53" || server.starts_with("127.") {
- continue;
- }
- if !server.is_empty() {
- return Some(server.to_string());
- }
- }
- }
- }
- None
-}
-
-/// Remove leftover `vmtap-*` interfaces from previous driver runs.
-///
-/// Called once at driver startup for interfaces that were not torn down
-/// (e.g. the launcher was `SIGKILL`-ed before teardown), so stale
-/// interfaces cannot cause subnet routing conflicts with newly allocated TAPs.
-pub fn cleanup_stale_tap_interfaces() {
- let Ok(entries) = std::fs::read_dir("/sys/class/net") else {
- return;
- };
- for entry in entries.flatten() {
- let name = entry.file_name();
- let Some(name) = name.to_str() else {
- continue;
- };
- if !name.starts_with("vmtap-") {
- continue;
- }
- // Read the IP address so we can clean up iptables rules too.
- // Port 0 tells teardown we don't know the original gateway port;
- // the blanket legacy rule is still cleaned up best-effort.
- let ip = read_tap_host_ip(name);
- if let Some(ref host_ip) = ip {
- teardown_tap_networking(name, host_ip, 0);
- } else {
- let _ = run_cmd("ip", &["link", "set", name, "down"]);
- let _ = run_cmd("ip", &["tuntap", "del", "dev", name, "mode", "tap"]);
- }
- tracing::warn!(interface = %name, "removed stale TAP interface from previous run");
- }
-}
-
-/// Read the first IPv4 address assigned to a network interface.
-fn read_tap_host_ip(device: &str) -> Option {
- let output = StdCommand::new("ip")
- .args(["-4", "-o", "addr", "show", "dev", device])
- .stdin(Stdio::null())
- .stdout(Stdio::piped())
- .stderr(Stdio::null())
- .output()
- .ok()?;
- let stdout = String::from_utf8_lossy(&output.stdout);
- // Format: "28: vmtap-xxx inet 10.0.128.1/30 ..."
- for token in stdout.split_whitespace() {
- if let Some((ip, _prefix)) = token.split_once('/')
- && ip.parse::().is_ok()
- {
- return Some(ip.to_string());
- }
- }
- None
-}
-
-fn setup_tap_networking(tap_device: &str, host_ip: &str, gateway_port: u16) -> Result<(), String> {
- run_cmd("ip", &["tuntap", "add", "dev", tap_device, "mode", "tap"])?;
- run_cmd(
- "ip",
- &["addr", "add", &format!("{host_ip}/30"), "dev", tap_device],
- )?;
- run_cmd("ip", &["link", "set", tap_device, "up"])?;
-
- // Deprioritize routes through down interfaces so a stale vmtap-*
- // that somehow survives cleanup cannot shadow the active one.
- let _ = std::fs::write(
- format!("/proc/sys/net/ipv4/conf/{tap_device}/ignore_routes_with_linkdown"),
- "1",
- );
-
- enable_ip_forwarding()?;
-
- let subnet = tap_subnet_from_host_ip(host_ip);
- let table_name = nft_ruleset::teardown_table_name(tap_device);
-
- // Delete any stale nftables table from a previous driver run.
- let _ = run_cmd("nft", &["delete", "table", "ip", &table_name]);
-
- // Clean up legacy iptables rules from older driver versions.
- let _ = run_cmd(
- "iptables",
- &[
- "-t",
- "nat",
- "-D",
- "POSTROUTING",
- "-s",
- &subnet,
- "-j",
- "MASQUERADE",
- ],
- );
- let _ = run_cmd(
- "iptables",
- &["-D", "FORWARD", "-i", tap_device, "-j", "ACCEPT"],
- );
- let _ = run_cmd(
- "iptables",
- &[
- "-D",
- "FORWARD",
- "-o",
- tap_device,
- "-m",
- "state",
- "--state",
- "RELATED,ESTABLISHED",
- "-j",
- "ACCEPT",
- ],
- );
- let port_str = gateway_port.to_string();
- let _ = run_cmd(
- "iptables",
- &[
- "-D", "INPUT", "-i", tap_device, "-p", "tcp", "--dport", &port_str, "-j", "ACCEPT",
- ],
- );
- let _ = run_cmd(
- "iptables",
- &["-D", "INPUT", "-i", tap_device, "-j", "ACCEPT"],
- );
-
- // Load nftables ruleset atomically.
- let ruleset = nft_ruleset::generate_tap_ruleset(tap_device, &subnet, gateway_port);
- run_nft_stdin(&ruleset)?;
-
- Ok(())
-}
-
-fn teardown_tap_networking(tap_device: &str, host_ip: &str, gateway_port: u16) {
- // Delete the entire nftables table — single atomic operation.
- let table_name = nft_ruleset::teardown_table_name(tap_device);
- let _ = run_cmd("nft", &["delete", "table", "ip", &table_name]);
-
- // Clean up legacy iptables rules from older driver versions.
- let subnet = tap_subnet_from_host_ip(host_ip);
- let _ = run_cmd(
- "iptables",
- &[
- "-D",
- "FORWARD",
- "-o",
- tap_device,
- "-m",
- "state",
- "--state",
- "RELATED,ESTABLISHED",
- "-j",
- "ACCEPT",
- ],
- );
- let _ = run_cmd(
- "iptables",
- &["-D", "FORWARD", "-i", tap_device, "-j", "ACCEPT"],
- );
- if gateway_port > 0 {
- let port_str = gateway_port.to_string();
- let _ = run_cmd(
- "iptables",
- &[
- "-D", "INPUT", "-i", tap_device, "-p", "tcp", "--dport", &port_str, "-j", "ACCEPT",
- ],
- );
- }
- let _ = run_cmd(
- "iptables",
- &["-D", "INPUT", "-i", tap_device, "-j", "ACCEPT"],
- );
- let _ = run_cmd(
- "iptables",
- &[
- "-t",
- "nat",
- "-D",
- "POSTROUTING",
- "-s",
- &subnet,
- "-j",
- "MASQUERADE",
- ],
- );
-
- let _ = run_cmd("ip", &["link", "set", tap_device, "down"]);
- let _ = run_cmd("ip", &["tuntap", "del", "dev", tap_device, "mode", "tap"]);
-}
-
-fn tap_subnet_from_host_ip(host_ip: &str) -> String {
- host_ip.parse::().map_or_else(
- |_| format!("{host_ip}/30"),
- |ip| {
- let base = u32::from(ip) & !3;
- let base_ip = std::net::Ipv4Addr::from(base);
- format!("{base_ip}/30")
- },
- )
-}
-
-fn enable_ip_forwarding() -> Result<(), String> {
- std::fs::write("/proc/sys/net/ipv4/ip_forward", "1")
- .map_err(|e| format!("enable ip_forward: {e}"))
-}
-
-fn run_cmd(cmd: &str, args: &[&str]) -> Result<(), String> {
- let output = StdCommand::new(cmd)
- .args(args)
- .stdin(Stdio::null())
- .stdout(Stdio::piped())
- .stderr(Stdio::piped())
- .output()
- .map_err(|e| format!("failed to run {cmd}: {e}"))?;
- if output.status.success() {
- Ok(())
- } else {
- let stderr = String::from_utf8_lossy(&output.stderr);
- Err(format!("{cmd} {} failed: {stderr}", args.join(" ")))
- }
-}
-
-fn run_nft_stdin(ruleset: &str) -> Result<(), String> {
- use std::io::Write;
-
- let mut child = StdCommand::new("nft")
- .args(["-f", "-"])
- .stdin(Stdio::piped())
- .stdout(Stdio::piped())
- .stderr(Stdio::piped())
- .spawn()
- .map_err(|e| format!("failed to run nft: {e}"))?;
-
- if let Some(mut stdin) = child.stdin.take() {
- stdin
- .write_all(ruleset.as_bytes())
- .map_err(|e| format!("failed to write nft ruleset: {e}"))?;
- }
-
- let output = child
- .wait_with_output()
- .map_err(|e| format!("failed to wait for nft: {e}"))?;
-
- if output.status.success() {
- Ok(())
- } else {
- let stderr = String::from_utf8_lossy(&output.stderr);
- Err(format!("nft -f - failed: {stderr}"))
- }
-}
-
-/// RAII guard that tears down TAP networking on drop.
-struct TapGuard {
- tap_device: String,
- host_ip: String,
- gateway_port: u16,
- disarmed: bool,
-}
-
-impl TapGuard {
- fn new(tap_device: String, host_ip: String, gateway_port: u16) -> Self {
- Self {
- tap_device,
- host_ip,
- gateway_port,
- disarmed: false,
- }
- }
-
- fn disarm(&mut self) {
- self.disarmed = true;
- }
-}
-
-impl Drop for TapGuard {
- fn drop(&mut self) {
- if !self.disarmed {
- teardown_tap_networking(&self.tap_device, &self.host_ip, self.gateway_port);
- }
- }
-}
-
/// Shared procguard cleanup callback for both libkrun and QEMU paths.
/// Only async-signal-safe calls: atomic loads and `kill(2)`.
fn procguard_kill_children() {
- let helper_pid = GVPROXY_PID.load(Ordering::Relaxed);
let child_pid = CHILD_PID.load(Ordering::Relaxed);
- if helper_pid > 0 {
- unsafe {
- libc::kill(helper_pid, libc::SIGTERM);
- }
- }
if child_pid > 0 {
unsafe {
libc::kill(child_pid, libc::SIGTERM);
}
}
std::thread::sleep(Duration::from_millis(200));
- if helper_pid > 0 {
- unsafe {
- libc::kill(helper_pid, libc::SIGKILL);
- }
- }
if child_pid > 0 {
unsafe {
libc::kill(child_pid, libc::SIGKILL);
@@ -677,13 +305,9 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
return Err(format!("image disk not found: {}", image_disk.display()));
}
- // Arm procguard first, BEFORE we spawn gvproxy or fork libkrun, so
- // that the launcher can't be orphaned during setup. The cleanup
- // callback reads the GVPROXY_PID atomic (initially 0 — no-op) and
- // the CHILD_PID atomic (the libkrun fork), so it stays correct as
- // those slots get populated later in this function. Only ONE arm
- // per process: racing two watchers for the same NOTE_EXIT event
- // would cause whichever wins to skip the cleanup.
+ // Arm procguard before forking libkrun so the VM worker cannot outlive
+ // the launcher. No network helper is started: the only host/guest data
+ // path is the protected vsock mapping below.
if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) {
return Err(format!("procguard arm failed: {err}"));
}
@@ -705,132 +329,12 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
)?;
vm.set_workdir(&config.workdir)?;
- // Run gvproxy strictly as the guest's virtual NIC / DHCP / router.
- //
- // After the supervisor-initiated relay migration (#867), the driver
- // no longer forwards any host-side ports into the guest — all ingress
- // traffic for SSH and exec rides the outbound `ConnectSupervisor`
- // gRPC stream the guest opens to the gateway. What gvproxy still
- // provides here is the TCP/IP *plane* the guest kernel needs:
- //
- // * a virtio-net backend attached to libkrun via a Unix
- // SOCK_STREAM (Linux) or SOCK_DGRAM (macOS vfkit), which
- // surfaces as `eth0` inside the guest;
- // * the DHCP server + default router the guest's udhcpc client
- // talks to on boot (IPs 192.168.127.1 / .2, defaults for
- // gvisor-tap-vsock);
- // * the host-facing gateway identity the guest uses for callbacks:
- // gvproxy installs a default NAT entry rewriting `192.168.127.254`
- // (the subnet's HostIP) to the host's `127.0.0.1`, and serves
- // `host.containers.internal` / `host.docker.internal` /
- // `host.openshell.internal` in its embedded DNS pointing at that
- // same HostIP. The guest init script seeds /etc/hosts with the
- // same mapping so the supervisor reaches the host gateway even
- // when gvproxy's DNS isn't in resolv.conf. The gateway IP
- // (192.168.127.1) is NOT a host-loopback proxy — it only listens
- // on its own service ports (DNS:53, DHCP, HTTP API:80).
- //
- // That network plane is also what the sandbox supervisor's
- // per-sandbox netns (veth pair + nftables, see
- // `openshell-sandbox/src/sandbox/linux/netns.rs`) branches off of;
- // libkrun's built-in TSI socket impersonation would not satisfy
- // those kernel-level primitives.
- //
- // The `-listen` API socket and `-ssh-port` forwarder are both
- // deliberately omitted: nothing in the driver enqueues port
- // forwards on the API any more, and the host-side SSH listener is
- // dead plumbing.
- let gvproxy_guard = {
- let gvproxy_binary = runtime_dir.join("gvproxy");
- if !gvproxy_binary.is_file() {
- return Err(format!(
- "missing runtime file: {}",
- gvproxy_binary.display()
- ));
- }
-
- let sock_base = gvproxy_socket_base(&config.overlay_disk)?;
- let net_sock = sock_base.with_extension("v");
- let _ = std::fs::remove_file(&net_sock);
- let _ = std::fs::remove_file(sock_base.with_extension("v-krun.sock"));
-
- let run_dir = config.overlay_disk.parent().unwrap_or(&config.overlay_disk);
- let gvproxy_log = run_dir.join("gvproxy.log");
- let gvproxy_log_file = std::fs::File::create(&gvproxy_log)
- .map_err(|e| format!("create gvproxy log {}: {e}", gvproxy_log.display()))?;
-
- #[cfg(target_os = "linux")]
- let (gvproxy_net_flag, gvproxy_net_url) =
- ("-listen-qemu", format!("unix://{}", net_sock.display()));
- #[cfg(target_os = "macos")]
- let (gvproxy_net_flag, gvproxy_net_url) = (
- "-listen-vfkit",
- format!("unixgram://{}", net_sock.display()),
- );
-
- // `-ssh-port -1` tells gvproxy to skip its default SSH forward
- // (127.0.0.1:2222 → guest:22). We don't use it — all gateway
- // ingress rides the supervisor-initiated relay — and leaving
- // the default on would bind a host-side TCP listener per
- // sandbox, racing concurrent sandboxes for port 2222 and
- // surfacing a misleading "sshd is reachable" endpoint. See
- // https://github.com/containers/gvisor-tap-vsock `cmd/gvproxy/main.go`
- // (`getForwardsMap` returns an empty map when `sshPort == -1`).
- let mut gvproxy_cmd = StdCommand::new(&gvproxy_binary);
- gvproxy_cmd
- .arg(gvproxy_net_flag)
- .arg(&gvproxy_net_url)
- .arg("-ssh-port")
- .arg("-1")
- .stdin(Stdio::null())
- .stdout(Stdio::null())
- .stderr(gvproxy_log_file);
-
- // On Linux the kernel will SIGKILL gvproxy the moment this
- // launcher dies (or is SIGKILLed). `pre_exec` runs in the child
- // between fork and execve, so the PR_SET_PDEATHSIG flag is
- // inherited across execve and applies to gvproxy proper. On
- // macOS/BSDs there is no equivalent; we fall back to killing
- // gvproxy explicitly from the launcher's procguard cleanup
- // callback (see `run_vm` above) and SIGTERM handler
- // (see `install_signal_forwarding` below).
- #[cfg(target_os = "linux")]
- {
- use nix::sys::signal::Signal;
- use std::os::unix::process::CommandExt as _;
- unsafe {
- gvproxy_cmd.pre_exec(|| {
- nix::sys::prctl::set_pdeathsig(Signal::SIGKILL)
- .map_err(|err| std::io::Error::other(format!("pdeathsig: {err}")))
- });
- }
- }
-
- let child = gvproxy_cmd
- .spawn()
- .map_err(|e| format!("failed to start gvproxy {}: {e}", gvproxy_binary.display()))?;
- // The procguard cleanup reads GVPROXY_PID atomically. Storing it
- // here makes the callback able to SIGTERM gvproxy if the driver
- // dies from this moment onward.
- GVPROXY_PID.store(child.id().cast_signed(), Ordering::Relaxed);
-
- wait_for_path(&net_sock, Duration::from_secs(5), "gvproxy data socket")?;
-
- vm.disable_implicit_vsock()?;
- vm.add_vsock(0)?;
-
- let mac: [u8; 6] = [0x5a, 0x94, 0xef, 0xe4, 0x0c, 0xee];
-
- #[cfg(target_os = "linux")]
- vm.add_net_unixstream(&net_sock, &mac, COMPAT_NET_FEATURES)?;
- #[cfg(target_os = "macos")]
- {
- const NET_FLAG_VFKIT: u32 = 1 << 0;
- vm.add_net_unixgram(&net_sock, &mac, COMPAT_NET_FEATURES, NET_FLAG_VFKIT)?;
- }
-
- Some(GvproxyGuard::new(child))
- };
+ vm.disable_implicit_vsock()?;
+ vm.add_vsock(0)?;
+ if let Some(port_map) = &config.vsock_port_map {
+ let _ = std::fs::remove_file(&port_map.host_socket);
+ vm.add_vsock_port(port_map)?;
+ }
vm.set_console_output(&config.console_output)?;
@@ -846,8 +350,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
// fires). Arm procguard so this fork is SIGKILLed if the
// parent launcher dies abruptly. On Linux this uses
// `PR_SET_PDEATHSIG`; on macOS this spawns a kqueue
- // NOTE_EXIT watcher thread. Either way it closes the same
- // leak gvproxy does above.
+ // NOTE_EXIT watcher thread.
//
// We also SIGKILL ourselves if arming fails — there's no
// safe way to continue if we can't guarantee cleanup.
@@ -864,9 +367,6 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
let status = wait_for_child(pid)?;
CHILD_PID.store(0, Ordering::Relaxed);
- cleanup_gvproxy(gvproxy_guard);
- GVPROXY_PID.store(0, Ordering::Relaxed);
-
if libc::WIFEXITED(status) {
match libc::WEXITSTATUS(status) {
0 => Ok(()),
@@ -1118,50 +618,18 @@ impl VmContext {
)
}
- #[cfg(target_os = "macos")]
- fn add_net_unixgram(
- &self,
- socket_path: &Path,
- mac: &[u8; 6],
- features: u32,
- flags: u32,
- ) -> Result<(), String> {
- let sock_c = path_to_cstring(socket_path)?;
- check(
- unsafe {
- (self.krun.krun_add_net_unixgram)(
- self.ctx_id,
- sock_c.as_ptr(),
- -1,
- mac.as_ptr(),
- features,
- flags,
- )
- },
- "krun_add_net_unixgram",
- )
- }
-
- #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode.
- fn add_net_unixstream(
- &self,
- socket_path: &Path,
- mac: &[u8; 6],
- features: u32,
- ) -> Result<(), String> {
- let sock_c = path_to_cstring(socket_path)?;
+ fn add_vsock_port(&self, port_map: &VsockPortMap) -> Result<(), String> {
+ let socket_c = path_to_cstring(&port_map.host_socket)?;
check(
unsafe {
- (self.krun.krun_add_net_unixstream)(
+ (self.krun.krun_add_vsock_port2)(
self.ctx_id,
- sock_c.as_ptr(),
- -1,
- mac.as_ptr(),
- features,
- 0,
+ port_map.guest_port,
+ socket_c.as_ptr(),
+ port_map.host_initiated,
)
},
- "krun_add_net_unixstream",
+ "krun_add_vsock_port2",
)
}
@@ -1210,109 +678,6 @@ impl Drop for VmContext {
}
}
-struct GvproxyGuard {
- child: Option,
-}
-
-impl GvproxyGuard {
- fn new(child: StdChild) -> Self {
- Self { child: Some(child) }
- }
-
- fn disarm(&mut self) -> Option {
- self.child.take()
- }
-}
-
-impl Drop for GvproxyGuard {
- fn drop(&mut self) {
- if let Some(mut child) = self.child.take() {
- let _ = child.kill();
- let _ = child.wait();
- }
- }
-}
-
-fn wait_for_path(path: &Path, timeout: Duration, label: &str) -> Result<(), String> {
- let deadline = Instant::now() + timeout;
- let mut interval = Duration::from_millis(5);
- while !path.exists() {
- if Instant::now() >= deadline {
- return Err(format!(
- "{label} did not appear within {:.1}s: {}",
- timeout.as_secs_f64(),
- path.display()
- ));
- }
- std::thread::sleep(interval);
- interval = (interval * 2).min(Duration::from_millis(200));
- }
- Ok(())
-}
-
-fn hash_path_id(path: &Path) -> String {
- let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
- for byte in path.to_string_lossy().as_bytes() {
- hash ^= u64::from(*byte);
- hash = hash.wrapping_mul(0x0100_0000_01b3);
- }
- format!("{:012x}", hash & 0x0000_ffff_ffff_ffff)
-}
-
-fn secure_socket_base(subdir: &str) -> Result {
- let base = std::env::var_os("XDG_RUNTIME_DIR").map_or_else(
- || {
- let fallback = PathBuf::from("/tmp");
- if fallback.is_dir() {
- fallback
- } else {
- std::env::temp_dir()
- }
- },
- PathBuf::from,
- );
- let dir = base.join(subdir);
-
- if dir.exists() {
- let meta = dir
- .symlink_metadata()
- .map_err(|e| format!("lstat {}: {e}", dir.display()))?;
- if meta.file_type().is_symlink() {
- return Err(format!(
- "socket directory {} is a symlink; refusing to use it",
- dir.display()
- ));
- }
- #[cfg(unix)]
- {
- use std::os::unix::fs::MetadataExt as _;
- let uid = unsafe { libc::getuid() };
- if meta.uid() != uid {
- return Err(format!(
- "socket directory {} is owned by uid {} but we are uid {}",
- dir.display(),
- meta.uid(),
- uid
- ));
- }
- }
- } else {
- std::fs::create_dir_all(&dir)
- .map_err(|e| format!("create socket dir {}: {e}", dir.display()))?;
- #[cfg(unix)]
- {
- use std::os::unix::fs::PermissionsExt as _;
- let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
- }
- }
-
- Ok(dir)
-}
-
-fn gvproxy_socket_base(overlay_disk: &Path) -> Result {
- Ok(secure_socket_base("osd-gv")?.join(hash_path_id(overlay_disk)))
-}
-
fn install_signal_forwarding(pid: i32) {
unsafe {
libc::signal(
@@ -1327,12 +692,7 @@ fn install_signal_forwarding(pid: i32) {
CHILD_PID.store(pid, Ordering::Relaxed);
}
-/// Async-signal-safe handler that forwards SIGTERM to every process we
-/// own: the libkrun VM worker and the gvproxy helper. We cannot rely on
-/// Rust destructors (`GvproxyGuard::drop`, `ManagedDriverProcess::drop`)
-/// running on signal-driven exit, so we explicitly deliver the signal
-/// here. The `wait_for_child` loop reaps libkrun and `cleanup_gvproxy`
-/// reaps gvproxy before `run_vm` returns.
+/// Async-signal-safe handler that forwards SIGTERM to the VM worker.
///
/// Only async-signal-safe libc calls are used — `kill(2)` is listed in
/// POSIX.1-2017 as async-signal-safe, atomic loads are lock-free on the
@@ -1344,13 +704,6 @@ extern "C" fn forward_signal(_sig: libc::c_int) {
libc::kill(vm_pid, libc::SIGTERM);
}
}
- let gv_pid = GVPROXY_PID.load(Ordering::Relaxed);
- if gv_pid > 0 {
- // gvproxy handles SIGTERM cleanly; no need for SIGKILL.
- unsafe {
- libc::kill(gv_pid, libc::SIGTERM);
- }
- }
}
fn wait_for_child(pid: i32) -> Result {
@@ -1365,15 +718,6 @@ fn wait_for_child(pid: i32) -> Result {
Ok(status)
}
-fn cleanup_gvproxy(mut guard: Option) {
- if let Some(mut guard) = guard.take()
- && let Some(mut child) = guard.disarm()
- {
- let _ = child.kill();
- let _ = child.wait();
- }
-}
-
fn check(ret: i32, func: &'static str) -> Result<(), String> {
if ret < 0 {
Err(format!("{func} failed with error code {ret}"))
@@ -1431,23 +775,17 @@ mod tests {
console_output: PathBuf::from("/console.log"),
backend: VmBackend::Qemu,
gpu_bdf: Some("0000:01:00.0".to_string()),
- tap_device: Some("vmtap-test".to_string()),
- guest_ip: Some("10.0.128.2".to_string()),
- host_ip: Some("10.0.128.1".to_string()),
vsock_cid: Some(4),
- guest_mac: Some("02:00:00:00:00:01".to_string()),
- gateway_port: Some(8080),
+ vsock_port_map: None,
}
}
#[test]
- fn qemu_guest_env_vars_include_driver_runtime_metadata() {
- let env = qemu_guest_env_vars(&qemu_config(), Some("1.1.1.1".to_string()));
+ fn qemu_guest_env_vars_omit_network_metadata() {
+ let env = qemu_guest_env_vars(&qemu_config());
assert!(env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()));
- assert!(env.contains(&"VM_NET_IP=10.0.128.2".to_string()));
- assert!(env.contains(&"VM_NET_GW=10.0.128.1".to_string()));
- assert!(env.contains(&"VM_NET_DNS=1.1.1.1".to_string()));
+ assert!(!env.iter().any(|value| value.starts_with("VM_NET_")));
assert!(env.contains(&"GPU_ENABLED=true".to_string()));
}
@@ -1490,13 +828,13 @@ mod tests {
}
#[test]
- fn kernel_cmdline_keeps_guest_init_metadata_out_of_proc_cmdline() {
+ fn kernel_cmdline_has_no_guest_network_configuration() {
let cmdline = build_kernel_cmdline(&qemu_config());
assert!(cmdline.contains("root=/dev/vda"));
assert!(cmdline.contains("rootfstype=ext4"));
assert!(cmdline.contains(" ro"));
- assert!(cmdline.contains("ip=10.0.128.2::10.0.128.1:255.255.255.252:sandbox::off"));
+ assert!(!cmdline.contains("ip="));
assert!(cmdline.contains("firmware_class.path=/lib/firmware"));
assert!(!cmdline.contains("VM_NET_IP="));
assert!(!cmdline.contains("VM_NET_GW="));
@@ -1524,6 +862,11 @@ mod tests {
assert!(args.contains(&"virtio-blk-pci,drive=overlay".to_string()));
}
+ #[test]
+ fn qemu_explicitly_disables_implicit_network_devices() {
+ assert_eq!(qemu_network_args(), ["-nic", "none"]);
+ }
+
#[test]
fn qemu_disk_args_attach_prepared_image_readonly_when_present() {
let mut config = qemu_config();
@@ -1536,29 +879,4 @@ mod tests {
));
assert!(args.contains(&"virtio-blk-pci,drive=image".to_string()));
}
-
- #[test]
- fn gvproxy_socket_base_is_per_sandbox_overlay_path() {
- let first =
- gvproxy_socket_base(Path::new("/tmp/openshell-vm/sandboxes/first/overlay.ext4"))
- .expect("first socket base");
- let second =
- gvproxy_socket_base(Path::new("/tmp/openshell-vm/sandboxes/second/overlay.ext4"))
- .expect("second socket base");
-
- assert_ne!(first, second);
- }
-
- #[test]
- fn tap_subnet_from_host_ip_calculates_slash30_base() {
- assert_eq!(tap_subnet_from_host_ip("10.0.128.1"), "10.0.128.0/30");
- assert_eq!(tap_subnet_from_host_ip("10.0.128.2"), "10.0.128.0/30");
- assert_eq!(tap_subnet_from_host_ip("10.0.128.5"), "10.0.128.4/30");
- }
-
- #[test]
- fn tap_subnet_from_host_ip_handles_invalid_ip() {
- let result = tap_subnet_from_host_ip("not-an-ip");
- assert_eq!(result, "not-an-ip/30");
- }
}
diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs
index f52bd9ddfa..c88b220824 100644
--- a/crates/openshell-gateway/src/vm.rs
+++ b/crates/openshell-gateway/src/vm.rs
@@ -104,9 +104,8 @@ pub struct VmComputeConfig {
/// for policy-approved TLS egress from VM sandboxes.
///
/// Deployment-level configuration, not a per-sandbox setting: it is passed
- /// to the driver, which puts it on the guest supervisor's argv. A proxy on
- /// this host's loopback is reachable from a guest only through the gvproxy
- /// host alias `host.openshell.internal`.
+ /// to the driver, which passes it to the host supervisor. The supervisor
+ /// resolves `host.openshell.internal` to host loopback.
pub https_proxy: Option,
/// Comma-separated `NO_PROXY` list. Bypasses only the corporate proxy,
diff --git a/deploy/docker/Dockerfile.driver-vm-macos b/deploy/docker/Dockerfile.driver-vm-macos
index 58317a52d8..438700eb09 100644
--- a/deploy/docker/Dockerfile.driver-vm-macos
+++ b/deploy/docker/Dockerfile.driver-vm-macos
@@ -8,7 +8,7 @@
#
# openshell-driver-vm loads libkrun/libkrunfw at runtime via dlopen, so it
# does NOT need Hypervisor.framework headers at build time. Pre-compressed
-# runtime artifacts (libkrun, libkrunfw, gvproxy, bundled supervisor) are injected via
+# runtime artifacts (libkrun, libkrunfw, bundled sandbox/supervisor) are injected via
# the vm-runtime-compressed build context and embedded into the binary via
# include_bytes!().
#
diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx
index 1c846c29a8..65f4871cd9 100644
--- a/docs/reference/gateway-config.mdx
+++ b/docs/reference/gateway-config.mdx
@@ -805,33 +805,17 @@ guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem"
# degrading to a direct dial. proxy_auth_file and proxy_ca_bundle are paths on
# the gateway host.
#
-# The sandbox cannot select or override these settings. They reach the guest
-# supervisor on its command line through a per-sandbox file the driver writes
-# into the overlay upperdir on every launch, so a sandbox image cannot supply
-# its own values or disable the operator's by baking a file at that path.
+# The sandbox cannot select or override these settings. The driver passes them
+# only to the host supervisor.
#
# Reachability: a proxy on the corporate network needs no special address and
-# works on every VM sandbox. The guest's callback to the gateway is unaffected
-# and never traverses the proxy.
+# works on every VM sandbox.
#
-# A proxy on the gateway host itself is reachable only from libkrun-backed
-# (non-GPU) sandboxes: their egress leaves through gvproxy, which NATs
-# 192.168.127.254 to the host's 127.0.0.1, so address it as
-# http://host.openshell.internal: rather than http://127.0.0.1:.
-# GPU sandboxes run on the QEMU/TAP backend, which has no such NAT —
-# host.openshell.internal resolves to the TAP host address, and the driver's
-# nftables rules let the guest reach only the gateway port on the host. A
-# gateway-host proxy URL is therefore rejected when the sandbox launches on
-# QEMU, rather than timing out on every CONNECT; give GPU sandboxes a proxy
-# address routable from the guest's masqueraded egress.
+# A proxy on the gateway host is addressed as
+# http://host.openshell.internal:. The host supervisor normalizes that
+# name to host loopback for both libkrun and QEMU sandboxes.
#
-# Because a microVM has no bind mounts or container secrets, the driver stages
-# the credential and the CA into the per-sandbox overlay disk: the credential
-# root-only inside the guest, and both removed with the sandbox. The
-# credential is therefore at rest in that overlay image on the gateway host —
-# the same delivery the per-sandbox gateway token already uses, and a
-# difference from the Podman secret model worth noting when choosing where to
-# keep proxy credentials.
+# Credentials and private CA material remain host-side with the supervisor.
# https_proxy = "http://host.openshell.internal:8080"
# no_proxy = "10.0.0.0/8,.internal.example"
# proxy_auth_file = "/etc/openshell/secrets/proxy-auth"
diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx
index 987e66b0d9..e025452e2d 100644
--- a/docs/reference/sandbox-compute-drivers.mdx
+++ b/docs/reference/sandbox-compute-drivers.mdx
@@ -354,23 +354,28 @@ The VM driver resolves sandbox images from a local container engine before falli
systemctl --user start podman.socket
```
-### Host Firewall
+### Network isolation
-The VM driver creates nftables rules on the host for each sandbox VM's TAP network interface. These rules provide NAT for VM connectivity and defense-in-depth isolation: unsolicited inbound connections to the VM are dropped, and the VM can only reach the gateway port on the host. Primary security enforcement (proxy-only egress and bypass detection) is handled by the sandbox supervisor inside the VM guest.
-
-On hosts with restrictive firewalls (e.g. firewalld), the host firewall may additionally block VM traffic that the driver's rules accept. If VM sandboxes cannot reach the network, verify that the host firewall allows forwarding and input for `vmtap-*` interfaces. See the [VM driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-vm/README.md#host-side-nftables-rules) for details.
+VM sandboxes boot without a virtual NIC. `openshell-sandbox` intercepts workload
+network syscalls inside the guest and carries mediated streams over virtio-vsock
+to the host `openshell-supervisor`, which owns DNS, policy evaluation, and
+external connections. The driver does not create TAP interfaces or host
+nftables rules.
### Corporate Proxy Egress
-For proxy-required networks, the VM driver accepts the same corporate egress proxy keys as the Podman driver: `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, and `proxy_ca_bundle`. The in-guest supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly.
+For proxy-required networks, the VM driver accepts the same corporate egress proxy keys as the Podman driver: `https_proxy`, `no_proxy`, `proxy_auth_file`, `proxy_auth_allow_insecure`, `proxy_connect_by_hostname`, and `proxy_ca_bundle`. The host supervisor chains policy-approved TLS tunnels through the proxy with HTTP CONNECT instead of dialing destinations directly.
-The settings reach the guest supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox cannot select, alter, or disable the proxy from inside the guest — including through image `ENV`, the sandbox environment, or files baked into the image at the paths the driver uses.
+The settings reach the host supervisor through driver-owned arguments, so a sandbox cannot select, alter, or disable the proxy from inside the guest.
A proxy on the corporate network needs no special address and works on every VM sandbox. The guest's callback to the gateway never traverses the proxy.
-A proxy on the gateway host itself works only for libkrun-backed (non-GPU) sandboxes, whose egress leaves through gvproxy: configure `https_proxy = "http://host.openshell.internal:"` rather than a `127.0.0.1` URL, because gvproxy NATs that alias to the host's `127.0.0.1`. GPU sandboxes use the QEMU/TAP backend, where `host.openshell.internal` resolves to the TAP host address and the driver's [host firewall rules](#host-firewall) allow the guest to reach only the gateway port on the host. The driver rejects a gateway-host proxy URL when a sandbox launches on QEMU instead of letting every CONNECT time out, so give GPU sandboxes a proxy address routable from the guest's masqueraded egress.
+A proxy on the gateway host works for both libkrun and QEMU sandboxes. Configure
+`https_proxy = "http://host.openshell.internal:"`; the host supervisor
+normalizes that name to host loopback.
-Because a microVM has no bind mounts or container secrets, the driver stages the credential (root-only) and the CA bundle into the per-sandbox overlay disk and removes them with the sandbox. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior.
+The credential and private CA material stay with the host supervisor rather
+than being staged into the guest. See the [Gateway Configuration File](./gateway-config) reference for the full contract, including the cleartext-credential acknowledgement and the validated-IP CONNECT behavior.
## Kubernetes Driver
diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh
index 96da2a879f..3b397d93d7 100755
--- a/e2e/rust/e2e-vm.sh
+++ b/e2e/rust/e2e-vm.sh
@@ -5,15 +5,10 @@
# Run the Rust e2e smoke test against an openshell-gateway running the
# standalone VM compute driver (`openshell-driver-vm`).
#
-# Architecture (post supervisor-initiated relay, PR #867):
-# * The gateway never dials the sandbox. Instead, the in-guest
-# supervisor opens an outbound `ConnectSupervisor` gRPC stream to
-# the gateway on startup and keeps it alive for the sandbox
-# lifetime. SSH (`/connect/ssh`) and `ExecSandbox` traffic ride the
-# same TCP+TLS+HTTP/2 connection as multiplexed HTTP/2 streams.
-# * There is no host-side SSH port forward. gvproxy still provides
-# guest egress so the supervisor can reach the gateway, but it no
-# longer forwards any TCP port back to the guest.
+# Architecture:
+# * `openshell-sandbox` runs inside a NIC-less guest.
+# * The host `openshell-supervisor` connects over virtio-vsock and owns
+# gateway registration, policy evaluation, DNS, and external networking.
# * Readiness is authoritative on the gateway: a sandbox's phase
# flips to `Ready` the moment `ConnectSupervisor` registers, and
# back to `Provisioning` when the session drops. The VM driver
@@ -24,11 +19,12 @@
#
# What the script does:
# 1. When no prebuilt VM driver is supplied, ensures the VM runtime
-# (libkrun + gvproxy) and bundled supervisor are staged.
-# 2. Builds `openshell-gateway`, `openshell-driver-vm`, and the
-# `openshell` CLI with the embedded runtime as needed. When CI supplies
-# OPENSHELL_GATEWAY_BIN, OPENSHELL_VM_DRIVER_BIN, or OPENSHELL_BIN, the
-# matching prebuilt binary is reused instead of rebuilt.
+# (libkrun) and bundled sandbox/supervisor binaries are staged.
+# 2. Builds `openshell-gateway`, `openshell-driver-vm`, the native host
+# `openshell-supervisor` host control, and the `openshell` CLI with the
+# embedded runtime as needed. When CI supplies OPENSHELL_GATEWAY_BIN,
+# OPENSHELL_VM_DRIVER_BIN, OPENSHELL_VM_SUPERVISOR_BIN, or OPENSHELL_BIN,
+# the matching prebuilt binary is reused instead of rebuilt.
# 3. On macOS, codesigns the VM driver (libkrun needs the
# `com.apple.security.hypervisor` entitlement).
# 4. Writes a per-run gateway config with `[openshell.drivers.vm]`
@@ -88,10 +84,10 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then
mise run vm:setup
fi
- if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then
- echo "==> Building bundled VM supervisor (mise run vm:supervisor)"
- mise run vm:supervisor
- fi
+ # Always rebuild the guest bundle so an e2e run cannot silently exercise a
+ # stale boundary binary after supervisor or isolation-interface changes.
+ echo "==> Building bundled VM supervisor (mise run vm:supervisor)"
+ mise run vm:supervisor
export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}"
else
@@ -116,6 +112,14 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then
else
echo "==> Using prebuilt openshell-driver-vm at ${DRIVER_BIN}"
fi
+if [ -z "${OPENSHELL_VM_SUPERVISOR_BIN:-}" ]; then
+ # The VM driver prefers a native sibling `openshell-supervisor`. Build it
+ # explicitly so a stale target/debug binary cannot disagree with the
+ # freshly embedded guest sandbox protocol.
+ build_packages+=(-p openshell-supervisor)
+else
+ echo "==> Using prebuilt VM host supervisor at ${OPENSHELL_VM_SUPERVISOR_BIN}"
+fi
if [ -z "${OPENSHELL_BIN:-}" ]; then
build_packages+=(-p openshell-cli)
else
@@ -226,7 +230,7 @@ cleanup() {
rm -f "${GATEWAY_LOG}" 2>/dev/null || true
# Only wipe the per-run state dir on success. On failure, leave it for
- # post-mortem (serial console logs, gvproxy logs, root disk images).
+ # post-mortem (serial console logs and root disk images).
if [ "${exit_code}" -eq 0 ]; then
rm -rf "${RUN_STATE_DIR}" 2>/dev/null || true
else
@@ -245,12 +249,8 @@ echo "==> Starting openshell-gateway on 127.0.0.1:${HOST_PORT} (state: ${RUN_STA
# `~/.local/libexec/openshell/openshell-driver-vm` when present,
# which silently shadows development builds — a subtle source of
# stale-binary bugs in e2e runs.
-# `grpc_endpoint` is the URL the VM driver passes into each guest as
-# OPENSHELL_ENDPOINT. The supervisor inside the VM dials this address.
-# Use `host.openshell.internal` rather than `127.0.0.1` so gvproxy's
-# host-loopback proxy carries the connection while keeping the endpoint aligned
-# with package-managed gateway certificates. gvproxy's bare gateway IP
-# (192.168.127.1) does NOT forward arbitrary host ports.
+# `grpc_endpoint` is consumed by the host supervisor. The host alias is
+# normalized to loopback while keeping package-managed certificate naming.
e2e_generate_gateway_jwt "${JWT_DIR}"
e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}"
diff --git a/e2e/rust/src/harness/host_process.rs b/e2e/rust/src/harness/host_process.rs
index f9fefd75a6..9e086e4718 100644
--- a/e2e/rust/src/harness/host_process.rs
+++ b/e2e/rust/src/harness/host_process.rs
@@ -4,11 +4,9 @@
//! Host-process TCP fixtures for e2e tests.
//!
//! [`HostSupportContainer`](super::container::HostSupportContainer) publishes
-//! the same shape of fixture through a container engine. VM sandboxes reach
-//! the host through gvproxy's `host.openshell.internal` alias and the VM e2e
-//! lane has no container runtime of its own, so this variant runs the fixture
-//! as a plain host process instead — keeping the lane free of a container
-//! dependency it does not otherwise need.
+//! the same shape of fixture through a container engine. The VM host supervisor
+//! reaches these fixtures directly, and the VM e2e lane has no container
+//! runtime of its own, so this variant runs the fixture as a plain host process.
use std::io::Read as _;
use std::path::PathBuf;
@@ -66,8 +64,7 @@ impl HostPythonFixture {
log_path,
};
// Bind to 127.0.0.1 for the readiness probe even though the fixture
- // listens on 0.0.0.0: the guest reaches it through gvproxy's NAT to
- // the host loopback, so loopback reachability is what matters.
+ // listens on 0.0.0.0; the host supervisor dials it over loopback.
wait_for_port("127.0.0.1", port, Duration::from_secs(60))
.await
.map_err(|err| {
diff --git a/e2e/rust/tests/vm_corporate_proxy.rs b/e2e/rust/tests/vm_corporate_proxy.rs
index 71c74c272c..1e34e71d0d 100644
--- a/e2e/rust/tests/vm_corporate_proxy.rs
+++ b/e2e/rust/tests/vm_corporate_proxy.rs
@@ -10,8 +10,7 @@
//! chain end to end:
//!
//! gateway TOML → VM driver config → driver subprocess argv → per-sandbox
-//! overlay staging (credential, CA, supervisor argument list) → guest init
-//! script → supervisor CLI parsing → policy evaluation → proxied CONNECT
+//! host supervisor configuration → policy evaluation → proxied CONNECT
//!
//! and asserts the properties only a real run can establish:
//!
@@ -25,12 +24,9 @@
//! 6. An incoherent setting is fatal at gateway startup rather than
//! degrading to a direct dial.
//!
-//! Fixtures run as host processes and are reached from the guest through
-//! gvproxy's `host.openshell.internal` alias, which is also what proves the
-//! documented host-loopback reachability rule for the libkrun backend. That
-//! rule is libkrun-specific: QEMU/TAP sandboxes (GPU) cannot reach a
-//! gateway-host proxy at all, and the driver rejects such a configuration at
-//! launch — see `qemu_backend_rejects_a_gateway_host_proxy` in the driver.
+//! Fixtures run as host processes and are reached by the host supervisor.
+//! `host.openshell.internal` is normalized to host loopback for both libkrun
+//! and QEMU guests because no workload networking leaves the VM.
use std::fmt::Write as _;
use std::io::Write as _;
@@ -45,12 +41,9 @@ use openshell_e2e::harness::sandbox::SandboxGuard;
use serial_test::serial;
use tempfile::NamedTempFile;
-/// The gvproxy host alias seeded into every guest's `/etc/hosts`. A host-bound
-/// fixture is reachable from the guest only through this name.
+/// The OpenShell alias for the gateway host.
const HOST_ALIAS: &str = "host.openshell.internal";
-/// The address `HOST_ALIAS` resolves to inside the guest, and therefore the
-/// CONNECT target the supervisor validates and sends to the proxy.
-const GVPROXY_HOST_LOOPBACK_IP: &str = "192.168.127.254";
+const HOST_LOOPBACK_IP: &str = "127.0.0.1";
const PROXY_USER: &str = "proxyuser";
const PROXY_PASS: &str = "proxypass";
@@ -79,18 +72,10 @@ impl FixturePorts {
}
}
-/// Python that maps the guest-side host address back to the host loopback.
-///
-/// The supervisor sends the *validated* CONNECT target, which inside the guest
-/// is gvproxy's host-loopback address. That address means nothing on the host
-/// where the proxy fixture runs, so the fixture rewrites it to `127.0.0.1`
-/// when dialing — standing in for the corporate network's own routing to the
-/// destination. The target the fixture *logs* is left untouched, so the
-/// validated-IP assertion still tests the supervisor's behaviour rather than
-/// the fixture's.
+/// Python helper retained by the shared proxy fixture shape.
const HOST_REWRITE: &str = "
def dial_host(host):
- return '127.0.0.1' if host == '192.168.127.254' else host
+ return host
";
/// A forward proxy that requires Basic auth and logs every CONNECT it sees.
@@ -671,7 +656,7 @@ fn assert_proxied_egress(output: &str, proxy_logs: &str, ports: &FixturePorts) {
// staged into the overlay reached the supervisor.
assert!(
proxy_logs.contains(&format!(
- "CONNECT {GVPROXY_HOST_LOOPBACK_IP}:{} auth=ok",
+ "CONNECT {HOST_LOOPBACK_IP}:{} auth=ok",
ports.allowed
)),
"proxy should have seen an authenticated validated-IP CONNECT to the approved upstream:\n{proxy_logs}"
@@ -713,7 +698,7 @@ async fn vm_corporate_proxy_routes_approved_tls_egress() {
let ports = FixturePorts::pick();
- // ── Host fixtures, reached from the guest through the gvproxy alias ──
+ // ── Host fixtures, reached by the host supervisor ──
let proxy = HostPythonFixture::start(&proxy_script(ports.proxy), ports.proxy)
.await
.expect("start fake corporate proxy");
@@ -832,10 +817,7 @@ async fn vm_corporate_proxy_trusts_ca_bundle_for_https_proxy() {
sandbox.create_output
);
assert!(
- proxy_logs.contains(&format!(
- "CONNECT {GVPROXY_HOST_LOOPBACK_IP}:{} ok",
- ports.allowed
- )),
+ proxy_logs.contains(&format!("CONNECT {HOST_LOOPBACK_IP}:{} ok", ports.allowed)),
"https proxy should have seen a validated-IP CONNECT to the approved upstream:\n{proxy_logs}"
);
assert!(
diff --git a/nix/pkgs/vm-runtime.nix b/nix/pkgs/vm-runtime.nix
index 963209c8a6..f48561a19b 100644
--- a/nix/pkgs/vm-runtime.nix
+++ b/nix/pkgs/vm-runtime.nix
@@ -13,38 +13,35 @@ let
{
x86_64-linux = {
platform = "linux-x86_64";
- hash = "sha256-dw3Lc7IapCyNeE7j6dnlgd/b8Yc91/7IOi3XJORyILQ=";
+ hash = "sha256-dJauQnv4L+rT003rJfFPbrJNsQwoWpX61X3GlBkuIog=";
artifacts = [
"libkrun.so"
"libkrunfw.so.5"
- "gvproxy"
"umoci"
];
};
aarch64-linux = {
platform = "linux-aarch64";
- hash = "sha256-aJDuDb7AsuH9R+AyXA/JIxE9fJmZ5kP0Lkhg6F0Ot5A=";
+ hash = "sha256-VvqnmAClehcU1IifoZsd4XrSI6N2Hlu1dskNuPQxME4=";
artifacts = [
"libkrun.so"
"libkrunfw.so.5"
- "gvproxy"
"umoci"
];
};
aarch64-darwin = {
platform = "darwin-aarch64";
- hash = "sha256-BDSeY5XGDozaBZzHTiQQX90jzsSc6shJZs5zdzludX0=";
+ hash = "sha256-orr16ZuCLQ5b2Uwg1NuobjLMpmxZjaGwmb+i/QZ5TZM=";
artifacts = [
"libkrun.dylib"
"libkrunfw.5.dylib"
- "gvproxy"
"umoci"
];
};
}
.${stdenv.hostPlatform.system};
archive = fetchurl {
- url = "https://github.com/NVIDIA/OpenShell/releases/download/vm-runtime/vm-runtime-${runtime.platform}.tar.zst";
+ url = "https://github.com/NVIDIA/OpenShell/releases/download/vm-runtime-capability-free/vm-runtime-${runtime.platform}.tar.zst";
inherit (runtime) hash;
};
in
@@ -59,7 +56,6 @@ stdenv.mkDerivation {
mkdir -p "$out"
tar --extract --file ${archive} --directory "$out"
-
mkdir -p "$out/compressed"
for artifact in ${lib.escapeShellArgs runtime.artifacts}; do
zstd -19 -T1 "$out/$artifact" -o "$out/compressed/$artifact.zst"
diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md
index 8e86bc0643..533a000aeb 100644
--- a/skills/debug-openshell-cluster/SKILL.md
+++ b/skills/debug-openshell-cluster/SKILL.md
@@ -636,41 +636,12 @@ grep -A20 '^\[openshell.drivers.vm\]' | grep -E 'https_proxy|no_p
ps -o args= -p "$(pgrep -f openshell-driver-vm | head -n1)" | tr ' ' '\n' | grep -A1 -- '--proxy\|--https-proxy\|--no-proxy'
```
-Reachability is the most common failure, and it depends on the VM backend.
-On libkrun (non-GPU sandboxes) guest egress leaves through gvproxy, so a proxy
-bound to the gateway host's loopback is **not** reachable at `127.0.0.1` from
-inside the guest: it must be addressed as
-`http://host.openshell.internal:`, which gvproxy NATs from
-`192.168.127.254` to the host's `127.0.0.1`. A `https_proxy` pointing at a
-loopback URL produces policy-approved CONNECT attempts that time out while
-public destinations still work.
-
-GPU sandboxes run on QEMU/TAP, where no gateway-host proxy is reachable at
-all: `host.openshell.internal` resolves to the TAP host address, and the
-driver's nftables `input` chain accepts only the gateway port from the guest.
-The driver rejects such a configuration at launch — a create failing with
-`https_proxy ... addresses the gateway host, which a QEMU/TAP sandbox ...
-cannot reach` means the proxy must move to an address routable from the
-guest's masqueraded egress (or the sandbox must run without a GPU).
-
-The settings reach the supervisor through a driver-written argument file in
-the per-sandbox overlay, not through the guest environment. The credential and
-CA bundle are staged into the same overlay at fixed guest paths. Inspect the
-guest side from the VM console log, which records how many driver-supplied
-arguments the init script read:
-
-```bash
-grep -E 'supervisor arguments from driver|supervisor argument list' /sandboxes//rootfs-console.log
-grep -Ei 'upstream|connect|proxy' /sandboxes//rootfs-console.log | tail -n 40
-```
-
-`FATAL: supervisor argument list ... is not readable` or `FATAL: empty entry in
-supervisor argument list` means the overlay is broken or was tampered with, and
-the guest deliberately aborts rather than starting a supervisor with a
-truncated egress configuration. If the guest logs no driver arguments at all
-while `gateway.toml` sets `https_proxy`, the running driver predates the
-configuration — check that the gateway spawned the driver binary you expect
-(`[openshell.drivers.vm].driver_dir`).
+Both libkrun and QEMU guests are NIC-less. Proxy settings, credentials, and
+private CA material stay with the host `openshell-supervisor`. For a proxy on
+the gateway host, use `http://host.openshell.internal:`; the supervisor
+normalizes that name to host loopback. Inspect `supervisor.log` and
+`supervisor.err.log` under the sandbox state directory for connection or
+credential failures.
## Common Failure Patterns
diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh
index 80d723eb11..d98670a58b 100755
--- a/tasks/scripts/gateway-vm.sh
+++ b/tasks/scripts/gateway-vm.sh
@@ -209,7 +209,7 @@ check_supervisor_cross_toolchain() {
fi
local missing=0
if ! command -v cargo-zigbuild >/dev/null 2>&1; then
- echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2
+ echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest sandbox)." >&2
echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2
missing=1
fi
@@ -287,16 +287,15 @@ VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFA
DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")"
-# Build prerequisites: VM runtime artifacts + bundled supervisor.
+# Build prerequisites: VM runtime artifacts + bundled sandbox/supervisor.
if [ ! -d "${COMPRESSED_DIR}" ] \
|| ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q . \
- || [ ! -f "${COMPRESSED_DIR}/gvproxy.zst" ] \
|| [ ! -f "${COMPRESSED_DIR}/umoci.zst" ]; then
echo "==> Preparing embedded VM runtime (mise run vm:setup)"
mise run vm:setup
fi
-if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then
+if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ] || [ ! -f "${COMPRESSED_DIR}/openshell-supervisor.zst" ]; then
check_supervisor_cross_toolchain
echo "==> Building bundled VM supervisor (mise run vm:supervisor)"
mise run vm:supervisor
@@ -309,9 +308,9 @@ if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then
CARGO_BUILD_JOBS_ARG=(-j "${CARGO_BUILD_JOBS}")
fi
-echo "==> Building openshell-gateway and openshell-driver-vm"
+echo "==> Building openshell-gateway, openshell-driver-vm, and native control supervisor"
cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \
- -p openshell-gateway -p openshell-driver-vm
+ -p openshell-gateway -p openshell-driver-vm -p openshell-supervisor
if [ "$(uname -s)" = "Darwin" ]; then
echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)"
diff --git a/tasks/scripts/vm/build-libkrun.sh b/tasks/scripts/vm/build-libkrun.sh
index 580f6bf4e5..e204dc0829 100755
--- a/tasks/scripts/vm/build-libkrun.sh
+++ b/tasks/scripts/vm/build-libkrun.sh
@@ -5,7 +5,7 @@
# Build libkrun and libkrunfw from source on Linux.
#
# This script builds libkrun (VMM) and libkrunfw (kernel firmware) from source
-# with OpenShell's custom kernel configuration for bridge/netfilter support.
+# with OpenShell's custom kernel configuration for sandbox enforcement.
#
# In addition to the platform's native .so artifacts, this script exports
# kernel.c and ABI_VERSION metadata so that other platforms (e.g. macOS) can
@@ -212,7 +212,11 @@ if [ -f openshell.kconfig ]; then
# Verify that the key options were actually applied.
all_ok=true
- for opt in CONFIG_BRIDGE CONFIG_NETFILTER CONFIG_NF_NAT; do
+ for opt in \
+ CONFIG_SECURITY \
+ CONFIG_SECURITY_LANDLOCK \
+ CONFIG_SECCOMP \
+ CONFIG_SECCOMP_FILTER; do
val="$(grep "^${opt}=" "${KERNEL_SOURCES}/.config" 2>/dev/null || true)"
if [ -n "$val" ]; then
echo " ${opt}: ${val#*=}"
@@ -221,6 +225,13 @@ if [ -f openshell.kconfig ]; then
all_ok=false
fi
done
+ lsm_order="$(grep '^CONFIG_LSM=' "${KERNEL_SOURCES}/.config" 2>/dev/null || true)"
+ if [[ "$lsm_order" == *landlock* ]]; then
+ echo " CONFIG_LSM: ${lsm_order#*=}"
+ else
+ echo " WARNING: CONFIG_LSM does not activate Landlock: ${lsm_order:-unset}" >&2
+ all_ok=false
+ fi
if [ "$all_ok" = false ]; then
echo "ERROR: kernel config fragment merge failed — required options missing" >&2
exit 1
diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh
index 0085c0619d..cadd5aae40 100755
--- a/tasks/scripts/vm/build-supervisor-bundle.sh
+++ b/tasks/scripts/vm/build-supervisor-bundle.sh
@@ -46,10 +46,10 @@ fi
case "${GUEST_ARCH}" in
aarch64|arm64)
- RUST_TARGET="aarch64-unknown-linux-gnu"
+ SANDBOX_RUST_TARGET="aarch64-unknown-linux-musl"
;;
x86_64|amd64)
- RUST_TARGET="x86_64-unknown-linux-gnu"
+ SANDBOX_RUST_TARGET="x86_64-unknown-linux-musl"
;;
*)
echo "ERROR: Unsupported guest architecture: ${GUEST_ARCH}" >&2
@@ -58,12 +58,17 @@ case "${GUEST_ARCH}" in
;;
esac
-SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox"
+SUPERVISOR_BIN="${ROOT}/target/${SANDBOX_RUST_TARGET}/release/openshell-sandbox"
SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst"
+GUEST_SUPERVISOR_BIN="${ROOT}/target/${SANDBOX_RUST_TARGET}/release/openshell-supervisor"
+HOST_SUPERVISOR_BIN="${ROOT}/target/release/openshell-supervisor"
+HOST_SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-supervisor.zst"
+SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst"
echo "==> Building openshell-sandbox supervisor bundle"
echo " Guest arch: ${GUEST_ARCH}"
-echo " Rust target: ${RUST_TARGET}"
+echo " Sandbox target: ${SANDBOX_RUST_TARGET} (static musl)"
+echo " Host supervisor target: native"
echo " Output: ${SUPERVISOR_OUTPUT}"
mkdir -p "${OUTPUT_DIR}"
@@ -79,13 +84,15 @@ run_supervisor_build() {
fi
if command -v cargo-zigbuild >/dev/null 2>&1; then
- ${cargo_prefix[@]+"${cargo_prefix[@]}"} cargo zigbuild --release -p openshell-sandbox --target "${RUST_TARGET}" \
+ ${cargo_prefix[@]+"${cargo_prefix[@]}"} cargo zigbuild --release -p openshell-sandbox -p openshell-supervisor --target "${SANDBOX_RUST_TARGET}" \
--manifest-path "${ROOT}/Cargo.toml"
else
echo " cargo-zigbuild not found, falling back to cargo build..."
- ${cargo_prefix[@]+"${cargo_prefix[@]}"} cargo build --release -p openshell-sandbox --target "${RUST_TARGET}" \
+ ${cargo_prefix[@]+"${cargo_prefix[@]}"} cargo build --release -p openshell-sandbox -p openshell-supervisor --target "${SANDBOX_RUST_TARGET}" \
--manifest-path "${ROOT}/Cargo.toml"
fi
+ ${cargo_prefix[@]+"${cargo_prefix[@]}"} cargo build --release -p openshell-supervisor \
+ --manifest-path "${ROOT}/Cargo.toml"
}
print_build_failure() {
@@ -116,13 +123,66 @@ else
fi
fi
-if [ ! -f "${SUPERVISOR_BIN}" ]; then
- echo "ERROR: supervisor binary not found at ${SUPERVISOR_BIN}" >&2
+if [ ! -f "${SUPERVISOR_BIN}" ] || [ ! -f "${GUEST_SUPERVISOR_BIN}" ] || [ ! -f "${HOST_SUPERVISOR_BIN}" ]; then
+ echo "ERROR: sandbox or supervisor binary not found after build" >&2
+ exit 1
+fi
+
+if readelf -l "${SUPERVISOR_BIN}" 2>/dev/null | grep -q 'Requesting program interpreter'; then
+ echo "ERROR: VM guest openshell-sandbox must be statically linked" >&2
exit 1
fi
zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}"
+zstd -19 -T0 -f "${HOST_SUPERVISOR_BIN}" -o "${HOST_SUPERVISOR_OUTPUT}"
+
+case "${GUEST_ARCH}" in
+ aarch64|arm64) DOCKER_ARCH="arm64" ;;
+ x86_64|amd64) DOCKER_ARCH="amd64" ;;
+esac
+
+echo "==> Building trusted supervisor helper runtime"
+STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox"
+STAGED_CONTROL="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-supervisor"
+RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$"
+mkdir -p "$(dirname "${STAGED_SUPERVISOR}")"
+cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}"
+cp "${GUEST_SUPERVISOR_BIN}" "${STAGED_CONTROL}"
+
+case "$(uname -m)" in
+ aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;;
+ x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;;
+ *) HOST_DOCKER_ARCH="" ;;
+esac
+
+if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then
+ docker build \
+ --build-arg "TARGETARCH=${DOCKER_ARCH}" \
+ --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \
+ --tag "${RUNTIME_IMAGE}" \
+ "${ROOT}"
+else
+ docker buildx build \
+ --load \
+ --platform "linux/${DOCKER_ARCH}" \
+ --build-arg "TARGETARCH=${DOCKER_ARCH}" \
+ --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \
+ --tag "${RUNTIME_IMAGE}" \
+ "${ROOT}"
+fi
+
+RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")"
+cleanup_runtime_image() {
+ docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true
+ docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true
+}
+trap cleanup_runtime_image EXIT
+docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \
+ | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}"
+cleanup_runtime_image
+trap - EXIT
echo "==> Bundled supervisor ready"
echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)"
echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)"
+echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)"
diff --git a/tasks/scripts/vm/compress-vm-runtime.sh b/tasks/scripts/vm/compress-vm-runtime.sh
index 598dc5505d..d2d5b5878b 100755
--- a/tasks/scripts/vm/compress-vm-runtime.sh
+++ b/tasks/scripts/vm/compress-vm-runtime.sh
@@ -4,7 +4,7 @@
# Gather VM runtime artifacts from local sources and compress for embedding.
#
-# This script collects libkrun, libkrunfw, gvproxy, and the guest OCI unpacker
+# This script collects libkrun, libkrunfw, and the guest OCI unpacker
# from local sources or pinned releases and compresses them with zstd for
# embedding into the openshell-driver-vm binary.
#
@@ -28,7 +28,6 @@ ROOT="$(vm_lib_root)"
# Source pins for runtime tool versions.
source "${ROOT}/crates/openshell-driver-vm/runtime/pins.env" 2>/dev/null || true
-GVPROXY_VERSION="${GVPROXY_VERSION:-v0.8.8}"
UMOCI_VERSION="${UMOCI_VERSION:-v0.6.0}"
# ── macOS dylib portability helpers ─────────────────────────────────────
@@ -72,12 +71,12 @@ _check_compressed_artifacts() {
platform="$(uname -s)-$(uname -m)"
case "$platform" in
Darwin-arm64)
- for f in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst; do
+ for f in libkrun.dylib.zst libkrunfw.5.dylib.zst umoci.zst; do
[ -f "${dir}/${f}" ] || return 1
done
;;
Linux-*)
- for f in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst; do
+ for f in libkrun.so.zst libkrunfw.so.5.zst umoci.zst; do
[ -f "${dir}/${f}" ] || return 1
done
;;
@@ -224,25 +223,13 @@ case "$(uname -s)-$(uname -m)" in
cp "$WORK_DIR/libkrunfw.dylib" "$WORK_DIR/libkrunfw.5.dylib"
fi
- # gvproxy - prefer Podman, fall back to Homebrew
- if [ -x /opt/podman/bin/gvproxy ]; then
- cp /opt/podman/bin/gvproxy "$WORK_DIR/"
- echo " Using gvproxy from Podman"
- elif [ -x "${BREW_PREFIX}/bin/gvproxy" ]; then
- cp "${BREW_PREFIX}/bin/gvproxy" "$WORK_DIR/"
- echo " Using gvproxy from Homebrew"
- else
- echo "Error: gvproxy not found. Install Podman Desktop or run: brew install gvproxy" >&2
- exit 1
- fi
download_umoci_for_guest "$WORK_DIR/umoci" "arm64"
;;
Linux-*)
ARCH="$(uname -m)"
case "$ARCH" in
- aarch64) GVPROXY_ARCH="arm64" ;;
- x86_64) GVPROXY_ARCH="amd64" ;;
+ aarch64|x86_64) ;;
*)
echo "Error: Unsupported Linux architecture: ${ARCH}" >&2
exit 1
@@ -274,13 +261,6 @@ case "$(uname -s)-$(uname -m)" in
fi
fi
- # Download gvproxy if not present
- if [ ! -f "$WORK_DIR/gvproxy" ]; then
- echo " Downloading gvproxy for linux-${GVPROXY_ARCH}..."
- curl -fsSL -o "$WORK_DIR/gvproxy" \
- "https://github.com/containers/gvisor-tap-vsock/releases/download/${GVPROXY_VERSION}/gvproxy-linux-${GVPROXY_ARCH}"
- chmod +x "$WORK_DIR/gvproxy"
- fi
download_umoci_for_guest "$WORK_DIR/umoci" "$ARCH"
;;
diff --git a/tasks/scripts/vm/package-vm-runtime.sh b/tasks/scripts/vm/package-vm-runtime.sh
index e9371bb509..fbbd482e55 100755
--- a/tasks/scripts/vm/package-vm-runtime.sh
+++ b/tasks/scripts/vm/package-vm-runtime.sh
@@ -4,8 +4,8 @@
# Package VM runtime artifacts into a release tarball.
#
-# Used by CI (release-vm-kernel.yml) to bundle libkrun, libkrunfw, gvproxy,
-# and the guest OCI unpacker into a platform-specific tarball for the
+# Used by CI (release-vm-kernel.yml) to bundle libkrun, libkrunfw, and the
+# guest OCI unpacker into a platform-specific tarball for the
# vm-runtime GitHub Release. Handles tool downloads, provenance metadata
# generation, and tarball creation.
#
@@ -31,7 +31,6 @@ ROOT="$(vm_lib_root)"
# Source pins for runtime tool versions.
source "${ROOT}/crates/openshell-driver-vm/runtime/pins.env" 2>/dev/null || true
-GVPROXY_VERSION="${GVPROXY_VERSION:-v0.8.8}"
UMOCI_VERSION="${UMOCI_VERSION:-v0.6.0}"
PLATFORM=""
@@ -106,18 +105,6 @@ case "$PLATFORM" in
;;
esac
-# ── Download gvproxy ────────────────────────────────────────────────────
-
-echo "==> Downloading gvproxy ${GVPROXY_VERSION} for ${PLATFORM}..."
-case "$PLATFORM" in
- linux-aarch64) GVPROXY_SUFFIX="linux-arm64" ;;
- linux-x86_64) GVPROXY_SUFFIX="linux-amd64" ;;
- darwin-aarch64) GVPROXY_SUFFIX="darwin" ;;
-esac
-
-curl -fsSL -o "${PACKAGE_DIR}/gvproxy" \
- "https://github.com/containers/gvisor-tap-vsock/releases/download/${GVPROXY_VERSION}/gvproxy-${GVPROXY_SUFFIX}"
-chmod +x "${PACKAGE_DIR}/gvproxy"
# ── Download umoci for the Linux guest ───────────────────────────────────
@@ -162,11 +149,10 @@ jq -n \
--arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--arg kfw_commit "$LIBKRUNFW_COMMIT" \
--arg kver "$KERNEL_VERSION" \
- --arg gvproxy "$GVPROXY_VERSION" \
--arg umoci "$UMOCI_VERSION" \
--arg sha "${GITHUB_SHA:-unknown}" \
--arg run "${GITHUB_RUN_ID:-unknown}" \
- '{artifact: $artifact, platform: $platform, build_timestamp: $ts, libkrunfw_commit: $kfw_commit, kernel_version: $kver, gvproxy_version: $gvproxy, umoci_version: $umoci, github_sha: $sha, github_run_id: $run}' \
+ '{artifact: $artifact, platform: $platform, build_timestamp: $ts, libkrunfw_commit: $kfw_commit, kernel_version: $kver, umoci_version: $umoci, github_sha: $sha, github_run_id: $run}' \
> "${PACKAGE_DIR}/provenance.json"
# ── Create tarball ──────────────────────────────────────────────────────
diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh
index 7d0b05334d..5cf3d58cee 100755
--- a/tasks/scripts/vm/smoke-orphan-cleanup.sh
+++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh
@@ -4,7 +4,7 @@
#
# Smoke test: start the gateway with the VM driver, create a sandbox, then
# signal the gateway (SIGTERM then SIGKILL) and verify that no driver,
-# launcher, gvproxy, or libkrun worker processes survive.
+# launcher or libkrun worker processes survive.
#
# Exit codes:
# 0 — both SIGTERM and SIGKILL cleanup passed
@@ -108,10 +108,8 @@ create_sandbox() {
for _ in $(seq 1 60); do
if pgrep -f "openshell-vm-orphan-$$|$STATE_DIR/sandboxes/" >/dev/null 2>&1; then
- if pgrep -f gvproxy >/dev/null 2>&1; then
- echo "sandbox came up (cli pid=$CLI_PID)"
- return 0
- fi
+ echo "sandbox came up (cli pid=$CLI_PID)"
+ return 0
fi
sleep 2
done
@@ -122,17 +120,14 @@ create_sandbox() {
snapshot_kids() {
# Return all PIDs whose --state-dir or --vm-rootfs references our
- # per-run directory, plus any gvproxy that mentions our socket base.
+ # per-run directory.
pgrep -fl "state-dir $STATE_DIR|$STATE_DIR/sandboxes" 2>/dev/null || true
- pgrep -fl "gvproxy" 2>/dev/null | grep "osd-gv" || true
}
count_alive() {
local alive
alive=$(pgrep -f "state-dir $STATE_DIR|$STATE_DIR/sandboxes" 2>/dev/null | wc -l | tr -d ' ')
- local gv
- gv=$(pgrep -f 'gvproxy' 2>/dev/null | xargs -r ps -o pid=,command= -p 2>/dev/null | grep -c 'osd-gv' || true)
- echo $((alive + gv))
+ echo "$alive"
}
verify_cleanup() {
@@ -176,8 +171,7 @@ run_scenario() {
# Belt-and-braces teardown between scenarios.
pkill -9 -f "$STATE_DIR/sandboxes|$STATE_DIR " 2>/dev/null || true
- pkill -9 -f 'gvproxy.*osd-gv' 2>/dev/null || true
- rm -rf "$STATE_DIR" /tmp/osd-gv "$XDG" 2>/dev/null || true
+ rm -rf "$STATE_DIR" "$XDG" 2>/dev/null || true
# CLI may still be running; reap it.
kill "${CLI_PID:-0}" 2>/dev/null || true
sleep 1
@@ -191,7 +185,6 @@ main() {
# Clean starting state.
pkill -9 -f 'openshell-gateway|openshell-driver-vm' 2>/dev/null || true
- pkill -9 -f 'gvproxy.*osd-gv' 2>/dev/null || true
sleep 1
if ! run_scenario TERM "graceful SIGTERM"; then
diff --git a/tasks/scripts/vm/vm-setup.sh b/tasks/scripts/vm/vm-setup.sh
index de568d8783..4b6bb297b3 100755
--- a/tasks/scripts/vm/vm-setup.sh
+++ b/tasks/scripts/vm/vm-setup.sh
@@ -4,7 +4,7 @@
# One-time setup for the openshell-driver-vm runtime.
#
-# Downloads pre-built runtime artifacts (libkrun, libkrunfw, gvproxy, umoci)
+# Downloads pre-built runtime artifacts (libkrun, libkrunfw, umoci)
# from the vm-runtime GitHub Release, or builds them from source when
# --from-source is set.
# After obtaining the runtime, compresses the artifacts for embedding into the
@@ -35,7 +35,7 @@ while [[ $# -gt 0 ]]; do
--help|-h)
echo "Usage: $0 [--from-source]"
echo ""
- echo "Set up the openshell-driver-vm runtime (libkrun, libkrunfw, gvproxy, umoci)."
+ echo "Set up the openshell-driver-vm runtime (libkrun, libkrunfw, umoci)."
echo ""
echo "Options:"
echo " --from-source Build runtime from source instead of downloading (~15-45min)"
@@ -101,7 +101,7 @@ OUTPUT_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${ROOT}/target/vm-runtime-com
missing=0
case "$PLATFORM" in
darwin-aarch64)
- for f in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst; do
+ for f in libkrun.dylib.zst libkrunfw.5.dylib.zst umoci.zst; do
if [ ! -f "${OUTPUT_DIR}/${f}" ]; then
echo "ERROR: Missing ${OUTPUT_DIR}/${f}" >&2
missing=1
@@ -109,7 +109,7 @@ case "$PLATFORM" in
done
;;
linux-aarch64|linux-x86_64)
- for f in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst; do
+ for f in libkrun.so.zst libkrunfw.so.5.zst umoci.zst; do
if [ ! -f "${OUTPUT_DIR}/${f}" ]; then
echo "ERROR: Missing ${OUTPUT_DIR}/${f}" >&2
missing=1