diff --git a/docs/INTEGRATION-TESTS.md b/docs/INTEGRATION-TESTS.md index b02c5ba69..224bc2ea3 100644 --- a/docs/INTEGRATION-TESTS.md +++ b/docs/INTEGRATION-TESTS.md @@ -237,7 +237,7 @@ following behavior: | `partial-start-cleanup` | Corrupt rootfs causes clean failure; no residue | | `cancellation` | `SIGTERM` cleans up residue within a non-flaky time ceiling; exits 143 | | `keep` | `--keep-containers` preserves namespace/run-directory; diagnostics ≤1 MiB | -| `security-assertions` **(CH-only)** | Live jailer-replacement boundary: non-root uid, `CapEff` limited to `CAP_NET_ADMIN` alone, `no_new_privs`, active seccomp filter, per-run cgroup membership/bounded memory, `landlock_enable` + exactly-minimal disk/net/vsock topology via `vm.info` | +| `security-assertions` **(CH-only)** | Live jailer-replacement boundary: non-root uid, empty `CapInh`/`CapPrm`/`CapEff`/`CapBnd`/`CapAmb`, `no_new_privs`, active seccomp filter, per-run cgroup membership/bounded memory, `landlock_enable` + exactly-minimal disk/net/vsock topology via `vm.info` | After every case, the suite asserts no `awfvm-*` namespaces, `vmh*`/`vmn*`/`vmt*` interfaces, `awf-cloud-hypervisor` cgroup entries, or diff --git a/docs/architecture.md b/docs/architecture.md index 88dbe6459..a76b59c7e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -252,8 +252,8 @@ runs the agent in a hardware-isolated microVM: proxy. - The mandatory API proxy keeps provider credentials out of the guest environment. -- The VMM runs as a non-root identity with `no_new_privs`, a minimal capability - set, Landlock filesystem rules, seccomp, and explicit cgroup v2 limits. +- The VMM runs as a non-root identity with `no_new_privs`, empty capability + sets, Landlock filesystem rules, seccomp, and explicit cgroup v2 limits. - The preview supports only GitHub-hosted Ubuntu x86_64 KVM runners and fails closed on unsupported hosts or missing artifacts. diff --git a/docs/cloud-hypervisor-foundation.md b/docs/cloud-hypervisor-foundation.md index c797003ec..7db53b0b5 100644 --- a/docs/cloud-hypervisor-foundation.md +++ b/docs/cloud-hypervisor-foundation.md @@ -114,7 +114,8 @@ shell. The process: - runs as the non-root identity recorded by `SUDO_UID` and `SUDO_GID`; - keeps only the KVM supplementary group; - sets `no_new_privs`; -- retains only `CAP_NET_ADMIN`, which the virtio-net TAP setup requires; +- has empty inheritable, permitted, effective, bounding, and ambient capability + sets; - uses Cloud Hypervisor's seccomp filter; - receives a minimal Landlock filesystem allowlist; and - belongs to a cgroup v2 leaf with explicit memory, CPU, and PID limits. @@ -503,9 +504,10 @@ recovery described above. ### VMM boot fails with TAP permission errors -Verify the launcher retained only `CAP_NET_ADMIN`, the TAP belongs to the -expected namespace, and the Landlock allowlist includes the TAP's -`/sys/class/net/` directory read-only. +Verify the TAP was pre-created with the VMM uid/gid and `vnet_hdr` in the +expected namespace, `/dev/net/tun` is accessible, and the Landlock allowlist +includes `/sys/class/net//tun_flags` read-only. Do not grant +`CAP_NET_ADMIN`; the VMM capability sets must remain empty. ## Related documentation diff --git a/scripts/ci/cloud-hypervisor-ci-scripts.test.ts b/scripts/ci/cloud-hypervisor-ci-scripts.test.ts index 6f2d1f5c7..ba7017941 100644 --- a/scripts/ci/cloud-hypervisor-ci-scripts.test.ts +++ b/scripts/ci/cloud-hypervisor-ci-scripts.test.ts @@ -119,10 +119,14 @@ describe('cloud-hypervisor-live-smoke.sh', () => { expect(source).toContain('/dev/vdb'); expect(source).toContain('eth0'); expect(source).toContain('sec_work='); - expect(source).toContain('CapEff'); + for (const field of ['CapInh', 'CapPrm', 'CapEff', 'CapBnd', 'CapAmb']) { + expect(source).toContain(field); + } + expect(source).toContain('0000000000000000'); expect(source).toContain('NoNewPrivs'); expect(source).toContain('Seccomp'); expect(source).toContain('landlock_enable'); + expect(source).toContain('"/tun_flags"'); expect(source).toContain('cgroup.procs'); }); diff --git a/scripts/ci/cloud-hypervisor-live-smoke.sh b/scripts/ci/cloud-hypervisor-live-smoke.sh index 7120c08c7..3b7884135 100755 --- a/scripts/ci/cloud-hypervisor-live-smoke.sh +++ b/scripts/ci/cloud-hypervisor-live-smoke.sh @@ -17,7 +17,7 @@ set -euo pipefail # - security-assertions: while a run is live, inspects the host-visible # Cloud Hypervisor process and its own vm.info response to confirm the # launcher's jailer-replacement boundary (netns join, non-root identity, -# capability set limited to CAP_NET_ADMIN alone, no_new_privs, active +# empty capability sets, no_new_privs, active # seccomp filter, per-run cgroup membership/limits, landlock_enable # reflected in vm.create, and an exactly-minimal disk/fs/net/vsock device # set with no path to the host-only API socket) — see @@ -496,8 +496,8 @@ assert_no_residue # --- Cloud Hypervisor-specific live security assertions ------------------- # # Reproduces the launcher's jailer-replacement boundary live, while a run is -# in flight: netns-join + non-root privilege drop + capability set limited -# to CAP_NET_ADMIN alone + no_new_privs + active seccomp filter + per-run +# in flight: netns-join + non-root privilege drop + empty capability sets + +# no_new_privs + active seccomp filter + per-run # cgroup membership/limits + landlock_enable reflected in vm.create + an # exactly-minimal disk/net/vsock device set (see # src/cloud-hypervisor/launcher.ts and manager.ts). @@ -556,13 +556,14 @@ proc_uid=$(sudo stat -c %u "/proc/$vmm_pid" 2>/dev/null || echo "") [ -n "$proc_uid" ] || fail_security "could not stat /proc/$vmm_pid" [ "$proc_uid" != "0" ] || fail_security "Cloud Hypervisor process is running as root" -# Capability set limited to CAP_NET_ADMIN alone (setpriv --inh-caps=-all, -# +net_admin --bounding-set=-all,+net_admin --ambient-caps=+net_admin). -# CAP_NET_ADMIN's bit is 12, so the expected 64-bit CapEff bitmask is -# exactly 0x1000: 0000000000001000. -cap_eff=$(sudo awk '/^CapEff:/{print $2}' "/proc/$vmm_pid/status" 2>/dev/null || echo "") -[ "$cap_eff" = "0000000000001000" ] \ - || fail_security "process capability set is not exactly CAP_NET_ADMIN: ${cap_eff:-unknown}" +# Every capability set is empty. In particular, a zero CapBnd prevents the +# non-root VMM from regaining CAP_NET_ADMIN after exec. +for cap_field in CapInh CapPrm CapEff CapBnd CapAmb; do + cap_value=$(sudo awk -v field="$cap_field:" '$1 == field { print $2 }' \ + "/proc/$vmm_pid/status" 2>/dev/null || echo "") + [ "$cap_value" = "0000000000000000" ] \ + || fail_security "$cap_field is not empty: ${cap_value:-unknown}" +done # no_new_privs set (setpriv --no-new-privs). no_new_privs=$(sudo awk '/^NoNewPrivs:/{print $2}' "/proc/$vmm_pid/status" 2>/dev/null || echo "") @@ -674,6 +675,17 @@ node -e ' if (config.landlock_enable !== true) { throw new Error("landlock_enable is not true in vm.info: " + JSON.stringify(config.landlock_enable)); } + const tapSysfsPath = "/sys/class/net/" + expectedTap + "/tun_flags"; + const tapSysfsRules = (config.landlock_rules || []) + .filter(rule => rule.path === tapSysfsPath); + if (tapSysfsRules.length !== 1 || tapSysfsRules[0].access !== "r") { + throw new Error("TAP tun_flags Landlock rule is not exactly read-only: " + + JSON.stringify(tapSysfsRules)); + } + if ((config.landlock_rules || []).some(rule => + rule.path === "/sys/class/net/" + expectedTap)) { + throw new Error("Landlock grants the VMM the broader TAP sysfs directory"); + } const disks = config.disks || []; if (disks.length !== 1) { diff --git a/src/cloud-hypervisor/launcher.test.ts b/src/cloud-hypervisor/launcher.test.ts index 18acf2c9b..668c632b5 100644 --- a/src/cloud-hypervisor/launcher.test.ts +++ b/src/cloud-hypervisor/launcher.test.ts @@ -16,7 +16,7 @@ describe('buildCloudHypervisorLaunchCommand', () => { logFilePath: '/run/awf/cloud-hypervisor.log', }; - it('joins the namespace, drops privileges but retains the kvm group and CAP_NET_ADMIN, then execs Cloud Hypervisor with no shell', () => { + it('joins the namespace, retains only the kvm group, empties all capability sets, and execs Cloud Hypervisor with no shell', () => { const result = buildCloudHypervisorLaunchCommand(baseOptions); expect(result.command).toBe('/usr/sbin/ip'); expect(result.args).toEqual([ @@ -26,9 +26,9 @@ describe('buildCloudHypervisorLaunchCommand', () => { '--regid=1000', '--groups=978', '--no-new-privs', - '--inh-caps=-all,+net_admin', - '--bounding-set=-all,+net_admin', - '--ambient-caps=+net_admin', + '--inh-caps=-all', + '--bounding-set=-all', + '--ambient-caps=-all', '--', '/opt/cloud-hypervisor', '--api-socket', 'path=/run/awf/api.socket', @@ -37,6 +37,7 @@ describe('buildCloudHypervisorLaunchCommand', () => { '--seccomp', 'true', ]); expect(result.args).not.toContain('--clear-groups'); + expect(result.args.some((arg) => arg.includes('+net_admin'))).toBe(false); // No argument contains shell metacharacters that would matter if ever // interpolated; more importantly, args are a plain array (never joined // into a shell string) so metacharacters have no special meaning here. @@ -77,7 +78,7 @@ describe('computeCloudHypervisorLandlockRules', () => { { path: '/run/awf/run', access: 'rw' }, { path: '/dev/kvm', access: 'rw' }, { path: '/dev/net/tun', access: 'rw' }, - { path: '/sys/class/net/vmtabc123', access: 'r' }, + { path: '/sys/class/net/vmtabc123/tun_flags', access: 'r' }, ]); expect(rules).not.toEqual(expect.arrayContaining([ expect.objectContaining({ path: '/host/workspace' }), @@ -97,7 +98,7 @@ describe('computeCloudHypervisorLandlockRules', () => { expect(rules.some((rule) => rule.path.includes('workspace'))).toBe(false); }); - it('grants read access to the TAP sysfs directory so tun_flags is readable under Landlock', () => { + it('grants read access only to the TAP tun_flags attribute required under Landlock', () => { // Regression test: /sys/class/net//tun_flags is a world-readable // (0444) kernel sysfs attribute with no capability requirement of its // own, but Landlock still blocks the read if the path isn't in the @@ -112,7 +113,8 @@ describe('computeCloudHypervisorLandlockRules', () => { tapName: 'vmtabc123', }); - expect(rules).toContainEqual({ path: '/sys/class/net/vmtabc123', access: 'r' }); + expect(rules).toContainEqual({ path: '/sys/class/net/vmtabc123/tun_flags', access: 'r' }); + expect(rules).not.toContainEqual({ path: '/sys/class/net/vmtabc123', access: 'r' }); }); }); diff --git a/src/cloud-hypervisor/launcher.ts b/src/cloud-hypervisor/launcher.ts index cf9588475..67b54d018 100644 --- a/src/cloud-hypervisor/launcher.ts +++ b/src/cloud-hypervisor/launcher.ts @@ -16,12 +16,12 @@ import type { CloudHypervisorLandlockRule } from './api-client'; * per-run namespace {@link https://man7.org/linux/man-pages/man8/ip-netns.8.html} * without an intermediate fork, so the resulting process keeps the PID * the host process observes. - * 2. **Privilege drop** — `setpriv --reuid --regid --clear-groups - * --no-new-privs --inh-caps=-all --bounding-set=-all` execs the Cloud - * Hypervisor binary as the non-root operator uid/gid with an empty - * capability bounding set and `no_new_privs` set, before any guest code - * runs. This requires operator preconditions including kvm-group membership, - * `/dev/kvm` access). + * 2. **Privilege drop** — `setpriv --reuid --regid --groups= + * --no-new-privs --inh-caps=-all --bounding-set=-all + * --ambient-caps=-all` execs the Cloud Hypervisor binary as the non-root + * operator uid/gid with empty capability sets and `no_new_privs` set, + * before any guest code runs. The sole supplementary group grants + * `/dev/kvm` access without a process capability. * 3. **Filesystem confinement** — Cloud Hypervisor has no chroot of its * own, and jailer's userspace chroot+pivot_root cannot be replicated * for a foreign static binary without reimplementing jailer itself. @@ -57,7 +57,7 @@ export interface CloudHypervisorLaunchPaths { readonly apiSocketPath: string; readonly vsockSocketPath: string; /** Host TAP interface name (e.g. `vmt`), for the - * `/sys/class/net/` Landlock rule — see + * `/sys/class/net//tun_flags` Landlock rule — see * {@link computeCloudHypervisorLandlockRules}. */ readonly tapName: string; } @@ -79,10 +79,10 @@ export interface CloudHypervisorLaunchToolPaths { /** * Builds the argv AWF spawns to launch Cloud Hypervisor: join the prepared - * network namespace, drop to the non-root operator identity retaining - * exactly two things it needs to configure its own virtio-net TAP device, - * then exec the pinned Cloud Hypervisor binary with only its API socket - * configured; the VM itself is created and booted afterwards over that socket. + * network namespace, drop to the non-root operator identity with no + * capabilities, then exec the pinned Cloud Hypervisor binary with only its API + * socket configured; the VM itself is created and booted afterwards over that + * socket. * * The launched process retains exactly one supplementary group: the group * that owns `/dev/kvm` (resolved by preflight). A blanket `--clear-groups` @@ -92,17 +92,13 @@ export interface CloudHypervisorLaunchToolPaths { * launch fail with EACCES opening `/dev/kvm` even though preflight (which * runs as root) passed. * - * It also retains exactly one capability: `CAP_NET_ADMIN`, via the - * bounding, inheritable, and ambient sets together (ambient capabilities - * are what let a specific capability survive `execve()` of a plain, - * non-file-capability-aware binary like `cloud-hypervisor` across a uid - * change, even under `--no-new-privs`). Cloud Hypervisor's virtio-net - * backend needs it to finish configuring the already-created, - * already-owned TAP device (observed live: `vm.boot` otherwise fails with - * "Failed to read the TAP flags from sysfs: Permission denied", even - * though the TAP device node itself is owned by the target uid/gid). This - * is a deliberate, minimal, single-capability exception to an otherwise - * fully empty capability set — not a broad grant. + * The network manager creates, configures, and brings up the TAP before this + * process starts, with the target uid/gid recorded as its owner. Cloud + * Hypervisor therefore only needs ordinary `/dev/net/tun` access to reopen that + * TAP and read-only Landlock access to its `tun_flags` sysfs attribute. The + * earlier `CAP_NET_ADMIN` requirement was a Landlock denial misdiagnosed as a + * TAP ownership failure; retaining it would let a compromised VMM reconfigure + * the namespace firewall and interfaces. */ export function buildCloudHypervisorLaunchCommand(options: { readonly tools: CloudHypervisorLaunchToolPaths; @@ -138,11 +134,9 @@ export function buildCloudHypervisorLaunchCommand(options: { // also drop kvm access). `--groups=${options.kvmGid}`, '--no-new-privs', - // CAP_NET_ADMIN is the sole exception to an otherwise fully empty - // capability set — see the function doc comment above for why. - '--inh-caps=-all,+net_admin', - '--bounding-set=-all,+net_admin', - '--ambient-caps=+net_admin', + '--inh-caps=-all', + '--bounding-set=-all', + '--ambient-caps=-all', '--', options.cloudHypervisorBinary, '--api-socket', `path=${options.apiSocketPath}`, @@ -160,7 +154,7 @@ export function buildCloudHypervisorLaunchCommand(options: { * read-write access to the private run directory (for the API and vsock * UNIX domain sockets it creates there), read-write access to the device * nodes it must reopen for virtio-net TAP attachment and KVM ioctls, and - * read access to the TAP's own sysfs device directory. Cloud Hypervisor's + * read access to the TAP's own sysfs flags attribute. Cloud Hypervisor's * virtio-net setup reads `/sys/class/net//tun_flags` (a * world-readable, `0444` file with no capability requirement of its own) * to detect multi-queue support; without a Landlock rule for it, that read @@ -180,7 +174,7 @@ export function computeCloudHypervisorLandlockRules( { path: paths.runDirectory, access: 'rw' }, { path: '/dev/kvm', access: 'rw' }, { path: '/dev/net/tun', access: 'rw' }, - { path: `/sys/class/net/${paths.tapName}`, access: 'r' }, + { path: `/sys/class/net/${paths.tapName}/tun_flags`, access: 'r' }, ]; return rules; }