Cloud VMs: retry DNS on a fallback resolver when the platform resolver returns SERVFAIL (#26640)

web/scripts/build-devbox-freestyle.ts

@@ -103,6 +103,10 @@ import {
   CMUX_TUI_STATE_RESERVATION_MARKER_PATH,
   CMUX_TUI_STATE_RESERVATION_BYTES,
   DEVBOX_DESKTOP_INSTALLS,
+  DEVBOX_DNS_PATH,
+  DEVBOX_DNS_UNITS,
+  devboxDnsCheckCommand,
+  devboxDnsInstallCommand,
   devboxTerminfoInstallCommand,
   DEVBOX_INSTANCE_ID_COMMAND,
   bakeMetadata,
@@ -633,6 +637,13 @@ try {
     "cmux-prompt-sync-unit",
     "python3 -m py_compile /usr/local/bin/cmux-prompt-sync && systemctl daemon-reload && systemctl enable cmux-prompt-sync && systemctl is-enabled cmux-prompt-sync",
   );
+  // The DNS fallback (devbox-image-common.ts, DEVBOX_DNS_UNITS): from here on
+  // the builder itself resolves through it, so the rest of the bake proves it.
+  await put("cmux-dns", DEVBOX_DNS_PATH, 0o755);
+  for (const [target, unit] of Object.entries(DEVBOX_DNS_UNITS)) {
+    await vm.fs.writeFile(target, unit, { mode: 0o644 });
+  }
+  await step("dns-fallback", `${devboxDnsInstallCommand} && ${devboxDnsCheckCommand}`);
   // Prove the daemon contract on the builder: the supervisor started the
   // daemon on its own, the session answers, and the listener is dual-stack.
   await step(

web/scripts/devbox-image-common.ts

@@ -49,6 +49,7 @@ export const DEVBOX_TEMPLATE_FILES = [
   "chrome-managed-policy.json",
   "cmux-bashrc",
   "cmux-devbox-boot",
+  "cmux-dns",
   "cmux-tui-state-mount",
   "cmux-motd",
   "cmux-prompt.bash",
@@ -521,6 +522,7 @@ export function devboxSourceManifest(
     cuaDriver: devboxCuaDriverVersion(dockerfile),
     ghosttyVersion: devboxGhosttyVersion(dockerfile),
     files,
+    dns: { units: DEVBOX_DNS_UNITS, install: devboxDnsInstallCommand },
   };
   if (schema >= 2) {
     shell.dockerfileInstructions = createHash("sha256").update(normalizedDockerfileInstructions(dockerfile)).digest("hex");
@@ -1625,3 +1627,83 @@ export const DEVBOX_PRE_SNAPSHOT_SETTLE_SECONDS = 10;
 export function devboxSettleBeforeSnapshotCommand(): string {
   return `sync && sleep ${DEVBOX_PRE_SNAPSHOT_SETTLE_SECONDS} && sync && echo "settled $(cut -d' ' -f1-3 /proc/loadavg)"`;
 }
+
+/**
+ * The DNS fallback (Freestyle only; services/vms/images/devbox/cmux-dns): a
+ * stub on 127.0.0.2:53 that asks the platform's resolver first and falls back
+ * to its last good answer, then DNS-over-HTTPS, when the platform resolver
+ * answers SERVFAIL or nothing (cmux issue 26640). Freestyle rewrites
+ * /etc/resolv.conf on every TLS rule change, so a path unit re-points it at
+ * the stub each time. The container recipe has none: its runtime owns
+ * resolv.conf.
+ */
+export const DEVBOX_DNS_PATH = "/usr/local/bin/cmux-dns";
+export const DEVBOX_DNS_UNITS: Readonly<Record<string, string>> = {
+  "/etc/systemd/system/cmux-dns.service": [
+    "[Unit]",
+    "Description=cmux DNS fallback stub (127.0.0.2:53)",
+    "Before=nss-lookup.target",
+    "Wants=nss-lookup.target",
+    "",
+    "[Service]",
+    "Type=notify",
+    // The distribution interpreter, not whichever python3 is first on PATH.
+    `ExecStart=/usr/bin/python3 ${DEVBOX_DNS_PATH} serve`,
+    "Restart=always",
+    "RestartSec=1",
+    "DynamicUser=yes",
+    "AmbientCapabilities=CAP_NET_BIND_SERVICE",
+    "CapabilityBoundingSet=CAP_NET_BIND_SERVICE",
+    "",
+    "[Install]",
+    "WantedBy=multi-user.target",
+  ].join("\n") + "\n",
+  "/etc/systemd/system/cmux-dns-guard.service": [
+    "[Unit]",
+    "Description=Point resolv.conf at the cmux DNS fallback",
+    "After=cmux-dns.service",
+    "",
+    "[Service]",
+    "Type=oneshot",
+    `ExecStart=/usr/bin/python3 ${DEVBOX_DNS_PATH} guard`,
+    "",
+    "[Install]",
+    "WantedBy=multi-user.target",
+  ].join("\n") + "\n",
+  "/etc/systemd/system/cmux-dns-guard.path": [
+    "[Unit]",
+    "Description=Re-point resolv.conf at the cmux DNS fallback whenever the platform rewrites it",
+    "",
+    "[Path]",
+    "PathChanged=/etc/resolv.conf",
+    "Unit=cmux-dns-guard.service",
+    "",
+    "[Install]",
+    "WantedBy=multi-user.target",
+  ].join("\n") + "\n",
+};
+
+/** Enables and starts the fallback once its files are in place, then waits until the stub answers. */
+export const devboxDnsInstallCommand = [
+  `python3 -m py_compile ${DEVBOX_DNS_PATH}`,
+  "systemctl daemon-reload",
+  "systemctl enable --now cmux-dns.service cmux-dns-guard.path",
+  "systemctl enable cmux-dns-guard.service",
+  "systemctl start cmux-dns-guard.service",
+  // The stub itself must answer: libc would hide a dead one behind the
+  // platform resolvers listed after it.
+  `/usr/bin/python3 ${DEVBOX_DNS_PATH} probe`,
+].join(" && ");
+
+/** The fallback is installed, answers on its own, and resolv.conf points at it first. */
+export const devboxDnsCheckCommand = [
+  `test -x ${DEVBOX_DNS_PATH}`,
+  `python3 -m py_compile ${DEVBOX_DNS_PATH}`,
+  "systemctl is-active cmux-dns.service >/dev/null",
+  "systemctl is-active cmux-dns-guard.path >/dev/null",
+  "head -1 /etc/resolv.conf | grep -qx '# cmux-dns'",
+  "[ \"$(awk '$1 == \"nameserver\" { print $2; exit }' /etc/resolv.conf)\" = 127.0.0.2 ]",
+  `/usr/bin/python3 ${DEVBOX_DNS_PATH} probe github.com`,
+  "getent ahostsv4 github.com >/dev/null",
+  "echo dns-fallback-ok",
+].join(" && ");

web/scripts/verify-devbox-image.ts

@@ -55,6 +55,7 @@ import {
   devboxTerminfoCheckCommand,
   devboxWaitForDaemonCommand,
   cmuxTuiWebsocketSmokeCommand,
+  devboxDnsCheckCommand,
   sha256File,
 } from "./devbox-image-common";
 import {
@@ -176,6 +177,7 @@ const DAEMON_CHECKS: readonly string[] = [
   `test -s /etc/cmux/model-plane.env && grep -q "^export OPENAI_BASE_URL='https://" /etc/cmux/model-plane.env && ! grep -q crt_ /etc/cmux/model-plane.env && env -i HOME=/tmp/mp-verify bash -c '. /etc/cmux/agent-config.sh; printf %s "$OPENAI_BASE_URL"' | grep -q '^https://' && rm -rf /tmp/mp-verify && echo model-plane-env-baked`,
   "systemctl is-active cmux-tui-daemon >/dev/null && echo systemd-supervisor-active",
   "test -x /usr/local/bin/cmux-prompt-sync && python3 -m py_compile /usr/local/bin/cmux-prompt-sync && systemctl is-enabled cmux-prompt-sync >/dev/null && echo prompt-sync-contract-ok",
+  devboxDnsCheckCommand,
   cmuxTuiWebsocketSmokeCommand(),
 ];
 

web/services/vms/drivers/freestyle.ts

@@ -154,7 +154,8 @@ export type { FreestylePreconnectOptions } from "./freestyleWarmup";
 // The coderouter model plane is edge-injected: the create carries an inline
 // `tls` rule for the coderouter host whose transform overwrites the signed
 // `x-cmux-authorization` plus compatibility bearer/VM headers on every request
-// the guest makes there. The platform steers the host to its edge (/etc/hosts) and
+// the guest makes there. The platform steers the host to its edge (its forwarding
+// resolver answers it with the edge address; see the devbox README) and
 // installs its CA at boot; rules added after boot never reach a running
 // guest, so the rule must be inline. The baked env file holds only base
 // URLs and placeholder keys: no token is ever written into the guest, and

web/services/vms/drivers/freestyleNetworkPolicy.ts

@@ -9,8 +9,11 @@ import { canonicalCidr, type NetworkRulePlan } from "../networkPolicy";
  * - A VM with no egress rule reaches nothing, DNS included: the guest's
  *   resolvers are public addresses.
  * - Firewall and TLS rule changes apply to a running VM within ~0.1 s.
- * - A TLS rule with a `public` destination steers its exact name through the
- *   guest's /etc/hosts to the edge and needs no firewall grant.
+ * - A TLS rule with a `public` destination steers its exact name to the edge
+ *   and needs no firewall grant. Steering was through the guest's /etc/hosts
+ *   at the spike; by 2026-10-28 the platform instead points resolv.conf at a
+ *   forwarding resolver on the edge address that answers steered names with
+ *   the edge (see the devbox README, "DNS fallback").
  * - A TLS rule with a `host` destination (the CodeRouter and reflection
  *   aliases) does NOT carry that grant: the guest needs a firewall path to the
  *   edge address on 443. {@link freestyleEdgeAddresses} supplies it.

web/services/vms/images/devbox/README.md

@@ -323,6 +323,66 @@ alone. The container recipe has no identity step: `docker build` mounts
 `/etc/hostname` and `/etc/hosts` from the daemon and the runtime names each
 container, so the hostname there is the runtime's.
 
+## DNS fallback (`cmux-dns`)
+
+Freestyle only. A machine with any TLS rule (every cmux machine: the
+model-plane and CodeRouter edge rules) resolves through the platform's
+forwarding resolver, `10.32.0.28`: the guest agent writes
+`/etc/resolv.conf` (`nameserver 10.32.0.28`) and
+`/etc/systemd/resolved.conf.d/61-freestyle-tls.conf`, and the platform
+answers every port-53 packet the guest sends, whatever address it names
+(a query to `8.8.8.8` comes back from the same forwarder). The forwarder
+answers the names TLS rules steer with the edge address and forwards the
+rest. It intermittently answers SERVFAIL (about 300 ms, then cached for a
+second or two) or nothing at all, for every name, on every machine at the
+same moment ([#26640](https://github.com/manaflow-ai/cmux/issues/26640));
+measured 2026-10-28, roughly 2% of queries over a few minutes. Pointing
+resolv.conf elsewhere does not help, and the agent rewrites it on every TLS
+rule change and on resume anyway.
+
+`cmux-dns serve` (`cmux-dns.service`, a `DynamicUser` with only
+`CAP_NET_BIND_SERVICE`) listens on `127.0.0.2:53`, UDP and TCP, and per
+query:
+
+1. asks the platform's resolvers (two tries of 1 s: the first two, IPv4 first,
+   or a lone one twice), so steered names keep resolving to the edge and
+   NXDOMAIN stays the platform's;
+2. on SERVFAIL, REFUSED or silence, answers with the platform's last good
+   answer for that exact question, at most a day old. The cache starts over
+   whenever resolv.conf is written (a TLS rule, a resume, a clone's first
+   boot), so a stale answer never outlives a steering change;
+3. while the provider's `/run/freestyle/tls-dns-servers` marker is nonempty,
+   returns SERVFAIL on a cache miss. The provider exports resolver IPs but no
+   steered-name list to the guest, so a public answer could bypass a required
+   TLS route or credential injection, even with full public egress. This
+   conservatively covers all names until the platform answers again;
+4. when TLS steering is inactive, asks DNS-over-HTTPS at `1.1.1.1` then `8.8.8.8`, addressed by IP
+   on 443, which the interception does not touch, and accepts only NOERROR:
+   a public NXDOMAIN for a name only the platform knows comes back as
+   SERVFAIL, never as a missing name. A machine whose network policy blocks
+   those addresses gets SERVFAIL from this step, as it would without the stub.
+
+Each exchange runs under a wall-clock deadline and the whole chain fits in
+3.5 s, under the stub's 4 s libc timeout. Replies for another question are
+dropped; UDP replies over the client's size (512, or its EDNS size up to
+1232) go out truncated so the client retries over TCP.
+
+`cmux-dns guard` (`cmux-dns-guard.path` on `PathChanged=/etc/resolv.conf`,
+plus once at boot) rewrites a resolv.conf the platform wrote to
+`nameserver 127.0.0.2` followed by two of the platform's resolvers (IPv4
+first), keeps its `search` line and options other than `timeout`/`attempts`
+(set to `timeout:4 attempts:1`; the stub retries itself), and records every
+resolver on a `# cmux-dns upstream` line the stub reads. A stopped stub
+therefore costs one refused packet before libc asks the platform directly.
+Between a platform rewrite and the guard (under a second) the machine
+resolves the old way. The bake installs it after the prompt sync; systemd
+waits for `READY=1` after both listeners bind, then checks that the stub itself
+answers (`cmux-dns probe`; a libc lookup would pass
+through the fallback resolvers), every later bake step resolves through it,
+and `verify-devbox-image.ts` checks the same contract. The container recipe
+has none: Docker owns its resolv.conf. Machines baked before it keep
+resolving straight through the platform.
+
 ## Terminal capabilities (`cmux-terminfo.src`)
 
 Daemon-spawned shells get the TERM the Mac exports (`xterm-256color`, or