Cloud: stop deterministic provider failure storms

ios/cmuxPackage/Sources/cmuxFeature/MobileAnalyticsComposition.swift

@@ -149,6 +149,10 @@ public struct MobileAnalyticsComposition {
         ]
         if let bundleIdentifier = Bundle.main.bundleIdentifier {
             properties["bundle_identifier"] = .string(bundleIdentifier)
+            let normalized = bundleIdentifier.lowercased()
+            let channel = normalized.contains("nightly") ? "nightly"
+                : normalized.contains("debug") ? "dev" : "production"
+            properties["client_channel"] = .string(channel)
         }
         if let version = info?["CFBundleShortVersionString"] as? String {
             properties["app_version"] = .string(version)

web/app/api/vm/[id]/exec/route.ts

@@ -55,6 +55,19 @@ export async function POST(
           details: { field: "command" },
         });
       }
+      const commandBytes = Buffer.byteLength(command, "utf8");
+      const MAX_PROVIDER_COMMAND_BYTES = 64 * 1024;
+      if (commandBytes > MAX_PROVIDER_COMMAND_BYTES) {
+        return vmErrorResponse({
+          error: "vm_command_too_large",
+          status: 413,
+          message: `Cloud VM commands must be 64 KiB or smaller. This command is ${commandBytes} bytes.`,
+          action: "Split the command into smaller requests or upload a script and execute the script path.",
+          phase: "exec",
+          retryable: false,
+          details: { commandBytes, maxCommandBytes: MAX_PROVIDER_COMMAND_BYTES },
+        });
+      }
       // Clamp the timeout so a client can't tie up provider quota on a runaway exec. Upper
       // bound matches the provider defaults (15 min on Freestyle); negative / non-number
       // values fall back to 30s.
@@ -69,7 +82,7 @@ export async function POST(
       if (!account.ok) return account.response;
       setSpanAttributes(span, {
         "cmux.vm.id": id,
-        "cmux.command_length": command.length,
+        "cmux.command_length": commandBytes,
         "cmux.timeout_ms": timeoutMs,
       });
       const run = await runVmRoute(execVm({

web/services/observability/mobileNetworkOutcome.ts

@@ -30,7 +30,7 @@ const transports = new Set(["unknown", "iroh", "tailscale", "websocket", "debugL
 
 const allowedPropertyKeys = new Set([
   "phase", "outcome", "duration_ms", "runtime_role", "user_usable",
-  "failure", "transport", "platform", "app_version", "build_number",
+  "failure", "transport", "platform", "client_channel", "app_version", "build_number",
   "bundle_identifier", "os_version", "device_model",
 ]);
 
@@ -44,6 +44,7 @@ export type MobileNetworkOutcome = {
   readonly failure?: string;
   readonly transport?: string;
   readonly platform?: "ios";
+  readonly clientChannel?: "dev" | "nightly" | "production" | "unknown";
   readonly appVersion?: string;
   readonly buildNumber?: string;
   readonly bundleIdentifier?: string;
@@ -67,7 +68,7 @@ export function parseMobileNetworkOutcome(candidate: unknown): MobileNetworkOutc
 }
 
 type CoreObservation = Pick<MobileNetworkOutcome, "phase" | "outcome" | "durationMs" | "userUsable" | "failure" | "transport">;
-type Metadata = Pick<MobileNetworkOutcome, "platform" | "appVersion" | "buildNumber" | "bundleIdentifier" | "osVersion" | "deviceModel">;
+type Metadata = Pick<MobileNetworkOutcome, "platform" | "clientChannel" | "appVersion" | "buildNumber" | "bundleIdentifier" | "osVersion" | "deviceModel">;
 
 function validTimestamp(value: unknown): value is string {
   return typeof value === "string"
@@ -100,14 +101,16 @@ function parseCore(properties: Record<string, unknown>): CoreObservation | null
 
 function parseMetadata(properties: Record<string, unknown>): Metadata | null {
   const platform = optionalExact(properties.platform, "ios");
+  const clientChannel = optionalSetValue(properties.client_channel, new Set(["dev", "nightly", "production", "unknown"]));
   const appVersion = optionalMachineString(properties.app_version);
   const buildNumber = optionalMachineString(properties.build_number);
   const bundleIdentifier = optionalMachineString(properties.bundle_identifier);
   const osVersion = optionalMachineString(properties.os_version);
   const deviceModel = optionalMachineString(properties.device_model, true);
-  if ([platform, appVersion, buildNumber, bundleIdentifier, osVersion, deviceModel].includes(false)) return null;
+  if ([platform, clientChannel, appVersion, buildNumber, bundleIdentifier, osVersion, deviceModel].includes(false)) return null;
   return {
     ...(platform === "ios" ? { platform } : {}),
+    ...(typeof clientChannel === "string" ? { clientChannel } : {}),
     ...(typeof appVersion === "string" ? { appVersion } : {}),
     ...(typeof buildNumber === "string" ? { buildNumber } : {}),
     ...(typeof bundleIdentifier === "string" ? { bundleIdentifier } : {}),
@@ -136,6 +139,7 @@ export async function emitMobileNetworkOutcomes(
       "cmux.mobile.failure": observation.failure,
       "cmux.mobile.transport": observation.transport,
       "cmux.mobile.platform": observation.platform,
+      "cmux.client.channel": observation.clientChannel,
       "cmux.mobile.app_version": observation.appVersion,
       "cmux.mobile.build_number": observation.buildNumber,
       "cmux.mobile.bundle_identifier": observation.bundleIdentifier,

web/services/vms/workflows.ts

@@ -383,7 +383,9 @@ export function reconcileVmProviderStatuses(input: {
           ensureNetwork(owner.provider, { slug: networkSlugForUser(owner.userId), heal: true }).pipe(
             Effect.catchAll(() => Effect.void),
           ),
-        { concurrency: 4, discard: true },
+        // Freestyle returns 429 when several VPC rule heals run together.
+        // One owner at a time keeps healing bounded.
+        { concurrency: 1, discard: true },
       );
     }
     let updated = 0;
@@ -2822,6 +2824,7 @@ export function getVmStats(input: {
   readonly providerVmId: string;
 }) {
   return Effect.gen(function* () {
+    const repo = yield* VmRepository;
     const providers = yield* VmProviderGateway;
     const vm = yield* requireUserVm(input);
     // No resume preflight on purpose: a reading must never wake a sleeping machine.
@@ -2834,7 +2837,19 @@ export function getVmStats(input: {
         }),
       );
     }
-    return yield* providers.getStats(vm.provider, input.providerVmId);
+    return yield* providers.getStats(vm.provider, input.providerVmId).pipe(
+      Effect.catchAll((error) => {
+        if (!isProviderNotFoundError(error)) return Effect.fail(error);
+        return Effect.gen(function* () {
+          yield* repo.markProviderObservedStatus({
+            id: vm.id,
+            providerVmId: input.providerVmId,
+            status: "destroyed",
+          }).pipe(Effect.catchAll(() => Effect.succeed(false)));
+          return yield* Effect.fail(new VmNotFoundError({ vmId: input.providerVmId }));
+        });
+      }),
+    );
   });
 }
 

web/tests/mobile-network-observability-route.test.ts

@@ -62,6 +62,7 @@ describe("iOS mobile network observability route", () => {
         duration_ms: 1_250,
         failure: "timedOut",
         transport: "iroh",
+        client_channel: "nightly",
       }),
     ]));
 
@@ -75,6 +76,7 @@ describe("iOS mobile network observability route", () => {
       durationMs: 1_250,
       failure: "timedOut",
       transport: "iroh",
+      clientChannel: "nightly",
     });
     expect(flushTimeouts).toEqual([1_000]);
   });

web/tests/vm-route-auth.test.ts

@@ -2206,6 +2206,28 @@ describe("VM REST auth", () => {
     expect(runVmWorkflow).not.toHaveBeenCalled();
   });
 
+  test("rejects commands larger than the Freestyle provider limit before workflow", async () => {
+    getUser.mockResolvedValue(authedStackUser());
+    const context = { params: Promise.resolve({ id: "provider-vm-1" }) };
+    const response = await execRoute.POST(
+      new Request("https://cmux.test/api/vm/provider-vm-1/exec", {
+        method: "POST",
+        headers: { origin: "https://cmux.test" },
+        body: JSON.stringify({ command: "x".repeat(64 * 1024 + 1) }),
+      }),
+      context,
+    );
+
+    expect(response.status).toBe(413);
+    const payload = await response.json();
+    expect(payload).toMatchObject({
+      error: "vm_command_too_large",
+      details: { maxCommandBytes: 64 * 1024 },
+    });
+    expect(payload.action).toContain("upload a script");
+    expect(runVmWorkflow).not.toHaveBeenCalled();
+  });
+
   test("does not echo unsupported VM service override values", async () => {
     getUser.mockResolvedValue(authedStackUser());