fix: preserve guest provisioning outcomes and safe rollback

.github/workflows/cloud-vm-guest-install.yml

@@ -0,0 +1,66 @@
+name: Cloud VM guest installation
+
+on:
+  pull_request:
+    paths:
+      - web/services/vms/drivers/**
+      - web/services/vms/guestCli*.ts
+      - web/services/vms/guestPrompt.ts
+      - web/services/vms/workflows.ts
+      - web/services/vms/repository.ts
+      - web/services/vms/routeHelpers.ts
+      - web/services/vms/vmErrorMessages.ts
+      - web/messages/**
+      - web/tests/vm-guest-*.test.ts
+      - web/tests/fixtures/freestyleGuest.ts
+      - web/tests/vm-workflows.test.ts
+      - web/package.json
+      - web/bun.lock
+      - .github/workflows/cloud-vm-guest-install.yml
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+concurrency:
+  group: cloud-vm-guest-install-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  guest-install:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: web
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_DB: cmux_test
+          POSTGRES_PASSWORD: cmux
+          POSTGRES_USER: cmux
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U cmux -d cmux_test"
+          --health-interval 1s --health-timeout 5s --health-retries 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
+        with:
+          bun-version: "1.3.14"
+      - run: bun install --frozen-lockfile
+      - name: SDK and guest filesystem regressions
+        run: bun test tests/vm-guest-install.test.ts tests/vm-guest-install-runtime.test.ts tests/vm-guest-install-deadline.test.ts tests/vm-guest-create-response.test.ts
+      - name: Migrate isolated database
+        env:
+          DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test
+        run: bunx drizzle-kit migrate --config drizzle.config.ts
+      - name: Guest provisioning and idempotency in PostgreSQL
+        env:
+          CMUX_DB_TEST: "1"
+          DATABASE_URL: postgres://cmux:cmux@localhost:5432/cmux_test
+        run: bun test --timeout=30000 --test-name-pattern 'guest bootstrap|unconfirmed guest' tests/vm-workflows.test.ts

web/messages/ar.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "تنظيف آلة Cloud الافتراضية معلق",
+      "message": "فشل إعداد آلة Cloud الافتراضية وما زال التنظيف معلقًا.",
+      "action": "تواصل مع الدعم وأرفق معرّف التتبع قبل إنشاء آلة أخرى."
+    },
     "unsupported": {
       "title": "عملية Cloud VM غير متاحة",
       "reason": "هذا النوع من الأجهزة لا يوفر هذه العملية.",

web/messages/bs.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Čišćenje Cloud VM-a na čekanju",
+      "message": "Postavljanje Cloud VM-a nije uspjelo i čišćenje je još na čekanju.",
+      "action": "Obratite se podršci s ID-jem praćenja prije kreiranja druge mašine."
+    },
     "unsupported": {
       "title": "Operacija Cloud VM-a nije dostupna",
       "reason": "Ova vrsta mašine ne pruža ovu operaciju.",

web/messages/da.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Oprydning af Cloud VM afventer",
+      "message": "Opsætningen af Cloud VM mislykkedes, og oprydningen afventer stadig.",
+      "action": "Kontakt support med sporings-ID’et, før du opretter en ny maskine."
+    },
     "unsupported": {
       "title": "Cloud VM-handling er ikke tilgængelig",
       "reason": "Denne maskintype tilbyder ikke denne handling.",

web/messages/de.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud-VM-Bereinigung ausstehend",
+      "message": "Die Einrichtung der Cloud-VM ist fehlgeschlagen. Die Bereinigung steht noch aus.",
+      "action": "Wenden Sie sich mit der Trace-ID an den Support, bevor Sie eine weitere Maschine erstellen."
+    },
     "unsupported": {
       "title": "Cloud-VM-Vorgang nicht verfügbar",
       "reason": "Dieser Maschinentyp stellt diesen Vorgang nicht bereit.",

web/messages/en.json

@@ -133,6 +133,11 @@
     "expiredBody": "{machine} and everything on it are fine — only this link's session ended. Reopen the desktop from cmux (right-click the machine, then Open Desktop) to get a fresh one."
   },
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud VM cleanup pending",
+      "message": "Cloud VM setup failed and cleanup is still pending.",
+      "action": "Contact support with the trace ID before creating another machine."
+    },
     "artifactUnavailable": {
       "title": "Cloud VM setup unavailable",
       "message": "Cloud VM setup is temporarily unavailable.",

web/messages/es.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Limpieza de la VM Cloud pendiente",
+      "message": "La configuración de la VM Cloud falló y la limpieza sigue pendiente.",
+      "action": "Contacta con soporte e indica el ID de seguimiento antes de crear otra máquina."
+    },
     "unsupported": {
       "title": "Operación de Cloud VM no disponible",
       "reason": "Este tipo de máquina no ofrece esta operación.",

web/messages/fr.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Nettoyage de la VM Cloud en attente",
+      "message": "La configuration de la VM Cloud a échoué et le nettoyage est encore en attente.",
+      "action": "Contactez le support avec l’identifiant de trace avant de créer une autre machine."
+    },
     "unsupported": {
       "title": "Opération de Cloud VM indisponible",
       "reason": "Ce type de machine ne propose pas cette opération.",

web/messages/it.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Pulizia della VM Cloud in sospeso",
+      "message": "La configurazione della VM Cloud non è riuscita e la pulizia è ancora in sospeso.",
+      "action": "Contatta il supporto con l’ID della traccia prima di creare un’altra macchina."
+    },
     "unsupported": {
       "title": "Operazione Cloud VM non disponibile",
       "reason": "Questo tipo di macchina non offre questa operazione.",

web/messages/ja.json

@@ -133,6 +133,11 @@
     "expiredBody": "{machine} とその中身は無事です。このリンクのセッションだけが終了しました。cmux でマシンを右クリックし、「デスクトップを開く」から新しく開き直してください。"
   },
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud VM のクリーンアップが保留中です",
+      "message": "Cloud VM のセットアップに失敗し、クリーンアップがまだ完了していません。",
+      "action": "別のマシンを作成する前に、トレース ID を添えてサポートにお問い合わせください。"
+    },
     "artifactUnavailable": {
       "title": "Cloud VM のセットアップを利用できません",
       "message": "Cloud VM のセットアップを一時的に利用できません。",

web/messages/km.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "ការសម្អាត Cloud VM កំពុងរង់ចាំ",
+      "message": "ការដំឡើង Cloud VM បានបរាជ័យ ហើយការសម្អាតមិនទាន់បញ្ចប់ទេ។",
+      "action": "សូមទាក់ទងផ្នែកគាំទ្រដោយផ្ដល់លេខសម្គាល់តាមដាន មុនពេលបង្កើតម៉ាស៊ីនផ្សេងទៀត។"
+    },
     "unsupported": {
       "title": "ប្រតិបត្តិការ Cloud VM មិនអាចប្រើបាន",
       "reason": "ប្រភេទម៉ាស៊ីននេះមិនផ្តល់ប្រតិបត្តិការនេះទេ។",

web/messages/ko.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud VM 정리 대기 중",
+      "message": "Cloud VM 설정에 실패했으며 정리가 아직 완료되지 않았습니다.",
+      "action": "다른 머신을 만들기 전에 추적 ID와 함께 지원팀에 문의하세요."
+    },
     "unsupported": {
       "title": "Cloud VM 작업을 사용할 수 없음",
       "reason": "이 머신 종류는 이 작업을 제공하지 않습니다.",

web/messages/no.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Opprydding av Cloud VM venter",
+      "message": "Oppsettet av Cloud VM mislyktes, og oppryddingen venter fortsatt.",
+      "action": "Kontakt kundestøtte med sporings-ID-en før du oppretter en ny maskin."
+    },
     "unsupported": {
       "title": "Cloud VM-operasjon er ikke tilgjengelig",
       "reason": "Denne maskintypen tilbyr ikke denne operasjonen.",

web/messages/pl.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Oczekiwanie na usunięcie pozostałości maszyny Cloud VM",
+      "message": "Konfiguracja maszyny Cloud VM nie powiodła się, a usuwanie pozostałości nadal oczekuje.",
+      "action": "Przed utworzeniem kolejnej maszyny skontaktuj się z pomocą techniczną, podając identyfikator śledzenia."
+    },
     "unsupported": {
       "title": "Operacja Cloud VM jest niedostępna",
       "reason": "Ten typ maszyny nie udostępnia tej operacji.",

web/messages/pt-BR.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Limpeza da VM Cloud pendente",
+      "message": "A configuração da VM Cloud falhou e a limpeza ainda está pendente.",
+      "action": "Entre em contato com o suporte e informe o ID de rastreamento antes de criar outra máquina."
+    },
     "unsupported": {
       "title": "Operação de Cloud VM indisponível",
       "reason": "Este tipo de máquina não oferece esta operação.",

web/messages/ru.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Очистка Cloud VM ожидается",
+      "message": "Не удалось настроить Cloud VM. Очистка ещё не завершена.",
+      "action": "Перед созданием другой машины обратитесь в поддержку и укажите идентификатор трассировки."
+    },
     "unsupported": {
       "title": "Операция Cloud VM недоступна",
       "reason": "Этот тип машины не поддерживает эту операцию.",

web/messages/th.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "รอการล้างข้อมูล Cloud VM",
+      "message": "ตั้งค่า Cloud VM ไม่สำเร็จและยังล้างข้อมูลไม่เสร็จ",
+      "action": "ติดต่อฝ่ายสนับสนุนพร้อมรหัสการติดตามก่อนสร้างเครื่องอื่น"
+    },
     "unsupported": {
       "title": "ไม่สามารถใช้การดำเนินการ Cloud VM ได้",
       "reason": "ประเภทเครื่องนี้ไม่มีการดำเนินการนี้",

web/messages/tr.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud VM temizliği bekleniyor",
+      "message": "Cloud VM kurulumu başarısız oldu ve temizlik henüz tamamlanmadı.",
+      "action": "Başka bir makine oluşturmadan önce izleme kimliğiyle destek ekibine başvurun."
+    },
     "unsupported": {
       "title": "Cloud VM işlemi kullanılamıyor",
       "reason": "Bu makine türü bu işlemi sağlamıyor.",

web/messages/uk.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Очищення Cloud VM очікується",
+      "message": "Не вдалося налаштувати Cloud VM. Очищення ще не завершено.",
+      "action": "Перш ніж створювати іншу машину, зверніться до підтримки та вкажіть ідентифікатор трасування."
+    },
     "unsupported": {
       "title": "Операція Cloud VM недоступна",
       "reason": "Цей тип машини не підтримує цю операцію.",

web/messages/zh-CN.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud VM 清理待完成",
+      "message": "Cloud VM 设置失败，清理仍未完成。",
+      "action": "创建另一台机器之前，请提供跟踪 ID 联系支持团队。"
+    },
     "unsupported": {
       "title": "云端虚拟机操作不可用",
       "reason": "此机器类型不提供此操作。",

web/messages/zh-TW.json

@@ -1,5 +1,10 @@
 {
   "vmErrors": {
+    "createCleanupPending": {
+      "title": "Cloud VM 清理待完成",
+      "message": "Cloud VM 設定失敗，清理仍未完成。",
+      "action": "建立另一台機器之前，請提供追蹤 ID 聯絡支援團隊。"
+    },
     "unsupported": {
       "title": "雲端虛擬機操作無法使用",
       "reason": "此機器類型不提供這項操作。",

web/services/vms/drivers/freestyle.ts

@@ -14,6 +14,8 @@ import { createHash, randomBytes } from "node:crypto";
 import { isIP } from "node:net";
 import { Effect } from "effect";
 import { announceFreestyleNetwork } from "./freestyleNetworkAnnouncement";
+import { installFreestyleGuestCli, type FreestyleClientFactory } from "./freestyleGuestCli";
+import { rollbackFreestyleCreate } from "./providerCreateCleanup";
 import { guestResourceReporterInstallCommand } from "../guestResourceReporter";
 import {
   ProviderError,
@@ -175,7 +177,7 @@ const EDGE_DOMAIN = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9
  * Production uses the env-configured client and the live manifest.
  */
 export type FreestyleProviderDependencies = {
-  readonly client: (timeoutMs?: number) => Freestyle;
+  readonly client: FreestyleClientFactory;
   readonly resolveDaemonSource: typeof resolveCmuxTuiSource;
 };
 
@@ -197,9 +199,12 @@ export function preconnectFreestyle(): void {
 }
 
 /** Exported for the publication provider, which shares this account-wide client. */
-export function freestyleClient(timeoutMs = DEFAULT_TIMEOUT_MS): Freestyle {
-  const longFetch = ((input: URL | RequestInfo, init?: RequestInit) =>
-    fetch(input as Request, { ...(init ?? {}), signal: AbortSignal.timeout(timeoutMs) })) as typeof fetch;
+export function freestyleClient(timeoutMs = DEFAULT_TIMEOUT_MS, signal?: AbortSignal): Freestyle {
+  const longFetch: typeof fetch = (input, init) =>
+    fetch(input as Request, {
+      ...(init ?? {}),
+      signal: AbortSignal.any([AbortSignal.timeout(timeoutMs), ...[signal, init?.signal].filter((value): value is AbortSignal => !!value)]),
+    });
   const baseUrl = process.env.FREESTYLE_API_URL?.trim() || undefined;
   const apiKey = process.env.FREESTYLE_API_KEY?.trim();
   if (apiKey) return new Freestyle({ apiKey, baseUrl, fetch: longFetch });
@@ -1004,9 +1009,8 @@ export class FreestyleProvider implements VMProvider {
             // A VM that failed to size or configure must not survive as an
             // orphan, and an undersized machine must not ship as if it were
             // the plan machine.
-            await vm.delete().catch((cleanupErr) => {
-              console.error(`[freestyle] create rollback failed; VM ${vmId} may be orphaned`, cleanupErr);
-            });
+            const rollback = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(this.deps.client, vmId, err)));
+            if (rollback._tag === "Left") throw rollback.left;
             throw err;
           }
           return {
@@ -1380,9 +1384,8 @@ export class FreestyleProvider implements VMProvider {
             await this.ensureCmuxTuiRunning(vm, vmId, false).catch(() => undefined);
             await this.announcePrivateAddresses(vm, data);
           } catch (err) {
-            await vm.delete().catch((cleanupErr) => {
-              console.error(`[freestyle] restore rollback failed; VM ${vmId} may be orphaned`, cleanupErr);
-            });
+            const rollback = await Effect.runPromise(Effect.either(rollbackFreestyleCreate(this.deps.client, vmId, err)));
+            if (rollback._tag === "Left") throw rollback.left;
             throw err;
           }
           return {
@@ -1706,23 +1709,20 @@ export class FreestyleProvider implements VMProvider {
    * as a failed heal.
    */
   private async installGuestCli(vm: Vm, vmId: string, promptIdentity?: GuestPromptIdentity): Promise<void> {
-    const temporaryPath = `${GUEST_CMUX_SHIM_PATH}.tmp-${randomBytes(12).toString("hex")}`;
-    try {
-      await vm.fs.writeTextFile(temporaryPath, GUEST_CMUX_SHIM, { mode: 0o755 });
-      const result = await vm.exec({
-        command: `chmod 0755 '${temporaryPath}' && mv -f '${temporaryPath}' '${GUEST_CMUX_SHIM_PATH}'`
-          + (promptIdentity ? ` && ${guestPromptInstallCommand(promptIdentity)}` : ""),
-        timeoutMs: 30_000,
-        linuxUser: GUEST_LINUX_USER,
-      });
-      const exitCode = result.statusCode ?? 124;
-      if (exitCode !== 0) {
-        throw new Error(`guest cmux shim install exited ${exitCode}`);
+    await withVmSpan("cmux.vm.guest_cli.install", "provider", {}, async (span) => {
+      const result = await Effect.runPromise(Effect.either(installFreestyleGuestCli(this.deps.client, vmId, promptIdentity)));
+      if (result._tag === "Left") {
+        setSpanAttributes(span, {
+          "cmux.vm.guest_install.outcome": result.left.outcome,
+          "cmux.vm.guest_install.stage": result.left.stage,
+          "cmux.vm.guest_install.exit_code": result.left.exitCode,
+          "cmux.duration_ms": result.left.elapsedMs,
+          "cmux.vm.guest_install.cleanup_failed": result.left.cleanupCause !== undefined,
+        });
+        throw result.left;
       }
-    } catch (error) {
-      await vm.fs.remove(temporaryPath).catch(() => undefined);
-      throw error;
-    }
+      setSpanAttributes(span, { "cmux.vm.guest_install.outcome": "success", "cmux.duration_ms": result.right.elapsedMs });
+    });
     await this.ensureResourceReporter(vm, vmId);
   }