fix(devices): keep devices listed when simulator discovery fails

apps/server/src/device/DeviceHost.ts

@@ -110,6 +110,8 @@ export interface DeviceOperations {
       readonly detail: string | undefined;
       /** Some Android virtual devices could not be listed; the rest are. */
       readonly androidDiscoveryIncomplete?: boolean;
+      /** Platforms that could not be listed; their last known devices stay listed. */
+      readonly incompletePlatforms?: ReadonlyArray<DevicePlatform>;
     },
     DeviceOperationError
   >;

apps/server/src/device/DeviceHubOperations.ts

@@ -6,13 +6,19 @@ import {
 } from "@t3tools/contracts";
 import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
 import * as Schema from "effect/Schema";
 import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
 
 import type { DeviceHostReady, DeviceOperations, HostDevice } from "./DeviceHost.ts";
 
 const BOOT_TIMEOUT = Duration.minutes(3);
 const SCREENSHOT_TIMEOUT = Duration.seconds(20);
+// Let the hub's 30 s simctl timeout report its cause first.
+const LIST_TIMEOUT_SECONDS = 40;
+// simctl could not list simulators at all, so the hub's empty iOS list says nothing.
+const SIMULATOR_LIST_FAILED =
+  /^\[apple-utils\] (?:Failed to (?:run `xcrun simctl list devices|parse JSON output:|list devices:)|`xcrun simctl list devices --json` did not respond within )/;
 
 const HubDevice = Schema.Struct({
   id: Schema.String,
@@ -69,25 +75,30 @@ export const makeDeviceHubOperations = (options: {
   const { hostId, origin, run } = options;
   const httpClient = options.http.pipe(HttpClient.withScope);
 
+  const requestFailed = (operation: string) => (cause: unknown) =>
+    new DeviceOperationError({ operation, reason: "request_failed", cause });
+
+  const fetchJson = <A, I>(
+    request: HttpClientRequest.HttpClientRequest,
+    schema: Schema.Codec<A, I>,
+  ) =>
+    httpClient
+      .execute(request)
+      .pipe(
+        Effect.flatMap(HttpClientResponse.filterStatusOk),
+        Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
+        Effect.scoped,
+      );
+
   const hubJson = <A, I>(
     request: HttpClientRequest.HttpClientRequest,
     schema: Schema.Codec<A, I>,
     operation: string,
     timeout: Duration.Input = Duration.seconds(15),
   ) =>
-    httpClient.execute(request).pipe(
-      Effect.flatMap(HttpClientResponse.filterStatusOk),
-      Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
-      Effect.scoped,
+    fetchJson(request, schema).pipe(
       Effect.timeout(timeout),
-      Effect.mapError(
-        (cause) =>
-          new DeviceOperationError({
-            operation,
-            reason: "request_failed",
-            cause,
-          }),
-      ),
+      Effect.mapError(requestFailed(operation)),
     );
 
   const postJson = (path: string, body: Record<string, string>, operation: string) =>
@@ -106,11 +117,21 @@ export const makeDeviceHubOperations = (options: {
     );
 
   const list: DeviceOperations["list"] = Effect.gen(function* () {
-    const hubList = yield* hubJson(
+    const listed = yield* fetchJson(
       HttpClientRequest.get(`${origin}/api/devices`),
       HubDeviceList,
-      "list",
+    ).pipe(
+      Effect.mapError(requestFailed("list")),
+      Effect.timeoutOption(Duration.seconds(LIST_TIMEOUT_SECONDS)),
     );
+    if (Option.isNone(listed)) {
+      return {
+        devices: [],
+        detail: `The device hub did not answer within ${LIST_TIMEOUT_SECONDS} seconds.`,
+        incompletePlatforms: ["ios", "android"] as const,
+      };
+    }
+    const hubList = listed.value;
     const devices: Array<HostDevice> = [...hubList.simulators, ...hubList.emulators];
     const androidAvailable = yield* options.androidAvailable;
     const warnings = hubList.errors?.filter(
@@ -155,10 +176,14 @@ export const makeDeviceHubOperations = (options: {
         }
       }
     }
+    const simulatorsUnlisted =
+      hubList.simulators.length === 0 &&
+      (hubList.errors ?? []).some((error) => SIMULATOR_LIST_FAILED.test(error.message));
     return {
       devices,
       detail: details.join("\n") || undefined,
       androidDiscoveryIncomplete,
+      ...(simulatorsUnlisted ? { incompletePlatforms: ["ios"] as const } : {}),
     };
   }).pipe(Effect.withSpan("DeviceHubOperations.list"));
 
@@ -183,7 +208,7 @@ export const makeDeviceHubOperations = (options: {
           diagnostics,
         )
           ? "disk_space"
-          : /timed? out|timeout/i.test(diagnostics)
+          : /timed? out|timeout|did not respond within/i.test(diagnostics)
             ? "timeout"
             : "launch_failed",
         ...(diagnostics ? { diagnostics: boundDiagnostics(diagnostics) } : {}),

apps/server/src/device/DeviceService.test.ts

@@ -26,6 +26,7 @@ import * as Ref from "effect/Ref";
 import * as Schema from "effect/Schema";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
+import * as TestClock from "effect/testing/TestClock";
 import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
 import { HttpClient, HttpClientResponse } from "effect/http";
 import * as ServerConfig from "../config.ts";
@@ -100,6 +101,7 @@ const fixture = Effect.fn("fixture")(function* (
   const discoveryErrors = yield* Ref.make<Array<{ message: string; error?: string }>>([]);
   const androidAvailable = yield* Ref.make(true);
   const encoderResult = yield* Ref.make({ code: 0, stdout: " V....D libx264", stderr: "" });
+  const listing = { hangs: false, requested: yield* Deferred.make<void>() };
   let booted = false;
   let shutDown = false;
   const connection = { online: true, kind: "local" as "local" | "ssh", failShutdown: false };
@@ -164,6 +166,10 @@ const fixture = Effect.fn("fixture")(function* (
       if (request.url.endsWith("/grid/api/start")) {
         return HttpClientResponse.fromWeb(request, Response.json({ ok: true }));
       }
+      if (listing.hangs) {
+        yield* Deferred.succeed(listing.requested, undefined);
+        return yield* Effect.never;
+      }
       return HttpClientResponse.fromWeb(
         request,
         Response.json({
@@ -326,6 +332,7 @@ const fixture = Effect.fn("fixture")(function* (
     discoveryErrors,
     androidAvailable,
     endDevice,
+    listing,
   };
 });
 
@@ -392,21 +399,40 @@ describe("device setup consent", () => {
       }).pipe(Effect.scoped),
   );
 
-  it.effect("boots a stopped Android AVD and uses its emulator serial without duplicating it", () =>
-    Effect.gen(function* () {
-      const { service, requests } = yield* fixture();
-      yield* service.configure({ enabled: true });
-      const session = yield* service.open({
-        threadId: ThreadId.make("thread-1"),
-        deviceId: "Pixel_API_35",
-        platform: "android",
-      });
-      expect(session.deviceId).toBe("emulator-5554");
-      expect(requests.filter((url) => url.endsWith("/boot"))).toHaveLength(1);
-      const state = yield* service.state;
-      expect(state.devices.map((device) => device.id)).toEqual(["emulator-5554"]);
-      expect(state.bootingDevices).toEqual([]);
-    }).pipe(Effect.scoped),
+  it.effect.each([1, 2])(
+    "opens a stopped Android AVD from %s threads using its emulator serial",
+    (count) =>
+      Effect.gen(function* () {
+        const booting = yield* Queue.unbounded<void>();
+        const allowBoot = yield* Deferred.make<void>();
+        const { service, requests } = yield* fixture(
+          Queue.offer(booting, undefined).pipe(Effect.andThen(Deferred.await(allowBoot))),
+        );
+        yield* service.configure({ enabled: true });
+        const opening = yield* Effect.forkChild(
+          Effect.all(
+            Array.from({ length: count }, (_, index) =>
+              service.open({
+                threadId: ThreadId.make(`thread-${index}`),
+                deviceId: "Pixel_API_35",
+                platform: "android",
+              }),
+            ),
+            { concurrency: "unbounded" },
+          ),
+        );
+        yield* Effect.forEach(Array.from({ length: count }), () => Queue.take(booting));
+        yield* Deferred.succeed(allowBoot, undefined);
+        const sessions = yield* Fiber.join(opening);
+        expect(sessions.map((session) => session.deviceId)).toEqual(
+          Array(count).fill("emulator-5554"),
+        );
+        expect(requests.filter((url) => url.endsWith("/boot"))).toHaveLength(count);
+        const state = yield* service.state;
+        expect(state.sessions).toHaveLength(count);
+        expect(state.devices.map((device) => device.id)).toEqual(["emulator-5554"]);
+        expect(state.bootingDevices).toEqual([]);
+      }).pipe(Effect.scoped),
   );
 
   it.effect("blocks only emulator video until ffmpeg is usable, then allows retry", () =>
@@ -689,6 +715,55 @@ it.effect("keeps the cause from the hub's error in discovery warnings, without t
   }).pipe(Effect.scoped),
 );
 
+const simulator = {
+  id: "iphone",
+  name: "iPhone",
+  platform: "ios",
+  version: "iOS 26",
+  booted: true,
+  physical: false,
+} as const;
+const androidPhone = { ...simulator, id: "pixel", name: "Pixel", platform: "android" } as const;
+
+it.effect.each([
+  "[apple-utils] Failed to run `xcrun simctl list devices --json`:",
+  "[apple-utils] `xcrun simctl list devices --json` did not respond within 30 s",
+])("keeps the last simulators after %s while Android still refreshes", (message) =>
+  Effect.gen(function* () {
+    const { service, devices, discoveryErrors } = yield* fixture();
+    devices.push(simulator, androidPhone);
+    yield* service.configure({ enabled: true });
+    devices.length = 0;
+    yield* Ref.set(discoveryErrors, [{ message }]);
+    const failed = yield* service.list;
+    expect(failed.devices.map((device) => device.id)).toEqual(["iphone", "Pixel_API_35"]);
+    expect(failed.hostStatuses.local).toEqual({ status: "ready", detail: message });
+
+    yield* Ref.set(discoveryErrors, []);
+    const emptied = yield* service.list;
+    expect(emptied.devices.map((device) => device.id)).toEqual(["Pixel_API_35"]);
+    expect(emptied.hostStatuses.local).toEqual({ status: "ready" });
+  }).pipe(Effect.scoped),
+);
+
+it.effect("keeps every device when the hub does not answer the listing within 40 seconds", () =>
+  Effect.gen(function* () {
+    const { service, devices, listing } = yield* fixture();
+    devices.push(simulator);
+    const before = yield* service.configure({ enabled: true });
+    listing.hangs = true;
+    const listed = yield* service.list.pipe(Effect.forkScoped);
+    yield* Deferred.await(listing.requested);
+    yield* TestClock.adjust("40 seconds");
+    const timedOut = yield* Fiber.join(listed);
+    expect(timedOut.devices).toEqual(before.devices);
+    expect(timedOut.hostStatuses.local).toEqual({
+      status: "ready",
+      detail: "The device hub did not answer within 40 seconds.",
+    });
+  }).pipe(Effect.scoped),
+);
+
 it.effect.each([
   {
     diagnostic: "Insufficient disk space at /private/user/path",
@@ -700,6 +775,12 @@ it.effect.each([
     reason: "timeout",
     message: "did not become ready in time",
   },
+  {
+    diagnostic:
+      "[apple-utils] `xcrun simctl boot iphone` did not respond within 120 s; CoreSimulatorService may need restarting",
+    reason: "timeout",
+    message: "did not become ready in time",
+  },
   {
     diagnostic:
       "The emulator process exited with code 1.\nTry manually: emulator -avd Pixel_API_35",
@@ -1733,6 +1814,7 @@ describe("locating a device id across hosts", () => {
   const listingHost = (
     host: { readonly id: string; readonly label: string },
     ids: ReadonlyArray<string>,
+    attach: DeviceHost.DeviceOperations["attach"] = () => Effect.void,
   ): DeviceHost.DeviceHost["Service"] => {
     const devices = ids.map((id) => ({
       id,
@@ -1748,7 +1830,7 @@ describe("locating a device id across hosts", () => {
       devices: {
         list: Effect.succeed({ devices, detail: undefined }),
         boot: (device) => Effect.succeed(device.id),
-        attach: () => Effect.void,
+        attach,
         shutdown: () => Effect.void,
         screenshot: () => Effect.succeed(new Uint8Array()),
       },
@@ -1803,6 +1885,60 @@ describe("locating a device id across hosts", () => {
       platform: deviceId.startsWith("Pixel") ? "android" : "ios",
     });
 
+  it.effect("rejects an open whose platform does not match the device", () =>
+    withTwoHosts((service) =>
+      Effect.gen(function* () {
+        const error = yield* service
+          .open({ threadId, hostId: macMini.id, deviceId: "MAC-UDID", platform: "android" })
+          .pipe(Effect.flip);
+        expect(error).toMatchObject({
+          _tag: "DeviceNotFoundError",
+          hostId: macMini.id,
+          deviceId: "MAC-UDID",
+        });
+        expect((yield* service.state).sessions).toEqual([]);
+      }),
+    ),
+  );
+
+  // The hub fails a stream start that overlaps another one on the same device.
+  it.effect.each([
+    { case: "one device take turns", other: "MAC-UDID", overlap: 1 },
+    { case: "different devices run together", other: "MAC-UDID-2", overlap: 2 },
+  ])("concurrent opens of $case", ({ other, overlap }) =>
+    Effect.gen(function* () {
+      const attaching = yield* Queue.unbounded<Deferred.Deferred<void>>();
+      let active = 0;
+      let most = 0;
+      const host = listingHost(macMini, ["MAC-UDID", "MAC-UDID-2"], () =>
+        Effect.gen(function* () {
+          most = Math.max(most, ++active);
+          const done = yield* Deferred.make<void>();
+          yield* Queue.offer(attaching, done);
+          yield* Deferred.await(done);
+          active--;
+        }),
+      );
+      const service = yield* DeviceService.makeWithHosts(new Map([[macMini.id, host]])).pipe(
+        Effect.provide(NodeCrypto.layer),
+      );
+      const opening = yield* Effect.forkChild(
+        Effect.all(
+          [
+            open(service, macMini.id, "MAC-UDID"),
+            open(service, macMini.id, other, ThreadId.make("thread-other")),
+          ],
+          { concurrency: "unbounded" },
+        ),
+      );
+      yield* Deferred.succeed(yield* Queue.take(attaching), undefined);
+      yield* Deferred.succeed(yield* Queue.take(attaching), undefined);
+      yield* Fiber.join(opening);
+      expect(most).toBe(overlap);
+      expect((yield* service.state).sessions).toHaveLength(2);
+    }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true })), Effect.scoped),
+  );
+
   it.effect("finds a device id that exists only on another host", () =>
     withTwoHosts((service) =>
       Effect.gen(function* () {