fix(builder): catch transient failures on getStateBuilders

packages/builder/src/identity.ts

@@ -3,7 +3,7 @@ import {ChainForkConfig} from "@lodestar/config";
 import {PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
 import {IClock} from "@lodestar/state-transition";
 import {BuilderIndex, BuilderStatus} from "@lodestar/types";
-import {ErrorAborted, Logger, sleep, toHex} from "@lodestar/utils";
+import {ErrorAborted, Logger, TimeoutError, isFetchError, sleep, toHex} from "@lodestar/utils";
 
 export const WAITING_FOR_BUILDER_POLL_MS = 10 * 1000;
 
@@ -38,7 +38,7 @@ export async function getBuilderStatus(
   id: routes.beacon.BuilderId
 ): Promise<{status: BuilderStatus; balance: number} | null> {
   try {
-    const builderEntry = await fetchBuilder(api, id);
+    const builderEntry = await fetchBuilder(api, logger, id);
     if (builderEntry) {
       return {
         status: builderEntry.status,
@@ -77,7 +77,7 @@ async function waitForBuilder(
 
     let builder: routes.beacon.BuilderResponse | null = null;
     try {
-      builder = await fetchBuilder(api, id);
+      builder = await fetchBuilder(api, logger, id);
     } catch (e) {
       // At the fork boundary getStateBuilders("head") can still 400: it serves the head block's
       // post-state, which stays pre-gloas until a gloas-epoch block is head. Keep polling on the transient.
@@ -111,28 +111,37 @@ async function waitForBuilder(
 
 async function fetchBuilder(
   api: ApiClient,
+  logger: Logger,
   id: routes.beacon.BuilderId
 ): Promise<routes.beacon.BuilderResponse | null> {
-  const builderRes = await api.beacon.getStateBuilders({
-    stateId: "head",
-    builderIds: [id],
-  });
+  try {
+    const builderRes = await api.beacon.getStateBuilders({
+      stateId: "head",
+      builderIds: [id],
+    });
 
-  const builders = builderRes.value();
+    const builders = builderRes.value();
 
-  if (builders.length === 0) {
-    return null;
-  }
+    if (builders.length === 0) {
+      return null;
+    }
 
-  const builder = builders[0];
+    const builder = builders[0];
 
-  if (typeof id === "number") {
-    if (id !== builder.index) {
-      throw Error(`Index mismatch: got=${builder.index} expected=${id}`);
+    if (typeof id === "number") {
+      if (id !== builder.index) {
+        throw Error(`Index mismatch: got=${builder.index} expected=${id}`);
+      }
+    } else if (id !== toHex(builder.builder.pubkey)) {
+      throw Error(`Pubkey mismatch: got=${toHex(builder.builder.pubkey)} expected=${id}`);
     }
-  } else if (id !== toHex(builder.builder.pubkey)) {
-    throw Error(`Pubkey mismatch: got=${toHex(builder.builder.pubkey)} expected=${id}`);
-  }
 
-  return builder;
+    return builder;
+  } catch (e) {
+    if (e instanceof TimeoutError || (isFetchError(e) && e.type !== "input")) {
+      logger.warn("Failed to fetch builder", {message: e.message});
+      return null;
+    }
+    throw e;
+  }
 }

packages/builder/test/unit/identity.test.ts

@@ -1,7 +1,7 @@
 import {afterEach, beforeEach, describe, expect, it, vi} from "vitest";
 import {createChainForkConfig} from "@lodestar/config";
 import {PAYLOAD_BUILDER_VERSION} from "@lodestar/params";
-import {ErrorAborted, toHex} from "@lodestar/utils";
+import {ErrorAborted, FetchError, TimeoutError, toHex} from "@lodestar/utils";
 import {WAITING_FOR_BUILDER_POLL_MS, getBuilderStatus, resolveBuilderIdentity} from "../../src/identity.js";
 import {getApiClientStub, mockApiErrorResponse, mockApiResponse} from "./utils/apiStub.js";
 import {ClockMock} from "./utils/clock.js";
@@ -179,4 +179,49 @@ describe("Identity", () => {
       resolveBuilderIdentity(api, logger, pubkeyString, abortController.signal, clock, config)
     ).rejects.toThrow(/status 500/);
   });
+
+  it("keeps polling on getStateBuilders request timeout", async () => {
+    vi.useFakeTimers();
+    api.beacon.getStateBuilders.mockRejectedValueOnce(new TimeoutError("getStateBuilders request"));
+    api.beacon.getStateBuilders.mockResolvedValue(
+      mockGetStateBuildersResponse(index, {status, pubkey, balance, version})
+    );
+    const promise = resolveBuilderIdentity(api, logger, pubkeyString, abortController.signal, clock, config);
+    await vi.advanceTimersByTimeAsync(WAITING_FOR_BUILDER_POLL_MS);
+    expect(await promise).toEqual(index);
+    expect(api.beacon.getStateBuilders).toHaveBeenCalledTimes(2);
+  });
+
+  it("keeps polling on fetch failure", async () => {
+    vi.useFakeTimers();
+    api.beacon.getStateBuilders.mockRejectedValueOnce(
+      new FetchError(
+        "http://127.0.0.1:9596",
+        new TypeError("fetch failed", {
+          cause: Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:9596"), {code: "ECONNREFUSED"}),
+        })
+      )
+    );
+    api.beacon.getStateBuilders.mockResolvedValue(
+      mockGetStateBuildersResponse(index, {status, pubkey, balance, version})
+    );
+    const promise = resolveBuilderIdentity(api, logger, pubkeyString, abortController.signal, clock, config);
+    await vi.advanceTimersByTimeAsync(WAITING_FOR_BUILDER_POLL_MS);
+    expect(await promise).toEqual(index);
+    expect(api.beacon.getStateBuilders).toHaveBeenCalledTimes(2);
+  });
+
+  it("throws on invalid beacon node url / invalid input", async () => {
+    api.beacon.getStateBuilders.mockRejectedValue(
+      new FetchError(
+        "invalid-url",
+        new TypeError("Failed to parse URL from invalid-url", {
+          cause: Object.assign(new Error("Invalid URL"), {input: "invalid-url", code: "ERR_INVALID_URL"}),
+        })
+      )
+    );
+    await expect(
+      resolveBuilderIdentity(api, logger, pubkeyString, abortController.signal, clock, config)
+    ).rejects.toThrow(/Failed to parse URL/);
+  });
 });