loadFgb: add retry on error

seasketch · Dec 11, 2024 · 170db00 · 170db00
1 parent b10e738
commit 170db00
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 34 deletions.
diff --git a/packages/geoprocessing/src/dataproviders/flatgeobuf.test.e2e.ts b/packages/geoprocessing/src/dataproviders/flatgeobuf.test.e2e.ts
@@ -6,31 +6,17 @@ import { deserialize } from "flatgeobuf/lib/mjs/geojson.js";
 import { readFileSync } from "node:fs";
 import path from "node:path";
 import { isFeatureCollection } from "../index.js";
+import fs from "fs-extra";
 
 describe("flatgeobuf", () => {
   test("flatgeobuf - local world fgb", async () => {
-    const canonicalStr = canonicalize([
-      {
-        id: 0, // this is not in the data, but fgb client automatically adds it on deserialize as of v3.36.0
-        type: "Feature",
-        properties: {
-          name: "World boundary",
-          description: "World",
-        },
-        geometry: {
-          coordinates: [
-            [
-              [-180, 90],
-              [-180, -90],
-              [180, -90],
-              [180, 90],
-              [-180, 90],
-            ],
-          ],
-          type: "Polygon",
-        },
-      },
-    ]);
+    const worldJson = fs.readJsonSync("data/in/world.json");
+    // pull features out of FC and add index based ID, just as the flatgeobuf client does on read
+    const worldFeatures = worldJson.features.map((f, index) => {
+      f.id = index;
+      return f;
+    });
+    const canonicalStr = canonicalize(worldFeatures);
     const url = "http://127.0.0.1:8080/data/in/world.fgb";
     const features = await loadFgb(url);
     expect(features.length).toEqual(1);

diff --git a/packages/geoprocessing/src/dataproviders/flatgeobuf.ts b/packages/geoprocessing/src/dataproviders/flatgeobuf.ts
@@ -21,8 +21,11 @@ export function fgBoundingBox(box: BBox): FgBoundingBox {
 }
 
 /**
- * Fetch features from flatgeobuf at url within bounding box
- * Awaits all features before returning, rather than streaming them.
+ * Fetch features from flatgeobuf at url that intersect with bounding box
+ * Retries up to 3 times if fetch fails in error
+ * @param url url of flatgeobuf file
+ * @param bbox optional bounding box to fetch features that intersect with
+ * @returns feature array
  * @deprecated Use `loadCog` instead.
  */
 export async function fgbFetchAll<F extends Feature<Geometry>>(
@@ -34,29 +37,53 @@ export async function fgbFetchAll<F extends Feature<Geometry>>(
 
 /**
  * Fetch features from flatgeobuf at url that intersect with bounding box
- * Awaits all features before returning, rather than streaming them.
+ * Retries up to 3 times if fetch fails in error
+ * @param url url of flatgeobuf file
+ * @param bbox optional bounding box to fetch features that intersect with
+ * @returns feature array
  */
 export async function loadFgb<F extends Feature<Geometry>>(
   url: string,
-  box?: BBox,
+  bbox?: BBox,
 ) {
   const fgBox = (() => {
-    if (!box && !Array.isArray(box)) {
+    if (!bbox && !Array.isArray(bbox)) {
       return fgBoundingBox([-180, -90, 180, 90]); // fallback to entire world
     } else {
-      return fgBoundingBox(box);
+      return fgBoundingBox(bbox);
     }
   })();
 
   if (process.env.NODE_ENV !== "test")
     console.log("loadFgb", `url: ${url}`, `box: ${JSON.stringify(fgBox)}`);
 
-  const features = (await takeAsync(
-    deserialize(url, fgBox) as AsyncGenerator,
-  )) as F[];
-  if (!Array.isArray(features))
-    throw new Error("Unexpected result from loadFgb");
-  return features;
+  const maxRetries = 3;
+  let attempt = 0;
+  let features: F[] | null = null;
+
+  while (attempt < maxRetries) {
+    try {
+      features = (await takeAsync(
+        deserialize(url, fgBox) as AsyncGenerator,
+      )) as F[];
+      if (!Array.isArray(features))
+        throw new Error("Unexpected result from loadFgb");
+      return features;
+    } catch (error: unknown) {
+      attempt++;
+      if (attempt >= maxRetries && error instanceof Error) {
+        throw new Error(
+          `Failed to load FGB after ${maxRetries} attempts: ${error.message}`,
+        );
+      }
+      const waitTime = attempt * 250; // Exponential backoff: 250ms, 500ms, 750ms, 1000ms, 1250ms
+      console.warn(
+        `Attempt ${attempt} failed. Retrying in ${waitTime / 1000} seconds...`,
+      );
+      await new Promise((resolve) => setTimeout(resolve, waitTime));
+    }
+  }
+  throw new Error("Failed to load FGB");
 }
 
 /**