Stabilize dataproviders: drop genClipLoader, getFeatures -> getDataso…

…urceFeatures, loadFgb retry on error, etc. (#384)

* Deprecate getFeatures for getDatasourceFeatures
getDatasourceFeatures - accept sketch option and use getFeaturesForSketchBBoxes
getFeaturesForSketchBBoxes - improve typing and docs, add lower-level getFeaturesForBBoxes
Drop genClipLoader

* rebuild docs

* Link to global datasources

* document getFeaturesForBBoxes

* migrate internally to getDatasourceFeatures

* add code examples to DataProviders doc

* add type narrowing to getFeatures and friends

* loadFgb: add retry on error

* add more tests to getFeaturesForSketchBBoxes, improve squareSketches, add squares.fgb with txt file showing layout, update squares.json

packages/base-project/project/datasources.json

@@ -1,42 +1,4 @@
 [
-  {
-    "datasourceId": "global-clipping-osm-land",
-    "geo_type": "vector",
-    "formats": ["subdivided"],
-    "classKeys": [],
-    "idProperty": "gid",
-    "nameProperty": "gid",
-    "url": "https://d3p1dsef9f0gjr.cloudfront.net",
-    "precalc": false
-  },
-  {
-    "datasourceId": "global-clipping-eez-land-union",
-    "geo_type": "vector",
-    "formats": ["subdivided"],
-    "classKeys": [],
-    "idProperty": "UNION",
-    "nameProperty": "UNION",
-    "url": "https://d3muy0hbwp5qkl.cloudfront.net",
-    "precalc": false
-  },
-  {
-    "datasourceId": "global-eez-mr-v12",
-    "geo_type": "vector",
-    "formats": ["fgb", "json"],
-    "metadata": {
-      "name": "World EEZ v11",
-      "description": "World EEZ boundaries and disputed areas",
-      "version": "11.0",
-      "publisher": "Flanders Marine Institute (VLIZ)",
-      "publishDate": "2019118",
-      "publishLink": "https://marineregions.org/"
-    },
-    "classKeys": [],
-    "url": "https://gp-global-datasources-datasets.s3.us-west-1.amazonaws.com/global-eez-mr-v12.fgb",
-    "idProperty": "GEONAME",
-    "nameProperty": "GEONAME",
-    "precalc": false
-  },
   {
     "datasourceId": "world",
     "geo_type": "vector",

packages/geoprocessing/data/in/squares.json

@@ -0,0 +1,106 @@
+{
+  "type": "FeatureCollection",
+  "name": "squares",
+  "crs": {
+    "type": "name",
+    "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" }
+  },
+  "features": [
+    {
+      "type": "Feature",
+      "properties": { "name": "outsideTwoByPolyBottomRight" },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [3.0, 0.0],
+            [4.0, 0.0],
+            [4.0, 1.0],
+            [3.0, 1.0],
+            [3.0, 0.0]
+          ]
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": { "name": "outsideTwoByPolyTopRight" },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [3.0, 3.0],
+            [4.0, 3.0],
+            [4.0, 4.0],
+            [3.0, 4.0],
+            [3.0, 3.0]
+          ]
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": { "name": "outsideTwoByPolyTopLeft" },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [0.0, 3.0],
+            [1.0, 3.0],
+            [1.0, 4.0],
+            [0.0, 4.0],
+            [0.0, 3.0]
+          ]
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": { "name": "halfInsideTwoByPoly" },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [1.0, 1.0],
+            [3.0, 1.0],
+            [3.0, 2.0],
+            [1.0, 2.0],
+            [1.0, 1.0]
+          ]
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": { "name": "twoByPoly" },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [0.0, 0.0],
+            [2.0, 0.0],
+            [2.0, 2.0],
+            [0.0, 2.0],
+            [0.0, 0.0]
+          ]
+        ]
+      }
+    },
+    {
+      "type": "Feature",
+      "properties": { "name": "insideTwoByPoly" },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [
+          [
+            [0.0, 0.0],
+            [1.0, 0.0],
+            [1.0, 1.0],
+            [0.0, 1.0],
+            [0.0, 0.0]
+          ]
+        ]
+      }
+    }
+  ]
+}

packages/geoprocessing/data/in/squaresReadme.txt

@@ -0,0 +1,54 @@
+
+
+
+
+squares.fgb polygons are rectangles in increments of 1 degree                                                                                                  
+
+    4      @@@@@@@@@@@@@@@@@@@@                                     @@@@@@@@@@@@@@@@@@@@            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+           @ ---------------- @                                     @ ---------------- @            
+    3      @@@@@@@@@@@@@@@@@@@@                                     @@@@@@@@@@@@@@@@@@@@            
+
+
+
+
+
+
+
+
+                              2 wide polygon overlaps top right of 2x2
+    2      @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+           @ ---------------- @ +++++++++++++++++@ ---------------- @                               
+    1      @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@           
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+           @+++++++++++++++++ @ ---------------- @                  @ ---------------- @            
+    0      @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                  @@@@@@@@@@@@@@@@@@@@                                                                                                                
+           1x1 poly overlaps
+           bottom left of
+           2x2 square                                                                              
+
+           0                  1                  2                  3                  4                                                                                         
+
+

packages/geoprocessing/scripts/base/datasources/precalcVectorDatasource.ts

@@ -14,7 +14,7 @@ import {
   datasourceConfig,
   genZodErrorMessage,
 } from "../../../src/index.js";
-import { getFeatures } from "../../../src/dataproviders/index.js";
+import { getDatasourceFeatures } from "../../../src/dataproviders/index.js";
 import { area, truncate, featureCollection } from "@turf/turf";
 import { getGeographyFeatures } from "../geographies/helpers.js";
 import { clipMultiMerge } from "../../../src/toolbox/clip.js";
@@ -75,7 +75,7 @@ export async function genVectorMetrics(
 
   const dsFeatureColl: FeatureCollection<Polygon | MultiPolygon> =
     await (async () => {
-      const feats = await getFeatures(datasource, url);
+      const feats = await getDatasourceFeatures(datasource, url);
       // Make sure only contains polygon or multipolygon in array
       const result = featuresSchema.safeParse(feats);
       if (!result.success) {

packages/geoprocessing/scripts/base/geographies/helpers.ts

@@ -12,7 +12,7 @@ import {
 } from "../../../src/index.js";
 import { featureCollection, truncate } from "@turf/turf";
 import { readDatasourceFgbById } from "../datasources/index.js";
-import { getFeatures } from "../../../src/dataproviders/index.js";
+import { getDatasourceFeatures } from "../../../src/dataproviders/index.js";
 
 /**
  * Given geography and its datasource, returns geography features with additional filter options
@@ -42,7 +42,7 @@ export async function getGeographyFeatures(
     return featureColl;
   } else if (isExternalVectorDatasource(datasource)) {
     // Fetch external datasource
-    const feats = await getFeatures(datasource, datasource.url, {
+    const feats = await getDatasourceFeatures(datasource, datasource.url, {
       bbox: geography.bboxFilter,
       propertyFilter: geography.propertyFilter,
     });

packages/geoprocessing/scripts/global/datasources/mr-eez-land-union-precalc.ts

@@ -1,7 +1,7 @@
 import fs from "fs-extra";
 import { area, bbox, featureCollection as fc } from "@turf/turf";
 import { $ } from "zx";
-import { getFeatures } from "../../../src/dataproviders/index.js";
+import { getDatasourceFeatures } from "../../../src/dataproviders/index.js";
 import { chunk, roundDecimal } from "../../../src/helpers/index.js";
 import project from "./ProjectClientGlobal.js";
 import { clip } from "../../../src/toolbox/clip.js";
@@ -35,7 +35,7 @@ const landUrl = project.getDatasourceUrl(landDs);
 for (const eezFeat of eezFeatures.features) {
   // Get land features that overlap with eez
   const eezBbox = bbox(eezFeat);
-  const landFeatures = await getFeatures(landDs, landUrl, {
+  const landFeatures = await getDatasourceFeatures(landDs, landUrl, {
     unionProperty: "gid", // gid is assigned per country
     bbox: eezBbox,
   });

packages/geoprocessing/src/dataproviders/flatgeobuf.test.e2e.ts

@@ -6,54 +6,42 @@ import { deserialize } from "flatgeobuf/lib/mjs/geojson.js";
 import { readFileSync } from "node:fs";
 import path from "node:path";
 import { isFeatureCollection } from "../index.js";
+import fs from "fs-extra";
 
-test("flatgeobuf - local world fgb", async () => {
-  const canonicalStr = canonicalize([
-    {
-      id: 0, // this is not in the data, but fgb client automatically adds it on deserialize as of v3.36.0
-      type: "Feature",
-      properties: {
-        name: "World boundary",
-        description: "World",
-      },
-      geometry: {
-        coordinates: [
-          [
-            [-180, 90],
-            [-180, -90],
-            [180, -90],
-            [180, 90],
-            [-180, 90],
-          ],
-        ],
-        type: "Polygon",
-      },
-    },
-  ]);
-  const url = "http://127.0.0.1:8080/data/in/world.fgb";
-  const features = await loadFgb(url);
-  expect(features.length).toEqual(1);
-  expect(canonicalize(features)).toEqual(canonicalStr);
-});
+describe("flatgeobuf", () => {
+  test("flatgeobuf - local world fgb", async () => {
+    const worldJson = fs.readJsonSync("data/in/world.json");
+    // pull features out of FC and add index based ID, just as the flatgeobuf client does on read
+    const worldFeatures = worldJson.features.map((f, index) => {
+      f.id = index;
+      return f;
+    });
+    const canonicalStr = canonicalize(worldFeatures);
+    const url = "http://127.0.0.1:8080/data/in/world.fgb";
+    const features = await loadFgb(url);
+    expect(features.length).toEqual(1);
+    expect(canonicalize(features)).toEqual(canonicalStr);
+  });
 
-test("flatgeobuf - file countries fgb from disk", async () => {
-  const filePath = path.join(
-    import.meta.dirname,
-    "../../data/in/countries.fgb",
-  );
-  // console.log(filePath);
-  const data = readFileSync(filePath);
-  const view = new Uint8Array(data.buffer);
-  const fc = deserialize(view);
-  if (isFeatureCollection(fc)) {
-    expect(fc.features.length).toEqual(179);
-  }
-});
+  test("flatgeobuf - file countries fgb from disk", async () => {
+    const filePath = path.join(
+      import.meta.dirname,
+      "../../data/in/countries.fgb",
+    );
+    // console.log(filePath);
+    const data = readFileSync(filePath);
+    const view = new Uint8Array(data.buffer);
+    const fc = deserialize(view);
+    if (isFeatureCollection(fc)) {
+      expect(fc.features.length).toEqual(179);
+    }
+  });
 
-test("flatgeobuf - external world fgb", async () => {
-  const url =
-    "https://gp-global-datasources-datasets.s3.us-west-1.amazonaws.com/world-unstable.fgb";
-  const features = await loadFgb(url);
-  expect(features.length).toEqual(1);
-  // console.log(JSON.stringify(features))
-}, 20_000);
+  test("flatgeobuf - external world fgb", async () => {
+    const url =
+      "https://gp-global-datasources-datasets.s3.us-west-1.amazonaws.com/world-unstable.fgb";
+    const features = await loadFgb(url);
+    expect(features.length).toEqual(1);
+    // console.log(JSON.stringify(features))
+  }, 20_000);
+});