Drop JSON format from data import and precalc, drop macos direct inst…

…all support from docs (#383) * drop macos direct install support * precalcDatasource - re-enable tests * drop json format from data import, add loadFgbFromDisk function, switch import vector to use it * fix bug await import vector datsource answers
seasketch · Dec 8, 2024 · d336d9b · d336d9b
1 parent e67aa1e
commit d336d9b
Show file tree

Hide file tree

Showing 9 changed files with 76 additions and 99 deletions.
diff --git a/packages/geoprocessing/scripts/base/datasources/datasources.ts b/packages/geoprocessing/scripts/base/datasources/datasources.ts
@@ -12,8 +12,8 @@ import {
   isInternalRasterDatasource,
   datasourceConfig,
 } from "../../../src/datasources/index.js";
-import { getJsonPath } from "./pathUtils.js";
-import { isFeatureCollection } from "../../../src/index.js";
+import { getJsonPath, getFlatGeobufPath } from "./pathUtils.js";
+import { isFeatureCollection, loadFgbFromDisk } from "../../../src/index.js";
 import { globalDatasources } from "../../../src/datasources/global.js";
 
 /**
@@ -134,3 +134,18 @@ export function readDatasourceGeojsonById(
       `GeoJSON at ${jsonPath} is not a FeatureCollection. Check datasource.`,
     );
 }
+
+export function readDatasourceFgbById(datasourceId: string, dstPath: string) {
+  const fgbPath = getFlatGeobufPath(dstPath, datasourceId);
+  if (!fs.existsSync(fgbPath))
+    throw new Error(
+      `Flatgeobuf form of datasource does not exist at ${fgbPath}`,
+    );
+  const polys = loadFgbFromDisk(fgbPath);
+  if (isFeatureCollection(polys)) {
+    return polys as FeatureCollection<Polygon | MultiPolygon>;
+  } else
+    throw new Error(
+      `GeoJSON at ${fgbPath} is not a FeatureCollection. Check datasource.`,
+    );
+}
diff --git a/packages/geoprocessing/scripts/base/datasources/genRasterConfig.ts b/packages/geoprocessing/scripts/base/datasources/genRasterConfig.ts
@@ -24,6 +24,6 @@ export function genRasterConfig<C extends ProjectClientBase>(
     band: options.band || 0,
     package: projectClient.package,
     gp: projectClient.geoprocessing,
-    formats: options.formats || datasourceConfig.importDefaultRasterFormats,
+    formats: ["tif"],
   };
 }
diff --git a/packages/geoprocessing/scripts/base/datasources/precalcRasterDatasource.test.e2e.ts b/packages/geoprocessing/scripts/base/datasources/precalcRasterDatasource.test.e2e.ts
@@ -27,7 +27,7 @@ describe("precalcRasterDatasource", () => {
     // Ensure test data folder
     fs.mkdirsSync(dstPath);
   });
-  test.skip("precalcRasterDatasource - single file, single class should write geography and precalc raster metrics", async () => {
+  test("precalcRasterDatasource - single file, single class should write geography and precalc raster metrics", async () => {
     const dsFilename = "datasources_precalc_raster_test_1.json";
     const dsFilePath = path.join(dstPath, dsFilename);
     const datasourceId = "samoa_benthic_reef_sand";
@@ -131,7 +131,7 @@ describe("precalcRasterDatasource", () => {
     fs.removeSync(precalcFilePath);
   }, 60_000);
 
-  test.skip("precalcRasterDatasource - multiple geog scenarios with external subdivided datasource", async () => {
+  test("precalcRasterDatasource - multiple geog scenarios with external subdivided datasource", async () => {
     const dsFilename = "datasources_precalc_raster_test_9.json";
     const dsFilePath = path.join(dstPath, dsFilename);
     const rasterDatasourceId = "samoa_benthic_reef_sand9";
@@ -284,7 +284,7 @@ describe("precalcRasterDatasource", () => {
     fs.removeSync(precalcFilePath);
   }, 10_000);
 
-  test.skip("precalcRasterDatasource - multiple geog scenarios with external flatgeobuf datasource", async () => {
+  test("precalcRasterDatasource - multiple geog scenarios with external flatgeobuf datasource", async () => {
     const dsFilename = "datasources_precalc_raster_test_8.json";
     const dsFilePath = path.join(dstPath, dsFilename);
     const rasterDatasourceId = "samoa_benthic_reef_sand8";
@@ -428,7 +428,7 @@ describe("precalcRasterDatasource", () => {
     fs.removeSync(precalcFilePath);
   }, 30_000);
 
-  test.skip("precalcRasterDatasource - multiple geog scenarios with internal geojson datasource", async () => {
+  test("precalcRasterDatasource - multiple geog scenarios with internal geojson datasource", async () => {
     const dsFilename = "datasources_precalc_raster_test_7.json";
     const dsFilePath = path.join(dstPath, dsFilename);
     const rasterDatasourceId = "samoa_benthic_reef_sand7";

diff --git a/packages/geoprocessing/scripts/base/datasources/precalcVectorDatasource.test.e2e.ts b/packages/geoprocessing/scripts/base/datasources/precalcVectorDatasource.test.e2e.ts
@@ -31,7 +31,7 @@ describe("precalcDatasources", () => {
       fs.mkdirsSync(dstPath);
     });
 
-    test.skip("precalcVectorDatasource - single geog, internal datasource, single class", async () => {
+    test("precalcVectorDatasource - single geog, internal datasource, single class", async () => {
       const dsFilename = "datasources_precalc_vector_test_1.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const datasourceId = "eez1";
@@ -51,7 +51,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezSrc}.json`),
           datasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: false,
@@ -118,7 +118,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(precalcFilePath);
     }, 20_000);
 
-    test.skip("precalcVectorDatasource - single geog, internal datasource, multi-class", async () => {
+    test("precalcVectorDatasource - single geog, internal datasource, multi-class", async () => {
       const dsFilename = "datasources_precalc_vector_test_2.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const classDatasourceId = "shelf_class2";
@@ -137,7 +137,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezSrc}.json`),
           datasourceId: geogDatasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: true,
@@ -155,7 +155,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${shelfSrc}.json`),
           datasourceId: classDatasourceId,
           classKeys: ["Class"],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: ["Class"],
           precalc: true,
           explodeMulti: true,
@@ -233,7 +233,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(geogFilePath);
       fs.removeSync(precalcFilePath);
     }, 20_000);
-    test.skip("precalcVectorDatasource - single geog, two datasources should write metrics", async () => {
+    test("precalcVectorDatasource - single geog, two datasources should write metrics", async () => {
       const dsFilename = "datasources_precalc_vector_test_3.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const classDatasourceId1 = "shelf_class3";
@@ -253,7 +253,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezSrc}.json`),
           datasourceId: geogDatasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: true,
@@ -271,7 +271,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${shelfSrc}.json`),
           datasourceId: classDatasourceId1,
           classKeys: ["Class"],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: ["Class"],
           precalc: true,
           explodeMulti: true,
@@ -310,7 +310,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${deepwaterSrc}.json`),
           datasourceId: classDatasourceId2,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: true,
@@ -392,7 +392,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(geogFilePath);
       fs.removeSync(precalcFilePath);
     }, 20_000);
-    test.skip("precalcVectorDatasource - single geog, update datasource", async () => {
+    test("precalcVectorDatasource - single geog, update datasource", async () => {
       const dsFilename = "datasources_precalc_vector_test_4.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const classDatasourceId1 = "shelf_class4";
@@ -412,7 +412,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezSrc}.json`),
           datasourceId: geogDatasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
 
           propertiesToKeep: [],
           precalc: true,
@@ -432,7 +432,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${shelfSrc}.json`),
           datasourceId: classDatasourceId1,
           classKeys: ["Class"],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: ["Class"],
           precalc: true,
           explodeMulti: true,
@@ -471,7 +471,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${shelfSrcUpdated}.json`),
           datasourceId: classDatasourceId1,
           classKeys: ["Class"],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: ["Class"],
           precalc: true,
           explodeMulti: true,
@@ -549,7 +549,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(precalcFilePath);
     }, 20_000);
 
-    test.skip("precalcVectorDatasource - multiple geog scenarios with external subdivided datasource", async () => {
+    test("precalcVectorDatasource - multiple geog scenarios with external subdivided datasource", async () => {
       const dsFilename = "datasources_precalc_vector_test_6.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const internalDatasourceId = "samoa-eez-cross";
@@ -581,7 +581,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezCrossSrc}.json`),
           datasourceId: internalDatasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: true,
@@ -670,21 +670,22 @@ describe("precalcDatasources", () => {
         (m) => m.geographyId === "geog-box-filter" && m.metricId === "area",
       );
       // Largest area value
-      expect(boxFilterMetric.value).toEqual(61_990_788_175.991_97);
+      expect(boxFilterMetric.value).toEqual(61_852_303_909.376_854);
 
       const singleFilterMetric = firstMatchingMetric(
         metrics,
         (m) => m.geographyId === "geog-single-filter" && m.metricId === "area",
       );
       // Smallest area value, samoa only
-      expect(singleFilterMetric.value).toEqual(37_822_608_708.983_15);
+
+      expect(singleFilterMetric.value).toEqual(37_738_114_925.589_806);
 
       const doubleFilterMetric = firstMatchingMetric(
         metrics,
         (m) => m.geographyId === "geog-double-filter" && m.metricId === "area",
       );
       // Slightly larger area value, both samoa
-      expect(doubleFilterMetric.value).toEqual(39_734_709_577.156_77);
+      expect(doubleFilterMetric.value).toEqual(39_645_944_257.713_905);
 
       fs.removeSync(dsFilePath);
       fs.removeSync(path.join(dstPath, `${internalDatasourceId}.fgb`));
@@ -693,7 +694,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(precalcFilePath);
     }, 20_000);
 
-    test.skip("precalcVectorDatasource - multiple geog scenarios with external flatgeobuf datasource", async () => {
+    test("precalcVectorDatasource - multiple geog scenarios with external flatgeobuf datasource", async () => {
       const dsFilename = "datasources_precalc_vector_test_7.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const internalDatasourceId = "samoa-eez-cross";
@@ -709,7 +710,7 @@ describe("precalcDatasources", () => {
           datasourceId: externalDatasourceId,
           geo_type: "vector",
           url: `https://gp-global-datasources-datasets.s3.us-west-1.amazonaws.com/${externalDatasourceId}.fgb`,
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           classKeys: [],
           idProperty: "GEONAME",
           nameProperty: "GEONAME",
@@ -725,7 +726,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezCrossSrc}.json`),
           datasourceId: internalDatasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: true,
@@ -817,21 +818,21 @@ describe("precalcDatasources", () => {
         (m) => m.geographyId === "geog-box-filter" && m.metricId === "area",
       );
       // Largest area value
-      expect(boxFilterMetric.value).toEqual(59_689_842_766.9754);
+      expect(boxFilterMetric.value).toEqual(59_556_498_695.328_66);
 
       const singleFilterMetric = firstMatchingMetric(
         metrics,
         (m) => m.geographyId === "geog-single-filter" && m.metricId === "area",
       );
       // Smallest area value, samoa only
-      expect(singleFilterMetric.value).toEqual(35_521_663_299.966_43);
+      expect(singleFilterMetric.value).toEqual(35_442_309_711.542_16);
 
       const doubleFilterMetric = firstMatchingMetric(
         metrics,
         (m) => m.geographyId === "geog-double-filter" && m.metricId === "area",
       );
       // Slightly larger area value, both samoa
-      expect(doubleFilterMetric.value).toEqual(37_433_764_168.140_05);
+      expect(doubleFilterMetric.value).toEqual(37_350_139_043.666_25);
 
       fs.removeSync(dsFilePath);
       fs.removeSync(path.join(dstPath, `${internalDatasourceId}.fgb`));
@@ -840,7 +841,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(precalcFilePath);
     }, 20_000);
 
-    test.skip("precalcVectorDatasource - world geog, external datasource", async () => {
+    test("precalcVectorDatasource - world geog, external datasource", async () => {
       const dsFilename = "datasources_precalc_vector_test_8.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const datasourceId = "world";
@@ -914,7 +915,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(precalcFilePath);
     }, 20_000);
 
-    test.skip("precalcVectorDatasource - world geog, internal datasource", async () => {
+    test("precalcVectorDatasource - world geog, internal datasource", async () => {
       const dsFilename = "datasources_precalc_vector_test_9.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const datasourceId = "world";
@@ -948,7 +949,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${internalFilename}`),
           datasourceId: internalDatasourceId,
           classKeys: ["Value"],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: true,
           explodeMulti: true,
@@ -993,7 +994,7 @@ describe("precalcDatasources", () => {
       fs.removeSync(precalcFilePath);
     }, 20_000);
 
-    test.skip("precalcDatasource - all precalc false should precalculate nothing", async () => {
+    test("precalcDatasource - all precalc false should precalculate nothing", async () => {
       const dsFilename = "datasources_precalc_false_test.json";
       const dsFilePath = path.join(dstPath, dsFilename);
       const internalDatasourceId = "samoa-eez-cross";
@@ -1009,7 +1010,7 @@ describe("precalcDatasources", () => {
           datasourceId: externalDatasourceId,
           geo_type: "vector",
           url: `https://gp-global-datasources-datasets.s3.us-west-1.amazonaws.com/${externalDatasourceId}.fgb`,
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           classKeys: [],
           idProperty: "GEONAME",
           nameProperty: "GEONAME",
@@ -1025,7 +1026,7 @@ describe("precalcDatasources", () => {
           src: path.join(srcPath, `${eezCrossSrc}.json`),
           datasourceId: internalDatasourceId,
           classKeys: [],
-          formats: ["fgb", "json"],
+          formats: ["fgb"],
           propertiesToKeep: [],
           precalc: false,
           explodeMulti: true,

diff --git a/packages/geoprocessing/scripts/base/geographies/helpers.ts b/packages/geoprocessing/scripts/base/geographies/helpers.ts
@@ -11,7 +11,7 @@ import {
   isInternalVectorDatasource,
 } from "../../../src/index.js";
 import { featureCollection, truncate } from "@turf/turf";
-import { readDatasourceGeojsonById } from "../datasources/index.js";
+import { readDatasourceFgbById } from "../datasources/index.js";
 import { getFeatures } from "../../../src/dataproviders/index.js";
 
 /**
@@ -26,7 +26,7 @@ export async function getGeographyFeatures(
   if (isInternalVectorDatasource(datasource)) {
     // Read local datasource
     let featureColl = truncate(
-      readDatasourceGeojsonById(geography.datasourceId, dstPath),
+      readDatasourceFgbById(geography.datasourceId, dstPath),
       { mutate: true },
     );
     if (geography.propertyFilter) {