From 83b517dfbc8f91ebc0a3be573574a7da6187a372 Mon Sep 17 00:00:00 2001 From: Greg Walker Date: Wed, 24 Jul 2024 13:17:43 -0500 Subject: [PATCH 1/3] collect zones as geometry collections; overwrite files on unzip --- spatial-data/lib/prep.js | 4 ++- spatial-data/sources/zones.js | 65 ++++++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/spatial-data/lib/prep.js b/spatial-data/lib/prep.js index bb3827bcc..7f0fa8819 100644 --- a/spatial-data/lib/prep.js +++ b/spatial-data/lib/prep.js @@ -41,5 +41,7 @@ module.exports.downloadAndUnzip = async (url) => { module.exports.unzip = async (path) => { console.log(` [${path}] decompressing...`); - await exec(`unzip -u ${path}`); + + // Use -o to overwrite existing files. + await exec(`unzip -o -u ${path}`); }; diff --git a/spatial-data/sources/zones.js b/spatial-data/sources/zones.js index 0fbcd337b..8ee5b131e 100644 --- a/spatial-data/sources/zones.js +++ b/spatial-data/sources/zones.js @@ -1,10 +1,11 @@ +const fs = require("node:fs/promises"); const shapefile = require("shapefile"); const { dropIndexIfExists, openDatabase } = require("../lib/db.js"); const metadata = { table: "weathergov_geo_zones", - version: 1, + version: 2, }; module.exports = async () => { @@ -24,6 +25,13 @@ module.exports = async () => { await dropIndexIfExists(db, "zones_spatial_idx", metadata.table); await db.query(`TRUNCATE TABLE ${metadata.table}`); + // Version 2: Change the shape column into a collection rather than a single + // multipolygon. This allows us to capture all of the polygons for a zone as + // a collection rather than trying to collect or union them into one entity. + await db.query( + `ALTER TABLE ${metadata.table} MODIFY shape GEOMETRYCOLLECTION`, + ); + const found = new Map(); const processFile = async (filename, zoneType) => { @@ -39,30 +47,23 @@ module.exports = async () => { geometry, } = value; - if (geometry.type === "Polygon") { - geometry.type = "MultiPolygon"; - geometry.coordinates = [geometry.coordinates]; - } - // These shapefiles are in NAD83, whose SRID is 4269. - geometry.crs = { type: "name", properties: { name: "EPSG:4269" } }; - const id = `https://api.weather.gov/zones/${zoneType}/${state}Z${zone}`; - // Some of the zones are duplicated. Dunno why. Don't put them in twice, - // the database will scream. + // Some of the zones are represented by multiple polygons. To handle that, + // we'll gather a list of all polygons and insert them into the database + // as a geometry collection. if (!found.has(id)) { - found.set(id, { state, zone, zoneType, filename, geometry }); - - await db.query( - `INSERT INTO weathergov_geo_zones - (id, state, shape) - VALUES( - '${id}', - '${state}', - ST_GeomFromGeoJSON('${JSON.stringify(geometry)}') - )`, - ); + found.set(id, { + state, + zone, + zoneType, + filename, + geometry: [geometry], + }); + } else { + found.get(id).geometry.push(geometry); } + return file.read().then(getSqlForShape); }; @@ -72,6 +73,28 @@ module.exports = async () => { await processFile(`./z_05mr24.shp`, "forecast"); await processFile(`./fz05mr24.shp`, "fire"); + // Our map now contains entries for every zone. Iterate over that to insert + // them into the database. + for await (const [id, { state, geometry }] of found) { + const featureCollection = { + type: "FeatureCollection", + features: geometry, + // Shapefiles are in NAD83, whose SRID is 4269. Set that at the collection + // level so that it automatically applies to all contained shapes. + crs: { type: "name", properties: { name: "EPSG:4269" } }, + }; + + await db.query( + `INSERT INTO ${metadata.table} + (id, state, shape) + VALUES( + '${id}', + '${state}', + ST_GeomFromGeoJSON('${JSON.stringify(featureCollection)}') + )`, + ); + } + db.end(); }; From 2be5c3fda80b9280af2b8c8574f45846dd7ec564 Mon Sep 17 00:00:00 2001 From: Greg Walker Date: Wed, 24 Jul 2024 16:32:24 -0500 Subject: [PATCH 2/3] rebuild spatial database as needed --- .github/actions/populate-database/action.yml | 2 +- .github/actions/setup-site/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/populate-database/action.yml b/.github/actions/populate-database/action.yml index 0fb0b5db1..dfc4f50da 100644 --- a/.github/actions/populate-database/action.yml +++ b/.github/actions/populate-database/action.yml @@ -5,7 +5,7 @@ runs: uses: actions/cache@v4 id: db-cache with: - key: drupal-database-${{ hashFiles('web/config/**/*.yml', 'web/scs-export/**/*', 'web/modules/weather_i18n/translations/*.po') }} + key: drupal-database-${{ hashFiles('web/config/**/*.yml', 'web/scs-export/**/*', 'web/modules/weather_i18n/translations/*.po', 'spatial/**/*.js') }} path: weathergov.sql - name: setup image cacheing diff --git a/.github/actions/setup-site/action.yml b/.github/actions/setup-site/action.yml index 3d228fe1a..ad79b46a7 100644 --- a/.github/actions/setup-site/action.yml +++ b/.github/actions/setup-site/action.yml @@ -11,7 +11,7 @@ runs: uses: actions/cache@v4 id: db-cache with: - key: drupal-database-${{ hashFiles('web/config/**/*.yml', 'web/scs-export/**/*', 'web/modules/weather_i18n/translations/*.po') }} + key: drupal-database-${{ hashFiles('web/config/**/*.yml', 'web/scs-export/**/*', 'web/modules/weather_i18n/translations/*.po', 'spatial/**/*.js') }} path: weathergov.sql - name: start the site From a9357ec952a12c2073da695bffc352a82e858d8f Mon Sep 17 00:00:00 2001 From: Greg Walker Date: Thu, 25 Jul 2024 09:48:41 -0500 Subject: [PATCH 3/3] fix handling of geojson geometry collections --- .../weather_data/src/Service/AlertUtility.php | 14 ++++++++--- .../Test/AlertUtilityGeometry.php.test | 25 +++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/web/modules/weather_data/src/Service/AlertUtility.php b/web/modules/weather_data/src/Service/AlertUtility.php index 8739c63b1..6cc7e9590 100644 --- a/web/modules/weather_data/src/Service/AlertUtility.php +++ b/web/modules/weather_data/src/Service/AlertUtility.php @@ -464,9 +464,17 @@ public static function getGeometryAsJSON($alert, $dataLayer) $polygon = json_decode($polygon->shape); - $polygon->coordinates = SpatialUtility::swapLatLon( - $polygon->coordinates, - ); + if ($polygon->type == "GeometryCollection") { + foreach ($polygon->geometries as $innerPolygon) { + $innerPolygon->coordinates = SpatialUtility::swapLatLon( + $innerPolygon->coordinates, + ); + } + } else { + $polygon->coordinates = SpatialUtility::swapLatLon( + $polygon->coordinates, + ); + } } return $polygon; diff --git a/web/modules/weather_data/src/Service/Test/AlertUtilityGeometry.php.test b/web/modules/weather_data/src/Service/Test/AlertUtilityGeometry.php.test index fa632faec..32a305553 100644 --- a/web/modules/weather_data/src/Service/Test/AlertUtilityGeometry.php.test +++ b/web/modules/weather_data/src/Service/Test/AlertUtilityGeometry.php.test @@ -88,7 +88,8 @@ final class AlertUtilityGeometryTest extends TestCase (object) ["shape" => "union 1,2,3"], ], - // Return the simplified shape + // Return the simplified shape as a geometry collection, so we + // can also test that those points are flipped properly. [ "SELECT ST_ASGEOJSON( ST_SIMPLIFY( @@ -96,12 +97,20 @@ final class AlertUtilityGeometryTest extends TestCase 0.003 ) ) as shape", - (object) ["shape" => '{"coordinates":[[0,1],[2,3],[4,5]]}'], + (object) [ + "shape" => + '{"type":"GeometryCollection","geometries":[{"coordinates":[[0,1],[2,3],[4,5]]}]}', + ], ], ]), ); - $expected = (object) ["coordinates" => [[1, 0], [3, 2], [5, 4]]]; + $expected = (object) [ + "type" => "GeometryCollection", + "geometries" => [ + (object) ["coordinates" => [[1, 0], [3, 2], [5, 4]]], + ], + ]; $actual = AlertUtility::getGeometryAsJSON($alert, $this->dataLayer); $this->assertEquals($expected, $actual); @@ -176,12 +185,18 @@ final class AlertUtilityGeometryTest extends TestCase 0.003 ) ) as shape", - (object) ["shape" => '{"coordinates":[[0,1],[2,3],[4,5]]}'], + (object) [ + "shape" => + '{"type":"Polygon","coordinates":[[0,1],[2,3],[4,5]]}', + ], ], ]), ); - $expected = (object) ["coordinates" => [[1, 0], [3, 2], [5, 4]]]; + $expected = (object) [ + "type" => "Polygon", + "coordinates" => [[1, 0], [3, 2], [5, 4]], + ]; $actual = AlertUtility::getGeometryAsJSON($alert, $this->dataLayer); $this->assertEquals($expected, $actual);