feat: simplify footprint geometry tde-1080 (#891)

* feat: simplify footprint geometry TDE-1080

* ci: add missing gsd argument with fake value

* ci: use real gsd values

* docs: update for new parameter gsd

.github/workflows/format-tests.yml

@@ -32,23 +32,23 @@ jobs:
 
       - name: End to end test - Aerial Imagery
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m
           cmp --silent "${{ runner.temp }}/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829.tiff
 
       - name: End to end test - Elevation
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 30m
           cmp --silent "${{ runner.temp }}/BK39_10000_0102.tiff" ./scripts/tests/data/output/BK39_10000_0102.tiff
           cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101.tiff
 
       - name: End to end test - Historical Aerial Imagery
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60m
           cmp --silent "${{ runner.temp }}/BQ31_5000_0608.tiff" ./scripts/tests/data/output/BQ31_5000_0608.tiff
 
       - name: End to end test - Cutline (Aerial Imagery)
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb --gsd 10m
           cmp --silent "${{ runner.temp }}/cutline/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829_cut.tiff
 
       - name: End to end test - Thumbnails (Topo50/Topo250)
@@ -59,7 +59,7 @@ jobs:
 
       - name: End to end test - Restandardise Aerial Imagery
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/restandardise.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/restandardise/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/restandardise.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/restandardise/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m
           cmp --silent "${{ runner.temp }}/restandardise/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829.tiff
 
       - name: End to end test - Translate Ascii Files (Elevation)
@@ -69,7 +69,7 @@ jobs:
 
       - name: End to end test - Remove empty files
         run: |
-          docker run  -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01
+          docker run  -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01 --gsd 60m
           empty_target_directory="$(find "${{ runner.temp }}/tmp-empty" -maxdepth 0 -type d -empty)"
           [[ -n "$empty_target_directory" ]]
 

README.md

@@ -53,7 +53,7 @@ Run `docker run topo-imagery python standardise_validate.py --help` to get the l
 - Example of local execution. This example uses the test data available on this repo and create the output will be created in a `~/tmp/` on the local machine (volume share with `Docker`):
 
 ```bash
-docker run -v ${HOME}/tmp/:/tmp/:rw topo-imagery python standardise_validate.py --preset webp --from-file ./tests/data/aerial.json --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --target /tmp/ --source-epsg 2193 --target-epsg 2193
+docker run -v ${HOME}/tmp/:/tmp/:rw topo-imagery python standardise_validate.py --preset webp --from-file ./tests/data/aerial.json --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --target /tmp/ --source-epsg 2193 --target-epsg 2193 --gsd 10m
 ```
 
 To use an AWS test dataset (input located in an AWS S3 bucket), log into the AWS account and add the following arguments to the `docker run` command:

scripts/stac/imagery/capture_area.py

@@ -10,6 +10,36 @@
 """
 
 
+def gsd_to_float(gsd: str) -> float:
+    """Transform the gsd from its string/human readable format to a float.
+
+    Args:
+        gsd: gsd as it's passed to the cli
+
+    Returns:
+        gsd as a float
+
+    Examples:
+        >>> gsd_to_float("0.2m")
+        0.2
+    """
+    return float(gsd.replace("m", ""))
+
+
+def get_buffer_distance(gsd: float) -> float:
+    """The `gsd` (in meters) is multiplied by 2 and then by the 1m degree of precision.
+    A `buffer factor` of 2 was decided on after experimenting with different outputs,
+    details of this can be found in TDE-1049.
+
+    Args:
+        gsd: Ground Sample Distance in meters
+
+    Returns:
+        buffer distance as a float
+    """
+    return gsd * 2 * DECIMAL_DEGREES_1M
+
+
 def to_feature(geometry: Geometry) -> Dict[str, Any]:
     """Transform a Geometry to a GeoJSON feature.
 
@@ -54,9 +84,7 @@ def generate_capture_area(polygons: List[Polygon], gsd: float) -> Dict[str, Any]
     creation process (before delivery).
     If we don't apply this rounding, we could get a very small gaps between tiffs
     which would result in a capture area having gaps.
-    The `gsd` (in meters) is multiplied by 2 and then by the 1m degree of precision.
-    A `buffer factor` of 2 was decided on after experimenting with different outputs,
-    details of this can be found in TDE-1049.
+
     Note that all the polygons are buffered which means a gap bigger than the gsd,
     but < buffer_factor*2 will be closed.
 
@@ -67,8 +95,6 @@ def generate_capture_area(polygons: List[Polygon], gsd: float) -> Dict[str, Any]
     Returns:
         The capture-area geojson document.
     """
-    buffer_factor = gsd * 2
-    buffer_distance = DECIMAL_DEGREES_1M * buffer_factor
-    merged_polygons = merge_polygons(polygons, buffer_distance)
+    merged_polygons = merge_polygons(polygons, get_buffer_distance(gsd))
 
     return to_feature(merged_polygons)

scripts/stac/imagery/collection.py

@@ -9,7 +9,7 @@
 
 from scripts.files.files_helper import ContentType
 from scripts.files.fs import write
-from scripts.stac.imagery.capture_area import generate_capture_area
+from scripts.stac.imagery.capture_area import generate_capture_area, gsd_to_float
 from scripts.stac.imagery.metadata_constants import (
     DATA_CATEGORIES,
     DEM,
@@ -95,7 +95,7 @@ def add_capture_area(self, polygons: List[shapely.geometry.shape], target: str,
         """
 
         # The GSD is measured in meters (e.g., `0.3m`)
-        capture_area_document = generate_capture_area(polygons, float(self.metadata["gsd"].replace("m", "")))
+        capture_area_document = generate_capture_area(polygons, gsd_to_float(self.metadata["gsd"]))
         capture_area_content: bytes = json.dumps(capture_area_document).encode("utf-8")
         file_checksum = checksum.multihash_as_hex(capture_area_content)
         capture_area = {

scripts/standardise_validate.py

@@ -28,6 +28,7 @@ def main() -> None:
         required=True,
         help="The target EPSG code. If different to source the imagery will be reprojected",
     )
+    parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True)
     parser.add_argument("--cutline", dest="cutline", help="Optional cutline to cut imagery to", required=False, nargs="?")
     parser.add_argument("--collection-id", dest="collection_id", help="Unique id for collection", required=True)
     parser.add_argument(
@@ -57,6 +58,7 @@ def main() -> None:
         concurrency,
         arguments.source_epsg,
         arguments.target_epsg,
+        arguments.gsd,
         arguments.target,
     )
 

scripts/standardising.py

@@ -22,6 +22,7 @@
     get_transform_srs_command,
 )
 from scripts.logging.time_helper import time_in_ms
+from scripts.stac.imagery.capture_area import get_buffer_distance, gsd_to_float
 from scripts.tile.tile_index import Bounds, get_bounds_from_name
 
 
@@ -32,6 +33,7 @@ def run_standardising(
     concurrency: int,
     source_epsg: str,
     target_epsg: str,
+    gsd: str,
     target_output: str = "/tmp/",
 ) -> List[FileTiff]:
     """Run `standardising()` in parallel (`concurrency`).
@@ -43,6 +45,7 @@ def run_standardising(
         concurrency: number of concurrent files to process
         source_epsg: EPSG code of the source file
         target_epsg: EPSG code of reprojection
+        gsd: Ground Sample Distance in meters
         target_output: output directory path. Defaults to "/tmp/"
 
     Returns:
@@ -64,6 +67,7 @@ def run_standardising(
                     source_epsg=source_epsg,
                     target_epsg=target_epsg,
                     target_output=target_output,
+                    gsd=gsd,
                     cutline=cutline,
                 ),
                 todo,
@@ -102,6 +106,7 @@ def standardising(
     preset: str,
     source_epsg: str,
     target_epsg: str,
+    gsd: str,
     cutline: Optional[str],
     target_output: str = "/tmp/",
 ) -> Optional[FileTiff]:
@@ -112,6 +117,7 @@ def standardising(
         preset: gdal preset to use. See `gdal.gdal_preset.py`
         source_epsg: EPSG code of the source file
         target_epsg: EPSG code of reprojection
+        gsd: Ground Sample Distance in meters
         cutline: path to the cutline. Must be `.fgb` or `.geojson`
         target_output: output directory path. Defaults to "/tmp/"
 
@@ -209,7 +215,15 @@ def standardising(
             if any(tile_byte_count != 0 for tile_byte_count in file_handle.pages.first.tags["TileByteCounts"].value):
                 # Create footprint GeoJSON
                 run_gdal(
-                    ["gdal_footprint", "-t_srs", EpsgCode.EPSG_4326, "-max_points", "unlimited"],
+                    [
+                        "gdal_footprint",
+                        "-t_srs",
+                        EpsgCode.EPSG_4326,
+                        "-max_points",
+                        "unlimited",
+                        "-simplify",
+                        str(get_buffer_distance(gsd_to_float(gsd))),
+                    ],
                     standardized_working_path,
                     footprint_tmp_path,
                 )