feat!: Auto-fill GSD unit TDE-1211

.github/workflows/format-tests.yml

@@ -31,23 +31,23 @@ jobs:
 
       - name: End to end test - Aerial Imagery
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m --create-footprints=true
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10 --create-footprints=true
           cmp --silent "${{ runner.temp }}/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829.tiff
 
       - name: End to end test - Elevation
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 30m --create-footprints=true
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 30 --create-footprints=true
           cmp --silent "${{ runner.temp }}/BK39_10000_0102.tiff" ./scripts/tests/data/output/BK39_10000_0102.tiff
           cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101.tiff
 
       - name: End to end test - Historical Aerial Imagery
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60m --create-footprints=true
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60 --create-footprints=true
           cmp --silent "${{ runner.temp }}/BQ31_5000_0608.tiff" ./scripts/tests/data/output/BQ31_5000_0608.tiff
 
       - name: End to end test - Cutline (Aerial Imagery)
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb --gsd 10m --create-footprints=true
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb --gsd 10 --create-footprints=true
           cmp --silent "${{ runner.temp }}/cutline/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829_cut.tiff
 
       - name: End to end test - Thumbnails (Topo50/Topo250)
@@ -58,7 +58,7 @@ jobs:
 
       - name: End to end test - Restandardise Aerial Imagery
         run: |
-          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/restandardise.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/restandardise/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10m --create-footprints=true
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/restandardise.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/restandardise/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 10 --create-footprints=true
           cmp --silent "${{ runner.temp }}/restandardise/BG35_1000_4829.tiff" ./scripts/tests/data/output/BG35_1000_4829.tiff
 
       - name: End to end test - Translate Ascii Files (Elevation)
@@ -68,7 +68,7 @@ jobs:
 
       - name: End to end test - Remove empty files
         run: |
-          docker run  -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01 --gsd 60m --create-footprints=true
+          docker run  -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01 --gsd 60 --create-footprints=true
           empty_target_directory="$(find "${{ runner.temp }}/tmp-empty" -maxdepth 0 -type d -empty)"
           [[ -n "$empty_target_directory" ]]
 

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [4.11.2](https://github.com/linz/topo-imagery/compare/v4.11.1...v4.11.2) (2024-09-24)
+
+
+### Bug Fixes
+
+* Use GeoJSON media type for item links TDE-1254 ([#1086](https://github.com/linz/topo-imagery/issues/1086)) ([57dd0a8](https://github.com/linz/topo-imagery/commit/57dd0a86566e739a16c124be6ef24aca99565b79))
+
 ## [4.11.1](https://github.com/linz/topo-imagery/compare/v4.11.0...v4.11.1) (2024-09-19)
 
 

README.md

@@ -53,7 +53,7 @@ Run `docker run topo-imagery python standardise_validate.py --help` to get the l
 - Example of local execution. This example uses the test data available on this repo and create the output will be created in a `~/tmp/` on the local machine (volume share with `Docker`):
 
 ```bash
-docker run -v ${HOME}/tmp/:/tmp/:rw topo-imagery python standardise_validate.py --preset webp --from-file ./tests/data/aerial.json --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --target /tmp/ --source-epsg 2193 --target-epsg 2193 --gsd 10m --create-footprints=true
+docker run -v ${HOME}/tmp/:/tmp/:rw topo-imagery python standardise_validate.py --preset webp --from-file ./tests/data/aerial.json --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --target /tmp/ --source-epsg 2193 --target-epsg 2193 --gsd 10 --create-footprints=true
 ```
 
 To use an AWS test dataset (input located in an AWS S3 bucket), log into the AWS account and add the following arguments to the `docker run` command:

pyproject.toml

@@ -25,7 +25,7 @@ ignore_missing_imports = true
 
 [tool.poetry]
 name = "topo-imagery"
-version = "4.11.1"
+version = "4.11.2"
 description = "A collection of scripts for processing imagery"
 authors = [
     "Blayne Chard <[email protected]>",

scripts/cli/cli_helper.py

@@ -1,6 +1,8 @@
 import argparse
 import json
+import warnings
 from datetime import datetime
+from decimal import Decimal
 from os import environ
 from typing import NamedTuple
 
@@ -120,3 +122,15 @@ def coalesce_multi_single(multi_items: str | None, single_item: str | None) -> l
     elif single_item:
         output.append(single_item)
     return output
+
+
+def str_to_gsd(value: str) -> Decimal:
+    number_value = value.removesuffix("m")
+    if number_value != value:
+        warnings.warn(
+            "Specifying GSD with a trailing 'm' character will not be supported in future versions. "
+            "Please use a plain decimal number like '0.3' instead.",
+            DeprecationWarning,
+        )
+
+    return Decimal(number_value)

scripts/collection_from_items.py

@@ -9,7 +9,7 @@
 from boto3 import client
 from linz_logger import get_log
 
-from scripts.cli.cli_helper import coalesce_multi_single, valid_date
+from scripts.cli.cli_helper import coalesce_multi_single, str_to_gsd, valid_date
 from scripts.datetimes import utc_now
 from scripts.files.files_helper import SUFFIX_FOOTPRINT, SUFFIX_JSON
 from scripts.files.fs_s3 import bucket_name_from_path, get_object_parallel_multithreading, list_files_in_uri
@@ -41,7 +41,7 @@ def parse_args(args: List[str] | None) -> Namespace:
         required=True,
         choices=HUMAN_READABLE_REGIONS.keys(),
     )
-    parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset", type=str, required=True)
+    parser.add_argument("--gsd", dest="gsd", help="GSD of imagery Dataset, for example 0.3", type=str_to_gsd, required=True)
     parser.add_argument(
         "--geographic-description",
         dest="geographic_description",

scripts/stac/imagery/capture_area.py

@@ -1,4 +1,5 @@
 import json
+from decimal import Decimal
 from typing import Any, Sequence
 
 from linz_logger import get_log
@@ -8,29 +9,13 @@
 
 from scripts.logging.time_helper import time_in_ms
 
-DECIMAL_DEGREES_1M = 0.00001
+DECIMAL_DEGREES_1M = Decimal("0.00001")
 """
 Degree precision of ~1m (decimal places 5, https://en.wikipedia.org/wiki/Decimal_degrees)
 """
 
 
-def gsd_to_float(gsd: str) -> float:
-    """Transform the gsd from its string/human readable format to a float.
-
-    Args:
-        gsd: gsd as it's passed to the cli
-
-    Returns:
-        gsd as a float
-
-    Examples:
-        >>> gsd_to_float("0.2m")
-        0.2
-    """
-    return float(gsd.replace("m", ""))
-
-
-def get_buffer_distance(gsd: float) -> float:
+def get_buffer_distance(gsd: Decimal) -> float:
     """The `gsd` (in meters) is multiplied by 2 and then by the 1m degree of precision.
     A `buffer factor` of 2 was decided on after experimenting with different outputs,
     details of this can be found in TDE-1049.
@@ -41,7 +26,7 @@ def get_buffer_distance(gsd: float) -> float:
     Returns:
         buffer distance as a float
     """
-    return gsd * 2 * DECIMAL_DEGREES_1M
+    return float(gsd * 2 * DECIMAL_DEGREES_1M)
 
 
 def to_feature(geometry: BaseGeometry) -> dict[str, Any]:
@@ -84,7 +69,7 @@ def merge_polygons(polygons: Sequence[BaseGeometry], buffer_distance: float) ->
     return oriented_union_simplified
 
 
-def generate_capture_area(polygons: Sequence[BaseGeometry], gsd: float) -> dict[str, Any]:
+def generate_capture_area(polygons: Sequence[BaseGeometry], gsd: Decimal) -> dict[str, Any]:
     """Generate the capture area from a list of BaseGeometries.
     Providing the `gsd` allows to round the geometry as we've seen some tiffs
     geometry being slightly off, sometimes due to rounding issue in their

scripts/stac/imagery/collection.py

@@ -10,7 +10,7 @@
 from scripts.files.files_helper import ContentType
 from scripts.files.fs import read, write
 from scripts.json_codec import dict_to_json_bytes
-from scripts.stac.imagery.capture_area import generate_capture_area, gsd_to_float
+from scripts.stac.imagery.capture_area import generate_capture_area
 from scripts.stac.imagery.metadata_constants import (
     DATA_CATEGORIES,
     DEM,
@@ -103,7 +103,7 @@ def add_capture_area(self, polygons: list[BaseGeometry], target: str, artifact_t
         """
 
         # The GSD is measured in meters (e.g., `0.3m`)
-        capture_area_document = generate_capture_area(polygons, gsd_to_float(self.metadata["gsd"]))
+        capture_area_document = generate_capture_area(polygons, self.metadata["gsd"])
         capture_area_content: bytes = dict_to_json_bytes(capture_area_document)
         file_checksum = checksum.multihash_as_hex(capture_area_content)
         capture_area = {
@@ -163,7 +163,7 @@ def add_item(self, item: dict[Any, Any]) -> None:
                 Link(
                     path=item_self_link["href"],
                     rel=Relation.ITEM,
-                    media_type=StacMediaType.JSON,
+                    media_type=StacMediaType.GEOJSON,
                     file_content=dict_to_json_bytes(item),
                 ).stac
             )
@@ -303,7 +303,13 @@ def _title(self) -> str:
                 raise MissingMetadataError("historic_survey_number")
             return " ".join(
                 value
-                for value in [imagery_name, self.metadata["gsd"], historic_survey_number, f"({date})", lifecycle_tag]
+                for value in [
+                    imagery_name,
+                    f"{self.metadata['gsd']}{GSD_UNIT}",
+                    historic_survey_number,
+                    f"({date})",
+                    lifecycle_tag,
+                ]
                 if value is not None
             )
 
@@ -316,7 +322,7 @@ def _title(self) -> str:
                 value
                 for value in [
                     imagery_name,
-                    self.metadata["gsd"],
+                    f"{self.metadata['gsd']}{GSD_UNIT}",
                     DATA_CATEGORIES[self.metadata["category"]],
                     f"({date})",
                     lifecycle_tag,
@@ -330,7 +336,7 @@ def _title(self) -> str:
                     region,
                     elevation_description,
                     "LiDAR",
-                    self.metadata["gsd"],
+                    f"{self.metadata['gsd']}{GSD_UNIT}",
                     DATA_CATEGORIES[self.metadata["category"]],
                     f"({date})",
                     lifecycle_tag,
@@ -376,3 +382,6 @@ def _description(self) -> str:
             desc = desc + f", published as a record of the {event} event"
 
         return desc + "."
+
+
+GSD_UNIT = "m"

scripts/stac/imagery/create_stac.py

@@ -54,7 +54,12 @@ def create_item(
                 start_datetime = min(start_datetime, derived_stac["properties"]["start_datetime"])
                 end_datetime = max(end_datetime, derived_stac["properties"]["end_datetime"])
             item.add_link(
-                Link(path=derived, rel=Relation.DERIVED_FROM, media_type=StacMediaType.JSON, file_content=derived_item_content)
+                Link(
+                    path=derived,
+                    rel=Relation.DERIVED_FROM,
+                    media_type=StacMediaType.GEOJSON,
+                    file_content=derived_item_content,
+                )
             )
 
     item.update_datetime(start_datetime, end_datetime)

scripts/stac/imagery/item.py

@@ -24,7 +24,7 @@ def __init__(self, id_: str, file: str, now: Callable[[], datetime]) -> None:
             "type": "Feature",
             "stac_version": STAC_VERSION,
             "id": id_,
-            "links": [Link(path=f"./{id_}.json", rel=Relation.SELF, media_type=StacMediaType.JSON).stac],
+            "links": [Link(path=f"./{id_}.json", rel=Relation.SELF, media_type=StacMediaType.GEOJSON).stac],
             "assets": {
                 "visual": {
                     "href": os.path.join(".", os.path.basename(file)),

scripts/stac/imagery/metadata_constants.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from decimal import Decimal
 from typing import TypedDict
 
 
@@ -19,7 +20,7 @@ class CollectionMetadata(TypedDict):
 
     category: str
     region: str
-    gsd: str
+    gsd: Decimal
     start_datetime: datetime
     end_datetime: datetime
     lifecycle: str

scripts/stac/imagery/tests/capture_area_test.py

@@ -1,3 +1,4 @@
+from decimal import Decimal
 from typing import cast
 
 from shapely import get_exterior_ring, is_ccw
@@ -84,9 +85,9 @@ def test_generate_capture_area_rounded() -> None:
 
     # Using GSD of 0.2m
     # Generate the capture area of the polygons that don't have a gap between each other
-    capture_area_expected = generate_capture_area(polygons_no_gap, 0.2)
+    capture_area_expected = generate_capture_area(polygons_no_gap, Decimal("0.2"))
     # Generate the capture area of the polygons that have a gap between each other
-    capture_area_result = generate_capture_area(polygon_with_gap, 0.2)
+    capture_area_result = generate_capture_area(polygon_with_gap, Decimal("0.2"))
     print(f"Polygon no gap A: {to_feature(polygons_no_gap[0])}")
     print(f"Polygon no gap B: {to_feature(polygons_no_gap[1])}")
     print(f"Polygon with gap A: {to_feature(polygon_with_gap[0])}")
@@ -110,9 +111,9 @@ def test_generate_capture_area_not_rounded() -> None:
 
     # Using GSD of 0.2m
     # Generate the capture area of the polygons that don't have a gap between each other
-    capture_area_expected = generate_capture_area(polygons_no_gap, 0.2)
+    capture_area_expected = generate_capture_area(polygons_no_gap, Decimal("0.2"))
     # Generate the capture area of the polygons that have a gap between each other
-    capture_area_result = generate_capture_area(polygon_with_gap, 0.2)
+    capture_area_result = generate_capture_area(polygon_with_gap, Decimal("0.2"))
 
     print(f"Polygon no gap A: {to_feature(polygons_no_gap[0])}")
     print(f"Polygon no gap B: {to_feature(polygons_no_gap[1])}")
@@ -146,7 +147,7 @@ def test_capture_area_orientation_polygon() -> None:
             }
         )
     )
-    capture_area = generate_capture_area(polygons, 0.05)
+    capture_area = generate_capture_area(polygons, Decimal("0.05"))
     assert is_ccw(get_exterior_ring(shape(capture_area["geometry"])))
 
 
@@ -185,6 +186,6 @@ def test_capture_area_orientation_multipolygon() -> None:
             }
         )
     )
-    capture_area = generate_capture_area(polygons, 0.05)
+    capture_area = generate_capture_area(polygons, Decimal("0.05"))
     mp_geom = cast(MultiPolygon, shape(capture_area["geometry"]))
     assert is_ccw(get_exterior_ring(mp_geom.geoms[0]))

scripts/stac/imagery/tests/collection_test.py

@@ -3,6 +3,7 @@
 import tempfile
 from collections.abc import Callable, Generator
 from datetime import datetime, timezone
+from decimal import Decimal
 from shutil import rmtree
 from tempfile import mkdtemp
 
@@ -31,7 +32,7 @@ def setup() -> Generator[CollectionMetadata, None, None]:
     metadata: CollectionMetadata = {
         "category": "urban-aerial-photos",
         "region": "auckland",
-        "gsd": "0.3m",
+        "gsd": Decimal("0.3"),
         "start_datetime": datetime(2022, 2, 2),
         "end_datetime": datetime(2022, 2, 2),
         "lifecycle": "completed",
@@ -155,7 +156,7 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None:
     with subtests.test(msg="Main links content"):
         assert [
             {"href": "./collection.json", "rel": "self", "type": "application/json"},
-            {"rel": "item", "href": "./BR34_5000_0304.json", "type": "application/json"},
+            {"rel": "item", "href": "./BR34_5000_0304.json", "type": "application/geo+json"},
         ] == links
 
     with subtests.test():

scripts/stac/imagery/tests/create_stac_test.py

@@ -23,7 +23,7 @@ def test_create_item_with_derived_from(tmp_path: Path) -> None:
     assert {
         "href": derived_from_path.as_posix(),
         "rel": "derived_from",
-        "type": "application/json",
+        "type": "application/geo+json",
         "file:checksum": "12209c3d50f21fdd739de5c76b3c7ca60ee7f5cf69c2cf92b1d0136308cf63d9c5d5",
     } in item.stac["links"]