test: add script to run end-to-end tests TDE-820

.github/workflows/format.yml

@@ -22,31 +22,6 @@ jobs:
         run: |
           poetry run pytest .
 
-      - name: Build containers
+      - name: E2E Tests
         run: |
-          docker build . --tag topo-imagery --label "github_run_id=${GITHUB_RUN_ID}"
-
-      - name: End to end test - Aerial Imagery
-        run: |
-          # Standardising test 
-          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
-          cmp --silent ${HOME}/tmp/BG35_1000_4829.tiff ./scripts/tests/data/output/BG35_1000_4829.tiff
-
-      - name: End to end test - Elevation
-        run: |
-          # Standardising test
-          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/dem.json --preset dem_lerc --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
-          cmp --silent ${HOME}/tmp/BK39_10000_0102.tiff ./scripts/tests/data/output/BK39_10000_0102.tiff
-          cmp --silent ${HOME}/tmp/BK39_10000_0101.tiff ./scripts/tests/data/output/BK39_10000_0101.tiff
-
-      - name: End to end test - Historical Aerial Imagery
-        run: |
-          # Standardising test 
-          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01
-          cmp --silent ${HOME}/tmp/BQ31_5000_0608.tiff ./scripts/tests/data/output/BQ31_5000_0608.tiff
-
-      - name: End to end test - Cutline (Aerial Imagery)
-        run: |
-          # Standardising test 
-          docker run  -v ${HOME}/tmp/:/tmp/ topo-imagery python3 standardise_validate.py --from-file ./tests/data/aerial.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/cutline/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --cutline ./tests/data/cutline_aerial.fgb
-          cmp --silent ${HOME}/tmp/cutline/BG35_1000_4829.tiff ./scripts/tests/data/output/BG35_1000_4829_cut.tiff
+          poetry run e2e --type all

Dockerfile

@@ -12,13 +12,13 @@ WORKDIR /app
 # Add Poetry config
 COPY poetry.lock pyproject.toml /app/
 
+# Copy Python scripts
+COPY ./scripts/ /app/scripts/
+
 # Install Python dependencies
 RUN poetry config virtualenvs.create false \
     && poetry install --only main --no-interaction --no-ansi
 
-# Copy Python scripts
-COPY ./scripts/ /app/scripts/
-
 ENV PYTHONPATH="/app"
 ENV GTIFF_SRS_SOURCE="EPSG"
 

pyproject.toml

@@ -27,6 +27,9 @@ ignore_missing_imports = true
 name = "topo-imagery"
 version = "3.0.1"
 description = "A collection of scripts for processing imagery"
+packages = [
+    { include = "scripts" }
+]
 authors = [
     "Blayne Chard <[email protected]>",
     "Daniel Silk <[email protected]>",
@@ -35,6 +38,9 @@ authors = [
     "Megan Davidson <[email protected]>"
 ]
 
+[tool.poetry.scripts]
+e2e = "scripts.tests.run_docker_tests:main"
+
 [tool.poetry.dependencies]
 python = "^3.10.6"
 boto3 = "^1.28.24"

scripts/tests/run_docker_tests.py

@@ -0,0 +1,137 @@
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from enum import Enum
+from typing import List, NamedTuple, Optional
+
+from linz_logger import get_log
+
+from scripts.cli.cli_helper import InputParameterError, TileFiles, get_tile_files
+from scripts.files.fs import read
+
+docker_tests_dir = "tests/data/"
+
+
+class DataType(str, Enum):
+    ALL = "all"
+    AERIAL = "aerial"
+    ELEVATION = "elevation"
+    HISTORICAL = "historical"
+
+
+class StandardisingArgs(NamedTuple):
+    file: str
+    preset: str
+    extra_args: Optional[List[str]] = []
+
+
+def get_preset(data_type: DataType) -> str:
+    """Get the preset corresponding to the Data Type.
+
+    Args:
+        data_type: type of the data
+
+    Returns:
+        a preset
+    """
+    preset = "webp"
+    if data_type == DataType.ELEVATION:
+        preset = "dem_lerc"
+
+    return preset
+
+
+def get_standardising_args(data_type: DataType, test_file_suffix: str) -> List[StandardisingArgs]:
+    """Get some specific arguments to run `standardise_validate.py`.
+
+    Args:
+        data_type: the data type to test
+        test_file_suffix: suffix of the input file
+
+    Returns:
+        a list of `StandardisingArgs`
+    """
+    standardising_args = []
+    if data_type == DataType.ALL:
+        for type_enum in DataType:
+            if type_enum == DataType.ALL:
+                continue
+            standardising_args.extend(get_standardising_args(type_enum, test_file_suffix))
+        return standardising_args
+
+    standardising_args.append(StandardisingArgs(file=data_type + test_file_suffix, preset=get_preset(data_type)))
+    if data_type == DataType.AERIAL:
+        # Add the cutline test
+        standardising_args.append(
+            StandardisingArgs(
+                file=data_type + test_file_suffix,
+                preset=get_preset(data_type),
+                extra_args=["--cutline", os.path.join(docker_tests_dir, "cutline.fgb")],
+            )
+        )
+    return standardising_args
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-t",
+        "--type",
+        dest="type",
+        choices=[DataType.ALL.value, DataType.AERIAL.value, DataType.ELEVATION.value, DataType.HISTORICAL.value],
+        required=True,
+        help="The data type to run the test on.",
+    )
+    parser.add_argument(
+        "--aws", dest="is_aws", required=False, action="store_true", help="Runs the test using files stored on AWS S3."
+    )
+    arguments = parser.parse_args()
+    data_type = arguments.type
+    test_file_suffix = ".json"
+    if arguments.is_aws:
+        test_file_suffix = "_aws.json"
+    local_tests_dir = "scripts/tests/data"
+
+    # Build Docker
+    subprocess.call("docker build -t topo-imagery-test .", shell=True)
+
+    # Get the standardise_validate.py arguments
+    standardising_args = get_standardising_args(data_type, test_file_suffix)
+
+    for arg in standardising_args:
+        with tempfile.TemporaryDirectory() as tmp_path:
+            # Parse the input JSON file in order to test the output
+            source = json.dumps(json.loads(read(os.path.join(local_tests_dir, arg.file))))
+            # pylint: disable-msg=duplicate-code
+            try:
+                tile_files: List[TileFiles] = get_tile_files(source)
+            except InputParameterError as e:
+                get_log().error("An error occurred while getting tile_files", error=str(e))
+                sys.exit(1)
+
+            extra_args = ""
+            if arg.extra_args:
+                " ".join(arg.extra_args)
+            # Run standardise_validate.py with test file
+            subprocess.call(
+                f"docker run  -v {tmp_path}:/tmp/ topo-imagery-test python3 standardise_validate.py \
+                    --from-file {os.path.join(docker_tests_dir, arg.file)} --preset {arg.preset}  \
+                        --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 \
+                            --start-datetime 2023-01-01 --end-datetime 2023-01-01 {extra_args}",
+                shell=True,
+            )
+
+            # Verify output(s)
+            for tile in tile_files:
+                output_file = f"{tile.output}.tiff"
+                return_code = subprocess.call(
+                    f"cmp --silent {os.path.join(tmp_path, output_file)} \
+                        {os.path.join(local_tests_dir, 'output',output_file)}",
+                    shell=True,
+                )
+                if return_code != 0:
+                    get_log().error(f"End-to-end test failed with data type {os.path.splitext(arg.file)[0]}")
+                    sys.exit(1)