fix: skip files which are not tiff (#53)

* fix: skip files which are not tiff

* fix: pylint

* fix: temporary fix pylint

* fix: log file is not tiff as trace level

scripts/create_polygons.py

@@ -6,6 +6,7 @@
 from urllib.parse import urlparse
 
 from aws_helper import get_bucket
+from file_helper import is_tiff
 from format_source import format_source
 from linz_logger import get_log
 
@@ -51,6 +52,9 @@ def main() -> None:  # pylint: disable=too-many-locals
     output_files = []
 
     for file in source:
+        if not is_tiff(file):
+            get_log().trace("create_polygon_file_not_tiff_skipped", file=file)
+            continue
         with tempfile.TemporaryDirectory() as tmp_dir:
             source_file_name = os.path.basename(file)
             uri_parse = file

scripts/file_helper.py

@@ -3,3 +3,7 @@
 
 def get_file_name_from_path(path: str) -> str:
     return os.path.basename(path)
+
+
+def is_tiff(path: str) -> bool:
+    return path.lower().endswith((".tiff", ".tif"))

scripts/non_visual_qa.py

@@ -2,6 +2,7 @@
 import json
 from typing import Any, Dict, List
 
+from file_helper import is_tiff
 from format_source import format_source
 from gdal_helper import run_gdal
 from linz_logger import get_log
@@ -84,6 +85,10 @@ def main() -> None:
     srs = gdalsrsinfo_result.stdout
 
     for file in source:
+        if not is_tiff(file):
+            get_log().trace("non_visual_qa_file_not_tiff_skipped", file=file)
+            continue
+
         gdalinfo_command = ["gdalinfo", "-stats", "-json"]
         gdalinfo_process = run_gdal(gdalinfo_command, file)
         gdalinfo_result = {}

scripts/standardising.py

@@ -2,7 +2,7 @@
 import os
 
 from aws_helper import parse_path
-from file_helper import get_file_name_from_path
+from file_helper import get_file_name_from_path, is_tiff
 from format_source import format_source
 from gdal_helper import run_gdal
 from linz_logger import get_log
@@ -17,10 +17,13 @@
 gdal_env = os.environ.copy()
 
 for file in source:
+    if not is_tiff(file):
+        get_log().trace("standardising_file_not_tiff_skipped", file=file)
+        continue
+
     src_bucket_name, src_file_path = parse_path(file)
     standardized_file_name = f"standardized_{get_file_name_from_path(src_file_path)}"
     tmp_file_path = os.path.join("/tmp/", standardized_file_name)
-
     command = [
         "gdal_translate",
         "-q",

scripts/tests/file_helper_test.py

@@ -0,0 +1,13 @@
+from file_helper import is_tiff
+
+
+def test_is_tiff() -> None:
+    file_a = "file.tiff"
+    file_b = "file.tif"
+    file_c = "file.TIFF"
+    file_d = "file.jpg"
+
+    assert is_tiff(file_a) is True
+    assert is_tiff(file_b) is True
+    assert is_tiff(file_c) is True
+    assert is_tiff(file_d) is False