Validate types and geometries #26

Author: m-mohr

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Converter for EuroCrops France dataset
 - `fiboa create-geojson`: Show conversion progress
 - `fiboa jsonschema` and `fiboa validate`: Support `geometryTypes` for `geometry` data type in GeoJSON
+- `fiboa validate`: Basic validation for geometries in GeoParquet files
 
 ### Changed
 
@@ -32,6 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - `fiboa validate`:
   - Is more robust against invalid collections and doesn't abort if not needed
   - Check NULL values correctly in case of arrays
+  - Validate geometry column metadata
 - `fiboa create-geojson`:
   - Handles GeoParquet bbox correctly
   - Converts numpy arrays

fiboa_cli/types.py

@@ -2,6 +2,7 @@
 import pyarrow.types as pat
 import pandas as pd
 import numpy as np
+from shapely.geometry.base import BaseGeometry
 
 def is_enum(schema):
     return isinstance(schema.get("enum"), list)
@@ -115,6 +116,7 @@ def get_pyarrow_type(schema):
         pa_subtype = get_pyarrow_type(schema.get("items", {}))
         return pa.list_(pa_subtype)
     elif dtype == "object":
+        # todo: add patternProperties
         additonal_properties = schema.get("additionalProperties", False)
         if additonal_properties is True:
             raise Exception("Additional properties for objects are not supported")
@@ -154,7 +156,6 @@ def get_pyarrow_type_for_geopandas(dtype):
     elif dtype == "datetime64":
         return pa.timestamp("ms", tz="UTC")
     else:
-        print(dtype)
         return None
 
 
@@ -174,10 +175,34 @@ def get_pyarrow_type_for_geopandas(dtype):
     "binary": pat.is_binary,
     "string": pat.is_string,
     "array": pat.is_list,
-    "object": pat.is_map,
+    "object": pat.is_struct,
     "date": pat.is_date32,
     "date-time": pat.is_timestamp,
-    "geometry": pat.is_binary, # todo: check more?
+    "geometry": pat.is_binary,
+    "bounding-box": pat.is_struct
+}
+
+
+# checks pyarrow datatypes
+PYTHON_TYPES = {
+    "boolean": bool,
+    "int8": int,
+    "uint8": int,
+    "int16": int,
+    "uint16": int,
+    "int32": int,
+    "uint32": int,
+    "int64": int,
+    "uint64": int,
+    "float": float,
+    "double": float,
+    "binary": None, # todo
+    "string": str,
+    "array": (list, np.ndarray),
+    "object": dict,
+    "date": None, # todo
+    "date-time": None, # todo
+    "geometry": BaseGeometry,
     "bounding-box": None # todo
 }
 

fiboa_cli/validate.py

@@ -259,6 +259,8 @@ def validate_parquet(file, config):
             if not pat.is_string(pq_field.key_type):
                 log(f"{key}: Map keys must be strings", "error")
                 valid = False
+        elif dtype == "geometry":
+            valid = validate_geometry_column(key, prop_schema, geo, valid)
 
         # Validate data of the column
         if gdf is not None:
@@ -275,6 +277,26 @@ def validate_parquet(file, config):
     return valid
 
 
+def validate_geometry_column(key, prop_schema, geo, valid = True):
+    columns = geo.get("columns", {})
+    if key not in columns:
+        log(f"{key}: Geometry column not found in GeoParquet metadata", "error")
+        valid = False
+
+    schema_geo_types = prop_schema.get("geometryTypes", [])
+    schema_geo_types.sort()
+    if len(schema_geo_types) > 0:
+        gp_geo_types = columns[key].get("geometry_types", [])
+        gp_geo_types.sort()
+        if len(gp_geo_types) == 0:
+            log(f"{key}: No geometry types specified in GeoParquet metadata", "warning")
+
+        if schema_geo_types != gp_geo_types:
+            log(f"{key}: GeoParquet geometry types differ, is {gp_geo_types} but must be {schema_geo_types}", "error")
+            valid = False
+
+    return valid
+
 # todo: use stac_validator instead of our own validation routine
 def validate_colletion_schema(obj):
     if "stac_version" in obj:

fiboa_cli/validate_data.py

@@ -1,6 +1,11 @@
 import re
 import pandas as pd
+
 from urllib.parse import urlparse
+from shapely.geometry.base import BaseGeometry
+from shapely.validation import explain_validity
+
+from .types import PYTHON_TYPES
 
 REGEX_EMAIL = re.compile("[^@]+@[^@]+\.[^@]+")
 REGEX_UUID = re.compile("^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\Z")
@@ -12,6 +17,11 @@ def validate_column(data, rules):
             # Skip validation for NaN values or implement special handling if required
             continue
 
+        dtype = rules.get('type')
+        python_type = PYTHON_TYPES.get(dtype)
+        if python_type is not None and not isinstance(value, python_type):
+            return [f"Value '{value}' is not of type {dtype}."]
+
         if isinstance(value, str):
             issues = validate_string(value, rules)
         elif isinstance(value, (int, float)):
@@ -20,6 +30,8 @@ def validate_column(data, rules):
             issues = validate_array(value, rules)
         elif isinstance(value, dict):
             issues = validate_object(value, rules)
+        elif isinstance(value, BaseGeometry):
+            issues = validate_geometry(value, rules)
         else:
             continue
 
@@ -28,6 +40,21 @@ def validate_column(data, rules):
 
     return []
 
+# Geometry validation
+def validate_geometry(value, rules):
+    issues = []
+
+    geom_types = rules.get("geometryTypes", [])
+    if len(geom_types) > 0 and value.geom_type not in geom_types:
+        allowed = ", ".join(geom_types)
+        issues.append(f"Geometry type '{value.geom_type}' is not one of the allowed types: {allowed}")
+
+    why = explain_validity(value)
+    if why != 'Valid Geometry':
+        issues.append(f"Geometry {value} is not valid: {why}")
+
+    return issues
+
 # String validation
 def validate_string(value, rules):
     issues = []