Add fiboa improve command #79 #21

Author: m-mohr

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- Command `fiboa improve` with helpers to
+  - change the CRS
+  - change the GeoParquet version and compression
+  - fill missing perimeter/area values
+  - fix invalid geometries
+  - rename columns
 - Converter for Switzerland
 
 ## [v0.8.0] - 2024-11-12

fiboa_cli/__init__.py

@@ -6,16 +6,18 @@
 import click
 import pandas as pd
 
+from .const import COMPRESSION_METHODS, CORE_COLUMNS
 from .convert import convert as convert_
 from .convert import list_all_converter_ids, list_all_converters
 from .create_geojson import create_geojson as create_geojson_
 from .create_geoparquet import create_geoparquet as create_geoparquet_
 from .describe import describe as describe_
-from .merge import merge as merge_, DEFAULT_COLUMNS, DEFAULT_CRS
+from .improve import improve as improve_
+from .merge import merge as merge_, DEFAULT_CRS
 from .jsonschema import jsonschema as jsonschema_
 from .rename_extension import rename_extension as rename_extension_
 from .util import (check_ext_schema_for_cli, log, parse_converter_input_files,
-                   valid_file_for_cli, valid_file_for_cli_with_ext,
+                   parse_map, valid_file_for_cli, valid_file_for_cli_with_ext,
                    valid_files_folders_for_cli, valid_folder_for_cli)
 from .validate import validate as validate_
 from .validate_schema import validate_schema as validate_schema_
@@ -376,7 +378,7 @@ def jsonschema(schema, out, fiboa_version, id_):
 )
 @click.option(
     '--compression', '-pc',
-    type=click.Choice(["brotli", "gzip", "lz4", "snappy", "zstd", "none"]),
+    type=click.Choice(COMPRESSION_METHODS),
     help='Compression method for the Parquet file.',
     show_default=True,
     default="brotli"
@@ -385,7 +387,7 @@ def jsonschema(schema, out, fiboa_version, id_):
     '--geoparquet1', '-gp1',
     is_flag=True,
     type=click.BOOL,
-    help='Enforces generating a GeoParquet 1.0 file bounding box. Defaults to GeoParquet 1.1 with bounding box.',
+    help='Enforces generating a GeoParquet 1.0 file. Defaults to GeoParquet 1.1 with bounding box.',
     default=False
 )
 @click.option(
@@ -518,7 +520,7 @@ def rename_extension(folder, title, slug, org = "fiboa", prefix = None):
     multiple=True,
     help='Additional column names to include.',
     show_default=True,
-    default=DEFAULT_COLUMNS,
+    default=CORE_COLUMNS,
 )
 @click.option(
     '--exclude', '-e',
@@ -536,7 +538,7 @@ def rename_extension(folder, title, slug, org = "fiboa", prefix = None):
 )
 @click.option(
     '--compression', '-pc',
-    type=click.Choice(["brotli", "gzip", "lz4", "snappy", "zstd", "none"]),
+    type=click.Choice(COMPRESSION_METHODS),
     help='Compression method for the Parquet file.',
     show_default=True,
     default="brotli"
@@ -545,7 +547,7 @@ def rename_extension(folder, title, slug, org = "fiboa", prefix = None):
     '--geoparquet1', '-gp1',
     is_flag=True,
     type=click.BOOL,
-    help='Enforces generating a GeoParquet 1.0 file bounding box. Defaults to GeoParquet 1.1 with bounding box.',
+    help='Enforces generating a GeoParquet 1.0 file. Defaults to GeoParquet 1.1 with bounding box.',
     default=False
 )
 def merge(datasets, out, crs, include, exclude, extension, compression, geoparquet1):
@@ -564,6 +566,69 @@ def merge(datasets, out, crs, include, exclude, extension, compression, geoparqu
         sys.exit(1)
 
 
+## IMPROVE (add area, perimeter, and fix geometries)
+@click.command()
+@click.argument('input', nargs=1, type=click.Path(exists=True))
+@click.option(
+    '--out', '-o',
+    type=click.Path(exists=False),
+    help='Path to write the GeoParquet file to. If not given, overwrites the input file.',
+    default=None
+)
+@click.option(
+    '--rename-column', '-r',
+    type=click.STRING,
+    callback=lambda ctx, param, value: parse_map(value),
+    multiple=True,
+    help='Renaming of columns. Provide the old name and the new name separated by an equal sign. Can be used multiple times.'
+)
+@click.option(
+    '--add-sizes', '-sz',
+    is_flag=True,
+    type=click.BOOL,
+    help='Computes missing sizes (area, perimeter)',
+    default=False
+)
+@click.option(
+    '--fix-geometries', '-g',
+    is_flag=True,
+    type=click.BOOL,
+    help='Tries to fix invalid geometries that are repored by the validator (uses shapely\'s make_valid method internally)',
+    default=False
+)
+@click.option(
+    '--crs',
+    type=click.STRING,
+    help='Coordinate Reference System (CRS) to use for the GeoParquet file.',
+    show_default=True,
+    default=DEFAULT_CRS,
+)
+@click.option(
+    '--compression', '-pc',
+    type=click.Choice(COMPRESSION_METHODS),
+    help='Compression method for the Parquet file.',
+    show_default=True,
+    default="brotli"
+)
+@click.option(
+    '--geoparquet1', '-gp1',
+    is_flag=True,
+    type=click.BOOL,
+    help='Enforces generating a GeoParquet 1.0 file. Defaults to GeoParquet 1.1 with bounding box.',
+    default=False
+)
+def improve(input, out, rename_column, add_sizes, fix_geometries, crs, compression, geoparquet1):
+    """
+    "Improves" a fiboa GeoParquet file according to the given parameters.
+    """
+    log(f"fiboa CLI {__version__} - Improve datasets\n", "success")
+    try:
+        improve_(input, out, rename_column, add_sizes, fix_geometries, crs, compression, geoparquet1)
+    except Exception as e:
+        log(e, "error")
+        sys.exit(1)
+
+
 cli.add_command(describe)
 cli.add_command(validate)
 cli.add_command(validate_schema)
@@ -574,6 +639,7 @@ def merge(datasets, out, crs, include, exclude, extension, compression, geoparqu
 cli.add_command(converters)
 cli.add_command(rename_extension)
 cli.add_command(merge)
+cli.add_command(improve)
 
 if __name__ == '__main__':
     cli()

fiboa_cli/const.py

@@ -10,3 +10,14 @@
 STAC_COLLECTION_SCHEMA = "http://schemas.stacspec.org/v{version}/collection-spec/json-schema/collection.json"
 GEOPARQUET_SCHEMA = "https://geoparquet.org/releases/v{version}/schema.json"
 STAC_TABLE_EXTENSION = "https://stac-extensions.github.io/table/v1.2.0/schema.json"
+
+COMPRESSION_METHODS = ["brotli", "gzip", "lz4", "snappy", "zstd", "none"]
+
+CORE_COLUMNS = [
+    "id",
+    "geometry",
+    "area",
+    "perimeter",
+    "determination_datetime",
+    "determination_method",
+]

fiboa_cli/convert.py

@@ -13,7 +13,7 @@ def convert(
         collection = False,
         compression = None,
         geoparquet1 = False,
-        mapping_file=None,
+        mapping_file = None,
     ):
     if dataset in IGNORED_DATASET_FILES:
         raise Exception(f"'{dataset}' is not a converter")

fiboa_cli/improve.py

@@ -0,0 +1,69 @@
+import os
+
+from pyproj import CRS
+
+from .const import CORE_COLUMNS
+from .parquet import create_parquet
+from .util import load_parquet_data, load_parquet_schema, log, parse_metadata
+
+
+def improve(input, out = None, rename_columns = {}, add_sizes = False, fix_geometries = False, crs = None, compression = None, geoparquet1 = False):
+    # Prepare and determine location of the output file
+    if not out:
+        out = input
+    else:
+        dir = os.path.dirname(out)
+        if dir:
+            os.makedirs(dir, exist_ok=True)
+
+    # Load the dataset
+    schema = load_parquet_schema(input)
+    collection = parse_metadata(schema, b"fiboa")
+    columns = list(schema.names)
+    # Remove the bbox column to avoid conflicts when writing GeoParquet file later
+    columns.remove("bbox")
+    gdf = load_parquet_data(input, columns = columns)
+
+    # Change the CRS
+    if crs is not None:
+        gdf.to_crs(crs=crs, inplace=True)
+        log(f"Changed CRS to {crs}", "info")
+
+    # Fix geometries
+    if fix_geometries:
+        gdf.geometry = gdf.geometry.make_valid()
+        log("Fixed geometries", "info")
+
+    # Rename columns
+    if len(rename_columns) > 0:
+        for col in rename_columns:
+            columns[columns.index(col)] = rename_columns[col]
+            if col in CORE_COLUMNS:
+                log(f"Column {col} is a fiboa core field - do you really want to rename it?", "warning")
+            if ":" in col:
+                log(f"Column {col} may be a fiboa extension field - do you really want to rename it?", "warning")
+        gdf.rename(columns=rename_columns, inplace=True)
+        log("Renamed columns", "info")
+
+    # Add sizes
+    if add_sizes:
+        # Add the area and perimeter columns
+        for name in ["area", "perimeter"]:
+            if name not in columns:
+                # Create column if not present
+                gdf[name] = None
+                columns.append(name)
+
+        gdf_m = gdf
+        # Determine whether the given CRS is in meters
+        if gdf.crs.axis_info[0].unit_name not in ["m", "metre", "meter"]:
+            # Reproject the geometries to an equal-area projection if needed
+            gdf_m = gdf.to_crs("EPSG:6933")
+
+        # Compute the missing area and perimeter values
+        gdf["area"] = gdf_m["area"].fillna(gdf_m.geometry.area * 0.0001)
+        gdf["perimeter"] = gdf_m["perimeter"].fillna(gdf_m.geometry.length)
+
+    # Write the merged dataset to the output file
+    create_parquet(gdf, columns, collection, out, {}, compression=compression, geoparquet1=geoparquet1)
+    log(f"Wrote data to {out}", "success")

fiboa_cli/merge.py

@@ -2,26 +2,19 @@
 
 import pandas as pd
 
+from .const import CORE_COLUMNS
 from .parquet import create_parquet
 from .util import load_parquet_data, load_parquet_schema, log, parse_metadata
 from .version import fiboa_version
 
-DEFAULT_COLUMNS = [
-    "id",
-    "geometry",
-    "area",
-    "perimeter",
-    "determination_datetime",
-    "determination_method",
-]
 DEFAULT_CRS = "EPSG:4326"
 
 def merge(datasets, out, crs = DEFAULT_CRS, includes = [], excludes = [], extensions = [], compression = None, geoparquet1 = False):
     dir = os.path.dirname(out)
     if dir:
         os.makedirs(dir, exist_ok=True)
 
-    columns = DEFAULT_COLUMNS.copy()
+    columns = CORE_COLUMNS.copy()
     columns.extend(includes)
     columns = list(set(columns) - set(excludes))
 

fiboa_cli/util.py

@@ -224,6 +224,22 @@ def parse_converter_input_files(ctx, param, value):
     return sources
 
 
+def parse_map(value, separator = "="):
+    if value is None:
+        return {}
+    elif not isinstance(value, tuple):
+        raise click.BadParameter('Input files must be a tuple')
+    elif len(value) == 0:
+         return {}
+
+    mapping = {}
+    for v in value:
+        key, value = v.split(separator, 2)
+        mapping[key] = value
+
+    return mapping
+
+
 def name_from_uri(url):
     if "://" in url:
         try: