Hotfix - grib to netcdf conversion (#91)

* hotfix for multi-adaptor format conversion * Auto chunking * default to field chunks * parsable format conversion kwargs * parsable format conversion kwargs * QA * remove debug * Update cads_adaptors/adaptors/mars.py Co-authored-by: James Varndell <[email protected]> --------- Co-authored-by: James Varndell <[email protected]>
ecmwf-projects · Mar 13, 2024 · 2c222ec · 2c222ec
1 parent 27e01a8
commit 2c222ec
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 4 deletions.
diff --git a/cads_adaptors/adaptors/mars.py b/cads_adaptors/adaptors/mars.py
@@ -12,6 +12,10 @@ def convert_format(
     context: Context,
     **kwargs,
 ) -> list:
+    if isinstance(data_format, (list, tuple)):
+        assert len(data_format) == 1, "Only one value of data_format is allowed"
+        data_format = data_format[0]
+
     # NOTE: The NetCDF compressed option will not be visible on the WebPortal, it is here for testing
     if data_format in ["netcdf", "nc", "netcdf_compressed"]:
         if data_format in ["netcdf_compressed"]:
@@ -114,7 +118,10 @@ def retrieve(self, request: Request) -> BinaryIO:
         data_format = request.pop("data_format", "grib")
 
         # Allow user to provide format conversion kwargs
-        convert_kwargs = request.pop("convert_kwargs", {})
+        convert_kwargs: dict[str, Any] = {
+            **self.config.get("format_conversion_kwargs", dict()),
+            **request.pop("format_conversion_kwargs", dict()),
+        }
 
         # To preserve existing ERA5 functionality the default download_format="as_source"
         request.setdefault("download_format", "as_source")

diff --git a/cads_adaptors/tools/convertors.py b/cads_adaptors/tools/convertors.py
@@ -8,11 +8,19 @@
 }
 
 
-def grib_to_netcdf_files(grib_file, compression_options=None, **to_netcdf_kwargs):
-    fname, extension = os.path.splitext(grib_file)
+def grib_to_netcdf_files(
+    grib_file, compression_options=None, open_datasets_kwargs=None, **to_netcdf_kwargs
+):
+    fname, _ = os.path.splitext(os.path.basename(grib_file))
+    grib_file = os.path.realpath(grib_file)
+
     import cfgrib
 
-    datasets = cfgrib.open_datasets(grib_file)
+    if open_datasets_kwargs is None:
+        open_datasets_kwargs = {
+            "chunks": {"time": 1, "step": 1, "plev": 1}  # Auto chunk by field
+        }
+    datasets = cfgrib.open_datasets(grib_file, **open_datasets_kwargs)
 
     if compression_options == "default":
         compression_options = DEFAULT_COMPRESSION_OPTIONS