HDF4 does not recognized as a supported file format

[alexxxroz]

Hello,
I'm trying to open HDF4 file using xarray function open_dataset.

import numpy as np
import pandas as pd
import xarray 
import rioxarray 
import earthpy as et
import earthpy.spatial as es
import earthpy.plot as ep
import rasterio

data = xarray.open_dataset('MYD09A1.A2018361.h20v03.006.2019010171535.hdf',
                            engine="rasterio")

Whatever I do, I get the same error: "not recognized as a supported file format":

KeyError                                  Traceback (most recent call last)
File ~/.local/lib/python3.10/site-packages/xarray/backends/file_manager.py:209, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    208 try:
--> 209     file = self._cache[self._key]
    210 except KeyError:

File ~/.local/lib/python3.10/site-packages/xarray/backends/lru_cache.py:55, in LRUCache.__getitem__(self, key)
     54 with self._lock:
---> 55     value = self._cache[key]
     56     self._cache.move_to_end(key)

KeyError: [<function open at 0x7f14eb2ba4d0>, ('MYD09A1.A2018361.h20v03.006.2019010171535.hdf',), 'r', (('sharing', False),), 'bd1a5f9b-b422-49ab-9a1d-46e4271a305e']

During handling of the above exception, another exception occurred:

CPLE_OpenFailedError                      Traceback (most recent call last)
File rasterio/_base.pyx:307, in rasterio._base.DatasetBase.__init__()

File rasterio/_base.pyx:218, in rasterio._base.open_dataset()

File rasterio/_err.pyx:221, in rasterio._err.exc_wrap_pointer()

CPLE_OpenFailedError: 'MYD09A1.A2018361.h20v03.006.2019010171535.hdf' not recognized as a supported file format.

During handling of the above exception, another exception occurred:

RasterioIOError                           Traceback (most recent call last)
Input In [1], in <cell line: 11>()
      7 import earthpy.plot as ep
      8 import rasterio
---> 11 data = xarray.open_dataset('MYD09A1.A2018361.h20v03.006.2019010171535.hdf',
     12                             engine="rasterio")

File ~/.local/lib/python3.10/site-packages/xarray/backends/api.py:539, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, backend_kwargs, **kwargs)
    527 decoders = _resolve_decoders_kwargs(
    528     decode_cf,
    529     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)
    535     decode_coords=decode_coords,
    536 )
    538 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 539 backend_ds = backend.open_dataset(
    540     filename_or_obj,
    541     drop_variables=drop_variables,
    542     **decoders,
    543     **kwargs,
    544 )
    545 ds = _dataset_from_backend_dataset(
    546     backend_ds,
    547     filename_or_obj,
   (...)
    555     **kwargs,
    556 )
    557 return ds

File ~/.local/lib/python3.10/site-packages/rioxarray/xarray_plugin.py:56, in RasterioBackend.open_dataset(self, filename_or_obj, drop_variables, parse_coordinates, chunks, cache, lock, masked, mask_and_scale, variable, group, default_name, decode_times, decode_timedelta, band_as_variable, open_kwargs)
     54 if open_kwargs is None:
     55     open_kwargs = {}
---> 56 rds = _io.open_rasterio(
     57     filename_or_obj,
     58     parse_coordinates=parse_coordinates,
     59     chunks=chunks,
     60     cache=cache,
     61     lock=lock,
     62     masked=masked,
     63     mask_and_scale=mask_and_scale,
     64     variable=variable,
     65     group=group,
     66     default_name=default_name,
     67     decode_times=decode_times,
     68     decode_timedelta=decode_timedelta,
     69     band_as_variable=band_as_variable,
     70     **open_kwargs,
     71 )
     72 if isinstance(rds, xr.DataArray):
     73     rds = rds.to_dataset()

File ~/.local/lib/python3.10/site-packages/rioxarray/_io.py:1069, in open_rasterio(filename, parse_coordinates, chunks, cache, lock, masked, mask_and_scale, variable, group, default_name, decode_times, decode_timedelta, band_as_variable, **open_kwargs)
   1067     else:
   1068         manager = URIManager(file_opener, filename, mode="r", kwargs=open_kwargs)
-> 1069     riods = manager.acquire()
   1070     captured_warnings = rio_warnings.copy()
   1072 if band_as_variable:

File ~/.local/lib/python3.10/site-packages/xarray/backends/file_manager.py:191, in CachingFileManager.acquire(self, needs_lock)
    176 def acquire(self, needs_lock=True):
    177     """Acquire a file object from the manager.
    178 
    179     A new file is only opened if it has expired from the
   (...)
    189         An open file object, as returned by ``opener(*args, **kwargs)``.
    190     """
--> 191     file, _ = self._acquire_with_cache_info(needs_lock)
    192     return file

File ~/.local/lib/python3.10/site-packages/xarray/backends/file_manager.py:215, in CachingFileManager._acquire_with_cache_info(self, needs_lock)
    213     kwargs = kwargs.copy()
    214     kwargs["mode"] = self._mode
--> 215 file = self._opener(*self._args, **kwargs)
    216 if self._mode == "w":
    217     # ensure file doesn't get overridden when opened again
    218     self._mode = "a"

File ~/.local/lib/python3.10/site-packages/rasterio/env.py:444, in ensure_env_with_credentials.<locals>.wrapper(*args, **kwds)
    441     session = DummySession()
    443 with env_ctor(session=session):
--> 444     return f(*args, **kwds)

File ~/.local/lib/python3.10/site-packages/rasterio/__init__.py:304, in open(fp, mode, driver, width, height, count, crs, transform, dtype, nodata, sharing, **kwargs)
    301 path = _parse_path(raw_dataset_path)
    303 if mode == "r":
--> 304     dataset = DatasetReader(path, driver=driver, sharing=sharing, **kwargs)
    305 elif mode == "r+":
    306     dataset = get_writer_for_path(path, driver=driver)(
    307         path, mode, driver=driver, sharing=sharing, **kwargs
    308     )

File rasterio/_base.pyx:309, in rasterio._base.DatasetBase.__init__()

RasterioIOError: 'MYD09A1.A2018361.h20v03.006.2019010171535.hdf' not recognized as a supported file format.

I also tried open_rasterio function, but nothing changes:

data = rioxarray.open_rasterio('MYD09A1.A2018361.h20v03.006.2019010171535.hdf', masked=True)

Some authors proposed this update for rasterio, but it didn't change anything for me as well.

pip3 install rasterio --force-reinstall --no-binary rasterio 

It seems, this problem happens only with hdf4 files. Do you have any ideas how I can fix it?

[dcherian]

HDF4 files are not supported directly. You'll have to read them in and create a Xarray object yourself. You could try asking at discourse.pangeo.io or at https://github.com/corteva/rioxarray/ to see if someone else has already tried it.