enable loading remote hdf5 files

xarray/backends/api.py

@@ -170,8 +170,8 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True,
         Strings and Path objects are interpreted as a path to a netCDF file
         or an OpenDAP URL and opened with python-netCDF4, unless the filename
         ends with .gz, in which case the file is gunzipped and opened with
-        scipy.io.netcdf (only netCDF3 supported). File-like objects are opened
-        with scipy.io.netcdf (only netCDF3 supported).
+        scipy.io.netcdf (only netCDF3 supported). Byte-strings or file-like
+        objects are opened by scipy.io.netcdf (netCDF3) or h5py (netCDF4/HDF).
     group : str, optional
         Path to the netCDF4 group in the given file to open (only works for
         netCDF4 files).
@@ -310,17 +310,9 @@ def maybe_decode_store(store, lock=False):
     if isinstance(filename_or_obj, backends.AbstractDataStore):
         store = filename_or_obj
         ds = maybe_decode_store(store)
-    elif isinstance(filename_or_obj, str):
 
-        if (isinstance(filename_or_obj, bytes) and
-                filename_or_obj.startswith(b'\x89HDF')):
-            raise ValueError('cannot read netCDF4/HDF5 file images')
-        elif (isinstance(filename_or_obj, bytes) and
-                filename_or_obj.startswith(b'CDF')):
-            # netCDF3 file images are handled by scipy
-            pass
-        elif isinstance(filename_or_obj, str):
-            filename_or_obj = _normalize_path(filename_or_obj)
+    elif isinstance(filename_or_obj, str):
+        filename_or_obj = _normalize_path(filename_or_obj)
 
         if engine is None:
             engine = _get_default_engine(filename_or_obj,
@@ -352,11 +344,25 @@ def maybe_decode_store(store, lock=False):
         with close_on_error(store):
             ds = maybe_decode_store(store)
     else:
-        if engine is not None and engine != 'scipy':
-            raise ValueError('can only read file-like objects with '
-                             "default engine or engine='scipy'")
-        # assume filename_or_obj is a file-like object
-        store = backends.ScipyDataStore(filename_or_obj)
+        if engine not in [None, 'scipy', 'h5netcdf']:
+            raise ValueError('can only read bytes or file-like objects with '
+                             "engine = None, 'scipy', or 'h5netcdf'")
+        else:
+            if isinstance(filename_or_obj, bytes):
+                filename_or_obj = BytesIO(filename_or_obj)
+            # read first bytes of file-like object to determine engine
+            magic_number = filename_or_obj.read(8)
+            filename_or_obj.seek(0)
+            if magic_number.startswith(b'CDF'):
+                store = backends.ScipyDataStore(filename_or_obj,
+                                                **backend_kwargs)
+            elif magic_number.startswith(b'\211HDF\r\n\032\n'):
+                store = backends.H5NetCDFStore(filename_or_obj, group=group,
+                                               lock=lock, **backend_kwargs)
+            else:
+                print(magic_number)
+                raise ValueError("file-like object is not a netCDF file: {}"
+                                 .format(filename_or_obj))
         ds = maybe_decode_store(store)
 
     # Ensure source filename always stored in dataset object (GH issue #2550)
@@ -383,8 +389,8 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True,
         Strings and Paths are interpreted as a path to a netCDF file or an
         OpenDAP URL and opened with python-netCDF4, unless the filename ends
         with .gz, in which case the file is gunzipped and opened with
-        scipy.io.netcdf (only netCDF3 supported). File-like objects are opened
-        with scipy.io.netcdf (only netCDF3 supported).
+        scipy.io.netcdf (only netCDF3 supported). Byte-strings or file-like
+        objects are opened by scipy.io.netcdf (netCDF3) or h5py (netCDF4/HDF).
     group : str, optional
         Path to the netCDF4 group in the given file to open (only works for
         netCDF4 files).

xarray/tests/__init__.py

@@ -77,6 +77,12 @@ def LooseVersion(vstring):
 has_cfgrib, requires_cfgrib = _importorskip('cfgrib')
 
 # some special cases
+has_h5netcdf07, requires_h5netcdf07 = _importorskip('h5netcdf',
+                                                    minversion='0.7')
+has_h5py29, requires_h5py29 = _importorskip('h5py', minversion='2.9.0')
+has_h5fileobj = has_h5netcdf07 and has_h5py29
+requires_h5fileobj = pytest.mark.skipif(
+    not has_h5fileobj, reason='requires h5py>2.9.0 & h5netcdf>0.7')
 has_scipy_or_netCDF4 = has_scipy or has_netCDF4
 requires_scipy_or_netCDF4 = pytest.mark.skipif(
     not has_scipy_or_netCDF4, reason='requires scipy or netCDF4')

xarray/tests/test_backends.py

@@ -35,7 +35,7 @@
     requires_cftime, requires_dask, requires_h5netcdf, requires_netCDF4,
     requires_pathlib, requires_pseudonetcdf, requires_pydap, requires_pynio,
     requires_rasterio, requires_scipy, requires_scipy_or_netCDF4,
-    requires_zarr)
+    requires_zarr, requires_h5fileobj)
 from .test_coding_times import (_STANDARD_CALENDARS, _NON_STANDARD_CALENDARS,
                                 _ALL_CALENDARS)
 from .test_dataset import create_test_data
@@ -1955,6 +1955,39 @@ def test_dump_encodings_h5py(self):
             assert actual.x.encoding['compression_opts'] is None
 
 
+@requires_h5fileobj
+class TestH5NetCDFFileObject(TestH5NetCDFData):
+    h5py = pytest.importorskip('h5py', minversion='2.9.0')
+    engine = 'h5netcdf'
+
+    @network
+    def test_h5remote(self):
+        # alternative: http://era5-pds.s3.amazonaws.com/2008/01/main.nc
+        import requests
+        url = ('https://www.unidata.ucar.edu/'
+               'software/netcdf/examples/test_hgroups.nc')
+        print(url)
+        bytes = requests.get(url).content
+        with xr.open_dataset(bytes) as ds:
+            assert len(ds['UTC_time']) == 74
+            assert ds['UTC_time'].attrs['name'] == 'time'
+
+    def test_h5binary(self):
+        expected = create_test_data().drop('dim3')
+        expected.attrs['foo'] = 'bar'
+        with create_tmp_file() as tmp_file:
+            expected.to_netcdf(tmp_file, engine='h5netcdf')
+
+            with open(tmp_file, 'rb') as f:
+                with open_dataset(f, engine='h5netcdf') as actual:
+                    assert_identical(expected, actual)
+
+                f.seek(0)
+                with BytesIO(f.read()) as bio:
+                    with open_dataset(bio, engine='h5netcdf') as actual:
+                        assert_identical(expected, actual)
+
+
 @requires_h5netcdf
 @requires_dask
 @pytest.mark.filterwarnings('ignore:deallocating CachingFileManager')