Error loading HCS plates from S3

[elyall]

The plugin loads an HCS plate (sparse or otherwise) fine when saved locally (though I'd prefer it not throw warnings for every missing well), however when I try to load the same file from an S3 bucket (viewer.open(s3_path, plugin="napari-ome-zarr")) it throws the error: Exception: Could not find first well.

Is it not possible to load an HCS plate from S3? I figured with zarr.storage.FSStore and potentially fsspec.mapping.FSMap this should be doable. I will dig into the code further but any help or insight would be appreciated.

Note the plugin is able to load an image from S3 when a specific field is defined (e.g. viewer.open(s3_path + "/A/2/0", plugin="napari-ome-zarr")).

Code

import zarr
import string
from numpy import ones
import napari
import subprocess

from ome_zarr.io import parse_url
from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata


local_path = "test.ome.zarr"
s3_path = "s3://test-bucket/test.ome.zarr"

# write file to local
def write_HCS_plate(
    file_path, 
    well_paths = ["A/2", "B/3"], 
    num_rows=2, 
    num_cols=3, 
    field_paths = ["0", "1", "2"],
    ):
    store = parse_url(str(file_path), mode="w").store
    root = zarr.group(store=store)

    row_names = string.ascii_uppercase[: num_rows]
    col_names = list(map(str, range(1, num_cols + 1)))
    write_plate_metadata(root, row_names, col_names, well_paths)
    for wp in well_paths:
        row, col = wp.split("/")
        row_group = root.require_group(row)
        well = row_group.require_group(col)
        write_well_metadata(well, field_paths)
        for field in field_paths:
            image = well.require_group(str(field))
            write_image(ones((1, 1, 1, 256, 256)), image)

write_HCS_plate(local_path)

# read file from local
viewer = napari.Viewer()
viewer.open(local_path, plugin="napari-ome-zarr")

# copy file to s3
p = subprocess.Popen(f'aws s3 sync {local_path} {s3_path}', shell=True)
p.wait()

# read file from s3
viewer = napari.Viewer()
viewer.open(s3_path, plugin="napari-ome-zarr")

One related gripe is the only place I can find the pattern for writing HCS plates is in ome_zarr.tests.test_reader.test_multiwells_plate. It would be helpful to (1) provide an example in the documentation or to (2) provide a wrapper function in ome_zarr.writer. I will see if I can provide a pull request for (1) given it's less intrusive.

UPDATE: added an example to the docs via this pull request: ome/ome-zarr-py#317

Error

Full Error

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
Cell In[19], line 13
     12 viewer = napari.Viewer()
---> 13 viewer.open(s3_path, plugin="napari-ome-zarr")

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/napari/components/viewer_model.py:1092], in ViewerModel.open(self, path, stack, plugin, layer_type, **kwargs)
   1089 _path = [_path] if not isinstance(_path, list) else _path
   1090 if plugin:
   1091     added.extend(
-> 1092         self._add_layers_with_plugins(
   1093             _path,
   1094             kwargs=kwargs,
   1095             plugin=plugin,
   1096             layer_type=layer_type,
   1097             stack=_stack,
   1098         )
   1099     )
   1100 # no plugin choice was made
   1101 else:
   1102     layers = self._open_or_raise_error(
   1103         _path, kwargs, layer_type, _stack
   1104     )

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/napari/components/viewer_model.py:1292], in ViewerModel._add_layers_with_plugins(self, paths, stack, kwargs, plugin, layer_type)
   1290 else:
   1291     assert len(paths) == 1
-> 1292     layer_data, hookimpl = read_data_with_plugins(
   1293         paths, plugin=plugin, stack=stack
   1294     )
   1296 # glean layer names from filename. These will be used as *fallback*
   1297 # names, if the plugin does not return a name kwarg in their meta dict.
   1298 filenames = []

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/napari/plugins/io.py:77], in read_data_with_plugins(paths, plugin, stack)
     74     assert len(paths) == 1
     75 hookimpl: Optional[HookImplementation]
---> 77 res = _npe2.read(paths, plugin, stack=stack)
     78 if res is not None:
     79     _ld, hookimpl = res

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/napari/plugins/_npe2.py:63], in read(paths, plugin, stack)
     61     npe1_path = paths[0]
     62 try:
---> 63     layer_data, reader = io_utils.read_get_reader(
     64         npe1_path, plugin_name=plugin
     65     )
     66 except ValueError as e:
     67     # plugin wasn't passed and no reader was found
     68     if 'No readers returned data' not in str(e):

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/npe2/io_utils.py:66], in read_get_reader(path, plugin_name, stack)
     62 if stack is None:
     63     # "npe1" old path
     64     # Napari 0.4.15 and older, hopefully we can drop this and make stack mandatory
     65     new_path, new_stack = v1_to_v2(path)
---> 66     return _read(
     67         new_path, plugin_name=plugin_name, return_reader=True, stack=new_stack
     68     )
     69 else:
     70     assert isinstance(path, list)

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/npe2/io_utils.py:165], in _read(paths, stack, plugin_name, return_reader, _pm)
    160     read_func = rdr.exec(
    161         kwargs={"path": paths, "stack": stack, "_registry": _pm.commands}
    162     )
    163     if read_func is not None:
    164         # if the reader function raises an exception here, we don't try to catch it
--> 165         if layer_data := read_func(paths, stack=stack):
    166             return (layer_data, rdr) if return_reader else layer_data
    168 if plugin_name:

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/npe2/manifest/contributions/_readers.py:60], in ReaderContribution.exec..npe1_compat(paths, stack)
     57 @wraps(callable_)
     58 def npe1_compat(paths, *, stack):
     59     path = v2_to_v1(paths, stack)
---> 60     return callable_(path)

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/napari_ome_zarr/_reader.py:105], in transform..f(*args, **kwargs)
    102 def f(*args: Any, **kwargs: Any) -> List[LayerData]:
    103     results: List[LayerData] = list()
--> 105     for node in nodes:
    106         data: List[Any] = node.data
    107         metadata: Dict[str, Any] = {}

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/ome_zarr/reader.py:625], in Reader.__call__(self)
    624 def __call__(self) -> Iterator[Node]:
--> 625     node = Node(self.zarr, self)
    626     if node.specs:  # Something has matched
    627         LOGGER.debug("treating %s as ome-zarr", self.zarr)

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/ome_zarr/reader.py:59], in Node.__init__(self, zarr, root, visibility, plate_labels)
     57     self.specs.append(PlateLabels(self))
     58 elif Plate.matches(zarr):
---> 59     self.specs.append(Plate(self))
     60     # self.add(zarr, plate_labels=True)
     61 if Well.matches(zarr):

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/ome_zarr/reader.py:478], in Plate.__init__(self, node)
    476 super().__init__(node)
    477 LOGGER.debug("Plate created with ZarrLocation fmt: %s", self.zarr.fmt)
--> 478 self.get_pyramid_lazy(node)

File [~/micromamba/envs/rb_p310/lib/python3.10/site-packages/ome_zarr/reader.py:504], in Plate.get_pyramid_lazy(self, node)
    502 well_spec: Optional[Well] = well_node.first(Well)
    503 if well_spec is None:
--> 504     raise Exception("Could not find first well")
    505 self.numpy_type = well_spec.numpy_type
    507 LOGGER.debug("img_pyramid_shapes: %s", well_spec.img_pyramid_shapes)

Exception: Could not find first well

Versions

napari-ome-zarr = 0.5.2
ome-zarr = 0.8.0
zarr = 2.16.0
napari = 0.4.18
python = 3.10.12

[imagesc-bot]

This issue has been mentioned on Image.sc Forum. There might be relevant details there:

https://forum.image.sc/t/should-it-be-possible-to-load-an-ome-zarr-hcs-plate-directly-from-s3/86956/1

[elyall]

Solved with ome/ome-zarr-py#322.