Add FsspecJsonWSIReader class. (#897)

The `FsspecJsonWSIReader` reads fsspec json file which represents SVS or TIFF whole slide image. The images are accessible by HTTP range requests, eg:

`https://api.gdc.cancer.gov/data/73c69d24-6f9e-44e2-bfe5-a608d4cf5c27`

The whole image can be downloaded like:

`curl -C - -o TCGA-22-1017-01Z-00-DX1.9562FE79-A261-42D3-B394-F3E0E2FF7DDA.svs   https://api.gdc.cancer.gov/data/73c69d24-6f9e-44e2-bfe5-a608d4cf5c27`


The `FsspecJsonWSIReader` class has a `_zarr_store` field which is created by reading json file using `fsspec`:

```
mapper = fsspec.get_mapper(
            "reference://", fo=str(input_img), target_protocol="file"
        )
self._zarr_array = zarr.open(mapper, mode="r")
self._zarr_store = self._zarr_array.store

self._zarr_lru_cache = zarr.LRUStoreCache(self._zarr_store, max_size=cache_size)
self._zarr_group = zarr.open(self._zarr_lru_cache)
```

This is equivalent to `TIFFWSIReader` code:

```
self._zarr_store = tifffile.imread(
            self.input_path,
            series=self.series_n,
            aszarr=True,
        )
        self._zarr_lru_cache = zarr.LRUStoreCache(self._zarr_store, max_size=cache_size)
        self._zarr_group = zarr.open(self._zarr_lru_cache)
```


Both FsspecJsonWSIReader and TIFFWSIReader forward calls to `read_bounds` and  `read_rect` methods of the`TIFFWSIReaderDelegate` delegate instance.


The method `_info` of the`TIFFWSIReaderDelegate` reads SVS metadata which is stored in the root group metadata like:
```
{
  ".zattrs": {
    "multiscales": [
      {
        "metadata": {
          "objective_power": 40,
          "vendor": "Aperio",
          "mpp": [0.2525, 0.2525]	
        }
      }
    ]
  }
}
```


To test, execute from the root dir:
```
pip install -r requirements/requirements_dev.txt
mkdir -p samples/slides
mkdir -p samples/fsspec
cd samples/slides
curl -C - -o TCGA-22-1017-01Z-00-DX1.9562FE79-A261-42D3-B394-F3E0E2FF7DDA.svs   https://api.gdc.cancer.gov/data/73c69d24-6f9e-44e2-bfe5-a608d4cf5c27
cd ../../
cp tiatoolbox/utils/tiff_to_fsspec.py .
python tiff_to_fsspec.py "samples/slides/TCGA-22-1017-01Z-00-DX1.9562FE79-A261-42D3-B394-F3E0E2FF7DDA.svs"  "samples/fsspec/73c69d24-6f9e-44e2-bfe5-a608d4cf5c27_fsspec.json" "https://api.gdc.cancer.gov/data/73c69d24-6f9e-44e2-bfe5-a608d4cf5c27"
```

Create `tileserver.py` inside of the project root:

```
from flask_cors import CORS

from tiatoolbox.visualization import TileServer
from tiatoolbox.wsicore.wsireader import FsspecJsonWSIReader

wsi = FsspecJsonWSIReader.open(
    "./samples/fsspec/73c69d24-6f9e-44e2-bfe5-a608d4cf5c27_fsspec.json"
)


# Initialize and run the TileServer
tile_server = TileServer(
    title="Tiatoolbox TileServer",
    layers={"layer": wsi},
)
CORS(tile_server, send_wildcard=True)


tile_server.run(host="127.0.0.1", port=5000)
```

Open `http://127.0.0.1:5000/` and verify that it works.
---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Shan E Ahmed Raza <[email protected]>

Author: 3 people

requirements/requirements.txt

@@ -1,5 +1,6 @@
 # torch installation
 --extra-index-url https://download.pytorch.org/whl/cu118; sys_platform != "darwin"
+aiohttp>=3.8.1
 albumentations>=1.3.0
 bokeh>=3.1.1, <3.6.0
 Click>=8.1.3

tests/test_wsireader.py

@@ -9,9 +9,8 @@
 import shutil
 from copy import deepcopy
 from pathlib import Path
-
-# When no longer supporting Python <3.9 this should be collections.abc.Iterable
 from typing import TYPE_CHECKING, Callable
+from unittest.mock import patch
 
 import cv2
 import glymur
@@ -27,7 +26,7 @@
 
 from tiatoolbox import cli, utils
 from tiatoolbox.annotation import SQLiteStore
-from tiatoolbox.utils import imread
+from tiatoolbox.utils import imread, tiff_to_fsspec
 from tiatoolbox.utils.exceptions import FileNotSupportedError
 from tiatoolbox.utils.magic import is_sqlite3
 from tiatoolbox.utils.transforms import imresize, locsize2bounds
@@ -37,6 +36,7 @@
     AnnotationStoreReader,
     ArrayView,
     DICOMWSIReader,
+    FsspecJsonWSIReader,
     JP2WSIReader,
     NGFFWSIReader,
     OpenSlideWSIReader,
@@ -221,6 +221,43 @@ def read_bounds_level_consistency(wsi: WSIReader, bounds: IntBounds) -> None:
 # Utility Test Classes & Functions
 # -------------------------------------------------------------------------------------
 
+_FSSPEC_WSI_CACHE = {}
+
+
+def fsspec_wsi(sample_svs: Path, tmp_path: Path) -> FsspecJsonWSIReader:
+    """Returns cached FsspecJsonWSIReader instance.
+
+    The reader instance opens CMU-1-Small-Region.svs image.
+
+    It's cached so the reader can be reused,
+
+    since loading the whole image using HTTP range requests from:
+
+    https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs
+
+    takes about 20 seconds.
+
+    """
+    cache_key = "sample_svs"
+
+    if cache_key in _FSSPEC_WSI_CACHE:
+        return _FSSPEC_WSI_CACHE[cache_key]  # Return cached instance
+
+    file_types = ("*.svs",)
+    files_all = utils.misc.grab_files_from_dir(
+        input_path=Path(sample_svs).parent,
+        file_types=file_types,
+    )
+    svs_file_path = str(files_all[0])
+    json_file_path = str(tmp_path / "fsspec.json")
+    final_url = (
+        "https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs"
+    )
+    tiff_to_fsspec.main(svs_file_path, json_file_path, final_url)
+
+    _FSSPEC_WSI_CACHE[cache_key] = wsireader.FsspecJsonWSIReader(json_file_path)
+    return _FSSPEC_WSI_CACHE[cache_key]
+
 
 class DummyMutableOpenSlideObject:
     """Dummy OpenSlide object with mutable properties."""
@@ -2812,3 +2849,133 @@ def test_read_multi_channel(source_image: Path) -> None:
     assert region.shape == (100, 50, (new_img_array.shape[-1]))
     assert np.abs(np.median(region.astype(int) - target.astype(int))) == 0
     assert np.abs(np.mean(region.astype(int) - target.astype(int))) < 0.2
+
+
+def test_fsspec_json_wsi_reader_instantiation() -> None:
+    """Test if FsspecJsonWSIReader is instantiated.
+
+    In case json is passed to  WSIReader.open, FsspecJsonWSIReader
+    should be instantiated.
+    """
+    input_path = "mock_path.json"
+    mpp = None
+    power = None
+
+    with (
+        patch(
+            "tiatoolbox.wsicore.wsireader.FsspecJsonWSIReader.is_valid_zarr_fsspec",
+            return_value=True,
+        ),
+        patch("tiatoolbox.wsicore.wsireader.FsspecJsonWSIReader") as mock_reader,
+    ):
+        WSIReader.open(input_path, mpp, power)
+        mock_reader.assert_called_once_with(input_path, mpp=mpp, power=power)
+
+
+def test_generate_fsspec_json_file_and_validate(
+    sample_svs: Path, tmp_path: Path
+) -> None:
+    """Test generate fsspec json file and validate it."""
+    file_types = ("*.svs",)
+
+    files_all = utils.misc.grab_files_from_dir(
+        input_path=Path(sample_svs).parent,
+        file_types=file_types,
+    )
+
+    svs_file_path = str(files_all[0])
+    json_file_path = str(tmp_path / "fsspec.json")
+    final_url = "https://example.com/some_id"
+
+    tiff_to_fsspec.main(svs_file_path, json_file_path, final_url)
+
+    assert Path(json_file_path).exists(), "Output JSON file was not created."
+
+    assert FsspecJsonWSIReader.is_valid_zarr_fsspec(json_file_path), (
+        "FSSPEC JSON file is invalid."
+    )
+
+
+def test_fsspec_wsireader_info_read(sample_svs: Path, tmp_path: Path) -> None:
+    """Test info read of the FsspecJsonWSIReader.
+
+    Generate fsspec json file and load image from:
+
+    https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+    info = wsi.info
+
+    assert info is not None, "info  should not be None."
+
+
+def test_read_bounds_fsspec_reader_baseline(sample_svs: Path, tmp_path: Path) -> None:
+    """Test FsspecJsonWSIReader read bounds at baseline.
+
+    Location coordinate is in baseline (level 0) reference frame.
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+
+    bounds = SVS_TEST_TISSUE_BOUNDS
+    size = SVS_TEST_TISSUE_SIZE
+    im_region = wsi.read_bounds(bounds, resolution=0, units="level")
+
+    assert isinstance(im_region, np.ndarray)
+    assert im_region.dtype == "uint8"
+    assert im_region.shape == (*size[::-1], 3)
+
+
+def test_read_rect_fsspec_reader_baseline(sample_svs: Path, tmp_path: Path) -> None:
+    """Test FsspecJsonWSIReader read rect at baseline.
+
+    Location coordinate is in baseline (level 0) reference frame.
+
+    """
+    wsi = fsspec_wsi(sample_svs, tmp_path)
+
+    location = SVS_TEST_TISSUE_LOCATION
+    size = SVS_TEST_TISSUE_SIZE
+    im_region = wsi.read_rect(location, size, resolution=0, units="level")
+
+    assert isinstance(im_region, np.ndarray)
+    assert im_region.dtype == "uint8"
+    assert im_region.shape == (*size[::-1], 3)
+
+
+def test_fsspec_reader_open_invalid_json_file(tmp_path: Path) -> None:
+    """Ensure JSONDecodeError is handled properly.
+
+    Pass invalid JSON to  FsspecJsonWSIReader.is_valid_zarr_fsspec.
+    """
+    json_path = tmp_path / "invalid.json"
+    json_path.write_text("{invalid json}")  # Corrupt JSON
+
+    assert not FsspecJsonWSIReader.is_valid_zarr_fsspec(str(json_path))
+
+
+def test_fsspec_reader_open_oserror_handling() -> None:
+    """Ensure OSError is handled properly.
+
+    Pass non existent JSON to  FsspecJsonWSIReader.is_valid_zarr_fsspec.
+
+    """
+    with patch("builtins.open", side_effect=OSError("File not found")):
+        result = FsspecJsonWSIReader.is_valid_zarr_fsspec("non_existent.json")
+
+    assert result is False, "Function should return False for OSError"
+
+
+def test_fsspec_reader_open_pass_empty_json(tmp_path: Path) -> None:
+    """Ensure empty JSON is handled properly.
+
+    Pass empty JSON to FsspecJsonWSIReader.is_valid_zarr_fsspec and
+
+    verify that it's not valid.
+
+    """
+    json_path = tmp_path / "empty.json"
+    json_path.write_text("{}")
+
+    assert not FsspecJsonWSIReader.is_valid_zarr_fsspec(str(json_path))

tiatoolbox/utils/tiff_to_fsspec.py

@@ -0,0 +1,110 @@
+"""Module for processing SVS metadata and generating fsspec zarr JSON file.
+
+The fsspec zarr json file is meant to be used in case SVS or TIFF files
+can be accessed using byte range HTTP API.
+
+The fsspec zarr json file can be opened using FsspecJsonWSIReader.
+
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from tifffile import TiffFile, tiff2fsspec
+
+from tiatoolbox.wsicore.wsireader import TIFFWSIReaderDelegate
+
+# Constants
+EXPECTED_KEY_VALUE_PAIRS = 2
+EXPECTED_ARG_COUNT = 4
+URL_PLACEHOLDER = "https://replace.me/"
+
+
+def convert_metadata(metadata: dict) -> dict:
+    """Convert metadata to JSON-compatible format."""
+    if isinstance(metadata, dict):
+        return {key: convert_metadata(value) for key, value in metadata.items()}
+    if isinstance(metadata, list):
+        return [convert_metadata(item) for item in metadata]
+    if isinstance(metadata, datetime):
+        return metadata.isoformat()  # Convert datetime to ISO 8601 string
+    return metadata
+
+
+def replace_url(
+    data: dict[str, Any], output_path: Path, old_url: str, new_url: str
+) -> None:
+    """Replace URL in the JSON file."""
+    for value in data.values():
+        if isinstance(value, list) and value[0] == old_url:
+            value[0] = new_url
+
+    with output_path.open("w") as json_file:
+        json.dump(data, json_file, indent=2)
+
+
+def main(svs_file_path: str, json_file_path: str, final_url: str) -> None:
+    """Main function to process an SVS file.
+
+    Args:
+        svs_file_path (str): The local file path of the SVS file to be processed.
+        json_file_path (str): The file path where the output JSON will be saved.
+        final_url (str): The URL where the SVS file is stored online
+        and can be accessed via HTTP byte range API.
+
+    Example:
+        main('/path/to/CMU-1-Small-Region.svs', '/path/to/CMU-1-Small-Region.json', 'https://tiatoolbox.dcs.warwick.ac.uk/sample_wsis/CMU-1-Small-Region.svs')
+
+    """
+    url_to_replace = f"{URL_PLACEHOLDER}{Path(svs_file_path).name}"
+
+    tiff = TiffFile(svs_file_path)
+
+    tiff_file_pages = tiff.pages
+
+    # Generate fsspec JSON
+    tiff2fsspec(svs_file_path, url=URL_PLACEHOLDER, out=json_file_path)
+
+    if tiff.is_svs:
+        metadata = TIFFWSIReaderDelegate.parse_svs_metadata(tiff_file_pages)
+    else:  # pragma: no cover
+        metadata = TIFFWSIReaderDelegate.parse_generic_tiff_metadata(tiff_file_pages)
+
+    # Convert metadata to JSON-compatible format
+    metadata_serializable = convert_metadata(metadata)
+
+    # Read the JSON data from the file
+    json_path = Path(json_file_path)
+    with json_path.open() as file:
+        json_data = json.load(file)
+
+    # Decode `.zattrs` JSON string into a dictionary
+    zattrs = json.loads(json_data[".zattrs"])
+
+    # Ensure "multiscales" exists and is a list
+    if "multiscales" not in zattrs or not isinstance(
+        zattrs["multiscales"], list
+    ):  # pragma: no cover
+        zattrs["multiscales"] = [{}]  # Initialize as a list with an empty dictionary
+
+    # Update metadata into `.zattrs`
+    zattrs["multiscales"][0]["metadata"] = metadata_serializable
+
+    # Convert back to a JSON string
+    json_data[".zattrs"] = json.dumps(zattrs)
+
+    # Replace URLs in the JSON file
+    replace_url(json_data, json_path, url_to_replace, final_url)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != EXPECTED_ARG_COUNT:
+        msg = " Usage: python script.py <svs_file_path> <json_file_path> <final_url>"
+        raise ValueError(msg)
+
+    main(sys.argv[1], sys.argv[2], sys.argv[3])