Test ROS3 streaming in CI and add ROS3 tutorial

.circleci/config.yml

@@ -230,6 +230,34 @@ jobs:
       - TEST_WHEELINSTALL_ENV: "wheelinstall"
     <<: *conda-steps
 
+  miniconda39-ros3:   # tox and conda do not play nicely so do not use tox here
+    <<: *conda-image
+    environment:
+      - CONDA_PYTHON_VER: "3.9"
+      - TEST_TOX_ENV: "ros3-py39"
+      - BUILD_TOX_ENV: "build-py39"
+      - TEST_WHEELINSTALL_ENV: "wheelinstall"
+    steps:
+      - checkout
+      - run: git submodule sync
+      - run: git submodule update --init
+      - run:
+          name: Configure conda
+          command: |
+            conda config --set always_yes yes --set changeps1 no
+            conda config --add channels conda-forge
+            conda install python=$CONDA_PYTHON_VER
+            conda install virtualenv
+            conda install tox
+      - run:
+          name: Run the tests
+          command: |
+            conda env create -f environment-ros3.yml
+            source activate ros3
+            pip install -r requirements-dev.txt
+            pip install -e .
+            python test.py --ros3
+
   gallery37:
     <<: *py37
     environment:
@@ -348,6 +376,8 @@ workflows:
           <<: *no_filters
       - gallery38:  # TODO replace with gallery39 after allensdk support py39
           <<: *no_filters
+      - miniconda39-ros3:
+          <<: *no_filters
       - deploy-dev:
           requires:
             - flake8
@@ -357,6 +387,7 @@ workflows:
             - miniconda39
             - gallery37-min-req
             - gallery38  # gallery39
+            - miniconda39-ros3
           filters:
             tags:
               ignore:
@@ -376,6 +407,7 @@ workflows:
             - miniconda39
             - gallery37-min-req
             - gallery38  # gallery39
+            - miniconda39-ros3
           filters:
             tags:
               only: /^[0-9]+(\.[0-9]+)*(\.post[0-9]+)?$/
@@ -424,3 +456,7 @@ workflows:
           <<: *no_filters
       - gallery39-upgrade-dev-pre:
           <<: *no_filters
+      - miniconda39-ros3:
+          <<: *no_filters
+      - test-validation:
+          <<: *no_filters

CHANGELOG.md

@@ -1,6 +1,6 @@
 # PyNWB Changelog
 
-## PyNWB 2.0.0 (Upcoming)
+## PyNWB 2.0.0 (August 13, 2021)
 
 ### Breaking changes:
 - ``SweepTable`` has been deprecated in favor of the new icephys metadata tables. Use of ``SweepTable``
@@ -23,6 +23,7 @@
   for more details. @oruebel, @rly (#1349)
 - Dropped Python 3.6 support, added Python 3.9 support. @rly (#1377)
 - Updated requirements to allow compatibility with HDMF 3 and h5py 3. @rly (#1377)
+  - When using HDMF 3 and h5py 3, users can now stream NWB files from an S3 bucket.
 
 ### Tutorial enhancements:
 - Added new tutorial for intracellular electrophysiology to describe the use of the new metadata tables
@@ -33,12 +34,15 @@
 - Used `sphinx.ext.extlinks` extension in docs to simplify linking to common targets. @oruebel (#1349)
 - Created new section for advanced I/O tutorials and moved parallel I/O tutorial to its own file. @oruebel (#1349)
 - Updated the optical physiology / Calcium imaging tutorial. @bendichter, @weiglszonja (#1375)
+- Added a tutorial on streaming using the ROS3 driver. @rly (#1393)
 
 ### Minor new features:
-- Add RRID for citing PyNWB to the docs. @oruebel (#1372)
-- Update CI and tests to handle deprecations in libraries. @rly (#1377)
-- Add test utilities for icephys (``pynwb.testing.icephys_testutils``) to ease creation of test data
+- Added RRID for citing PyNWB to the docs. @oruebel (#1372)
+- Updated CI and tests to handle deprecations in libraries. @rly (#1377)
+- Added test utilities for icephys (``pynwb.testing.icephys_testutils``) to ease creation of test data
   for tests and tutorials. @oruebel (#1349, #1383)
+- Added on-push and nightly tests of streaming using the ROS3 driver. @rly (#1393)
+  - These tests make use of a new dandiset for testing the API: https://gui.dandiarchive.org/#/dandiset/000126
 
 ### Bug fixes:
 - Updated behavior of ``make clean`` command for docs to ensure tutorial files are cleaned up.  @oruebel (#1349)

azure-pipelines-nightly.yml

@@ -129,3 +129,71 @@ jobs:
   - bash: |
       tox -e $(testWheelInstallEnv) --recreate --installpkg dist/*-none-any.whl
     displayName: 'Run wheel install tests'
+
+
+- job: 'Test_ROS3_Windows'
+  displayName: "Test PyNWB with Conda and ROS3 on Windows"
+
+  pool:
+    vmImage: 'vs2017-win2016'
+
+  steps:
+
+  - checkout: self
+    submodules: true
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.9'
+      architecture: 'x64'
+
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: 'Add conda to PATH'
+
+  - powershell: conda env create -f environment-ros3.yml
+    displayName: 'Create conda environment'
+
+  - script: |
+      call activate ros3
+      pip install -r requirements-dev.txt
+      pip install -e .
+    displayName: 'Install PyNWB and dependencies'
+
+  - script: |
+      call activate ros3
+      python test.py --ros3
+    displayName: 'Run ROS3 tests'
+
+
+- job: 'Test_ROS3_macOS'
+  displayName: "Test PyNWB with Conda and ROS3 on macOS"
+
+  pool:
+    vmImage: 'macos-10.15'
+
+  steps:
+
+  - checkout: self
+    submodules: true
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.9'
+      architecture: 'x64'
+
+  - bash: |
+      echo "##vso[task.prependpath]$CONDA/bin"
+      sudo chown -R $USER $CONDA  # Take ownership of conda installation
+    displayName: 'Add conda to PATH'
+
+  - bash: |
+      conda env create -f environment-ros3.yml
+      source activate ros3
+      pip install -r requirements-dev.txt
+      pip install -e .
+    displayName: 'Create conda environment and install PyNWB and dependencies'
+
+  - bash: |
+      source activate ros3
+      python test.py --ros3
+    displayName: 'Run ROS3 tests'

docs/gallery/advanced_io/streaming.py

@@ -0,0 +1,55 @@
+'''
+Streaming from an S3 Bucket
+===========================
+
+Using PyNWB 2, HDMF 3, and h5py 3.2+, you can now stream data from an NWB file stored in an S3 bucket, such as data
+from the `DANDI Archive <https://dandiarchive.org/>`_. This is especially useful for reading small pieces of data
+from a large NWB file stored remotely.
+'''
+
+####################
+# Streaming data from an S3 bucket requires having HDF5 installed with the ROS3 (read-only S3) driver.
+# You can install HDF5 with the ROS3 driver from `conda-forge <https://conda-forge.org/>`_ using ``conda``.
+# You may first need to uninstall a currently installed version of h5py.
+#
+# .. code-block:: bash
+#
+#   pip uninstall h5py
+#   conda install -c conda-forge "h5py>=3.2"
+#
+
+####################
+# Next, use the ``DandiAPIClient`` to get the S3 URL to an NWB file of interest stored in the DANDI Archive.
+# If you have not already, install the latest release of the ``dandi`` package.
+#
+# .. code-block:: bash
+#
+#   pip install dandi
+#
+# .. code-block:: python
+#
+#   from dandi.dandiapi import DandiAPIClient
+#
+#   dandiset_id = '000006'  # ephys dataset from the Svoboda Lab
+#   filepath = 'sub-anm372795/sub-anm372795_ses-20170718.nwb'  # 450 kB file
+#   with DandiAPIClient() as client:
+#       asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath)
+#       s3_path = asset.get_content_url(follow_redirects=1, strip_query=True)
+
+####################
+# Finally, instantiate a :py:class:`~pynwb.NWBHDF5IO` object with the S3 URL and specify the driver as "ros3". This
+# will download metadata about the file from the S3 bucket to memory. The values of datasets are accessed lazily,
+# just like when reading an NWB file stored locally. So, slicing into a dataset will require additional time to
+# download the sliced data (and only the sliced data) to memory.
+#
+# .. code-block:: python
+#
+#   from pynwb import NWBHDF5IO
+#
+#   with NWBHDF5IO(s3_path, mode='r', load_namespaces=True, driver='ros3') as io:
+#       nwbfile = io.read()
+#       print(nwbfile)
+#       print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:])
+#
+
+# sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_streaming.png'

environment-ros3.yml

@@ -0,0 +1,13 @@
+# pinned dependencies to reproduce an entire development environment to use PyNWB with ROS3 support
+name: ros3
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.9
+  - h5py==3.3.0
+  - hdmf==3.1.1
+  - numpy==1.21.0
+  - pandas==1.3.0
+  - python-dateutil==2.8.1
+  - setuptools

test.py

@@ -12,7 +12,7 @@
 import traceback
 import unittest
 
-flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5, 'validation': 6}
+flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5, 'validation': 6, 'ros3': 7}
 
 TOTAL = 0
 FAILURES = 0
@@ -47,7 +47,9 @@ def run_test_suite(directory, description="", verbose=True):
     logging.info("running %s" % description)
     directory = os.path.join(os.path.dirname(__file__), directory)
     runner = unittest.TextTestRunner(verbosity=verbose, resultclass=SuccessRecordingResult)
-    test_result = runner.run(unittest.TestLoader().discover(directory))
+    # set top_level_dir below to prevent import name clashes between
+    # tests/unit/test_base.py and tests/integration/hdf5/test_base.py
+    test_result = runner.run(unittest.TestLoader().discover(directory, top_level_dir='tests'))
 
     TOTAL += test_result.testsRun
     FAILURES += len(test_result.failures)
@@ -161,7 +163,7 @@ def get_namespaces(nwbfile):
 
 
 def run_integration_tests(verbose=True):
-    pynwb_test_result = run_test_suite("tests/integration", "integration tests", verbose=verbose)
+    pynwb_test_result = run_test_suite("tests/integration/hdf5", "integration tests", verbose=verbose)
     test_cases = pynwb_test_result.get_all_cases_run()
 
     import pynwb
@@ -208,11 +210,14 @@ def main():
                         help='run backwards compatibility tests')
     parser.add_argument('-w', '--validation', action='append_const', const=flags['validation'], dest='suites',
                         help='run validation tests')
+    parser.add_argument('-r', '--ros3', action='append_const', const=flags['ros3'], dest='suites',
+                        help='run ros3 streaming tests')
     args = parser.parse_args()
     if not args.suites:
         args.suites = list(flags.values())
         args.suites.pop(args.suites.index(flags['example']))  # remove example as a suite run by default
         args.suites.pop(args.suites.index(flags['validation']))  # remove validation as a suite run by default
+        args.suites.pop(args.suites.index(flags['ros3']))  # remove ros3 as a suite run by default (different reqs)
 
     # set up logger
     root = logging.getLogger()
@@ -251,6 +256,10 @@ def main():
     if flags['backwards'] in args.suites:
         run_test_suite("tests/back_compat", "pynwb backwards compatibility tests", verbose=args.verbosity)
 
+    # Run ros3 streaming tests
+    if flags['ros3'] in args.suites:
+        run_test_suite("tests/integration/ros3", "pynwb ros3 streaming tests", verbose=args.verbosity)
+
     final_message = 'Ran %s tests' % TOTAL
     exitcode = 0
     if ERRORS > 0 or FAILURES > 0:

tests/integration/ros3/test_ros3.py

@@ -0,0 +1,23 @@
+from pynwb import NWBHDF5IO
+from pynwb.testing import TestCase
+
+
+class TestRos3Streaming(TestCase):
+    # requires h5py to be built with the ROS3 driver: conda install -c conda-forge h5py
+
+    def test_read(self):
+        s3_path = 'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'
+
+        with NWBHDF5IO(s3_path, mode='r', load_namespaces=True, driver='ros3') as io:
+            nwbfile = io.read()
+            test_data = nwbfile.acquisition['ts_name'].data[:]
+            self.assertEqual(len(test_data), 3)
+
+    def test_dandi_read(self):
+        # this is the NWB Test Data dandiset #000126 sub-1/sub-1.nwb
+        s3_path = 'https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991'
+
+        with NWBHDF5IO(s3_path, mode='r', load_namespaces=True, driver='ros3') as io:
+            nwbfile = io.read()
+            test_data = nwbfile.acquisition['TestData'].data[:]
+            self.assertEqual(len(test_data), 3)