diff --git a/.circleci/config.yml b/.circleci/config.yml index 379790738..ec86e28ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -314,7 +314,7 @@ jobs: python test.py --example-ros3 test-validation: - <<: *py39 + <<: *py38 steps: - checkout - run: git submodule sync @@ -326,7 +326,7 @@ jobs: command: | . ../venv/bin/activate pip install tox - tox -e validation-py39 + tox -e validation-py38 # Install is expected to be quick. Increase timeout in case there are some network issues. # While pip installing tox does not output by default. Circle thinks task is dead after 10 min. no_output_timeout: 30m diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md index 698263510..bf70a7223 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ b/.github/PULL_REQUEST_TEMPLATE/release.md @@ -10,9 +10,9 @@ Prepare for release of PyNWB [version] - [ ] Update `README.rst` as needed - [ ] Update `src/pynwb/nwb-schema` submodule as needed. Check the version number and commit SHA manually - [ ] Update changelog (set release date) in `CHANGELOG.md` and any other docs as needed -- [ ] Run tests locally including gallery tests and validation tests, and inspect all warnings and outputs - (`python test.py -v > out.txt`) -- [ ] Test docs locally (`make apidoc`, `make html`) +- [ ] Run tests locally including gallery, validation, and streaming tests, and inspect all warnings and outputs + (`python test.py -v -p -i -b -w -r > out.txt 2>&1`) +- [ ] Test docs locally (`make clean`, `make html`) - [ ] Push changes to this PR and make sure all PRs to be included in this release have been merged - [ ] Check that the readthedocs build for this PR succeeds (build latest to pull the new branch, then activate and build docs for new branch): https://readthedocs.org/projects/pynwb/builds/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc978431..18e819fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,8 @@ # PyNWB Changelog -## PyNWB 2.1.0 (Upcoming) +## PyNWB 2.1.0 (July 6, 2022) ### Breaking changes: -- A warning is now raised if `SpatialSeries.data` has more than 3 columns. @bendichter, @rly (#1455, #1480) - Updated ``TimeIntervals`` to use the new ``TimeSeriesReferenceVectorData`` type. This does not alter the overall structure of ``TimeIntervals`` in a major way aside from changing the value of the ``neurodata_type`` attribute of the ``TimeIntervals.timeseries`` column from ``VectorData`` to ``TimeSeriesReferenceVectorData``. This change facilitates @@ -16,7 +15,9 @@ behavior of the API is largely consistent with existing behavior. @oruebel, @rly (#1390) # Enhancements and minor changes +- A warning is now raised if `SpatialSeries.data` has more than 3 columns. @bendichter, @rly (#1455, #1480) - The arguments x, y, z, imp, location, filtering are no longer required in the electrodes table. + @h-mayorquin, @rly (#1448) - Added `cell_id` attribute to `IntracellularElectrode`. @bendichter (#1459) - Added `offset` field to `TimeSeries` and its subtypes. @codycbakerphd (#1424) - Added support for NWB 2.5.0. @@ -33,11 +34,11 @@ ### Documentation and tutorial enhancements: - Added tutorial on annotating data via ``TimeIntervals``. @oruebel (#1390) -- Added copy button to code blocks @weiglszonja (#1460) -- Created behavioral tutorial @weiglszonja (#1464) +- Added copy button to code blocks. @weiglszonja (#1460) +- Created behavioral tutorial. @weiglszonja (#1464) - Enhanced display of icephys pandas tutorial by using ``dataframe_image`` to render and display large tables as images. @oruebel (#1469) -- Created tutorial about reading and exploring an existing `NWBFile` @weiglszonja (#1453) +- Created tutorial about reading and exploring an existing `NWBFile`. @weiglszonja (#1453) - Added new logo for PyNWB. @oruebel (#1461) - Minor text fixes. @oruebel @bendichter (#1443, #1462, #1463, #1466, #1472, #1473) diff --git a/docs/gallery/general/plot_timeintervals.py b/docs/gallery/general/plot_timeintervals.py index 2bae57c06..fa59617ef 100644 --- a/docs/gallery/general/plot_timeintervals.py +++ b/docs/gallery/general/plot_timeintervals.py @@ -69,6 +69,8 @@ # tables for :py:meth:`~pynwb.file.NWBFile.epochs`, :py:meth:`~pynwb.file.NWBFile.trials`, and # :py:meth:`~pynwb.file.NWBFile.invalid_times`. # +# .. _basic_trials: +# # Trials # ^^^^^^ # diff --git a/docs/gallery/general/read_basics.py b/docs/gallery/general/read_basics.py index a97178259..4e6dba70d 100644 --- a/docs/gallery/general/read_basics.py +++ b/docs/gallery/general/read_basics.py @@ -183,7 +183,8 @@ # # (200, 400, 300, 3) # -# This :py:class:`~pynwb.image.OpticalSeries` data contains 200 images of size 400x300 pixels with three channels (red, green, and blue). +# This :py:class:`~pynwb.image.OpticalSeries` data contains 200 images of size 400x300 pixels with three channels +# (red, green, and blue). # # Slicing datasets # ---------------- diff --git a/docs/source/conf.py b/docs/source/conf.py index 8340287ce..a2a48d11b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,9 +74,9 @@ } intersphinx_mapping = { - 'python': ('https://docs.python.org/3.8', None), + 'python': ('https://docs.python.org/3.9', None), 'numpy': ('https://numpy.org/doc/stable/', None), - 'matplotlib': ('https://matplotlib.org', None), + 'matplotlib': ('https://matplotlib.org/stable/', None), 'h5py': ('https://docs.h5py.org/en/latest/', None), 'hdmf': ('https://hdmf.readthedocs.io/en/latest/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), diff --git a/environment-ros3.yml b/environment-ros3.yml index d22a1a2f2..d267e5461 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -6,10 +6,10 @@ channels: dependencies: - python=3.9 - h5py==3.6.0 - - hdmf==3.3.1 + - hdmf==3.3.2 - matplotlib==3.5.1 - - numpy==1.21.0 - - pandas==1.3.0 - - python-dateutil==2.8.1 + - numpy==1.21.5 + - pandas==1.3.5 + - python-dateutil==2.8.2 - setuptools - - dandi==0.39.4 + - dandi==0.43.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 27b099d30..88e66e21d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,11 @@ # pinned dependencies to reproduce an entire development environment to use PyNWB, run PyNWB tests, check code style, # compute coverage, and create test environments -codecov==2.1.11 -coverage==5.5 -flake8==3.9.2 -flake8-debugger==4.0.0 -flake8-print==4.0.0 -importlib-metadata==4.6.1 -tox==3.23.1 +codecov==2.1.12 +coverage==6.3.2 +flake8==4.0.1 +flake8-debugger==4.1.2 +flake8-print==5.0.0 +importlib-metadata==4.2.0 +pytest==7.1.2 +pytest-cov==3.0.0 +tox==3.25.0 diff --git a/requirements-doc.txt b/requirements-doc.txt index d03314b3b..2e70bd4c8 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,7 +1,8 @@ -# dependencies to generate the documentation for PyNWB -sphinx +# dependencies to generate the documentation for PyNWB and test run the sphinx gallery examples that do not require +# ROS3 support in h5py. For those examples, use the conda env specified in environment-ros3.yml +sphinx>=4 # improved support for docutils>=0.17 +sphinx_rtd_theme>=1 # <1 does not work with docutils>=0.17 matplotlib -sphinx_rtd_theme sphinx-gallery allensdk>=2.11.0 # python 3.8 is not supported in allensdk<2.11 MarkupSafe==2.0.1 # resolve incompatibility between jinja2 and markupsafe: https://github.com/AllenInstitute/AllenSDK/issues/2308 diff --git a/requirements-min.txt b/requirements-min.txt index 74671d51f..b79f26568 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # minimum versions of package dependencies for installing PyNWB h5py==2.10 # support for selection of datasets with list of indices added in 2.10 -hdmf==3.3.1 +hdmf==3.3.2 numpy==1.16 pandas==1.0.5 python-dateutil==2.7 diff --git a/requirements.txt b/requirements.txt index 2f8f7fd55..75ad8fc6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # pinned dependencies to reproduce an entire development environment to use PyNWB -h5py==3.3.0 -hdmf==3.3.1 -numpy==1.21.0 -pandas==1.3.0 -python-dateutil==2.8.1 -setuptools==57.1.0 +h5py==3.6.0 +hdmf==3.3.2 +numpy==1.21.5 # note that numpy 1.22 dropped python 3.7 support +pandas==1.3.5 # note that pandas 1.4 dropped python 3.7 support +python-dateutil==2.8.2 +setuptools==62.2.0 diff --git a/setup.py b/setup.py index 39e758249..448ada2f9 100755 --- a/setup.py +++ b/setup.py @@ -19,9 +19,9 @@ schema_dir = 'nwb-schema/core' reqs = [ - 'h5py>=2.9,<4', - 'hdmf>=3.1.1,<4', - 'numpy>=1.16,<1.22', + 'h5py>=2.10,<4', + 'hdmf>=3.3.2,<4', + 'numpy>=1.16,<1.23', 'pandas>=1.0.5,<2', 'python-dateutil>=2.7,<3', 'setuptools' diff --git a/src/pynwb/nwb-schema b/src/pynwb/nwb-schema index 79b07bed2..761a0d783 160000 --- a/src/pynwb/nwb-schema +++ b/src/pynwb/nwb-schema @@ -1 +1 @@ -Subproject commit 79b07bed2a402b03c598796866d64edbf8818564 +Subproject commit 761a0d7838304864643f8bc3ab88c93bfd437f2a diff --git a/test.py b/test.py index c3e1fd410..7c1364283 100755 --- a/test.py +++ b/test.py @@ -73,6 +73,7 @@ def _import_from_file(script): def run_example_tests(): + """Run the Sphinx gallery example files, excluding ROS3-dependent ones, to check for errors.""" logging.info('running example tests') examples_scripts = list() for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), "docs", "gallery")): @@ -88,6 +89,7 @@ def run_example_tests(): def run_example_ros3_tests(): + """Run the Sphinx gallery example files that depend on ROS3 to check for errors.""" logging.info('running example ros3 tests') examples_scripts = list() for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), "docs", "gallery")): @@ -149,7 +151,7 @@ def validate_nwbs(): def get_namespaces(nwbfile): comp = run(["python", "-m", "pynwb.validate", "--list-namespaces", "--cached-namespace", nwb], - stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=20) + stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=30) if comp.returncode != 0: return [] @@ -172,7 +174,7 @@ def get_namespaces(nwbfile): for cmd in cmds: logging.info("Validating with \"%s\"." % (" ".join(cmd[:-1]))) - comp = run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=20) + comp = run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=30) TOTAL += 1 if comp.returncode != 0: @@ -273,17 +275,17 @@ def main(): run_test_suite("tests/unit", "pynwb unit tests", verbose=args.verbosity) # Run example tests - if flags['example'] in args.suites: + if flags['example'] in args.suites or flags['validation'] in args.suites: run_example_tests() # Run example tests with ros3 streaming examples + # NOTE this requires h5py to be built with ROS3 support and the dandi package to be installed + # this is most easily done by creating a conda environment using environment-ros3.yml if flags['example-ros3'] in args.suites: run_example_ros3_tests() - # Run validation tests + # Run validation tests on the example NWB files generated above if flags['validation'] in args.suites: - run_example_tests() - run_example_ros3_tests() validate_nwbs() # Run integration tests diff --git a/tox.ini b/tox.ini index 390204ad6..a77ea510a 100644 --- a/tox.ini +++ b/tox.ini @@ -99,7 +99,7 @@ commands = {[testenv:build]commands} deps = null commands = python -c "import hdmf" -# Envs that will execute gallery tests +# Envs that will execute gallery tests that do not require ROS3 [testenv:gallery] install_command = pip install -U {opts} {packages} @@ -110,7 +110,7 @@ deps = commands = pip install -r requirements-doc.txt - # installing allensdk may downgrade certain requirements so + # NOTE: installing allensdk may downgrade certain requirements so # reinstall the repo with its requirements pip install -r requirements-dev.txt -r requirements.txt pip install -U -e . @@ -143,7 +143,7 @@ deps = commands = pip install -r requirements-doc.txt - # installing allensdk may downgrade certain requirements so + # NOTE: installing allensdk may downgrade certain requirements so # reinstall the repo with its requirements pip install -r requirements-dev.txt -r requirements.txt pip install -U -e . @@ -161,7 +161,7 @@ deps = commands = pip install -r requirements-doc.txt - # installing allensdk may downgrade certain requirements so + # NOTE: installing allensdk may downgrade certain requirements so # reinstall the repo with its requirements pip install -r requirements-dev.txt -r requirements.txt pip install -U --pre -e . @@ -175,8 +175,8 @@ deps = -rrequirements-min.txt commands = {[testenv:gallery]commands} -[testenv:validation-py39] -basepython = python3.9 +[testenv:validation-py38] +basepython = python3.8 install_command = pip install -U {opts} {packages} deps =