diff --git a/.github/workflows/generate_test_files.yml b/.github/workflows/generate_test_files.yml index 5c3648c0f..49bc31b14 100644 --- a/.github/workflows/generate_test_files.yml +++ b/.github/workflows/generate_test_files.yml @@ -1,6 +1,7 @@ name: Generate test files on: workflow_dispatch: + # pull_request: jobs: gen-test-files: diff --git a/CHANGELOG.md b/CHANGELOG.md index bd61d96e1..b579f7076 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ # PyNWB Changelog -## PyNWB 2.2.0 (October 11, 2022) +## PyNWB 2.2.0 (October 19, 2022) -### Internal enhancements: +### Enhancements and minor changes +- Enhanced `pynwb.validate` API function to accept a list of file paths as well as the ability to operate on cached + namespaces. Also adjusted the validate CLI to directly use the API function. @CodyCBakerPhD + [#1511](https://github.com/NeurodataWithoutBorders/pynwb/pull/1511) + +### Internal enhancements - Moved CI to GitHub Actions. @rly [#1560](https://github.com/NeurodataWithoutBorders/pynwb/pull/1560), [#1566](https://github.com/NeurodataWithoutBorders/pynwb/pull/1566) diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 19611f776..6cf7d7656 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -11,15 +11,14 @@ from hdmf.utils import docval, getargs, popargs, get_docval from hdmf.backends.io import HDMFIO from hdmf.backends.hdf5 import HDF5IO as _HDF5IO -from hdmf.validate import ValidatorMap from hdmf.build import BuildManager, TypeMap import hdmf.common - CORE_NAMESPACE = 'core' __core_ns_file_name = 'nwb.namespace.yaml' from .spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace # noqa E402 +from .validate import validate # noqa: F401, E402 def __get_resources(): @@ -186,18 +185,6 @@ def get_sum(self, a, b): return __TYPE_MAP.get_dt_container_cls(neurodata_type, namespace) -@docval({'name': 'io', 'type': HDMFIO, 'doc': 'the HDMFIO object to read from'}, - {'name': 'namespace', 'type': str, 'doc': 'the namespace to validate against', 'default': CORE_NAMESPACE}, - returns="errors in the file", rtype=list, - is_method=False) -def validate(**kwargs): - """Validate an NWB file against a namespace""" - io, namespace = getargs('io', 'namespace', kwargs) - builder = io.read_builder() - validator = ValidatorMap(io.manager.namespace_catalog.get_namespace(name=namespace)) - return validator.validate(builder) - - class NWBHDF5IO(_HDF5IO): @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, diff --git a/src/pynwb/testing/make_test_files.py b/src/pynwb/testing/make_test_files.py index 052b8501e..301381688 100644 --- a/src/pynwb/testing/make_test_files.py +++ b/src/pynwb/testing/make_test_files.py @@ -1,9 +1,10 @@ +from datetime import datetime import numpy as np from pathlib import Path - -from datetime import datetime -from pynwb import NWBFile, NWBHDF5IO, __version__, TimeSeries +from pynwb import NWBFile, NWBHDF5IO, __version__, TimeSeries, get_class, load_namespaces from pynwb.image import ImageSeries +from pynwb.spec import NWBNamespaceBuilder, export_spec, NWBGroupSpec, NWBAttributeSpec + # pynwb 1.0.2 should be installed with hdmf 1.0.3 # pynwb 1.0.3 should be installed with hdmf 1.0.5 @@ -149,12 +150,61 @@ def _make_imageseries_nonmatch_starting_frame(): _write(test_name, nwbfile) +def _make_empty_with_extension(): + ns_builder = NWBNamespaceBuilder( + doc="An NWB test extension", + name="ndx-testextension", + version="0.1.0", + author="PyNWB Test File Generator", + contact="my_email@example.com", + ) + + ns_builder.include_type('TimeSeries', namespace='core') + tetrode_series = NWBGroupSpec( + neurodata_type_def='TimeSeriesWithID', + neurodata_type_inc='TimeSeries', + doc=('An extension of TimeSeries to include an ID.'), + attributes=[ + NWBAttributeSpec( + name='id', + doc='The time series ID.', + dtype='int32' + ), + ], + ) + + new_data_types = [tetrode_series] + + # export the spec to yaml files in the current directory + export_spec(ns_builder, new_data_types, output_dir=".") + + nwbfile = NWBFile(session_description='ADDME', + identifier='ADDME', + session_start_time=datetime.now().astimezone()) + + load_namespaces("ndx-testextension.namespace.yaml") # load from the current directory + TimeSeriesWithID = get_class("TimeSeriesWithID", "ndx-testextension") + ts = TimeSeriesWithID( + name="test_ts", + data=[1., 2., 3.], + description="ADDME", + unit="ADDME", + rate=1., + id=1, + ) + nwbfile.add_acquisition(ts) + test_name = 'nwbfile_with_extension' + _write(test_name, nwbfile) + + if __name__ == '__main__': # install these versions of PyNWB and run this script to generate new files # python src/pynwb/testing/make_test_files.py # files will be made in src/pynwb/testing/ # files should be moved to tests/back_compat/ + # NOTE: this script is run in the GitHub Actions workflow generate_test_files.yml + if __version__ == '1.1.2': _make_empty() _make_str_experimenter() @@ -170,3 +220,4 @@ def _make_imageseries_nonmatch_starting_frame(): _make_imageseries_no_data() _make_imageseries_non_external_format() _make_imageseries_nonmatch_starting_frame() + _make_empty_with_extension() diff --git a/src/pynwb/validate.py b/src/pynwb/validate.py index a6ad3e546..a5a313481 100644 --- a/src/pynwb/validate.py +++ b/src/pynwb/validate.py @@ -1,137 +1,228 @@ -"""Command line tool to Validate an NWB file against a namespace""" -import os +"""Command line tool to Validate an NWB file against a namespace.""" import sys from argparse import ArgumentParser +from typing import Tuple, List, Dict from hdmf.spec import NamespaceCatalog from hdmf.build import BuildManager from hdmf.build import TypeMap as TypeMap +from hdmf.utils import docval, getargs +from hdmf.backends.io import HDMFIO +from hdmf.validate import ValidatorMap -from pynwb import validate, CORE_NAMESPACE, NWBHDF5IO +from pynwb import CORE_NAMESPACE from pynwb.spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace -def _print_errors(validation_errors): +def _print_errors(validation_errors: list): if validation_errors: - print(' - found the following errors:', file=sys.stderr) + print(" - found the following errors:", file=sys.stderr) for err in validation_errors: print(str(err), file=sys.stderr) else: - print(' - no errors found.') + print(" - no errors found.") -def _validate_helper(**kwargs): - errors = validate(**kwargs) - _print_errors(errors) +def _validate_helper(io: HDMFIO, namespace: str = CORE_NAMESPACE) -> list: + builder = io.read_builder() + validator = ValidatorMap(io.manager.namespace_catalog.get_namespace(name=namespace)) + return validator.validate(builder) - return (errors is not None and len(errors) > 0) - -def main(): # noqa: C901 - - ep = """ - If --ns is not specified, validate against all namespaces in the NWB file. +def _get_cached_namespaces_to_validate(path: str) -> Tuple[List[str], BuildManager, Dict[str, str]]: """ + Determine the most specific namespace(s) that are cached in the given NWBFile that can be used for validation. + + Example + ------- + The following example illustrates how we can use this function to validate against namespaces + cached in a file. This is useful, e.g., when a file was created using an extension + >>> from pynwb import validate + >>> from pynwb.validate import _get_cached_namespaces_to_validate + >>> path = "my_nwb_file.nwb" + >>> validate_namespaces, manager, cached_namespaces = _get_cached_namespaces_to_validate(path) + >>> with NWBHDF5IO(path, "r", manager=manager) as reader: + >>> errors = [] + >>> for ns in validate_namespaces: + >>> errors += validate(io=reader, namespace=ns) + :param path: Path for the NWB file + :return: Tuple with: + - List of strings with the most specific namespace(s) to use for validation. + - BuildManager object for opening the file for validation + - Dict with the full result from NWBHDF5IO.load_namespaces + """ + from . import NWBHDF5IO # TODO: modularize to avoid circular import - parser = ArgumentParser(description="Validate an NWB file", epilog=ep) - parser.add_argument("paths", type=str, nargs='+', help="NWB file paths") - # parser.add_argument('-p', '--nspath', type=str, help="the path to the namespace YAML file") - parser.add_argument("-n", "--ns", type=str, help="the namespace to validate against") - parser.add_argument("-lns", "--list-namespaces", dest="list_namespaces", - action='store_true', help="List the available namespaces and exit.") + catalog = NamespaceCatalog( + group_spec_cls=NWBGroupSpec, dataset_spec_cls=NWBDatasetSpec, spec_namespace_cls=NWBNamespace + ) + namespace_dependencies = NWBHDF5IO.load_namespaces(namespace_catalog=catalog, path=path) - feature_parser = parser.add_mutually_exclusive_group(required=False) - feature_parser.add_argument("--cached-namespace", dest="cached_namespace", action='store_true', - help="Use the cached namespace (default).") - feature_parser.add_argument('--no-cached-namespace', dest="cached_namespace", action='store_false', - help="Don't use the cached namespace.") - parser.set_defaults(cached_namespace=True) + # Determine which namespaces are the most specific (i.e. extensions) and validate against those + candidate_namespaces = set(namespace_dependencies.keys()) + for namespace_dependency in namespace_dependencies: + candidate_namespaces -= namespace_dependencies[namespace_dependency].keys() - args = parser.parse_args() - ret = 0 - - # TODO Validation against a specific namespace file is currently broken. See pynwb#1396 - # if args.nspath: - # if not os.path.isfile(args.nspath): - # print("The namespace file {} is not a valid file.".format(args.nspath), file=sys.stderr) - # sys.exit(1) - # - # if args.cached_namespace: - # print("Turning off validation against cached namespace information " - # "as --nspath was passed.", file=sys.stderr) - # args.cached_namespace = False - - for path in args.paths: - - if not os.path.isfile(path): - print("The file {} does not exist.".format(path), file=sys.stderr) - ret = 1 - continue + # TODO: remove this workaround for issue https://github.com/NeurodataWithoutBorders/pynwb/issues/1357 + candidate_namespaces.discard("hdmf-experimental") # remove validation of hdmf-experimental for now + cached_namespaces = sorted(candidate_namespaces) - if args.cached_namespace: - catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) - ns_deps = NWBHDF5IO.load_namespaces(catalog, path) - s = set(ns_deps.keys()) # determine which namespaces are the most - for k in ns_deps: # specific (i.e. extensions) and validate - s -= ns_deps[k].keys() # against those - # TODO remove this workaround for issue https://github.com/NeurodataWithoutBorders/pynwb/issues/1357 - if 'hdmf-experimental' in s: - s.remove('hdmf-experimental') # remove validation of hdmf-experimental for now - namespaces = list(sorted(s)) - if len(namespaces) > 0: - tm = TypeMap(catalog) - manager = BuildManager(tm) - specloc = "cached namespace information" + if len(cached_namespaces) > 0: + type_map = TypeMap(namespaces=catalog) + manager = BuildManager(type_map=type_map) + else: + manager = None + + return cached_namespaces, manager, namespace_dependencies + + +@docval( + { + "name": "io", + "type": HDMFIO, + "doc": "An open IO to an NWB file.", + "default": None, + }, # For back-compatability + { + "name": "namespace", + "type": str, + "doc": "A specific namespace to validate against.", + "default": None, + }, # Argument order is for back-compatability + { + "name": "paths", + "type": list, + "doc": "List of NWB file paths.", + "default": None, + }, + { + "name": "use_cached_namespaces", + "type": bool, + "doc": "Whether to use namespaces cached within the file for validation.", + "default": True, + }, + { + "name": "verbose", + "type": bool, + "doc": "Whether or not to print messages to stdout.", + "default": False, + }, + returns="Validation errors in the file.", + rtype=(list, (list, bool)), + is_method=False, +) +def validate(**kwargs): + """Validate NWB file(s) against a namespace or its cached namespaces.""" + from . import NWBHDF5IO # TODO: modularize to avoid circular import + + io, paths, use_cached_namespaces, namespace, verbose = getargs( + "io", "paths", "use_cached_namespaces", "namespace", "verbose", kwargs + ) + assert io != paths, "Both 'io' and 'paths' were specified! Please choose only one." + + if io is not None: + validation_errors = _validate_helper(io=io, namespace=namespace or CORE_NAMESPACE) + return validation_errors + + status = 0 + validation_errors = list() + for path in paths: + namespaces_to_validate = [] + namespace_message = "PyNWB namespace information" + io_kwargs = dict(path=path, mode="r") + + if use_cached_namespaces: + cached_namespaces, manager, namespace_dependencies = _get_cached_namespaces_to_validate(path=path) + io_kwargs.update(manager=manager) + + if any(cached_namespaces): + namespaces_to_validate = cached_namespaces + namespace_message = "cached namespace information" else: - manager = None - namespaces = [CORE_NAMESPACE] - specloc = "pynwb namespace information" - print("The file {} has no cached namespace information. " - "Falling back to {}.".format(path, specloc), file=sys.stderr) - # elif args.nspath: - # catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) - # namespaces = catalog.load_namespaces(args.nspath) - # - # if len(namespaces) == 0: - # print("Could not load namespaces from file {}.".format(args.nspath), file=sys.stderr) - # sys.exit(1) - # - # tm = TypeMap(catalog) - # manager = BuildManager(tm) - # specloc = "--nspath namespace information" + namespaces_to_validate = [CORE_NAMESPACE] + if verbose: + print( + f"The file {path} has no cached namespace information. Falling back to {namespace_message}.", + file=sys.stderr, + ) else: - manager = None - namespaces = [CORE_NAMESPACE] - specloc = "pynwb namespace information" - - if args.list_namespaces: - print("\n".join(namespaces)) - ret = 0 + namespaces_to_validate = [CORE_NAMESPACE] + + if namespace is not None: + if namespace in namespaces_to_validate: + namespaces_to_validate = [namespace] + elif use_cached_namespaces and namespace in namespace_dependencies: # validating against a dependency + for namespace_dependency in namespace_dependencies: + if namespace in namespace_dependencies[namespace_dependency]: + status = 1 + print( + f"The namespace '{namespace}' is included by the namespace " + f"'{namespace_dependency}'. Please validate against that namespace instead.", + file=sys.stderr, + ) + else: + status = 1 + print( + f"The namespace '{namespace}' could not be found in {namespace_message} as only " + f"{namespaces_to_validate} is present.", + file=sys.stderr, + ) + + if status == 1: continue - if args.ns: - if args.ns in namespaces: - namespaces = [args.ns] - elif args.cached_namespace and args.ns in ns_deps: # validating against a dependency - for k in ns_deps: - if args.ns in ns_deps[k]: - print(("The namespace '{}' is included by the namespace '{}'. Please validate against " - "that namespace instead.").format(args.ns, k), file=sys.stderr) - ret = 1 - continue - else: - print("The namespace '{}' could not be found in {} as only {} is present.".format( - args.ns, specloc, namespaces), file=sys.stderr) - ret = 1 - continue + with NWBHDF5IO(**io_kwargs) as io: + for validation_namespace in namespaces_to_validate: + if verbose: + print(f"Validating {path} against {namespace_message} using namespace '{validation_namespace}'.") + validation_errors += _validate_helper(io=io, namespace=validation_namespace) + return validation_errors, status + + +def validate_cli(): + """CLI wrapper around pynwb.validate.""" + parser = ArgumentParser( + description="Validate an NWB file", + epilog="If --ns is not specified, validate against all namespaces in the NWB file.", + ) + + # Special arg specific to CLI + parser.add_argument( + "-lns", + "--list-namespaces", + dest="list_namespaces", + action="store_true", + help="List the available namespaces and exit.", + ) + + # Common args to the API validate + parser.add_argument("paths", type=str, nargs="+", help="NWB file paths") + parser.add_argument("-n", "--ns", type=str, help="the namespace to validate against") + feature_parser = parser.add_mutually_exclusive_group(required=False) + feature_parser.add_argument( + "--no-cached-namespace", + dest="no_cached_namespace", + action="store_true", + help="Use the PyNWB loaded namespace (true) or use the cached namespace (false; default).", + ) + parser.set_defaults(no_cached_namespace=False) + args = parser.parse_args() + status = 0 - with NWBHDF5IO(path, mode='r', manager=manager) as io: - for ns in namespaces: - print("Validating {} against {} using namespace '{}'.".format(path, specloc, ns)) - ret = ret or _validate_helper(io=io, namespace=ns) + if args.list_namespaces: + for path in args.paths: + cached_namespaces, _, _ = _get_cached_namespaces_to_validate(path=path) + print("\n".join(cached_namespaces)) + else: + validation_errors, validation_status = validate( + paths=args.paths, use_cached_namespaces=not args.no_cached_namespace, namespace=args.ns, verbose=True + ) + if not validation_status: + _print_errors(validation_errors=validation_errors) + status = status or validation_status or (validation_errors is not None and len(validation_errors) > 0) - sys.exit(ret) + sys.exit(status) -if __name__ == '__main__': # pragma: no cover - main() +if __name__ == "__main__": # pragma: no cover + validate_cli() diff --git a/test.py b/test.py index f218f49be..401a75e5c 100755 --- a/test.py +++ b/test.py @@ -228,7 +228,7 @@ def run_integration_tests(verbose=True): logging.info('all classes have integration tests') # also test the validation script - run_test_suite("tests/validation", "validation CLI tests", verbose=verbose) + run_test_suite("tests/validation", "validation tests", verbose=verbose) def clean_up_tests(): @@ -293,7 +293,7 @@ def main(): parser.add_argument('-b', '--backwards', action='append_const', const=flags['backwards'], dest='suites', help='run backwards compatibility tests') parser.add_argument('-w', '--validation', action='append_const', const=flags['validation'], dest='suites', - help='run validation tests') + help='run example tests and validation tests on example NWB files') parser.add_argument('-r', '--ros3', action='append_const', const=flags['ros3'], dest='suites', help='run ros3 streaming tests') args = parser.parse_args() diff --git a/tests/back_compat/2.1.0_nwbfile_with_extension.nwb b/tests/back_compat/2.1.0_nwbfile_with_extension.nwb new file mode 100644 index 000000000..4471a57e6 Binary files /dev/null and b/tests/back_compat/2.1.0_nwbfile_with_extension.nwb differ diff --git a/tests/back_compat/test_import_structure.py b/tests/back_compat/test_import_structure.py new file mode 100644 index 000000000..dba11a48a --- /dev/null +++ b/tests/back_compat/test_import_structure.py @@ -0,0 +1,89 @@ +from unittest import TestCase + +import pynwb + + +class TestImportStructure(TestCase): + """Test whether the classes/modules imported from pynwb in version 2.1.1 are still accessible. + + NOTE: this test was needed to ensure backward compatibility of "import pynwb" after changes to the package file + hierarchy in PyNWB 2.2.0 around validate.py (see https://github.com/NeurodataWithoutBorders/pynwb/pull/1511). + """ + def test_outer_import_structure(self): + current_structure = dir(pynwb) + expected_structure = [ + "BuildManager", + "CORE_NAMESPACE", + "DataChunkIterator", + "H5DataIO", + "HDMFIO", + "NWBContainer", + "NWBData", + "NWBDatasetSpec", + "NWBFile", + "NWBGroupSpec", + "NWBHDF5IO", + "NWBNamespace", + "NamespaceCatalog", + "Path", + "ProcessingModule", + "TimeSeries", + "TypeMap", + "_HDF5IO", + "__NS_CATALOG", + "__TYPE_MAP", + "__builtins__", + "__cached__", + "__core_ns_file_name", + "__doc__", + "__file__", + "__get_resources", + "__io", + "__loader__", + "__name__", + "__package__", + "__path__", + "__resources", + "__spec__", + "__version__", + "_due", + "_get_resources", + "_version", + "available_namespaces", + "base", + "behavior", + "core", + "deepcopy", + "device", + "docval", + "ecephys", + "epoch", + "file", + "get_class", + "get_docval", + "get_manager", + "get_type_map", + "getargs", + "h5py", + "hdmf", + "hdmf_typemap", + "icephys", + "image", + "io", + "legacy", + "load_namespaces", + "misc", + "ogen", + "ophys", + "os", + "popargs", + "register_class", + "register_map", + "retinotopy", + "spec", + "testing", + "validate", + "warn", + ] + for member in expected_structure: + self.assertIn(member=member, container=current_structure) diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index 568a53c81..0d3b0d7c9 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -42,7 +42,7 @@ def test_read(self): for f in nwb_files: with self.subTest(file=f.name): with warnings.catch_warnings(record=True) as warnings_on_read: - with NWBHDF5IO(str(f), 'r') as io: + with NWBHDF5IO(str(f), 'r', load_namespaces=True) as io: errors = validate(io) io.read() for w in warnings_on_read: diff --git a/tests/validation/test_validate.py b/tests/validation/test_validate.py index 327034e59..4cf9bea33 100644 --- a/tests/validation/test_validate.py +++ b/tests/validation/test_validate.py @@ -1,11 +1,13 @@ import subprocess import re +from unittest.mock import patch +from io import StringIO from pynwb.testing import TestCase from pynwb import validate, NWBHDF5IO -class TestValidateScript(TestCase): +class TestValidateCLI(TestCase): # 1.0.2_nwbfile.nwb has no cached specifications # 1.0.3_nwbfile.nwb has cached "core" specification @@ -18,6 +20,8 @@ class TestValidateScript(TestCase): # simplify collecting and merging coverage data from multiple subprocesses. if "-p" # is not used, then each "coverage run" will overwrite the .coverage file from a # previous "coverage run". + # NOTE the run_coverage.yml GitHub Action runs "python -m coverage combine" to + # combine the individual coverage reprots into one .coverage file. def test_validate_file_no_cache(self): """Test that validating a file with no cached spec against the core namespace succeeds.""" @@ -28,12 +32,12 @@ def test_validate_file_no_cache(self): r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*" r"warnings.warn\(msg\)\s*" r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " - r"Falling back to pynwb namespace information\.\s*" + r"Falling back to PyNWB namespace information\.\s*" ) self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) stdout_regex = re.compile( - r"Validating tests/back_compat/1\.0\.2_nwbfile\.nwb against pynwb namespace information using namespace " + r"Validating tests/back_compat/1\.0\.2_nwbfile\.nwb against PyNWB namespace information using namespace " r"'core'\.\s* - no errors found\.\s*") self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) @@ -46,8 +50,8 @@ def test_validate_file_no_cache_bad_ns(self): r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*" r"warnings.warn\(msg\)\s*" r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " - r"Falling back to pynwb namespace information\.\s*" - r"The namespace 'notfound' could not be found in pynwb namespace information as only " + r"Falling back to PyNWB namespace information\.\s*" + r"The namespace 'notfound' could not be found in PyNWB namespace information as only " r"\['core'\] is present\.\s*" ) self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) @@ -79,6 +83,43 @@ def test_validate_file_cached_bad_ns(self): self.assertEqual(result.stdout.decode('utf-8'), '') + def test_validate_file_cached_extension(self): + """Test that validating a file with cached spec against the cached namespaces succeeds.""" + result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", + "tests/back_compat/2.1.0_nwbfile_with_extension.nwb"], capture_output=True) + + self.assertEqual(result.stderr.decode('utf-8'), '') + + stdout_regex = re.compile( + r"Validating tests/back_compat/2\.1\.0_nwbfile_with_extension\.nwb against cached namespace information " + r"using namespace 'ndx-testextension'\.\s* - no errors found\.\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_cached_extension_pass_ns(self): + """Test that validating a file with cached spec against the extension namespace succeeds.""" + result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", + "tests/back_compat/2.1.0_nwbfile_with_extension.nwb", + "--ns", "ndx-testextension"], capture_output=True) + + self.assertEqual(result.stderr.decode('utf-8'), '') + + stdout_regex = re.compile( + r"Validating tests/back_compat/2\.1\.0_nwbfile_with_extension\.nwb against cached namespace information " + r"using namespace 'ndx-testextension'\.\s* - no errors found\.\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_cached_core(self): + """Test that validating a file with cached spec against the core namespace succeeds.""" + result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", + "tests/back_compat/2.1.0_nwbfile_with_extension.nwb", + "--ns", "core"], capture_output=True) + + stdout_regex = re.compile( + r"The namespace 'core' is included by the namespace 'ndx-testextension'. " + r"Please validate against that namespace instead\.\s*" + ) + self.assertRegex(result.stderr.decode('utf-8'), stdout_regex) + def test_validate_file_cached_hdmf_common(self): """Test that validating a file with cached spec against the hdmf-common namespace fails.""" result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb", @@ -98,42 +139,167 @@ def test_validate_file_cached_ignore(self): self.assertEqual(result.stderr.decode('utf-8'), '') stdout_regex = re.compile( - r"Validating tests/back_compat/1\.1\.2_nwbfile\.nwb against pynwb namespace information using namespace " + r"Validating tests/back_compat/1\.1\.2_nwbfile\.nwb against PyNWB namespace information using namespace " r"'core'\.\s* - no errors found\.\s*") self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + def test_validate_file_invalid(self): + """Test that validating an invalid file outputs errors.""" + result = subprocess.run( + [ + "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.0.2_str_experimenter.nwb", + "--no-cached-namespace" + ], + capture_output=True + ) + + stderr_regex = re.compile( + r" - found the following errors:\s*" + r"root/general/experimenter \(general/experimenter\): incorrect shape - expected an array of shape " + r"'\[None\]', got non-array data 'one experimenter'\s*" + ) + self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) + + stdout_regex = re.compile( + r"Validating tests/back_compat/1\.0\.2_str_experimenter\.nwb against PyNWB namespace information using " + r"namespace 'core'\.\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_list_namespaces_core(self): + """Test listing namespaces from a file""" + result = subprocess.run( + [ + "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb", + "--list-namespaces" + ], + capture_output=True + ) + + self.assertEqual(result.stderr.decode('utf-8'), '') + + stdout_regex = re.compile(r"core\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_list_namespaces_extension(self): + """Test listing namespaces from a file with an extension""" + result = subprocess.run( + [ + "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/2.1.0_nwbfile_with_extension.nwb", + "--list-namespaces" + ], + capture_output=True + ) + + self.assertEqual(result.stderr.decode('utf-8'), '') + + stdout_regex = re.compile(r"ndx-testextension\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + class TestValidateFunction(TestCase): # 1.0.2_nwbfile.nwb has no cached specifications # 1.0.3_nwbfile.nwb has cached "core" specification - # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications + # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specificaitions - def test_validate_file_no_cache(self): + def test_validate_io_no_cache(self): """Test that validating a file with no cached spec against the core namespace succeeds.""" with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: errors = validate(io) self.assertEqual(errors, []) - def test_validate_file_no_cache_bad_ns(self): + def test_validate_io_no_cache_bad_ns(self): """Test that validating a file with no cached spec against a specified, unknown namespace fails.""" with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): validate(io, 'notfound') - def test_validate_file_cached(self): + def test_validate_io_cached(self): """Test that validating a file with cached spec against its cached namespace succeeds.""" with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: errors = validate(io) self.assertEqual(errors, []) - def test_validate_file_cached_bad_ns(self): + def test_validate_io_cached_extension(self): + """Test that validating a file with cached spec against its cached namespaces succeeds.""" + with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r', load_namespaces=True) as io: + errors = validate(io) + self.assertEqual(errors, []) + + def test_validate_io_cached_extension_pass_ns(self): + """Test that validating a file with cached extension spec against the extension namespace succeeds.""" + with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r', load_namespaces=True) as io: + errors = validate(io, 'ndx-testextension') + self.assertEqual(errors, []) + + def test_validate_io_cached_core_with_io(self): + """ + For back-compatability, test that validating a file with cached extension spec against the core + namespace succeeds when using the `io` + `namespace` keywords. + """ + with NWBHDF5IO( + path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb', mode='r', load_namespaces=True + ) as io: + results = validate(io=io, namespace="core") + self.assertEqual(results, []) + + def test_validate_file_cached_extension(self): + """ + Test that validating a file with cached extension spec against the core + namespace raises an error with the new CLI-mimicing paths keyword. + """ + nwbfile_path = "tests/back_compat/2.1.0_nwbfile_with_extension.nwb" + with patch("sys.stderr", new=StringIO()) as fake_err: + with patch("sys.stdout", new=StringIO()) as fake_out: + results, status = validate(paths=[nwbfile_path], namespace="core", verbose=True) + self.assertEqual(results, []) + self.assertEqual(status, 1) + self.assertEqual( + fake_err.getvalue(), + ( + "The namespace 'core' is included by the namespace 'ndx-testextension'. " + "Please validate against that namespace instead.\n" + ) + ) + self.assertEqual(fake_out.getvalue(), "") + + def test_validate_file_cached_core(self): + """ + Test that validating a file with cached core spec with verbose=False. + """ + nwbfile_path = "tests/back_compat/1.1.2_nwbfile.nwb" + with patch("sys.stderr", new=StringIO()) as fake_err: + with patch("sys.stdout", new=StringIO()) as fake_out: + results, status = validate(paths=[nwbfile_path], namespace="core") + self.assertEqual(results, []) + self.assertEqual(status, 0) + self.assertEqual(fake_err.getvalue(), "") + self.assertEqual(fake_out.getvalue(), "") + + def test_validate_file_cached_no_cache_bad_ns(self): + """ + Test that validating a file with no cached namespace, a namespace that is not found, and verbose=False. + """ + nwbfile_path = "tests/back_compat/1.0.2_nwbfile.nwb" + with patch("sys.stderr", new=StringIO()) as fake_err: + with patch("sys.stdout", new=StringIO()) as fake_out: + results, status = validate(paths=[nwbfile_path], namespace="notfound") + self.assertEqual(results, []) + self.assertEqual(status, 1) + stderr_regex = ( + r"The namespace 'notfound' could not be found in PyNWB namespace information as only " + r"\['core'\] is present.\n" + ) + self.assertRegex(fake_err.getvalue(), stderr_regex) + self.assertEqual(fake_out.getvalue(), "") + + def test_validate_io_cached_bad_ns(self): """Test that validating a file with cached spec against a specified, unknown namespace fails.""" with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): validate(io, 'notfound') - def test_validate_file_cached_hdmf_common(self): + def test_validate_io_cached_hdmf_common(self): """Test that validating a file with cached spec against the hdmf-common namespace fails.""" with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: # TODO this error should not be different from the error when using the validate script above