Apply shape constraints (#706)

* apply shape to all datasets
* fix tests

src/pynwb/behavior.py

@@ -25,7 +25,7 @@ class SpatialSeries(TimeSeries):
     _help = "Stores points in space over time. The data[] array structure is [num samples][num spatial dimensions]"
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this SpatialSeries dataset'},
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, None),
              'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames'},
             {'name': 'reference_frame', 'type': str, 'doc': 'description defining what the zero-position is'},
             {'name': 'conversion', 'type': float,
@@ -34,7 +34,7 @@ class SpatialSeries(TimeSeries):
             {'name': 'resolution', 'type': float,
              'doc': 'The smallest meaningful difference (in specified unit) between values in data',
              'default': _default_resolution},
-            {'name': 'timestamps', 'type': ('array_data', 'data', 'TimeSeries'),
+            {'name': 'timestamps', 'type': ('array_data', 'data', 'TimeSeries'), 'shape': (None, ),
              'doc': 'Timestamps for samples stored in data', 'default': None},
             {'name': 'starting_time', 'type': float, 'doc': 'The timestamp of the first sample', 'default': None},
             {'name': 'rate', 'type': float, 'doc': 'Sampling rate in Hz', 'default': None},

src/pynwb/data/nwb.behavior.yaml

@@ -13,7 +13,7 @@ groups:
   attributes:
   - name: help
     dtype: text
-    doc: Value is 'Stores points in space over time. The data[] array structure is
+    doc: 'Value is: Stores points in space over time. The data[] array structure is
       [num samples][num spatial dimensions]'
     value: Stores points in space over time. The data[] array structure is [num samples][num
       spatial dimensions]
@@ -42,7 +42,7 @@ groups:
     quantity: '?'
 - neurodata_type_def: BehavioralEpochs
   neurodata_type_inc: NWBDataInterface
-  doc: 'TimeSeries for storing behavoioral epochs.  The objective of this and the other
+  doc: 'TimeSeries for storing behavioral epochs.  The objective of this and the other
     two Behavioral interfaces (e.g. BehavioralEvents and BehavioralTimeSeries) is
     to provide generic hooks for software tools/scripts. This allows a tool/script
     to take the output one specific interface (e.g., UnitTimes) and plot that data

src/pynwb/data/nwb.image.yaml

@@ -53,8 +53,6 @@ groups:
     doc: Either binary data containing image or empty.
     quantity: '?'
     dims:
-    - - x
-      - y
     - - frame
       - y
       - x
@@ -63,8 +61,6 @@ groups:
       - y
       - x
     shape:
-    - - null
-      - null
     - - null
       - null
       - null
@@ -145,13 +141,14 @@ groups:
     quantity: '?'
   - name: field_of_view
     dtype: float32
-    doc: Width, height and depto of image, or imaged area (meters).
+    doc: Width, height and depth of image, or imaged area (meters).
     dims:
     - - width|height
-      - width|height|depth
+    - - width|height|depth
     quantity: '?'
     shape:
-    - 2
+    - - 2
+    - - 3
   - name: orientation
     dtype: text
     doc: Description of image relative to some reference frame (e.g., which way is

src/pynwb/data/nwb.misc.yaml

@@ -30,11 +30,13 @@ groups:
       default_value: see 'feature_units'
       required: false
     dims:
-    - num_times
-    - num_features
+    - - num_times
+    - - num_times
+      - num_features
     shape:
-    - null
-    - null
+    - - null
+    - - null
+      - null
   - name: feature_units
     dtype: text
     doc: Units of each feature.

src/pynwb/data/nwb.ophys.yaml

@@ -22,11 +22,13 @@ groups:
     dtype: float32
     doc: Width, height and depth of image, or imaged area (meters).
     dims:
+    - width|height
     - width|height|depth
     quantity: '?'
     required: false
     shape:
-    - 3
+    - - 2
+    - - 3
   links:
   - name: imaging_plane
     doc: link to ImagingPlane group from which this TimeSeries data was generated

src/pynwb/ecephys.py

@@ -220,8 +220,8 @@ class Clustering(NWBDataInterface):
     @docval({'name': 'description', 'type': str,
              'doc': 'Description of clusters or clustering, (e.g. cluster 0 is noise, \
              clusters curated using Klusters, etc).'},
-            {'name': 'num', 'type': ('array_data', 'data'), 'doc': 'Cluster number of each event.', 'shape': (None,)},
-            {'name': 'peak_over_rms', 'type': Iterable,
+            {'name': 'num', 'type': ('array_data', 'data'), 'doc': 'Cluster number of each event.', 'shape': (None, )},
+            {'name': 'peak_over_rms', 'type': Iterable, 'shape': (None, ),
              'doc': 'Maximum ratio of waveform peak to RMS on any channel in the cluster\
              (provides a basic clustering metric).'},
             {'name': 'times', 'type': ('array_data', 'data'), 'doc': 'Times of clustered events, in seconds.',
@@ -256,8 +256,10 @@ class ClusterWaveforms(NWBDataInterface):
              'doc': 'the clustered spike data used as input for computing waveforms'},
             {'name': 'waveform_filtering', 'type': str,
              'doc': 'filter applied to data before calculating mean and standard deviation'},
-            {'name': 'waveform_mean', 'type': Iterable, 'doc': 'the mean waveform for each cluster'},
-            {'name': 'waveform_sd', 'type': Iterable, 'doc': 'the standard deviations of waveforms for each cluster'},
+            {'name': 'waveform_mean', 'type': Iterable, 'shape': (None, None),
+             'doc': 'the mean waveform for each cluster'},
+            {'name': 'waveform_sd', 'type': Iterable, 'shape': (None, None),
+            'doc': 'the standard deviations of waveforms for each cluster'},
             {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'ClusterWaveforms'})
     def __init__(self, **kwargs):
         import warnings
@@ -334,10 +336,10 @@ class FeatureExtraction(NWBDataInterface):
     @docval({'name': 'electrodes', 'type': DynamicTableRegion,
              'doc': 'the table region corresponding to the electrodes from which this series was recorded'},
             {'name': 'description', 'type': (list, tuple, np.ndarray, DataChunkIterator),
-             'doc': 'A description for each feature extracted', 'ndim': 1},
-            {'name': 'times', 'type': ('array_data', 'data'),
-             'doc': 'The times of events that features correspond to', 'ndim': 1},
-            {'name': 'features', 'type': ('array_data', 'data'),
+             'doc': 'A description for each feature extracted', 'shape': (None, )},
+            {'name': 'times', 'type': ('array_data', 'data'), 'shape': (None, ),
+             'doc': 'The times of events that features correspond to'},
+            {'name': 'features', 'type': ('array_data', 'data'), 'shape': (None, None, None),
              'doc': 'Features for each channel', 'ndim': 3},
             {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': 'FeatureExtraction'})
     def __init__(self, **kwargs):

src/pynwb/form/utils.py

@@ -176,12 +176,12 @@ def __parse_args(validator, args, kwargs, enforce_type=True, enforce_shape=True,
                 argsi += 1
             else:
                 ret[argname] = arg['default']
+            argval = ret[argname]
             if enforce_type:
-                argval = ret[argname]
                 if not __type_okay(argval, arg['type'], arg['default'] is None):
                     fmt_val = (argname, type(argval).__name__, __format_type(arg['type']))
                     type_errors.append("incorrect type for '%s' (got '%s', expected '%s')" % fmt_val)
-            if enforce_shape and 'shape' in arg:
+            if enforce_shape and 'shape' in arg and argval is not None:
                 if not __shape_okay_multi(argval, arg['shape']):
                     fmt_val = (argname, get_data_shape(argval), arg['shape'])
                     value_errors.append("incorrect shape for '%s' (got '%s, expected '%s')" % fmt_val)

src/pynwb/image.py

@@ -23,7 +23,7 @@ class ImageSeries(TimeSeries):
     _help = "Storage object for time-series 2-D image data"
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ([None] * 3, [None] * 4),
              'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames',
              'default': None},
             {'name': 'unit', 'type': str,
@@ -88,7 +88,7 @@ class IndexSeries(TimeSeries):
     an arbitrary order. The data[] field stores frame number in reference stack."
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ),
              'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames'},
             {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)'},
 
@@ -203,7 +203,7 @@ class OpticalSeries(ImageSeries):
             {'name': 'format', 'type': str,
              'doc': 'Format of image. Three types: 1) Image format; tiff, png, jpg, etc. 2) external 3) raw.'},
             {'name': 'distance', 'type': float, 'doc': 'Distance from camera/monitor to target/eye.'},
-            {'name': 'field_of_view', 'type': (list, np.ndarray, 'TimeSeries'),
+            {'name': 'field_of_view', 'type': (list, np.ndarray, 'TimeSeries'), 'shape': ((2, ), (3, )),
              'doc': 'Width, height and depth of image, or imaged area (meters).'},
             {'name': 'orientation', 'type': str,
              'doc': 'Description of image relative to some reference frame (e.g., which way is up). \

src/pynwb/misc.py

@@ -26,7 +26,7 @@ class AnnotationSeries(TimeSeries):
             {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
              'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames',
              'default': list()},
-            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ),
              'doc': 'Timestamps for samples stored in data', 'default': None},
             {'name': 'comments', 'type': str,
              'doc': 'Human-readable comments about this TimeSeries dataset', 'default': 'no comments'},
@@ -68,19 +68,18 @@ class AbstractFeatureSeries(TimeSeries):
     _help = "Features of an applied stimulus. This is useful when storing the raw stimulus is impractical."
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
-            {'name': 'feature_units', 'type': (str, Iterable), 'doc': 'The unit of each feature'},
-            {'name': 'features', 'type': (str, Iterable), 'doc': 'Description of each feature'},
-
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
-             'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames',
-             'default': list()},
+            {'name': 'feature_units', 'type': Iterable, 'shape': (None, ), 'doc': 'The unit of each feature'},
+            {'name': 'features', 'type': Iterable, 'shape': (None, ), 'doc': 'Description of each feature'},
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ((None,), (None, None)),
+             'default': list(),
+             'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames'},
             {'name': 'resolution', 'type': float,
              'doc': 'The smallest meaningful difference (in specified unit) between values in data',
              'default': _default_resolution},
             {'name': 'conversion', 'type': float,
              'doc': 'Scalar to multiply each element in data to convert it to the specified unit',
              'default': _default_conversion},
-            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ),
              'doc': 'Timestamps for samples stored in data', 'default': None},
             {'name': 'starting_time', 'type': float, 'doc': 'The timestamp of the first sample', 'default': None},
             {'name': 'rate', 'type': float, 'doc': 'Sampling rate in Hz', 'default': None},
@@ -105,8 +104,11 @@ def __init__(self, **kwargs):
             {'name': 'features', 'type': (list, np.ndarray), 'doc': 'the feature values for this time point'})
     def add_features(self, **kwargs):
         time, features = getargs('time', 'features', kwargs)
-        self.timestamps.append(time)
-        self.data.append(features)
+        if type(self.timestamps) == list and type(self.data) is list:
+            self.timestamps.append(time)
+            self.data.append(features)
+        else:
+            raise ValueError('Can only add feature if timestamps and data are lists')
 
 
 @register_class('IntervalSeries', CORE_NAMESPACE)
@@ -116,7 +118,7 @@ class IntervalSeries(TimeSeries):
     data field stores whether the interval just started (>0 value) or ended (<0 value). Different interval
     types can be represented in the same series by using multiple key values (eg, 1 for feature A, 2
     for feature B, 3 for feature C, etc). The field data stores an 8-bit integer. This is largely an alias
-    of a standard TimeSeries but that is identifiable as representing time intervals in a machinereadable
+    of a standard TimeSeries but that is identifiable as representing time intervals in a machine-readable
     way.
     """
 
@@ -125,9 +127,9 @@ class IntervalSeries(TimeSeries):
     _help = "Stores the start and stop times for events."
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None,),
              'doc': '>0 if interval started, <0 if interval ended.', 'default': list()},
-            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries), 'shape': (None,),
              'doc': 'Timestamps for samples stored in data', 'default': list()},
             {'name': 'comments', 'type': str,
              'doc': 'Human-readable comments about this TimeSeries dataset', 'default':  'no comments'},

src/pynwb/ogen.py

@@ -45,7 +45,7 @@ class OptogeneticSeries(TimeSeries):
     _help = "Optogenetic stimulus."
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ),
              'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames'},
             {'name': 'unit', 'type': str, 'doc': 'Value is the string "Watt".', 'default': 'Watt'},
             {'name': 'site', 'type': OptogeneticStimulusSite,
@@ -55,7 +55,7 @@ class OptogeneticSeries(TimeSeries):
             {'name': 'conversion', 'type': float,
              'doc': 'Scalar to multiply each element by to conver to volts', 'default': _default_conversion},
 
-            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'timestamps', 'type': ('array_data', 'data', TimeSeries), 'shape': (None, ),
              'doc': 'Timestamps for samples stored in data', 'default': None},
             {'name': 'starting_time', 'type': float, 'doc': 'The timestamp of the first sample', 'default': None},
             {'name': 'rate', 'type': float, 'doc': 'Sampling rate in Hz', 'default': None},

src/pynwb/ophys.py

@@ -106,14 +106,14 @@ class TwoPhotonSeries(ImageSeries):
 
     @docval({'name': 'name', 'type': str, 'doc': 'The name of this TimeSeries dataset'},
             {'name': 'imaging_plane', 'type': ImagingPlane, 'doc': 'Imaging plane class/pointer.'},
-            {'name': 'data', 'type': ('array_data', 'data', TimeSeries),
+            {'name': 'data', 'type': ('array_data', 'data', TimeSeries), 'shape': ([None] * 3, [None] * 4),
              'doc': 'The data this TimeSeries dataset stores. Can also store binary data e.g. image frames',
              'default': None},
             {'name': 'unit', 'type': str, 'doc': 'The base unit of measurement (should be SI unit)', 'default': None},
             {'name': 'format', 'type': str,
              'doc': 'Format of image. Three types: 1) Image format; tiff, png, jpg, etc. 2) external 3) raw.',
              'default': None},
-            {'name': 'field_of_view', 'type': (Iterable, TimeSeries),
+            {'name': 'field_of_view', 'type': (Iterable, TimeSeries), 'shape': ((2, ), (3, )),
              'doc': 'Width, height and depth of image, or imaged area (meters).', 'default': None},
             {'name': 'pmt_gain', 'type': float, 'doc': 'Photomultiplier gain.', 'default': None},
             {'name': 'scan_line_rate', 'type': float,
@@ -245,11 +245,14 @@ def __init__(self, **kwargs):
         self.reference_images = reference_images
 
     @docval({'name': 'pixel_mask', 'type': 'array_data', 'default': None,
-             'doc': 'pixel mask for 2D ROIs: [(x1, y1, weight1), (x2, y2, weight2), ...]'},
+             'doc': 'pixel mask for 2D ROIs: [(x1, y1, weight1), (x2, y2, weight2), ...]',
+             'shape': (None, 3)},
             {'name': 'voxel_mask', 'type': 'array_data', 'default': None,
-             'doc': 'voxel mask for 3D ROIs: [(x1, y1, z1, weight1), (x2, y2, z1, weight2), ...]'},
+             'doc': 'voxel mask for 3D ROIs: [(x1, y1, z1, weight1), (x2, y2, z1, weight2), ...]',
+             'shape': (None, 4)},
             {'name': 'image_mask', 'type': 'array_data', 'default': None,
-             'doc': 'image with the same size of image where positive values mark this ROI', },
+             'doc': 'image with the same size of image where positive values mark this ROI',
+             'shape': [[None]*2, [None]*3]},
             {'name': 'id', 'type': int, 'help': 'the ID for the ROI', 'default': None},
             allow_extra=True)
     def add_roi(self, **kwargs):

src/pynwb/retinotopy.py

@@ -47,10 +47,11 @@ class AxisMap(NWBContainer):
                      'dimension')
 
     @docval({'name': 'name', 'type': str, 'doc': 'the name of this axis map'},
-            {'name': 'data', 'type': Iterable, 'doc': 'data field.'},
+            {'name': 'data', 'type': Iterable, 'shape': (None, None), 'doc': 'data field.'},
             {'name': 'field_of_view', 'type': Iterable, 'doc': 'Size of viewing area, in meters.'},
             {'name': 'unit', 'type': str, 'doc': 'Unit that axis data is stored in (e.g., degrees)'},
-            {'name': 'dimension', 'type': Iterable, 'doc': 'Number of rows and columns in the image'})
+            {'name': 'dimension', 'type': Iterable, 'shape': (None, ),
+             'doc': 'Number of rows and columns in the image'})
     def __init__(self, **kwargs):
         data, field_of_view, unit, dimension = popargs('data', 'field_of_view', 'unit', 'dimension', kwargs)
         pargs, pkwargs = fmt_docval_args(super(AxisMap, self).__init__, kwargs)

tests/integration/ui_write/test_ecephys.py

@@ -338,8 +338,8 @@ def setUpContainer(self):
         num = [3, 4]
         peak_over_rms = [5.3, 6.3]
         self.clustering = Clustering('description', num, peak_over_rms, times)
-        means = [7.3, 7.3]
-        stdevs = [8.3, 8.3]
+        means = [[7.3, 7.3]]
+        stdevs = [[8.3, 8.3]]
         cw = ClusterWaveforms(self.clustering, 'filtering', means, stdevs)
         return cw