From af7deb7675577c8191d317546191e224379b96b6 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 16 Aug 2022 14:33:43 +0200 Subject: [PATCH 01/18] black create_extension_spec.py and add author --- src/spec/create_extension_spec.py | 494 +++++++++++++++--------------- 1 file changed, 246 insertions(+), 248 deletions(-) diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index e0746d0..bd3ec41 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -5,394 +5,392 @@ export_spec, NWBGroupSpec, NWBDatasetSpec, - NWBAttributeSpec + NWBAttributeSpec, ) -def main(): - name = 'ndx-extract' - ns_path = name + ".namespace.yaml" - ext_source = name + ".extensions.yaml" +def main(): ns_builder = NWBNamespaceBuilder( - doc='NWB:N Extension for storage of EXTRACT parameters and output', - name='ndx-extract', - version='0.1.1', - author=list(map(str.strip, 'Cesar Echavarria'.split(','))), - contact=list(map(str.strip, 'cesar.echavarria@catalystneuro.com'.split(','))) + doc="NWB Extension for storage of EXTRACT parameters and output", + name="ndx-extract", + version="0.2.0", + author=list(map(str.strip, "Cesar Echavarria, Szonja Weigl".split(","))), + contact=list(map(str.strip, "cesar.echavarria@catalystneuro.com, szonja.weigl@catalystneuro.com".split(","))), ) - ns_builder.include_type('ImageSegmentation', namespace='core') + ns_builder.include_type("ImageSegmentation", namespace="core") - configs = NWBGroupSpec( - doc='EXTRACT configuration parameters', - neurodata_type_def='EXTRACTSegmentation', - neurodata_type_inc='ImageSegmentation', - attributes = [ + ExtractSegmentationExtension = NWBGroupSpec( + doc="EXTRACT configuration parameters", + neurodata_type_def="EXTRACTSegmentation", + neurodata_type_inc="ImageSegmentation", + attributes=[ # preprocessing parameters NWBAttributeSpec( - name='preprocess', - doc='int32ean flag indicating data preprocessing before main EXTRACT function', - dtype='int32' + name="preprocess", + doc="int32ean flag indicating data preprocessing before main EXTRACT function", + dtype="int32", ), NWBAttributeSpec( - name = 'fix_zero_FOV_strips', - doc = 'int32ean flag. Find and fix spatial slices that are occasionally zero due to '\ - 'frame registration during preprocessing.', - dtype = 'int32' + name="fix_zero_FOV_strips", + doc="int32ean flag. Find and fix spatial slices that are occasionally zero due to " + "frame registration during preprocessing.", + dtype="int32", ), NWBAttributeSpec( - name = 'medfilt_outlier_pixels', - doc = 'int32ean flag determining whether outlier pixels in the movie should be replaced with their '\ - 'neighborhood median.', - dtype = 'int32' + name="medfilt_outlier_pixels", + doc="int32ean flag determining whether outlier pixels in the movie should be replaced with their " + "neighborhood median.", + dtype="int32", ), NWBAttributeSpec( - name = 'skip_dff', - doc = 'int32ean flag. Skip Df/F calculation in preprocessing.', - dtype = 'int32' + name="skip_dff", + doc="int32ean flag. Skip Df/F calculation in preprocessing.", + dtype="int32", ), NWBAttributeSpec( - name = 'baseline_quantile', - doc = 'Baseline quantile for Df/F calculation in preprocessing.', - dtype = 'float32' + name="baseline_quantile", + doc="Baseline quantile for Df/F calculation in preprocessing.", + dtype="float32", ), NWBAttributeSpec( - name = 'skip_highpass', - doc = 'int32ean flag. Skip highpass filtering in preprocessing.', - dtype = 'int32' + name="skip_highpass", + doc="int32ean flag. Skip highpass filtering in preprocessing.", + dtype="int32", ), NWBAttributeSpec( - name = 'spatial_highpass_cutoff', - doc = 'Cutoff determining the strength of butterworth spatial filtering of the movie. '\ - 'Values defined relative to the average cell radius. ', - dtype = 'float32' + name="spatial_highpass_cutoff", + doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius. ", + dtype="float32", ), NWBAttributeSpec( - name = 'temporal_denoising', - doc = 'int32ean flag that determines whether to apply temporal wavelet denoising.', - dtype = 'int32' + name="temporal_denoising", + doc="int32ean flag that determines whether to apply temporal wavelet denoising.", + dtype="int32", ), NWBAttributeSpec( - name = 'remove_background', - doc = 'int32ean flag that determines whether to subtract the (spatial) background '\ - '(largest spatiotemporal mode of the movie matrix).', - dtype = 'int32' + name="remove_background", + doc="int32ean flag that determines whether to subtract the (spatial) background " + "(largest spatiotemporal mode of the movie matrix).", + dtype="int32", ), # general control parameters NWBAttributeSpec( - name = 'avg_cell_radius', - doc = 'Radius estimate for an average cell in the movie.', - dtype = 'float32' + name="avg_cell_radius", + doc="Radius estimate for an average cell in the movie.", + dtype="float32", ), NWBAttributeSpec( - name = 'num_partitions_x', - doc = 'Number of movie partitions in x dimension.', - dtype = 'uint32', - required = False + name="num_partitions_x", + doc="Number of movie partitions in x dimension.", + dtype="uint32", + required=False, ), NWBAttributeSpec( - name = 'num_partitions_y', - doc = 'Number of movie partitions in y dimension.', - dtype = 'uint32', - required = False + name="num_partitions_y", + doc="Number of movie partitions in y dimension.", + dtype="uint32", + required=False, ), NWBAttributeSpec( - name = 'trace_output_option', - doc = 'Raw or non-negative output traces', - dtype = 'text' + name="trace_output_option", + doc="Raw or non-negative output traces", + dtype="text", ), NWBAttributeSpec( - name = 'dendrite_aware', - doc = 'int32ean flag, set it to true if dendrites exist in the movie & are desired in the output.', - dtype = 'int32' + name="dendrite_aware", + doc="int32ean flag, set it to true if dendrites exist in the movie & are desired in the output.", + dtype="int32", ), NWBAttributeSpec( - name = 'crop_circular', - doc = 'int32ean flag. For microendoscopic movies, set it to true for automatically cropping out '\ - 'the region outside the circular imaging region.', - dtype = 'int32' + name="crop_circular", + doc="int32ean flag. For microendoscopic movies, set it to true for automatically cropping out " + "the region outside the circular imaging region.", + dtype="int32", ), NWBAttributeSpec( - name = 'use_sparse_arrays', - doc = 'int32ean flag. If set to true, then the output cell images will be saved as sparse arrays', - dtype = 'int32' + name="use_sparse_arrays", + doc="int32ean flag. If set to true, then the output cell images will be saved as sparse arrays", + dtype="int32", ), NWBAttributeSpec( - name = 'compact_output', - doc = 'int32ean flag. If set to true, then the output will not include bad components that were '\ - 'found but then eliminated', - dtype = 'int32' + name="compact_output", + doc="int32ean flag. If set to true, then the output will not include bad components that were " + "found but then eliminated", + dtype="int32", ), NWBAttributeSpec( - name = 'hyperparameter_tuning_flag', - doc = 'int32ean flag indicating internal hyperparameter tuning.', - dtype = 'int32' + name="hyperparameter_tuning_flag", + doc="int32ean flag indicating internal hyperparameter tuning.", + dtype="int32", ), NWBAttributeSpec( - name = 'remove_duplicate_cells', - doc = 'For movies processed in multiple partitions, this flag controls duplicate removal in the '\ - 'overlap regions.', - dtype = 'int32' + name="remove_duplicate_cells", + doc="For movies processed in multiple partitions, this flag controls duplicate removal in the " + "overlap regions.", + dtype="int32", ), NWBAttributeSpec( - name = 'max_iter', - doc = 'Maximum number of alternating estimation iterations.', - dtype = 'uint32' + name="max_iter", + doc="Maximum number of alternating estimation iterations.", + dtype="uint32", ), - #cell finding parameters + # cell finding parameters NWBAttributeSpec( - name = 'cellfind_min_snr', - doc = 'Minimum peak SNR value for an object to be considered as a cell.', - dtype = 'float32' + name="cellfind_min_snr", + doc="Minimum peak SNR value for an object to be considered as a cell.", + dtype="float32", ), NWBAttributeSpec( - name = 'cellfind_max_steps', - doc = 'Maximum number of cell candidate initialization during cell finding step.', - dtype = 'int' + name="cellfind_max_steps", + doc="Maximum number of cell candidate initialization during cell finding step.", + dtype="int", ), NWBAttributeSpec( - name = 'cellfind_kappa_std_ratio', - doc = 'Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.', - dtype = 'float32' + name="cellfind_kappa_std_ratio", + doc="Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.", + dtype="float32", ), NWBAttributeSpec( - name = 'cellfind_filter_type', - doc = 'Type of the spatial smoothing filter used for cell finding.', - dtype = 'text' + name="cellfind_filter_type", + doc="Type of the spatial smoothing filter used for cell finding.", + dtype="text", ), NWBAttributeSpec( - name = 'cellfind_numpix_threshold', - doc = 'During cell finding, objects with an area < cellfind_numpix_threshold are discarded.', - dtype = 'float32' + name="cellfind_numpix_threshold", + doc="During cell finding, objects with an area < cellfind_numpix_threshold are discarded.", + dtype="float32", ), NWBAttributeSpec( - name = 'moving_radius', - doc = 'Radius of moving average filter in the case when cellfind_filter_type = moveavg '\ - '(moving average)', - dtype = 'float32' + name="moving_radius", + doc="Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average)", + dtype="float32", ), NWBAttributeSpec( - name = 'init_with_gaussian', - doc = 'int32ean flag. If true, then during cell finding, each cell is initialized with a gaussian shape '\ - 'prior to robust estimation. If false, then initialization is done with a '\ - 'correlation image (preferred for movies with dendrites).', - dtype = 'int32' + name="init_with_gaussian", + doc="int32ean flag. If true, then during cell finding, each cell is initialized with a gaussian shape " + "prior to robust estimation. If false, then initialization is done with a " + "correlation image (preferred for movies with dendrites).", + dtype="int32", ), NWBAttributeSpec( - name = 'high2low_brightness_ratio', - doc = 'Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to '\ - 'stop cell finding process.', - dtype = 'float32' + name="high2low_brightness_ratio", + doc="Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to " + "stop cell finding process.", + dtype="float32", ), # cell refinement parameters NWBAttributeSpec( - name = 'kappa_std_ratio', - doc = 'Kappa will be set to this times the noise std during the cell refinement process. '\ - 'Cell refinement parameter.', - dtype = 'float32' + name="kappa_std_ratio", + doc="Kappa will be set to this times the noise std during the cell refinement process. " + "Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'T_dup_corr_thresh', - doc = 'Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh '\ - 'are eliminated. Cell refinement parameter.', - dtype = 'float32' + name="T_dup_corr_thresh", + doc="Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh " + "are eliminated. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'S_dup_corr_thresh', - doc = 'Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh '\ - 'are eliminated. Cell refinement parameter.', - dtype = 'float32' + name="S_dup_corr_thresh", + doc="Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh " + "are eliminated. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'temporal_corrupt_thresh', - doc = 'Threshold for temporal corruption index. Traces above this threshold are eliminated duirng '\ - 'alternating minimization routine. Cell refinement parameter.', - dtype = 'float32' + name="temporal_corrupt_thresh", + doc="Threshold for temporal corruption index. Traces above this threshold are eliminated duirng " + "alternating minimization routine. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'spatial_corrupt_thresh', - doc = 'Threshold for spatial corruption index. Images above this threshold are eliminated duirng '\ - 'alternating minimization routine. Cell refinement parameter.', - dtype = 'float32' + name="spatial_corrupt_thresh", + doc="Threshold for spatial corruption index. Images above this threshold are eliminated duirng " + "alternating minimization routine. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'T_min_snr', - doc = 'Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. '\ - 'Cell refinement parameter.', - dtype = 'float32' + name="T_min_snr", + doc="Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. " + "Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'size_lower_limit', - doc = 'Lower size limit for found cells. Cell refinement parameter.', - dtype = 'float32' + name="size_lower_limit", + doc="Lower size limit for found cells. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'size_upper_limit', - doc = 'Lower size limit for found cells. Cell refinement parameter.', - dtype = 'float32' + name="size_upper_limit", + doc="Lower size limit for found cells. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'eccent_thresh', - doc = 'Upper limit of eccentricity for found cells. Cell refinement parameter.', - dtype = 'float32' + name="eccent_thresh", + doc="Upper limit of eccentricity for found cells. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'low_ST_index_thresh', - doc = 'Lower limit of spatiotemporal activity index. Cell refinement parameter.', - dtype = 'float32' + name="low_ST_index_thresh", + doc="Lower limit of spatiotemporal activity index. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'high_ST_index_thresh', - doc = 'Upper limit limit of spatiotemporal activity index. Cell refinement parameter.', - dtype = 'float32' + name="high_ST_index_thresh", + doc="Upper limit limit of spatiotemporal activity index. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'low_ST_corr_thresh', - doc = 'Lower limit of spatiotemporal corelation. Cell refinement parameter.', - dtype = 'float32' + name="low_ST_corr_thresh", + doc="Lower limit of spatiotemporal corelation. Cell refinement parameter.", + dtype="float32", ), NWBAttributeSpec( - name = 'confidence_thresh', - doc = 'Confidence threhsold for found cells. Cell refinement parameter.', - dtype = 'float32' + name="confidence_thresh", + doc="Confidence threhsold for found cells. Cell refinement parameter.", + dtype="float32", ), - #other parameters + # other parameters NWBAttributeSpec( - name='downsample_time_by', - doc='Time downsampling factor. If set to auto downsampling factor based on avg cell radius and '\ - 'avg calcium event time constant.', - dtype='float32', - required = False + name="downsample_time_by", + doc="Time downsampling factor. If set to auto downsampling factor based on avg cell radius and " + "avg calcium event time constant.", + dtype="float32", + required=False, ), NWBAttributeSpec( - name = 'min_tau_after_downsampling', - doc = 'Minimum event tau after downsampling. Used when downsample_time_by = auto', - dtype = 'float32' + name="min_tau_after_downsampling", + doc="Minimum event tau after downsampling. Used when downsample_time_by = auto", + dtype="float32", ), NWBAttributeSpec( - name='downsample_space_by', - doc='Spatial downsampling factor. If set to auto downsampling factor based on avg cell radius and '\ - 'avg calcium event time constant.', - dtype='float32', - required = False + name="downsample_space_by", + doc="Spatial downsampling factor. If set to auto downsampling factor based on avg cell radius and " + "avg calcium event time constant.", + dtype="float32", + required=False, ), NWBAttributeSpec( - name='min_radius_after_downsampling', - doc='Minimum avg radius after downsampling. Used when downsample_space_by = auto.', - dtype='float32' + name="min_radius_after_downsampling", + doc="Minimum avg radius after downsampling. Used when downsample_space_by = auto.", + dtype="float32", ), NWBAttributeSpec( - name = 'reestimate_S_if_downsampled', - doc = 'int32ean flag. When set to true, images are re-estimated from full movie at the end. '\ - 'When false, images are upsampled by interpolation.', - dtype = 'int32' + name="reestimate_S_if_downsampled", + doc="int32ean flag. When set to true, images are re-estimated from full movie at the end. " + "When false, images are upsampled by interpolation.", + dtype="int32", ), NWBAttributeSpec( - name = 'reestimate_T_if_downsampled', - doc = 'int32ean flag. When set to true, traces are re-estimated from full movie at the end. '\ - 'When false, traces are upsampled by interpolation.', - dtype = 'int32' + name="reestimate_T_if_downsampled", + doc="int32ean flag. When set to true, traces are re-estimated from full movie at the end. " + "When false, traces are upsampled by interpolation.", + dtype="int32", ), NWBAttributeSpec( - name = 'adaptive_kappa', - doc = 'int32ean flag. If true, then during cell finding, the robust esimation loss will adaptively '\ - 'set its robustness parameter', - dtype = 'int32' + name="adaptive_kappa", + doc="int32ean flag. If true, then during cell finding, the robust esimation loss will adaptively " + "set its robustness parameter", + dtype="int32", ), NWBAttributeSpec( - name='smoothing_ratio_x2y', - doc='If the movie contains mainly objects that are elongated in one dimension '\ - '(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.', - dtype='float32' + name="smoothing_ratio_x2y", + doc="If the movie contains mainly objects that are elongated in one dimension " + "(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.", + dtype="float32", ), NWBAttributeSpec( - name = 'spatial_lowpass_cutoff', - doc = 'Cutoff determining the strength of butterworth spatial filtering of the movie. '\ - 'Values defined relative to the average cell radius. ', - dtype = 'float32' + name="spatial_lowpass_cutoff", + doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius. ", + dtype="float32", ), NWBAttributeSpec( - name = 'smooth_T', - doc = 'int32ean flag indicating whether calculated traces are smoothed using median filtering.', - dtype = 'int32' + name="smooth_T", + doc="int32ean flag indicating whether calculated traces are smoothed using median filtering.", + dtype="int32", ), NWBAttributeSpec( - name = 'smooth_S', - doc = 'int32ean flag indicating whether calculated images are smoothed using a 2-D Gaussian filter.', - dtype = 'int32' + name="smooth_S", + doc="int32ean flag indicating whether calculated images are smoothed using a 2-D Gaussian filter.", + dtype="int32", ), NWBAttributeSpec( - name = 'l1_penalty_factor', - doc = 'Strength of l1 regularization penalty to be applied when estimating the temporal components.', - dtype = 'float32' + name="l1_penalty_factor", + doc="Strength of l1 regularization penalty to be applied when estimating the temporal components.", + dtype="float32", ), NWBAttributeSpec( - name = 'max_iter_S', - doc = 'Maximum number of iterations for S estimation steps', - dtype = 'uint32' + name="max_iter_S", + doc="Maximum number of iterations for S estimation steps", + dtype="uint32", ), NWBAttributeSpec( - name = 'max_iter_T', - doc = 'Maximum number of iterations for T estimation steps', - dtype = 'uint32' + name="max_iter_T", + doc="Maximum number of iterations for T estimation steps", + dtype="uint32", ), NWBAttributeSpec( - name = 'TOL_sub', - doc = 'If the 1-step relative change in the objective within each T and S optimization is less than this, '\ - 'the respective optimization is terminated.', - dtype = 'float32' + name="TOL_sub", + doc="If the 1-step relative change in the objective within each T and S optimization is less than this, " + "the respective optimization is terminated.", + dtype="float32", ), NWBAttributeSpec( - name = 'TOL_main', - doc = ' If the relative change in the main objective function between 2 consecutive alternating '\ - 'minimization steps is less than this, cell extraction is terminated.', - dtype = 'float32' + name="TOL_main", + doc=" If the relative change in the main objective function between 2 consecutive alternating " + "minimization steps is less than this, cell extraction is terminated.", + dtype="float32", ), NWBAttributeSpec( - name = 'T_lower_snr_threshold', - doc = 'Lower SNR threshold for found traces.', - dtype = 'float32' + name="T_lower_snr_threshold", + doc="Lower SNR threshold for found traces.", + dtype="float32", ), NWBAttributeSpec( - name = 'save_all_found', - doc = 'int32ean flag. Save all spatial and temporal components found', - dtype = 'int32' + name="save_all_found", + doc="int32ean flag. Save all spatial and temporal components found", + dtype="int32", ), ], - datasets = [ + datasets=[ NWBDatasetSpec( - name = 'S_init', - doc = 'Cell images provided to algorithm as the initial set of cells, skipping its native initialization', - shape = (None, None), - dtype = 'double', - dims = ('heightxwidth','num_cells'), - quantity = '?' + name="S_init", + doc="Cell images provided to algorithm as the initial set of cells, skipping its native initialization", + shape=(None, None), + dtype="double", + dims=("heightxwidth", "num_cells"), + quantity="?", ), NWBDatasetSpec( - name = 'T_init', - doc = 'Cell traces provided to algorithm as the initial set of traces, skipping its native initialization', - shape = (None, None), - dtype = 'double', - dims = ('num_cells','timepoints'), - quantity = '?' + name="T_init", + doc="Cell traces provided to algorithm as the initial set of traces, skipping its native initialization", + shape=(None, None), + dtype="double", + dims=("num_cells", "timepoints"), + quantity="?", ), NWBDatasetSpec( - name = 'movie_mask', - doc = 'Movie mask.', - shape = (None, None), - dtype = 'double', - dims = ('height', 'width'), - quantity = '?' + name="movie_mask", + doc="Movie mask.", + shape=(None, None), + dtype="double", + dims=("height", "width"), + quantity="?", ), ], ) - # export the spec to yaml files in the spec folder - output_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'spec')) - export_spec(ns_builder, [configs], output_dir) + output_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "spec") + ) + export_spec(ns_builder, [ExtractSegmentationExtension], output_dir) -if __name__ == '__main__': +if __name__ == "__main__": # usage: python create_extension_spec.py main() From 7277d8a4c4fd27db6c7cbe5cc393c05d1c029ba3 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 10:40:13 +0200 Subject: [PATCH 02/18] update extension spec --- src/spec/create_extension_spec.py | 351 ++++++++++++++++++++++-------- 1 file changed, 265 insertions(+), 86 deletions(-) diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index bd3ec41..b60f6b6 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -28,129 +28,185 @@ def main(): # preprocessing parameters NWBAttributeSpec( name="preprocess", - doc="int32ean flag indicating data preprocessing before main EXTRACT function", - dtype="int32", + doc="Boolean flag indicating data preprocessing before main EXTRACT function", + dtype="bool", + ), + NWBAttributeSpec( + name="pre_mask_on", + doc="Boolean flag indicating whether to use an image mask for preprocessing.", + dtype="bool", + ), + NWBAttributeSpec( + name="pre_mask_radius", + doc="The radius of the image mask.", + dtype="float64", ), NWBAttributeSpec( name="fix_zero_FOV_strips", - doc="int32ean flag. Find and fix spatial slices that are occasionally zero due to " + doc="Boolean flag. Find and fix spatial slices that are occasionally zero due to " "frame registration during preprocessing.", - dtype="int32", + dtype="bool", ), NWBAttributeSpec( name="medfilt_outlier_pixels", - doc="int32ean flag determining whether outlier pixels in the movie should be replaced with their " + doc="Boolean flag determining whether outlier pixels in the movie should be replaced with their " "neighborhood median.", - dtype="int32", + dtype="bool", ), NWBAttributeSpec( name="skip_dff", - doc="int32ean flag. Skip Df/F calculation in preprocessing.", - dtype="int32", + doc="Boolean flag indicating whether to skip Df/F calculation in preprocessing.", + dtype="bool", ), NWBAttributeSpec( - name="baseline_quantile", - doc="Baseline quantile for Df/F calculation in preprocessing.", - dtype="float32", + name="second_df", + doc="No description available.", + dtype="float64", ), NWBAttributeSpec( name="skip_highpass", - doc="int32ean flag. Skip highpass filtering in preprocessing.", - dtype="int32", + doc="Boolean flag. Skip highpass filtering in preprocessing.", + dtype="bool", ), NWBAttributeSpec( name="spatial_highpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " "Values defined relative to the average cell radius. ", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="temporal_denoising", - doc="int32ean flag that determines whether to apply temporal wavelet denoising.", - dtype="int32", + doc="Boolean flag that determines whether to apply temporal wavelet denoising.", + dtype="bool", ), NWBAttributeSpec( name="remove_background", - doc="int32ean flag that determines whether to subtract the (spatial) background " + doc="Boolean flag that determines whether to subtract the (spatial) background " "(largest spatiotemporal mode of the movie matrix).", - dtype="int32", + dtype="float64", ), # general control parameters + NWBAttributeSpec( + name="use_default_gpu", + doc="Boolean flag indicating whether to use the default GPU.", + dtype="bool", + ), + NWBAttributeSpec( + name="use_gpu", + doc="Boolean flag indicating whether to run EXTRACT on GPU." + "If False, EXTRACT was run on CPU.", + dtype="bool", + ), + NWBAttributeSpec( + name="multi_gpu", + doc="Boolean flag indicating whether multiple GPUs were used.", + dtype="bool", + ), + NWBAttributeSpec( + name="parallel_cpu", + doc="Boolean flag indicating whether to run EXTRACT on parallel CPU.", + dtype="bool", + ), NWBAttributeSpec( name="avg_cell_radius", doc="Radius estimate for an average cell in the movie.", - dtype="float32", + dtype="float64", + ), + NWBAttributeSpec( + name="avg_event_tau", + doc="Determines the average event tau.", + dtype="float64", + ), + NWBAttributeSpec( + name="avg_yield_threshold", + doc="Determines the average yield threshold.", + dtype="float64", ), NWBAttributeSpec( name="num_partitions_x", doc="Number of movie partitions in x dimension.", - dtype="uint32", + dtype="float64", required=False, ), NWBAttributeSpec( name="num_partitions_y", doc="Number of movie partitions in y dimension.", - dtype="uint32", + dtype="float64", required=False, ), NWBAttributeSpec( name="trace_output_option", - doc="Raw or non-negative output traces", + doc="Raw or non-negative output traces.", dtype="text", ), + NWBAttributeSpec( + name="trace_quantile", + doc="No description available.", + dtype="float64", + ), NWBAttributeSpec( name="dendrite_aware", - doc="int32ean flag, set it to true if dendrites exist in the movie & are desired in the output.", - dtype="int32", + doc="Boolean flag, set it to true if dendrites exist in the movie & are desired in the output.", + dtype="bool", ), NWBAttributeSpec( name="crop_circular", - doc="int32ean flag. For microendoscopic movies, set it to true for automatically cropping out " + doc="Boolean flag. For microendoscopic movies, set it to true for automatically cropping out " "the region outside the circular imaging region.", - dtype="int32", + dtype="bool", ), NWBAttributeSpec( name="use_sparse_arrays", - doc="int32ean flag. If set to true, then the output cell images will be saved as sparse arrays", - dtype="int32", + doc="Boolean flag. If set to true, then the output cell images will be saved as sparse arrays.", + dtype="bool", + ), + NWBAttributeSpec( + name="verbose", + doc="Indicates the level of verbosity.", + dtype="float64", ), NWBAttributeSpec( name="compact_output", - doc="int32ean flag. If set to true, then the output will not include bad components that were " - "found but then eliminated", - dtype="int32", + doc="Boolean flag. If set to true, then the output will not include bad components that were " + "found but then eliminated.", + dtype="bool", ), NWBAttributeSpec( name="hyperparameter_tuning_flag", - doc="int32ean flag indicating internal hyperparameter tuning.", - dtype="int32", + doc="Boolean flag indicating internal hyperparameter tuning.", + dtype="bool", ), NWBAttributeSpec( name="remove_duplicate_cells", doc="For movies processed in multiple partitions, this flag controls duplicate removal in the " "overlap regions.", - dtype="int32", + dtype="bool", ), NWBAttributeSpec( name="max_iter", doc="Maximum number of alternating estimation iterations.", - dtype="uint32", + dtype="float64", + ), + NWBAttributeSpec( + name="minimal_checks", + doc="Minimum number of checks that are performed.", + dtype="float64", ), # cell finding parameters NWBAttributeSpec( name="cellfind_min_snr", doc="Minimum peak SNR value for an object to be considered as a cell.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="cellfind_max_steps", doc="Maximum number of cell candidate initialization during cell finding step.", - dtype="int", + dtype="float64", ), NWBAttributeSpec( name="cellfind_kappa_std_ratio", doc="Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="cellfind_filter_type", @@ -160,209 +216,300 @@ def main(): NWBAttributeSpec( name="cellfind_numpix_threshold", doc="During cell finding, objects with an area < cellfind_numpix_threshold are discarded.", - dtype="float32", + dtype="float64", + ), + NWBAttributeSpec( + name="cellfind_adaptive_kappa", + doc="Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " + "set its robustness parameter", + dtype="bool", + ), + NWBAttributeSpec( + name="cellfind_spatial_highpass_cutoff", + doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius. ", + dtype="float64", ), NWBAttributeSpec( name="moving_radius", - doc="Radius of moving average filter in the case when cellfind_filter_type = moveavg " + doc="Deprecated variable for older EXTRACT file versions. Radius of moving average filter in the case when cellfind_filter_type = moveavg " "(moving average)", dtype="float32", + required=False, + ), + NWBAttributeSpec( + name="moving_radius_spatial", + doc="Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average) for the image.", + dtype="float64", + required=False, + ), + NWBAttributeSpec( + name="moving_radius_temporal", + doc="Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average) for the traces.", + dtype="float64", + required=False, ), NWBAttributeSpec( name="init_with_gaussian", - doc="int32ean flag. If true, then during cell finding, each cell is initialized with a gaussian shape " - "prior to robust estimation. If false, then initialization is done with a " + doc="Boolean flag. If True, then during cell finding, each cell is initialized with a gaussian shape " + "prior to robust estimation. If False, then initialization is done with a " "correlation image (preferred for movies with dendrites).", - dtype="int32", + dtype="bool", ), NWBAttributeSpec( name="high2low_brightness_ratio", doc="Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to " "stop cell finding process.", - dtype="float32", + dtype="float64", + ), + # Visualizations during cell finding + NWBAttributeSpec( + name="visualize_cellfinding", + doc="The visualization setting for cell finding.", + dtype="float64", + ), + NWBAttributeSpec( + name="visualize_cellfinding_full_range", + doc="The visualization setting for cell finding.", + dtype="float64", + ), + NWBAttributeSpec( + name="visualize_cellfinding_max", + doc="The visualization setting for cell finding.", + dtype="float64", + ), + NWBAttributeSpec( + name="visualize_cellfinding_min", + doc="The visualization setting for cell finding.", + dtype="float64", + ), + NWBAttributeSpec( + name="visualize_cellfinding_show_bad_cells", + doc="The visualization setting for cell finding.", + dtype="float64", ), # cell refinement parameters NWBAttributeSpec( name="kappa_std_ratio", doc="Kappa will be set to this times the noise std during the cell refinement process. " "Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="T_dup_corr_thresh", doc="Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh " "are eliminated. Cell refinement parameter.", - dtype="float32", + dtype="float64", + ), + NWBAttributeSpec( + name="T_dup_thresh", + doc="Threshold for traces. Cell refinement parameter.", + dtype="float64", ), NWBAttributeSpec( name="S_dup_corr_thresh", doc="Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh " "are eliminated. Cell refinement parameter.", - dtype="float32", + dtype="float64", + ), + NWBAttributeSpec( + name="S_corr_thresh", + doc="Image correlation threshold. Cell refinement parameter.", + dtype="float64", + ), + NWBAttributeSpec( + name="T_corr_thresh", + doc="Trace correlation threshold. Cell refinement parameter.", + dtype="float64", ), NWBAttributeSpec( name="temporal_corrupt_thresh", doc="Threshold for temporal corruption index. Traces above this threshold are eliminated duirng " "alternating minimization routine. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="spatial_corrupt_thresh", doc="Threshold for spatial corruption index. Images above this threshold are eliminated duirng " "alternating minimization routine. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="T_min_snr", doc="Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. " "Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="size_lower_limit", doc="Lower size limit for found cells. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="size_upper_limit", doc="Lower size limit for found cells. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="eccent_thresh", doc="Upper limit of eccentricity for found cells. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="low_ST_index_thresh", doc="Lower limit of spatiotemporal activity index. Cell refinement parameter.", - dtype="float32", + dtype="float64", + ), + NWBAttributeSpec( + name="low_cell_area_flag", + doc="Boolean flag indicating lower limit of cell area.", + dtype="bool", ), NWBAttributeSpec( name="high_ST_index_thresh", doc="Upper limit limit of spatiotemporal activity index. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="low_ST_corr_thresh", doc="Lower limit of spatiotemporal corelation. Cell refinement parameter.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="confidence_thresh", - doc="Confidence threhsold for found cells. Cell refinement parameter.", - dtype="float32", + doc="Confidence threshold for found cells. Cell refinement parameter.", + dtype="float64", ), # other parameters NWBAttributeSpec( name="downsample_time_by", doc="Time downsampling factor. If set to auto downsampling factor based on avg cell radius and " "avg calcium event time constant.", - dtype="float32", + dtype="float64", required=False, ), NWBAttributeSpec( name="min_tau_after_downsampling", doc="Minimum event tau after downsampling. Used when downsample_time_by = auto", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="downsample_space_by", doc="Spatial downsampling factor. If set to auto downsampling factor based on avg cell radius and " "avg calcium event time constant.", - dtype="float32", + dtype="float64", required=False, ), NWBAttributeSpec( name="min_radius_after_downsampling", doc="Minimum avg radius after downsampling. Used when downsample_space_by = auto.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="reestimate_S_if_downsampled", - doc="int32ean flag. When set to true, images are re-estimated from full movie at the end. " - "When false, images are upsampled by interpolation.", - dtype="int32", + doc="Boolean flag. When set to True, images are re-estimated from full movie at the end. " + "When False, images are upsampled by interpolation.", + dtype="bool", ), NWBAttributeSpec( name="reestimate_T_if_downsampled", - doc="int32ean flag. When set to true, traces are re-estimated from full movie at the end. " - "When false, traces are upsampled by interpolation.", - dtype="int32", + doc="Boolean flag. When set to True, traces are re-estimated from full movie at the end. " + "When False, traces are upsampled by interpolation.", + dtype="bool", ), NWBAttributeSpec( name="adaptive_kappa", - doc="int32ean flag. If true, then during cell finding, the robust esimation loss will adaptively " + doc="Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " "set its robustness parameter", - dtype="int32", + dtype="bool", + ), + NWBAttributeSpec( + name="arbitrary_mask", + doc="Boolean flag indicating whether to use an arbitraty mask on the images.", + dtype="bool", ), NWBAttributeSpec( name="smoothing_ratio_x2y", doc="If the movie contains mainly objects that are elongated in one dimension " "(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="spatial_lowpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " "Values defined relative to the average cell radius. ", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="smooth_T", - doc="int32ean flag indicating whether calculated traces are smoothed using median filtering.", - dtype="int32", + doc="Boolean flag indicating whether calculated traces are smoothed using median filtering.", + dtype="bool", ), NWBAttributeSpec( name="smooth_S", - doc="int32ean flag indicating whether calculated images are smoothed using a 2-D Gaussian filter.", - dtype="int32", + doc="Boolean flag indicating whether calculated images are smoothed using a 2-D Gaussian filter.", + dtype="bool", ), NWBAttributeSpec( name="l1_penalty_factor", doc="Strength of l1 regularization penalty to be applied when estimating the temporal components.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="max_iter_S", doc="Maximum number of iterations for S estimation steps", - dtype="uint32", + dtype="float64", ), NWBAttributeSpec( name="max_iter_T", doc="Maximum number of iterations for T estimation steps", - dtype="uint32", + dtype="float64", ), NWBAttributeSpec( name="TOL_sub", doc="If the 1-step relative change in the objective within each T and S optimization is less than this, " "the respective optimization is terminated.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="TOL_main", doc=" If the relative change in the main objective function between 2 consecutive alternating " "minimization steps is less than this, cell extraction is terminated.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="T_lower_snr_threshold", doc="Lower SNR threshold for found traces.", - dtype="float32", + dtype="float64", ), NWBAttributeSpec( name="save_all_found", - doc="int32ean flag. Save all spatial and temporal components found", - dtype="int32", + doc="Boolean flag that determines whether to save all spatial and temporal components found.", + dtype="bool", + ), + NWBAttributeSpec( + name="plot_loss", + doc="Boolean flag indicating whether empirical risk was plotted against iterations during alternating estimation.", + dtype="bool", ), ], datasets=[ + NWBDatasetSpec( + name="baseline_quantile", + doc="Baseline quantile for Df/F calculation in preprocessing.", + shape=(None, None), + dtype="double", + dims=("height", "width"), + quantity="?", + ), NWBDatasetSpec( name="S_init", doc="Cell images provided to algorithm as the initial set of cells, skipping its native initialization", shape=(None, None), dtype="double", - dims=("heightxwidth", "num_cells"), + dims=("height", "width"), quantity="?", ), NWBDatasetSpec( @@ -375,7 +522,39 @@ def main(): ), NWBDatasetSpec( name="movie_mask", - doc="Movie mask.", + doc="The circular mask to apply for microendoscopic movies during preprocessing.", + shape=(None, None), + dtype="double", + dims=("height", "width"), + quantity="?", + ), + NWBDatasetSpec( + name="is_pixel_valid", + doc="No description available", + shape=(None, None), + dtype="double", + dims=("height", "width"), + quantity="?", + ), + NWBDatasetSpec( + name="num_frames", + doc="No description available", + shape=(None, None), + dtype="double", + dims=("height", "width"), + quantity="?", + ), + NWBDatasetSpec( + name="num_iter_stop_quality_checks", + doc="No description available", + shape=(None, None), + dtype="double", + dims=("height", "width"), + quantity="?", + ), + NWBDatasetSpec( + name="pick_gpu", + doc="No description available", shape=(None, None), dtype="double", dims=("height", "width"), From 307278775358691a5ed5a8fb21cdbaa4ea0ec9e9 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 12:06:06 +0200 Subject: [PATCH 03/18] refactor descriptions --- src/spec/create_extension_spec.py | 79 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index b60f6b6..4b55ac2 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -28,12 +28,12 @@ def main(): # preprocessing parameters NWBAttributeSpec( name="preprocess", - doc="Boolean flag indicating data preprocessing before main EXTRACT function", + doc="Indicates whether or not data preprocessing was applied before main EXTRACT function.", dtype="bool", ), NWBAttributeSpec( name="pre_mask_on", - doc="Boolean flag indicating whether to use an image mask for preprocessing.", + doc="Indicates whether or not an image mask was applied for preprocessing.", dtype="bool", ), NWBAttributeSpec( @@ -43,68 +43,67 @@ def main(): ), NWBAttributeSpec( name="fix_zero_FOV_strips", - doc="Boolean flag. Find and fix spatial slices that are occasionally zero due to " + doc="Indicates whether or not find and fix spatial slices that are occasionally zero due to " "frame registration during preprocessing.", dtype="bool", ), NWBAttributeSpec( name="medfilt_outlier_pixels", - doc="Boolean flag determining whether outlier pixels in the movie should be replaced with their " + doc="Determines whether outlier pixels in the movie should be replaced with their " "neighborhood median.", dtype="bool", ), NWBAttributeSpec( name="skip_dff", - doc="Boolean flag indicating whether to skip Df/F calculation in preprocessing.", + doc="Indicates whether to skip Df/F calculation in preprocessing.", dtype="bool", ), NWBAttributeSpec( name="second_df", - doc="No description available.", + doc="The setting for second df calculation.", dtype="float64", ), NWBAttributeSpec( name="skip_highpass", - doc="Boolean flag. Skip highpass filtering in preprocessing.", + doc="Indicates whether to skip highpass filtering in preprocessing.", dtype="bool", ), NWBAttributeSpec( name="spatial_highpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " - "Values defined relative to the average cell radius. ", + "Values defined relative to the average cell radius.", dtype="float64", ), NWBAttributeSpec( name="temporal_denoising", - doc="Boolean flag that determines whether to apply temporal wavelet denoising.", + doc="Determines whether to apply temporal wavelet denoising.", dtype="bool", ), NWBAttributeSpec( name="remove_background", - doc="Boolean flag that determines whether to subtract the (spatial) background " + doc="Determines whether to subtract the (spatial) background " "(largest spatiotemporal mode of the movie matrix).", - dtype="float64", + dtype="bool", ), # general control parameters NWBAttributeSpec( name="use_default_gpu", - doc="Boolean flag indicating whether to use the default GPU.", + doc="Indicates whether or not use the default GPU.", dtype="bool", ), NWBAttributeSpec( name="use_gpu", - doc="Boolean flag indicating whether to run EXTRACT on GPU." - "If False, EXTRACT was run on CPU.", + doc="Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU.", dtype="bool", ), NWBAttributeSpec( name="multi_gpu", - doc="Boolean flag indicating whether multiple GPUs were used.", + doc="Indicates whether multiple GPUs were used.", dtype="bool", ), NWBAttributeSpec( name="parallel_cpu", - doc="Boolean flag indicating whether to run EXTRACT on parallel CPU.", + doc="Indicates whether parallel CPUs were used.", dtype="bool", ), NWBAttributeSpec( @@ -146,18 +145,19 @@ def main(): ), NWBAttributeSpec( name="dendrite_aware", - doc="Boolean flag, set it to true if dendrites exist in the movie & are desired in the output.", + doc="Determines whether or not dendrites are preserved in the output " + "for movies where dendrites are present.", dtype="bool", ), NWBAttributeSpec( name="crop_circular", - doc="Boolean flag. For microendoscopic movies, set it to true for automatically cropping out " + doc="For microendoscopic movies, whether or not automatically cropping out " "the region outside the circular imaging region.", dtype="bool", ), NWBAttributeSpec( name="use_sparse_arrays", - doc="Boolean flag. If set to true, then the output cell images will be saved as sparse arrays.", + doc="Determines whether not the output cell images were saved as sparse arrays.", dtype="bool", ), NWBAttributeSpec( @@ -167,13 +167,12 @@ def main(): ), NWBAttributeSpec( name="compact_output", - doc="Boolean flag. If set to true, then the output will not include bad components that were " - "found but then eliminated.", + doc="Indicates whether or not leave out bad components that were found but then eliminated from the output.", dtype="bool", ), NWBAttributeSpec( name="hyperparameter_tuning_flag", - doc="Boolean flag indicating internal hyperparameter tuning.", + doc="Indicates whether or not use internal hyperparameter tuning.", dtype="bool", ), NWBAttributeSpec( @@ -220,7 +219,7 @@ def main(): ), NWBAttributeSpec( name="cellfind_adaptive_kappa", - doc="Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " + doc="If True, then during cell finding, the robust esimation loss will adaptively " "set its robustness parameter", dtype="bool", ), @@ -253,7 +252,7 @@ def main(): ), NWBAttributeSpec( name="init_with_gaussian", - doc="Boolean flag. If True, then during cell finding, each cell is initialized with a gaussian shape " + doc="If True, then during cell finding, each cell is initialized with a gaussian shape " "prior to robust estimation. If False, then initialization is done with a " "correlation image (preferred for movies with dendrites).", dtype="bool", @@ -385,26 +384,24 @@ def main(): # other parameters NWBAttributeSpec( name="downsample_time_by", - doc="Time downsampling factor. If set to auto downsampling factor based on avg cell radius and " - "avg calcium event time constant.", + doc="Time downsampling factor.", dtype="float64", required=False, ), NWBAttributeSpec( name="min_tau_after_downsampling", - doc="Minimum event tau after downsampling. Used when downsample_time_by = auto", + doc="Minimum event tau after downsampling.", dtype="float64", ), NWBAttributeSpec( name="downsample_space_by", - doc="Spatial downsampling factor. If set to auto downsampling factor based on avg cell radius and " - "avg calcium event time constant.", + doc="Spatial downsampling factor.", dtype="float64", required=False, ), NWBAttributeSpec( name="min_radius_after_downsampling", - doc="Minimum avg radius after downsampling. Used when downsample_space_by = auto.", + doc="Minimum avg radius after downsampling.", dtype="float64", ), NWBAttributeSpec( @@ -427,7 +424,7 @@ def main(): ), NWBAttributeSpec( name="arbitrary_mask", - doc="Boolean flag indicating whether to use an arbitraty mask on the images.", + doc="Indicates whether to use an arbitrary mask on the image.", dtype="bool", ), NWBAttributeSpec( @@ -439,17 +436,17 @@ def main(): NWBAttributeSpec( name="spatial_lowpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " - "Values defined relative to the average cell radius. ", + "Values defined relative to the average cell radius.", dtype="float64", ), NWBAttributeSpec( name="smooth_T", - doc="Boolean flag indicating whether calculated traces are smoothed using median filtering.", + doc="Indicates whether calculated traces are smoothed using median filtering.", dtype="bool", ), NWBAttributeSpec( name="smooth_S", - doc="Boolean flag indicating whether calculated images are smoothed using a 2-D Gaussian filter.", + doc="Indicates whether calculated images are smoothed using a 2-D Gaussian filter.", dtype="bool", ), NWBAttributeSpec( @@ -459,12 +456,12 @@ def main(): ), NWBAttributeSpec( name="max_iter_S", - doc="Maximum number of iterations for S estimation steps", + doc="Maximum number of iterations for S estimation steps.", dtype="float64", ), NWBAttributeSpec( name="max_iter_T", - doc="Maximum number of iterations for T estimation steps", + doc="Maximum number of iterations for T estimation steps.", dtype="float64", ), NWBAttributeSpec( @@ -475,7 +472,7 @@ def main(): ), NWBAttributeSpec( name="TOL_main", - doc=" If the relative change in the main objective function between 2 consecutive alternating " + doc="If the relative change in the main objective function between 2 consecutive alternating " "minimization steps is less than this, cell extraction is terminated.", dtype="float64", ), @@ -486,12 +483,12 @@ def main(): ), NWBAttributeSpec( name="save_all_found", - doc="Boolean flag that determines whether to save all spatial and temporal components found.", + doc="Determines whether to save all spatial and temporal components found.", dtype="bool", ), NWBAttributeSpec( name="plot_loss", - doc="Boolean flag indicating whether empirical risk was plotted against iterations during alternating estimation.", + doc="Indicates whether empirical risk was plotted against iterations during alternating estimation.", dtype="bool", ), ], @@ -506,7 +503,7 @@ def main(): ), NWBDatasetSpec( name="S_init", - doc="Cell images provided to algorithm as the initial set of cells, skipping its native initialization", + doc="Cell images provided to algorithm as the initial set of cells, skipping its native initialization.", shape=(None, None), dtype="double", dims=("height", "width"), @@ -514,7 +511,7 @@ def main(): ), NWBDatasetSpec( name="T_init", - doc="Cell traces provided to algorithm as the initial set of traces, skipping its native initialization", + doc="Cell traces provided to algorithm as the initial set of traces, skipping its native initialization.", shape=(None, None), dtype="double", dims=("num_cells", "timepoints"), From 39d37a68e5951bab9a7878bf4dd3dfed006e63df Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 12:14:51 +0200 Subject: [PATCH 04/18] make all attributes optional --- src/spec/create_extension_spec.py | 78 +++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index 4b55ac2..6cabaad 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -30,96 +30,114 @@ def main(): name="preprocess", doc="Indicates whether or not data preprocessing was applied before main EXTRACT function.", dtype="bool", + required=False, ), NWBAttributeSpec( name="pre_mask_on", doc="Indicates whether or not an image mask was applied for preprocessing.", dtype="bool", + required=False, ), NWBAttributeSpec( name="pre_mask_radius", doc="The radius of the image mask.", dtype="float64", + required=False, ), NWBAttributeSpec( name="fix_zero_FOV_strips", doc="Indicates whether or not find and fix spatial slices that are occasionally zero due to " "frame registration during preprocessing.", dtype="bool", + required=False, ), NWBAttributeSpec( name="medfilt_outlier_pixels", doc="Determines whether outlier pixels in the movie should be replaced with their " "neighborhood median.", dtype="bool", + required=False, ), NWBAttributeSpec( name="skip_dff", doc="Indicates whether to skip Df/F calculation in preprocessing.", dtype="bool", + required=False, ), NWBAttributeSpec( name="second_df", doc="The setting for second df calculation.", dtype="float64", + required=False, ), NWBAttributeSpec( name="skip_highpass", doc="Indicates whether to skip highpass filtering in preprocessing.", dtype="bool", + required=False, ), NWBAttributeSpec( name="spatial_highpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " "Values defined relative to the average cell radius.", dtype="float64", + required=False, ), NWBAttributeSpec( name="temporal_denoising", doc="Determines whether to apply temporal wavelet denoising.", dtype="bool", + required=False, ), NWBAttributeSpec( name="remove_background", doc="Determines whether to subtract the (spatial) background " "(largest spatiotemporal mode of the movie matrix).", dtype="bool", + required=False, ), # general control parameters NWBAttributeSpec( name="use_default_gpu", doc="Indicates whether or not use the default GPU.", dtype="bool", + required=False, ), NWBAttributeSpec( name="use_gpu", doc="Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU.", dtype="bool", + required=False, ), NWBAttributeSpec( name="multi_gpu", doc="Indicates whether multiple GPUs were used.", dtype="bool", + required=False, ), NWBAttributeSpec( name="parallel_cpu", doc="Indicates whether parallel CPUs were used.", dtype="bool", + required=False, ), NWBAttributeSpec( name="avg_cell_radius", doc="Radius estimate for an average cell in the movie.", dtype="float64", + required=False, ), NWBAttributeSpec( name="avg_event_tau", doc="Determines the average event tau.", dtype="float64", + required=False, ), NWBAttributeSpec( name="avg_yield_threshold", doc="Determines the average yield threshold.", dtype="float64", + required=False, ), NWBAttributeSpec( name="num_partitions_x", @@ -137,97 +155,115 @@ def main(): name="trace_output_option", doc="Raw or non-negative output traces.", dtype="text", + required=False, ), NWBAttributeSpec( name="trace_quantile", doc="No description available.", dtype="float64", + required=False, ), NWBAttributeSpec( name="dendrite_aware", doc="Determines whether or not dendrites are preserved in the output " "for movies where dendrites are present.", dtype="bool", + required=False, ), NWBAttributeSpec( name="crop_circular", doc="For microendoscopic movies, whether or not automatically cropping out " "the region outside the circular imaging region.", dtype="bool", + required=False, ), NWBAttributeSpec( name="use_sparse_arrays", doc="Determines whether not the output cell images were saved as sparse arrays.", dtype="bool", + required=False, ), NWBAttributeSpec( name="verbose", doc="Indicates the level of verbosity.", dtype="float64", + required=False, ), NWBAttributeSpec( name="compact_output", doc="Indicates whether or not leave out bad components that were found but then eliminated from the output.", dtype="bool", + required=False, ), NWBAttributeSpec( name="hyperparameter_tuning_flag", doc="Indicates whether or not use internal hyperparameter tuning.", dtype="bool", + required=False, ), NWBAttributeSpec( name="remove_duplicate_cells", doc="For movies processed in multiple partitions, this flag controls duplicate removal in the " "overlap regions.", dtype="bool", + required=False, ), NWBAttributeSpec( name="max_iter", doc="Maximum number of alternating estimation iterations.", dtype="float64", + required=False, ), NWBAttributeSpec( name="minimal_checks", doc="Minimum number of checks that are performed.", dtype="float64", + required=False, ), # cell finding parameters NWBAttributeSpec( name="cellfind_min_snr", doc="Minimum peak SNR value for an object to be considered as a cell.", dtype="float64", + required=False, ), NWBAttributeSpec( name="cellfind_max_steps", doc="Maximum number of cell candidate initialization during cell finding step.", dtype="float64", + required=False, ), NWBAttributeSpec( name="cellfind_kappa_std_ratio", doc="Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.", dtype="float64", + required=False, ), NWBAttributeSpec( name="cellfind_filter_type", doc="Type of the spatial smoothing filter used for cell finding.", dtype="text", + required=False, ), NWBAttributeSpec( name="cellfind_numpix_threshold", doc="During cell finding, objects with an area < cellfind_numpix_threshold are discarded.", dtype="float64", + required=False, ), NWBAttributeSpec( name="cellfind_adaptive_kappa", doc="If True, then during cell finding, the robust esimation loss will adaptively " "set its robustness parameter", dtype="bool", + required=False, ), NWBAttributeSpec( name="cellfind_spatial_highpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " "Values defined relative to the average cell radius. ", dtype="float64", + required=False, ), NWBAttributeSpec( name="moving_radius", @@ -256,38 +292,45 @@ def main(): "prior to robust estimation. If False, then initialization is done with a " "correlation image (preferred for movies with dendrites).", dtype="bool", + required=False, ), NWBAttributeSpec( name="high2low_brightness_ratio", doc="Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to " "stop cell finding process.", dtype="float64", + required=False, ), # Visualizations during cell finding NWBAttributeSpec( name="visualize_cellfinding", doc="The visualization setting for cell finding.", dtype="float64", + required=False, ), NWBAttributeSpec( name="visualize_cellfinding_full_range", doc="The visualization setting for cell finding.", dtype="float64", + required=False, ), NWBAttributeSpec( name="visualize_cellfinding_max", doc="The visualization setting for cell finding.", dtype="float64", + required=False, ), NWBAttributeSpec( name="visualize_cellfinding_min", doc="The visualization setting for cell finding.", dtype="float64", + required=False, ), NWBAttributeSpec( name="visualize_cellfinding_show_bad_cells", doc="The visualization setting for cell finding.", dtype="float64", + required=False, ), # cell refinement parameters NWBAttributeSpec( @@ -295,91 +338,108 @@ def main(): doc="Kappa will be set to this times the noise std during the cell refinement process. " "Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="T_dup_corr_thresh", doc="Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh " "are eliminated. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="T_dup_thresh", doc="Threshold for traces. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="S_dup_corr_thresh", doc="Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh " "are eliminated. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="S_corr_thresh", doc="Image correlation threshold. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="T_corr_thresh", doc="Trace correlation threshold. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="temporal_corrupt_thresh", doc="Threshold for temporal corruption index. Traces above this threshold are eliminated duirng " "alternating minimization routine. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="spatial_corrupt_thresh", doc="Threshold for spatial corruption index. Images above this threshold are eliminated duirng " "alternating minimization routine. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="T_min_snr", doc="Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. " "Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="size_lower_limit", doc="Lower size limit for found cells. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="size_upper_limit", doc="Lower size limit for found cells. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="eccent_thresh", doc="Upper limit of eccentricity for found cells. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="low_ST_index_thresh", doc="Lower limit of spatiotemporal activity index. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="low_cell_area_flag", doc="Boolean flag indicating lower limit of cell area.", dtype="bool", + required=False, ), NWBAttributeSpec( name="high_ST_index_thresh", doc="Upper limit limit of spatiotemporal activity index. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="low_ST_corr_thresh", doc="Lower limit of spatiotemporal corelation. Cell refinement parameter.", dtype="float64", + required=False, ), NWBAttributeSpec( name="confidence_thresh", doc="Confidence threshold for found cells. Cell refinement parameter.", dtype="float64", + required=False, ), # other parameters NWBAttributeSpec( @@ -392,6 +452,7 @@ def main(): name="min_tau_after_downsampling", doc="Minimum event tau after downsampling.", dtype="float64", + required=False, ), NWBAttributeSpec( name="downsample_space_by", @@ -403,93 +464,110 @@ def main(): name="min_radius_after_downsampling", doc="Minimum avg radius after downsampling.", dtype="float64", + required=False, ), NWBAttributeSpec( name="reestimate_S_if_downsampled", doc="Boolean flag. When set to True, images are re-estimated from full movie at the end. " "When False, images are upsampled by interpolation.", dtype="bool", + required=False, ), NWBAttributeSpec( name="reestimate_T_if_downsampled", doc="Boolean flag. When set to True, traces are re-estimated from full movie at the end. " "When False, traces are upsampled by interpolation.", dtype="bool", + required=False, ), NWBAttributeSpec( name="adaptive_kappa", doc="Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " "set its robustness parameter", dtype="bool", + required=False, ), NWBAttributeSpec( name="arbitrary_mask", doc="Indicates whether to use an arbitrary mask on the image.", dtype="bool", + required=False, ), NWBAttributeSpec( name="smoothing_ratio_x2y", doc="If the movie contains mainly objects that are elongated in one dimension " "(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.", dtype="float64", + required=False, ), NWBAttributeSpec( name="spatial_lowpass_cutoff", doc="Cutoff determining the strength of butterworth spatial filtering of the movie. " "Values defined relative to the average cell radius.", dtype="float64", + required=False, ), NWBAttributeSpec( name="smooth_T", doc="Indicates whether calculated traces are smoothed using median filtering.", dtype="bool", + required=False, ), NWBAttributeSpec( name="smooth_S", doc="Indicates whether calculated images are smoothed using a 2-D Gaussian filter.", dtype="bool", + required=False, ), NWBAttributeSpec( name="l1_penalty_factor", doc="Strength of l1 regularization penalty to be applied when estimating the temporal components.", dtype="float64", + required=False, ), NWBAttributeSpec( name="max_iter_S", doc="Maximum number of iterations for S estimation steps.", dtype="float64", + required=False, ), NWBAttributeSpec( name="max_iter_T", doc="Maximum number of iterations for T estimation steps.", dtype="float64", + required=False, ), NWBAttributeSpec( name="TOL_sub", doc="If the 1-step relative change in the objective within each T and S optimization is less than this, " "the respective optimization is terminated.", dtype="float64", + required=False, ), NWBAttributeSpec( name="TOL_main", doc="If the relative change in the main objective function between 2 consecutive alternating " "minimization steps is less than this, cell extraction is terminated.", dtype="float64", + required=False, ), NWBAttributeSpec( name="T_lower_snr_threshold", doc="Lower SNR threshold for found traces.", dtype="float64", + required=False, ), NWBAttributeSpec( name="save_all_found", doc="Determines whether to save all spatial and temporal components found.", dtype="bool", + required=False, ), NWBAttributeSpec( name="plot_loss", doc="Indicates whether empirical risk was plotted against iterations during alternating estimation.", dtype="bool", + required=False, ), ], datasets=[ From 9249b610a392a354752fed1df89904be00df75c0 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 12:26:08 +0200 Subject: [PATCH 05/18] add version attribute --- src/spec/create_extension_spec.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/spec/create_extension_spec.py b/src/spec/create_extension_spec.py index 6cabaad..da019ea 100644 --- a/src/spec/create_extension_spec.py +++ b/src/spec/create_extension_spec.py @@ -26,6 +26,12 @@ def main(): neurodata_type_inc="ImageSegmentation", attributes=[ # preprocessing parameters + NWBAttributeSpec( + name="version", + doc="The version of EXTRACT.", + dtype="text", + required=False, + ), NWBAttributeSpec( name="preprocess", doc="Indicates whether or not data preprocessing was applied before main EXTRACT function.", From 1b13d3c12b41ba1805d203a2e95636509b6e7c05 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:41:07 +0200 Subject: [PATCH 06/18] add EXTRACTSegmentation class --- src/pynwb/ndx_extract/extract.py | 567 +++++++++++++++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 src/pynwb/ndx_extract/extract.py diff --git a/src/pynwb/ndx_extract/extract.py b/src/pynwb/ndx_extract/extract.py new file mode 100644 index 0000000..a82d822 --- /dev/null +++ b/src/pynwb/ndx_extract/extract.py @@ -0,0 +1,567 @@ +from hdmf import docval +from hdmf.utils import get_docval, popargs +from pynwb import register_class +from pynwb.ophys import ImageSegmentation + + +@register_class('EXTRACTSegmentation', 'ndx-extract') +class EXTRACTSegmentation(ImageSegmentation): + """EXTRACT configuration parameters added to an ImageSegmentation object.""" + + __nwbfields__ = ('version', 'preprocess', 'pre_mask_on', 'pre_mask_radius', + 'fix_zero_FOV_strips', 'medfilt_outlier_pixels', 'skip_dff', + 'second_df', 'skip_highpass', 'spatial_highpass_cutoff', + 'temporal_denoising', 'remove_background', 'use_default_gpu', + 'use_gpu', 'multi_gpu', 'parallel_cpu', 'avg_cell_radius', + 'avg_event_tau', 'avg_yield_threshold', 'num_partitions_x', + 'num_partitions_y', 'trace_output_option', 'trace_quantile', + 'dendrite_aware', 'crop_circular', 'use_sparse_arrays', 'verbose', + 'compact_output', 'hyperparameter_tuning_flag', + 'remove_duplicate_cells', 'max_iter', 'minimal_checks', + 'cellfind_min_snr', 'cellfind_max_steps', 'cellfind_kappa_std_ratio', + 'cellfind_filter_type', 'cellfind_numpix_threshold', + 'cellfind_adaptive_kappa', 'cellfind_spatial_highpass_cutoff', + 'moving_radius', 'moving_radius_spatial', 'moving_radius_temporal', + 'init_with_gaussian', 'high2low_brightness_ratio', + 'visualize_cellfinding', 'visualize_cellfinding_full_range', + 'visualize_cellfinding_max', 'visualize_cellfinding_min', + 'visualize_cellfinding_show_bad_cells', 'kappa_std_ratio', + 'T_dup_corr_thresh', 'T_dup_thresh', 'S_dup_corr_thresh', + 'S_corr_thresh', 'T_corr_thresh', 'temporal_corrupt_thresh', + 'spatial_corrupt_thresh', 'T_min_snr', 'size_lower_limit', + 'size_upper_limit', 'eccent_thresh', 'low_ST_index_thresh', + 'low_cell_area_flag', 'high_ST_index_thresh', + 'low_ST_corr_thresh', 'confidence_thresh', 'downsample_time_by', + 'min_tau_after_downsampling', 'downsample_space_by', + 'min_radius_after_downsampling', 'reestimate_S_if_downsampled', + 'reestimate_T_if_downsampled', 'adaptive_kappa', 'arbitrary_mask', + 'smoothing_ratio_x2y', 'spatial_lowpass_cutoff', 'smooth_T', + 'smooth_S', 'l1_penalty_factor', 'max_iter_S', 'max_iter_T', + 'TOL_sub', 'TOL_main', 'T_lower_snr_threshold', 'save_all_found', + 'plot_loss', 'baseline_quantile', 'S_init', 'T_init', 'movie_mask', + 'is_pixel_valid', 'num_frames', 'num_iter_stop_quality_checks', 'pick_gpu') + + @docval( + *get_docval( + ImageSegmentation.__init__, + "name", + ), # required + {'name': 'version', + 'type': str, + 'doc': 'The version of EXTRACT.', + 'default': None, + }, + {'name': 'preprocess', + 'type': bool, + 'doc': 'Indicates whether or not data preprocessing was applied before main EXTRACT function.', + 'default': None, + }, + {'name': 'pre_mask_on', + 'type': bool, + 'doc': 'Indicates whether or not an image mask was applied for preprocessing.', + 'default': None, + }, + {'name': 'pre_mask_radius', + 'type': float, + 'doc': 'If an image mask was applied for preprocessing, indicates the radius of the image mask.', + 'default': None, + }, + {'name': 'fix_zero_FOV_strips', + 'type': bool, + 'doc': "Indicates whether or not find and fix spatial slices that are occasionally zero due to " + "frame registration during preprocessing.", + 'default': None, + }, + {'name': 'medfilt_outlier_pixels', + 'type': bool, + 'doc': "Determines whether outlier pixels in the movie should be replaced with their " + "neighborhood median.", + 'default': None, + }, + {'name': 'skip_dff', + 'type': bool, + 'doc': "Indicates whether to skip Df/F calculation in preprocessing.", + 'default': None, + }, + {'name': 'second_df', + 'type': float, + 'doc': "The setting for second df calculation.", + 'default': None, + }, + {'name': 'skip_highpass', + 'type': bool, + 'doc': "Indicates whether to skip highpass filtering in preprocessing.", + 'default': None, + }, + {'name': 'spatial_highpass_cutoff', + 'type': float, + 'doc': "Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius.", + 'default': None, + }, + {'name': 'temporal_denoising', + 'type': bool, + 'doc': "Determines whether to apply temporal wavelet denoising.", + 'default': None, + }, + {'name': 'remove_background', + 'type': bool, + 'doc': "Determines whether to subtract the (spatial) background " + "(largest spatiotemporal mode of the movie matrix).", + 'default': None, + }, + {'name': 'use_default_gpu', + 'type': bool, + 'doc': "Indicates whether or not use the default GPU.", + 'default': None, + }, + {'name': 'use_gpu', + 'type': bool, + 'doc': "Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU.", + 'default': None, + }, + {'name': 'multi_gpu', + 'type': bool, + 'doc': "Indicates whether multiple GPUs were used.", + 'default': None, + }, + {'name': 'parallel_cpu', + 'type': bool, + 'doc': "Indicates whether parallel CPUs were used.", + 'default': None, + }, + {'name': 'avg_cell_radius', + 'type': float, + 'doc': "Radius estimate for an average cell in the movie.", + 'default': None, + }, + {'name': 'avg_event_tau', + 'type': float, + 'doc': "Determines the average event tau.", + 'default': None, + }, + {'name': 'avg_yield_threshold', + 'type': float, + 'doc': "Determines the average yield threshold.", + 'default': None, + }, + {'name': 'num_partitions_x', + 'type': float, + 'doc': "Number of movie partitions in x dimension.", + 'default': None, + }, + {'name': 'num_partitions_y', + 'type': float, + 'doc': "Number of movie partitions in y dimension.", + 'default': None, + }, + {'name': 'trace_output_option', + 'type': str, + 'doc': "Raw or non-negative output traces.", + 'default': None, + }, + {'name': 'trace_quantile', + 'type': float, + 'doc': "No description available.", + 'default': None, + }, + {'name': 'dendrite_aware', + 'type': bool, + 'doc': "Determines whether or not dendrites are preserved in the output " + "for movies where dendrites are present.", + 'default': None, + }, + {'name': 'crop_circular', + 'type': bool, + 'doc': "For microendoscopic movies, whether or not automatically cropping out " + "the region outside the circular imaging region.", + 'default': None, + }, + {'name': 'use_sparse_arrays', + 'type': bool, + 'doc': "Determines whether not the output cell images were saved as sparse arrays.", + 'default': None, + }, + {'name': 'verbose', + 'type': float, + 'doc': "Indicates the level of verbosity.", + 'default': None, + }, + {'name': 'compact_output', + 'type': bool, + 'doc': "Indicates whether or not leave out bad components that were found but " + "then eliminated from the output.", + 'default': None, + }, + {'name': 'hyperparameter_tuning_flag', + 'type': bool, + 'doc': "Indicates whether or not use internal hyperparameter tuning.", + 'default': None, + }, + {'name': 'remove_duplicate_cells', + 'type': bool, + 'doc': "For movies processed in multiple partitions, " + "this flag controls duplicate removal in the overlap regions.", + 'default': None, + }, + {'name': 'max_iter', + 'type': float, + 'doc': "Maximum number of alternating estimation iterations.", + 'default': None, + }, + {'name': 'minimal_checks', + 'type': float, + 'doc': "Minimum number of checks that are performed.", + 'default': None, + }, + {'name': 'cellfind_min_snr', + 'type': float, + 'doc': "Minimum peak SNR value for an object to be considered as a cell.", + 'default': None, + }, + {'name': 'cellfind_max_steps', + 'type': float, + 'doc': "Maximum number of cell candidate initialization during cell finding step.", + 'default': None, + }, + {'name': 'cellfind_kappa_std_ratio', + 'type': float, + 'doc': "Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.", + 'default': None, + }, + {'name': 'cellfind_filter_type', + 'type': str, + 'doc': "Type of the spatial smoothing filter used for cell finding.", + 'default': None, + }, + {'name': 'cellfind_numpix_threshold', + 'type': float, + 'doc': "During cell finding, objects with an area < cellfind_numpix_threshold are discarded.", + 'default': None, + }, + {'name': 'cellfind_adaptive_kappa', + 'type': bool, + 'doc': "If True, then during cell finding, the robust esimation loss will adaptively " + "set its robustness parameter", + 'default': None, + }, + {'name': 'cellfind_spatial_highpass_cutoff', + 'type': float, + 'doc': "Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius. ", + 'default': None, + }, + {'name': 'moving_radius', + 'type': float, + 'doc': "Deprecated variable for older EXTRACT file versions. " + "Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average)", + 'default': None, + }, + {'name': 'moving_radius_spatial', + 'type': float, + 'doc': "Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average) for the image.", + 'default': None, + }, + {'name': 'moving_radius_temporal', + 'type': float, + 'doc': "Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average) for the traces.", + 'default': None, + }, + {'name': 'init_with_gaussian', + 'type': bool, + 'doc': "If True, then during cell finding, each cell is initialized with a gaussian shape " + "prior to robust estimation. If False, then initialization is done with a " + "correlation image (preferred for movies with dendrites).", + 'default': None, + }, + {'name': 'high2low_brightness_ratio', + 'type': float, + 'doc': "Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to " + "stop cell finding process.", + 'default': None, + }, + {'name': 'visualize_cellfinding', + 'type': float, + 'doc': "The visualization setting for cell finding.", + 'default': None, + }, + {'name': 'visualize_cellfinding_full_range', + 'type': float, + 'doc': "The visualization setting for cell finding.", + 'default': None, + }, + {'name': 'visualize_cellfinding_max', + 'type': float, + 'doc': "The visualization setting for cell finding.", + 'default': None, + }, + {'name': 'visualize_cellfinding_min', + 'type': float, + 'doc': "The visualization setting for cell finding.", + 'default': None, + }, + {'name': 'visualize_cellfinding_show_bad_cells', + 'type': float, + 'doc': "The visualization setting for cell finding.", + 'default': None, + }, + {'name': 'kappa_std_ratio', + 'type': float, + 'doc': "Kappa will be set to this times the noise std during the cell refinement process. " + "Cell refinement parameter.", + 'default': None, + }, + {'name': 'T_dup_corr_thresh', + 'type': float, + 'doc': "Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh " + "are eliminated. Cell refinement parameter.", + 'default': None, + }, + {'name': 'T_dup_thresh', + 'type': float, + 'doc': "Threshold for traces. Cell refinement parameter.", + 'default': None, + }, + {'name': 'S_dup_corr_thresh', + 'type': float, + 'doc': "Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh " + "are eliminated. Cell refinement parameter.", + 'default': None, + }, + {'name': 'S_corr_thresh', + 'type': float, + 'doc': "Image correlation threshold. Cell refinement parameter.", + 'default': None, + }, + {'name': 'T_corr_thresh', + 'type': float, + 'doc': "Trace correlation threshold. Cell refinement parameter.", + 'default': None, + }, + {'name': 'temporal_corrupt_thresh', + 'type': float, + 'doc': "Threshold for temporal corruption index. Traces above this threshold are eliminated duirng " + "alternating minimization routine. Cell refinement parameter.", + 'default': None, + }, + {'name': 'spatial_corrupt_thresh', + 'type': float, + 'doc': "Threshold for spatial corruption index. Images above this threshold are eliminated duirng " + "alternating minimization routine. Cell refinement parameter.", + 'default': None, + }, + {'name': 'T_min_snr', + 'type': float, + 'doc': "Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. " + "Cell refinement parameter.", + 'default': None, + }, + {'name': 'size_lower_limit', + 'type': float, + 'doc': "Lower size limit for found cells. Cell refinement parameter.", + 'default': None, + }, + {'name': 'size_upper_limit', + 'type': float, + 'doc': "Lower size limit for found cells. Cell refinement parameter.", + 'default': None, + }, + {'name': 'eccent_thresh', + 'type': float, + 'doc': "Upper limit of eccentricity for found cells. Cell refinement parameter.", + 'default': None, + }, + {'name': 'low_ST_index_thresh', + 'type': float, + 'doc': "Lower limit of spatiotemporal activity index. Cell refinement parameter.", + 'default': None, + }, + {'name': 'low_cell_area_flag', + 'type': bool, + 'doc': "Boolean flag indicating lower limit of cell area.", + 'default': None, + }, + {'name': 'high_ST_index_thresh', + 'type': float, + 'doc': "Upper limit limit of spatiotemporal activity index. Cell refinement parameter.", + 'default': None, + }, + {'name': 'low_ST_corr_thresh', + 'type': float, + 'doc': "Lower limit of spatiotemporal corelation. Cell refinement parameter.", + 'default': None, + }, + {'name': 'confidence_thresh', + 'type': float, + 'doc': "Confidence threshold for found cells. Cell refinement parameter.", + 'default': None, + }, + {'name': 'downsample_time_by', + 'type': float, + 'doc': "Time downsampling factor.", + 'default': None, + }, + {'name': 'min_tau_after_downsampling', + 'type': float, + 'doc': "Minimum event tau after downsampling.", + 'default': None, + }, + {'name': 'downsample_space_by', + 'type': float, + 'doc': "Spatial downsampling factor.", + 'default': None, + }, + {'name': 'min_radius_after_downsampling', + 'type': float, + 'doc': "Minimum avg radius after downsampling.", + 'default': None, + }, + {'name': 'reestimate_S_if_downsampled', + 'type': bool, + 'doc': "Boolean flag. When set to True, images are re-estimated from full movie at the end. " + "When False, images are upsampled by interpolation.", + 'default': None, + }, + {'name': 'reestimate_T_if_downsampled', + 'type': bool, + 'doc': "Boolean flag. When set to True, traces are re-estimated from full movie at the end. " + "When False, traces are upsampled by interpolation.", + 'default': None, + }, + {'name': 'adaptive_kappa', + 'type': bool, + 'doc': "Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " + "set its robustness parameter", + 'default': None, + }, + {'name': 'arbitrary_mask', + 'type': bool, + 'doc': "Indicates whether to use an arbitrary mask on the image.", + 'default': None, + }, + {'name': 'smoothing_ratio_x2y', + 'type': float, + 'doc': "If the movie contains mainly objects that are elongated in one dimension " + "(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.", + 'default': None, + }, + {'name': 'spatial_lowpass_cutoff', + 'type': float, + 'doc': "Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius.", + 'default': None, + }, + {'name': 'smooth_T', + 'type': bool, + 'doc': "Indicates whether calculated traces are smoothed using median filtering.", + 'default': None, + }, + {'name': 'smooth_S', + 'type': bool, + 'doc': "Indicates whether calculated images are smoothed using a 2-D Gaussian filter.", + 'default': None, + }, + {'name': 'l1_penalty_factor', + 'type': float, + 'doc': "Strength of l1 regularization penalty to be applied when estimating the temporal components.", + 'default': None, + }, + {'name': 'max_iter_S', + 'type': float, + 'doc': "Maximum number of iterations for S estimation steps.", + 'default': None, + }, + {'name': 'max_iter_T', + 'type': float, + 'doc': "Maximum number of iterations for T estimation steps.", + 'default': None, + }, + {'name': 'TOL_sub', + 'type': float, + 'doc': "If the 1-step relative change in the objective within each T and S optimization is less than this, " + "the respective optimization is terminated.", + 'default': None, + }, + {'name': 'TOL_main', + 'type': float, + 'doc': "If the relative change in the main objective function between 2 consecutive alternating " + "minimization steps is less than this, cell extraction is terminated.", + 'default': None, + }, + {'name': 'T_lower_snr_threshold', + 'type': float, + 'doc': "Lower SNR threshold for found traces.", + 'default': None, + }, + {'name': 'save_all_found', + 'type': bool, + 'doc': "Determines whether to save all spatial and temporal components found.", + 'default': None, + }, + {'name': 'plot_loss', + 'type': bool, + 'doc': "Indicates whether empirical risk was plotted against iterations during alternating estimation.", + 'default': None, + }, + {'name': 'baseline_quantile', + 'type': ('array_data', 'data'), + 'doc': "Baseline quantile for Df/F calculation in preprocessing." + "The first dimension denotes the height of the image, the second dimension " + "corresponds to the width of the image.", + 'default': None, + 'shape': (None, None), + }, + {'name': 'S_init', + 'type': ('array_data', 'data'), + 'doc': "Cell images provided to algorithm as the initial set of cells, skipping its native initialization." + "The first dimension denotes the height of the image, the second dimension " + "corresponds to the width of the image.", + 'default': None, + 'shape': (None, None), + }, + {'name': 'T_init', + 'type': ('array_data', 'data'), + 'doc': "Cell traces provided to algorithm as the initial set of traces, skipping its native initialization." + "The first dimension denotes the number of cells, the second dimension " + "corresponds to time.", + 'default': None, + 'shape': (None, None), + }, + {'name': 'movie_mask', + 'type': ('array_data', 'data'), + 'doc': "The circular mask to apply for microendoscopic movies during preprocessing.", + 'default': None, + 'shape': (None, None), + }, + {'name': 'is_pixel_valid', + 'type': ('array_data', 'data'), + 'doc': "No description available", + 'default': None, + 'shape': (None, None), + }, + {'name': 'num_frames', + 'type': ('array_data', 'data'), + 'doc': "No description available", + 'default': None, + }, + {'name': 'num_iter_stop_quality_checks', + 'type': ('array_data', 'data'), + 'doc': "No description available", + 'default': None, + 'shape': (None, None), + }, + {'name': 'pick_gpu', + 'type': ('array_data', 'data'), + 'doc': "No description available", + 'default': None, + 'shape': (None, None), + }, + ) + def __init__(self, **kwargs): + name = popargs('name', kwargs) + super().__init__(name=name) + for extract_config_name, extract_config_value in kwargs.items(): + setattr(self, extract_config_name, extract_config_value) From 3b72ab60d713ab43cb678eed5ff3aed7713fe2ee Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:41:56 +0200 Subject: [PATCH 07/18] import EXTRACTSegmentation from class definition --- src/pynwb/ndx_extract/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pynwb/ndx_extract/__init__.py b/src/pynwb/ndx_extract/__init__.py index 1ef26c7..274c62f 100644 --- a/src/pynwb/ndx_extract/__init__.py +++ b/src/pynwb/ndx_extract/__init__.py @@ -21,5 +21,4 @@ # Load the namespace load_namespaces(ndx_extract_specpath) - -EXTRACTSegmentation = get_class('EXTRACTSegmentation', 'ndx-extract') +from .extract import EXTRACTSegmentation From 179b41378e1e712956e47b603f871e2365115399 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:42:28 +0200 Subject: [PATCH 08/18] add basic tests for EXTRACTSegmentation --- src/pynwb/tests/test_extract.py | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/pynwb/tests/test_extract.py diff --git a/src/pynwb/tests/test_extract.py b/src/pynwb/tests/test_extract.py new file mode 100644 index 0000000..5d75594 --- /dev/null +++ b/src/pynwb/tests/test_extract.py @@ -0,0 +1,85 @@ +from datetime import datetime +from pathlib import Path +from shutil import rmtree +from tempfile import mkdtemp + +import numpy as np +from hdmf.testing import TestCase +from numpy.testing import assert_array_equal +from pynwb import NWBFile, NWBHDF5IO + +from ndx_extract import EXTRACTSegmentation + + +class TestExtractSegmentation(TestCase): + @classmethod + def setUpClass(cls): + cls.file_path = "extract_public_output.mat" + cls.output_struct_name = "output" + + cls.session_start_time = datetime.now().astimezone() + + cls.image_segmentation_name = "ExtractSegmentation" + + def setUp(self): + self.nwbfile = NWBFile( + session_description="session_description", + identifier="file_id", + session_start_time=self.session_start_time, + ) + + self.tmpdir = Path(mkdtemp()) + self.nwbfile_path = self.tmpdir / "test_nwb.nwb" + + def tearDown(self): + rmtree(self.tmpdir) + + def test_extract_defaults(self): + image_segmentation = EXTRACTSegmentation() + self.assertEqual(image_segmentation.name, "ImageSegmentation") + + def test_extract_name_propagates(self): + image_segmentation = EXTRACTSegmentation(name=self.image_segmentation_name) + self.assertEqual(image_segmentation.name, self.image_segmentation_name) + + def test_extract_roundtrip(self): + image_segmentation = EXTRACTSegmentation( + name="test_name", + version="1.1.0", + preprocess=True, + movie_mask=np.ones((15, 15)), + ) + + ophys_module = self.nwbfile.create_processing_module( + name="ophys", + description="optical physiology processed data", + ) + + ophys_module.add(image_segmentation) + + with NWBHDF5IO(self.nwbfile_path, mode="w") as io: + io.write(self.nwbfile) + + with NWBHDF5IO(self.nwbfile_path, mode="r") as io: + nwbfile_in = io.read() + + movie_mask = ( + nwbfile_in.processing["ophys"] + .data_interfaces["test_name"] + .movie_mask[:] + ) + + assert_array_equal( + movie_mask, + np.ones((15, 15)), + ) + + self.assertEqual( + nwbfile_in.processing["ophys"].data_interfaces["test_name"].version, + "1.1.0", + ) + + self.assertEqual( + nwbfile_in.processing["ophys"].data_interfaces["test_name"].preprocess, + True, + ) From 8f7d0646d9ebbc37f165b50b30ba9197bdbb0b33 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:50:13 +0200 Subject: [PATCH 09/18] update yaml files --- spec/ndx-EXTRACT.namespace.yaml | 6 ++++-- src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/spec/ndx-EXTRACT.namespace.yaml b/spec/ndx-EXTRACT.namespace.yaml index 8306bf8..bb0d2de 100644 --- a/spec/ndx-EXTRACT.namespace.yaml +++ b/spec/ndx-EXTRACT.namespace.yaml @@ -1,13 +1,15 @@ namespaces: - author: - Cesar Echavarria + - Szonja Weigl contact: - cesar.echavarria@catalystneuro.com - doc: NWB:N Extension for storage of EXTRACT parameters and output + - szonja.weigl@catalystneuro.com + doc: NWB Extension for storage of EXTRACT parameters and output name: ndx-extract schema: - namespace: core neurodata_types: - ImageSegmentation - source: ndx-extract.extensions.yaml - version: 0.1.1 + version: 0.2.0 diff --git a/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml b/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml index 8306bf8..bb0d2de 100644 --- a/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml +++ b/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml @@ -1,13 +1,15 @@ namespaces: - author: - Cesar Echavarria + - Szonja Weigl contact: - cesar.echavarria@catalystneuro.com - doc: NWB:N Extension for storage of EXTRACT parameters and output + - szonja.weigl@catalystneuro.com + doc: NWB Extension for storage of EXTRACT parameters and output name: ndx-extract schema: - namespace: core neurodata_types: - ImageSegmentation - source: ndx-extract.extensions.yaml - version: 0.1.1 + version: 0.2.0 From cac91507357688956fd5e27df38714bcda34f865 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:50:22 +0200 Subject: [PATCH 10/18] update setup.py --- setup.py | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/setup.py b/setup.py index 0fc19fe..92a867b 100644 --- a/setup.py +++ b/setup.py @@ -5,37 +5,25 @@ from setuptools import setup, find_packages from shutil import copy2 -# load README.md/README.rst file -try: - if os.path.exists('README.md'): - with open('README.md', 'r') as fp: - readme = fp.read() - readme_type = 'text/markdown; charset=UTF-8' - elif os.path.exists('README.rst'): - with open('README.rst', 'r') as fp: - readme = fp.read() - readme_type = 'text/x-rst; charset=UTF-8' - else: - readme = "" -except Exception: - readme = "" +here = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(here, "requirements.txt")) as f: + install_requires = f.read().strip().split("\n") + +with open("README.md", "r") as f: + long_description = f.read() setup_args = { 'name': 'ndx-extract', - 'version': '0.1.0', - 'description': 'NWB:N extension for storage of parameters and output of EXTRACT pipeline', - 'long_description': readme, - 'long_description_content_type': readme_type, - 'author': 'Cesar Echavarria', - 'author_email': 'cechavarroa60@gmail.com', - 'url': '', + 'version': '0.2.0', + 'description': 'NWB extension for storage of parameters and output of EXTRACT pipeline.', + 'long_description': long_description, + 'long_description_content_type': "text/markdown", + 'author': 'Ben Dichter, Szonja Weigl and Cesar Echavarria', + 'author_email': 'ben.dichter@catalystneuro.com', + 'url': "https://github.com/catalystneuro/ndx-extract", 'license': 'BSD 3-Clause', - 'install_requires': [ - 'pynwb>=1.5.0,<3', - 'hdmf>=2.5.6,<4', - 'hdmf-docutils>=0.4.4,<1' - ], - 'packages': find_packages('src/pynwb', exclude=["tests", "tests.*"]), + 'install_requires': install_requires, + 'packages': find_packages(where='src/pynwb', exclude=["tests", "tests.*"]), 'package_dir': {'': 'src/pynwb'}, 'package_data': {'ndx_extract': [ 'spec/ndx-extract.namespace.yaml', From ae29798909d78cf6bc4705126843d355155ca7bc Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:50:55 +0200 Subject: [PATCH 11/18] update yaml files --- spec/ndx-EXTRACT.extensions.yaml | 438 +++++++++++++----- .../spec/ndx-extract.extensions.yaml | 400 ++++++++++++---- 2 files changed, 634 insertions(+), 204 deletions(-) diff --git a/spec/ndx-EXTRACT.extensions.yaml b/spec/ndx-EXTRACT.extensions.yaml index 91099cd..86c8cab 100644 --- a/spec/ndx-EXTRACT.extensions.yaml +++ b/spec/ndx-EXTRACT.extensions.yaml @@ -3,232 +3,406 @@ groups: neurodata_type_inc: ImageSegmentation doc: EXTRACT configuration parameters attributes: + - name: version + dtype: text + doc: The version of EXTRACT. + required: false - name: preprocess - dtype: int32 - doc: int32ean flag indicating data preprocessing before main EXTRACT function + dtype: bool + doc: Indicates whether or not data preprocessing was applied before main EXTRACT + function. + required: false + - name: pre_mask_on + dtype: bool + doc: Indicates whether or not an image mask was applied for preprocessing. + required: false + - name: pre_mask_radius + dtype: float64 + doc: The radius of the image mask. + required: false - name: fix_zero_FOV_strips - dtype: int32 - doc: int32ean flag. Find and fix spatial slices that are occasionally zero due - to frame registration during preprocessing. + dtype: bool + doc: Indicates whether or not find and fix spatial slices that are occasionally + zero due to frame registration during preprocessing. + required: false - name: medfilt_outlier_pixels - dtype: int32 - doc: int32ean flag determining whether outlier pixels in the movie should be replaced - with their neighborhood median. + dtype: bool + doc: Determines whether outlier pixels in the movie should be replaced with their + neighborhood median. + required: false - name: skip_dff - dtype: int32 - doc: int32ean flag. Skip Df/F calculation in preprocessing. - - name: baseline_quantile - dtype: float32 - doc: Baseline quantile for Df/F calculation in preprocessing. + dtype: bool + doc: Indicates whether to skip Df/F calculation in preprocessing. + required: false + - name: second_df + dtype: float64 + doc: The setting for second df calculation. + required: false - name: skip_highpass - dtype: int32 - doc: int32ean flag. Skip highpass filtering in preprocessing. + dtype: bool + doc: Indicates whether to skip highpass filtering in preprocessing. + required: false - name: spatial_highpass_cutoff - dtype: float32 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' + dtype: float64 + doc: Cutoff determining the strength of butterworth spatial filtering of the movie. + Values defined relative to the average cell radius. + required: false - name: temporal_denoising - dtype: int32 - doc: int32ean flag that determines whether to apply temporal wavelet denoising. + dtype: bool + doc: Determines whether to apply temporal wavelet denoising. + required: false - name: remove_background - dtype: int32 - doc: int32ean flag that determines whether to subtract the (spatial) background - (largest spatiotemporal mode of the movie matrix). + dtype: bool + doc: Determines whether to subtract the (spatial) background (largest spatiotemporal + mode of the movie matrix). + required: false + - name: use_default_gpu + dtype: bool + doc: Indicates whether or not use the default GPU. + required: false + - name: use_gpu + dtype: bool + doc: Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU. + required: false + - name: multi_gpu + dtype: bool + doc: Indicates whether multiple GPUs were used. + required: false + - name: parallel_cpu + dtype: bool + doc: Indicates whether parallel CPUs were used. + required: false - name: avg_cell_radius - dtype: float32 + dtype: float64 doc: Radius estimate for an average cell in the movie. + required: false + - name: avg_event_tau + dtype: float64 + doc: Determines the average event tau. + required: false + - name: avg_yield_threshold + dtype: float64 + doc: Determines the average yield threshold. + required: false - name: num_partitions_x - dtype: uint32 + dtype: float64 doc: Number of movie partitions in x dimension. required: false - name: num_partitions_y - dtype: uint32 + dtype: float64 doc: Number of movie partitions in y dimension. required: false - name: trace_output_option dtype: text - doc: Raw or non-negative output traces + doc: Raw or non-negative output traces. + required: false + - name: trace_quantile + dtype: float64 + doc: No description available. + required: false - name: dendrite_aware - dtype: int32 - doc: int32ean flag, set it to true if dendrites exist in the movie & are desired - in the output. + dtype: bool + doc: Determines whether or not dendrites are preserved in the output for movies + where dendrites are present. + required: false - name: crop_circular - dtype: int32 - doc: int32ean flag. For microendoscopic movies, set it to true for automatically - cropping out the region outside the circular imaging region. + dtype: bool + doc: For microendoscopic movies, whether or not automatically cropping out the + region outside the circular imaging region. + required: false - name: use_sparse_arrays - dtype: int32 - doc: int32ean flag. If set to true, then the output cell images will be saved - as sparse arrays + dtype: bool + doc: Determines whether not the output cell images were saved as sparse arrays. + required: false + - name: verbose + dtype: float64 + doc: Indicates the level of verbosity. + required: false - name: compact_output - dtype: int32 - doc: int32ean flag. If set to true, then the output will not include bad components - that were found but then eliminated + dtype: bool + doc: Indicates whether or not leave out bad components that were found but then + eliminated from the output. + required: false - name: hyperparameter_tuning_flag - dtype: int32 - doc: int32ean flag indicating internal hyperparameter tuning. + dtype: bool + doc: Indicates whether or not use internal hyperparameter tuning. + required: false - name: remove_duplicate_cells - dtype: int32 + dtype: bool doc: For movies processed in multiple partitions, this flag controls duplicate removal in the overlap regions. + required: false - name: max_iter - dtype: uint32 + dtype: float64 doc: Maximum number of alternating estimation iterations. + required: false + - name: minimal_checks + dtype: float64 + doc: Minimum number of checks that are performed. + required: false - name: cellfind_min_snr - dtype: float32 + dtype: float64 doc: Minimum peak SNR value for an object to be considered as a cell. + required: false - name: cellfind_max_steps - dtype: int + dtype: float64 doc: Maximum number of cell candidate initialization during cell finding step. + required: false - name: cellfind_kappa_std_ratio - dtype: float32 + dtype: float64 doc: Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization. + required: false - name: cellfind_filter_type dtype: text doc: Type of the spatial smoothing filter used for cell finding. + required: false - name: cellfind_numpix_threshold - dtype: float32 + dtype: float64 doc: During cell finding, objects with an area < cellfind_numpix_threshold are discarded. + required: false + - name: cellfind_adaptive_kappa + dtype: bool + doc: If True, then during cell finding, the robust esimation loss will adaptively + set its robustness parameter + required: false + - name: cellfind_spatial_highpass_cutoff + dtype: float64 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' + required: false - name: moving_radius dtype: float32 + doc: Deprecated variable for older EXTRACT file versions. Radius of moving average + filter in the case when cellfind_filter_type = moveavg (moving average) + required: false + - name: moving_radius_spatial + dtype: float64 + doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg + (moving average) for the image. + required: false + - name: moving_radius_temporal + dtype: float64 doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) + (moving average) for the traces. + required: false - name: init_with_gaussian - dtype: int32 - doc: int32ean flag. If true, then during cell finding, each cell is initialized - with a gaussian shape prior to robust estimation. If false, then initialization - is done with a correlation image (preferred for movies with dendrites). + dtype: bool + doc: If True, then during cell finding, each cell is initialized with a gaussian + shape prior to robust estimation. If False, then initialization is done with + a correlation image (preferred for movies with dendrites). + required: false - name: high2low_brightness_ratio - dtype: float32 + dtype: float64 doc: Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to stop cell finding process. + required: false + - name: visualize_cellfinding + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_full_range + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_max + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_min + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_show_bad_cells + dtype: float64 + doc: The visualization setting for cell finding. + required: false - name: kappa_std_ratio - dtype: float32 + dtype: float64 doc: Kappa will be set to this times the noise std during the cell refinement process. Cell refinement parameter. + required: false - name: T_dup_corr_thresh - dtype: float32 + dtype: float64 doc: Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh are eliminated. Cell refinement parameter. + required: false + - name: T_dup_thresh + dtype: float64 + doc: Threshold for traces. Cell refinement parameter. + required: false - name: S_dup_corr_thresh - dtype: float32 + dtype: float64 doc: Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh are eliminated. Cell refinement parameter. + required: false + - name: S_corr_thresh + dtype: float64 + doc: Image correlation threshold. Cell refinement parameter. + required: false + - name: T_corr_thresh + dtype: float64 + doc: Trace correlation threshold. Cell refinement parameter. + required: false - name: temporal_corrupt_thresh - dtype: float32 + dtype: float64 doc: Threshold for temporal corruption index. Traces above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. + required: false - name: spatial_corrupt_thresh - dtype: float32 + dtype: float64 doc: Threshold for spatial corruption index. Images above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. + required: false - name: T_min_snr - dtype: float32 + dtype: float64 doc: Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. Cell refinement parameter. + required: false - name: size_lower_limit - dtype: float32 + dtype: float64 doc: Lower size limit for found cells. Cell refinement parameter. + required: false - name: size_upper_limit - dtype: float32 + dtype: float64 doc: Lower size limit for found cells. Cell refinement parameter. + required: false - name: eccent_thresh - dtype: float32 + dtype: float64 doc: Upper limit of eccentricity for found cells. Cell refinement parameter. + required: false - name: low_ST_index_thresh - dtype: float32 + dtype: float64 doc: Lower limit of spatiotemporal activity index. Cell refinement parameter. + required: false + - name: low_cell_area_flag + dtype: bool + doc: Boolean flag indicating lower limit of cell area. + required: false - name: high_ST_index_thresh - dtype: float32 + dtype: float64 doc: Upper limit limit of spatiotemporal activity index. Cell refinement parameter. + required: false - name: low_ST_corr_thresh - dtype: float32 + dtype: float64 doc: Lower limit of spatiotemporal corelation. Cell refinement parameter. + required: false - name: confidence_thresh - dtype: float32 - doc: Confidence threhsold for found cells. Cell refinement parameter. + dtype: float64 + doc: Confidence threshold for found cells. Cell refinement parameter. + required: false - name: downsample_time_by - dtype: float32 - doc: Time downsampling factor. If set to auto downsampling factor based on avg - cell radius and avg calcium event time constant. + dtype: float64 + doc: Time downsampling factor. required: false - name: min_tau_after_downsampling - dtype: float32 - doc: Minimum event tau after downsampling. Used when downsample_time_by = auto + dtype: float64 + doc: Minimum event tau after downsampling. + required: false - name: downsample_space_by - dtype: float32 - doc: Spatial downsampling factor. If set to auto downsampling factor based on - avg cell radius and avg calcium event time constant. + dtype: float64 + doc: Spatial downsampling factor. required: false - name: min_radius_after_downsampling - dtype: float32 - doc: Minimum avg radius after downsampling. Used when downsample_space_by = auto. + dtype: float64 + doc: Minimum avg radius after downsampling. + required: false - name: reestimate_S_if_downsampled - dtype: int32 - doc: int32ean flag. When set to true, images are re-estimated from full movie - at the end. When false, images are upsampled by interpolation. + dtype: bool + doc: Boolean flag. When set to True, images are re-estimated from full movie at + the end. When False, images are upsampled by interpolation. + required: false - name: reestimate_T_if_downsampled - dtype: int32 - doc: int32ean flag. When set to true, traces are re-estimated from full movie - at the end. When false, traces are upsampled by interpolation. + dtype: bool + doc: Boolean flag. When set to True, traces are re-estimated from full movie at + the end. When False, traces are upsampled by interpolation. + required: false - name: adaptive_kappa - dtype: int32 - doc: int32ean flag. If true, then during cell finding, the robust esimation loss + dtype: bool + doc: Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively set its robustness parameter + required: false + - name: arbitrary_mask + dtype: bool + doc: Indicates whether to use an arbitrary mask on the image. + required: false - name: smoothing_ratio_x2y - dtype: float32 + dtype: float64 doc: If the movie contains mainly objects that are elongated in one dimension (e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension. + required: false - name: spatial_lowpass_cutoff - dtype: float32 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' + dtype: float64 + doc: Cutoff determining the strength of butterworth spatial filtering of the movie. + Values defined relative to the average cell radius. + required: false - name: smooth_T - dtype: int32 - doc: int32ean flag indicating whether calculated traces are smoothed using median - filtering. + dtype: bool + doc: Indicates whether calculated traces are smoothed using median filtering. + required: false - name: smooth_S - dtype: int32 - doc: int32ean flag indicating whether calculated images are smoothed using a 2-D - Gaussian filter. + dtype: bool + doc: Indicates whether calculated images are smoothed using a 2-D Gaussian filter. + required: false - name: l1_penalty_factor - dtype: float32 + dtype: float64 doc: Strength of l1 regularization penalty to be applied when estimating the temporal components. + required: false - name: max_iter_S - dtype: uint32 - doc: Maximum number of iterations for S estimation steps + dtype: float64 + doc: Maximum number of iterations for S estimation steps. + required: false - name: max_iter_T - dtype: uint32 - doc: Maximum number of iterations for T estimation steps + dtype: float64 + doc: Maximum number of iterations for T estimation steps. + required: false - name: TOL_sub - dtype: float32 + dtype: float64 doc: If the 1-step relative change in the objective within each T and S optimization is less than this, the respective optimization is terminated. + required: false - name: TOL_main - dtype: float32 - doc: ' If the relative change in the main objective function between 2 consecutive - alternating minimization steps is less than this, cell extraction is terminated.' + dtype: float64 + doc: If the relative change in the main objective function between 2 consecutive + alternating minimization steps is less than this, cell extraction is terminated. + required: false - name: T_lower_snr_threshold - dtype: float32 + dtype: float64 doc: Lower SNR threshold for found traces. + required: false - name: save_all_found - dtype: int32 - doc: int32ean flag. Save all spatial and temporal components found + dtype: bool + doc: Determines whether to save all spatial and temporal components found. + required: false + - name: plot_loss + dtype: bool + doc: Indicates whether empirical risk was plotted against iterations during alternating + estimation. + required: false datasets: + - name: baseline_quantile + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: Baseline quantile for Df/F calculation in preprocessing. + quantity: '?' - name: S_init dtype: double dims: - - heightxwidth - - num_cells + - height + - width shape: - null - null doc: Cell images provided to algorithm as the initial set of cells, skipping its - native initialization + native initialization. quantity: '?' - name: T_init dtype: double @@ -239,7 +413,7 @@ groups: - null - null doc: Cell traces provided to algorithm as the initial set of traces, skipping - its native initialization + its native initialization. quantity: '?' - name: movie_mask dtype: double @@ -249,5 +423,45 @@ groups: shape: - null - null - doc: Movie mask. + doc: The circular mask to apply for microendoscopic movies during preprocessing. + quantity: '?' + - name: is_pixel_valid + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: num_frames + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: num_iter_stop_quality_checks + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: pick_gpu + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available quantity: '?' diff --git a/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml b/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml index b6f0a1c..86c8cab 100644 --- a/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml +++ b/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml @@ -3,230 +3,406 @@ groups: neurodata_type_inc: ImageSegmentation doc: EXTRACT configuration parameters attributes: + - name: version + dtype: text + doc: The version of EXTRACT. + required: false - name: preprocess dtype: bool - doc: Boolean flag indicating data preprocessing before main EXTRACT function + doc: Indicates whether or not data preprocessing was applied before main EXTRACT + function. + required: false + - name: pre_mask_on + dtype: bool + doc: Indicates whether or not an image mask was applied for preprocessing. + required: false + - name: pre_mask_radius + dtype: float64 + doc: The radius of the image mask. + required: false - name: fix_zero_FOV_strips dtype: bool - doc: Boolean flag. Find and fix spatial slices that are occasionally zero due - to frame registration during preprocessing. + doc: Indicates whether or not find and fix spatial slices that are occasionally + zero due to frame registration during preprocessing. + required: false - name: medfilt_outlier_pixels dtype: bool - doc: Boolean flag determining whether outlier pixels in the movie should be replaced - with their neighborhood median. + doc: Determines whether outlier pixels in the movie should be replaced with their + neighborhood median. + required: false - name: skip_dff dtype: bool - doc: Boolean flag. Skip Df/F calculation in preprocessing. - - name: baseline_quantile - dtype: float32 - doc: Baseline quantile for Df/F calculation in preprocessing. + doc: Indicates whether to skip Df/F calculation in preprocessing. + required: false + - name: second_df + dtype: float64 + doc: The setting for second df calculation. + required: false - name: skip_highpass dtype: bool - doc: Boolean flag. Skip highpass filtering in preprocessing. + doc: Indicates whether to skip highpass filtering in preprocessing. + required: false - name: spatial_highpass_cutoff - dtype: float32 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' + dtype: float64 + doc: Cutoff determining the strength of butterworth spatial filtering of the movie. + Values defined relative to the average cell radius. + required: false - name: temporal_denoising dtype: bool - doc: Boolean flag that determines whether to apply temporal wavelet denoising. + doc: Determines whether to apply temporal wavelet denoising. + required: false - name: remove_background dtype: bool - doc: Boolean flag that determines whether to subtract the (spatial) background - (largest spatiotemporal mode of the movie matrix). + doc: Determines whether to subtract the (spatial) background (largest spatiotemporal + mode of the movie matrix). + required: false + - name: use_default_gpu + dtype: bool + doc: Indicates whether or not use the default GPU. + required: false + - name: use_gpu + dtype: bool + doc: Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU. + required: false + - name: multi_gpu + dtype: bool + doc: Indicates whether multiple GPUs were used. + required: false + - name: parallel_cpu + dtype: bool + doc: Indicates whether parallel CPUs were used. + required: false - name: avg_cell_radius - dtype: float32 + dtype: float64 doc: Radius estimate for an average cell in the movie. + required: false + - name: avg_event_tau + dtype: float64 + doc: Determines the average event tau. + required: false + - name: avg_yield_threshold + dtype: float64 + doc: Determines the average yield threshold. + required: false - name: num_partitions_x - dtype: uint + dtype: float64 doc: Number of movie partitions in x dimension. required: false - name: num_partitions_y - dtype: uint + dtype: float64 doc: Number of movie partitions in y dimension. required: false - name: trace_output_option dtype: text - doc: Raw or non-negative output traces + doc: Raw or non-negative output traces. + required: false + - name: trace_quantile + dtype: float64 + doc: No description available. + required: false - name: dendrite_aware dtype: bool - doc: Boolean flag, set it to true if dendrites exist in the movie & are desired - in the output. + doc: Determines whether or not dendrites are preserved in the output for movies + where dendrites are present. + required: false - name: crop_circular dtype: bool - doc: Boolean flag. For microendoscopic movies, set it to true for automatically - cropping out the region outside the circular imaging region. + doc: For microendoscopic movies, whether or not automatically cropping out the + region outside the circular imaging region. + required: false - name: use_sparse_arrays dtype: bool - doc: Boolean flag. If set to true, then the output cell images will be saved as - sparse arrays + doc: Determines whether not the output cell images were saved as sparse arrays. + required: false + - name: verbose + dtype: float64 + doc: Indicates the level of verbosity. + required: false - name: compact_output dtype: bool - doc: Boolean flag. If set to true, then the output will not include bad components - that were found but then eliminated + doc: Indicates whether or not leave out bad components that were found but then + eliminated from the output. + required: false - name: hyperparameter_tuning_flag dtype: bool - doc: Boolean flag indicating internal hyperparameter tuning. + doc: Indicates whether or not use internal hyperparameter tuning. + required: false - name: remove_duplicate_cells dtype: bool doc: For movies processed in multiple partitions, this flag controls duplicate removal in the overlap regions. + required: false - name: max_iter - dtype: uint + dtype: float64 doc: Maximum number of alternating estimation iterations. + required: false + - name: minimal_checks + dtype: float64 + doc: Minimum number of checks that are performed. + required: false - name: cellfind_min_snr - dtype: float32 + dtype: float64 doc: Minimum peak SNR value for an object to be considered as a cell. + required: false - name: cellfind_max_steps - dtype: int + dtype: float64 doc: Maximum number of cell candidate initialization during cell finding step. + required: false - name: cellfind_kappa_std_ratio - dtype: float32 + dtype: float64 doc: Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization. + required: false - name: cellfind_filter_type dtype: text doc: Type of the spatial smoothing filter used for cell finding. + required: false - name: cellfind_numpix_threshold - dtype: float32 + dtype: float64 doc: During cell finding, objects with an area < cellfind_numpix_threshold are discarded. + required: false + - name: cellfind_adaptive_kappa + dtype: bool + doc: If True, then during cell finding, the robust esimation loss will adaptively + set its robustness parameter + required: false + - name: cellfind_spatial_highpass_cutoff + dtype: float64 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' + required: false - name: moving_radius dtype: float32 + doc: Deprecated variable for older EXTRACT file versions. Radius of moving average + filter in the case when cellfind_filter_type = moveavg (moving average) + required: false + - name: moving_radius_spatial + dtype: float64 + doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg + (moving average) for the image. + required: false + - name: moving_radius_temporal + dtype: float64 doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) + (moving average) for the traces. + required: false - name: init_with_gaussian dtype: bool - doc: Boolean flag. If true, then during cell finding, each cell is initialized - with a gaussian shape prior to robust estimation. If false, then initialization - is done with a correlation image (preferred for movies with dendrites). + doc: If True, then during cell finding, each cell is initialized with a gaussian + shape prior to robust estimation. If False, then initialization is done with + a correlation image (preferred for movies with dendrites). + required: false - name: high2low_brightness_ratio - dtype: float32 + dtype: float64 doc: Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to stop cell finding process. + required: false + - name: visualize_cellfinding + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_full_range + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_max + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_min + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_show_bad_cells + dtype: float64 + doc: The visualization setting for cell finding. + required: false - name: kappa_std_ratio - dtype: float32 + dtype: float64 doc: Kappa will be set to this times the noise std during the cell refinement process. Cell refinement parameter. + required: false - name: T_dup_corr_thresh - dtype: float32 + dtype: float64 doc: Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh are eliminated. Cell refinement parameter. + required: false + - name: T_dup_thresh + dtype: float64 + doc: Threshold for traces. Cell refinement parameter. + required: false - name: S_dup_corr_thresh - dtype: float32 + dtype: float64 doc: Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh are eliminated. Cell refinement parameter. + required: false + - name: S_corr_thresh + dtype: float64 + doc: Image correlation threshold. Cell refinement parameter. + required: false + - name: T_corr_thresh + dtype: float64 + doc: Trace correlation threshold. Cell refinement parameter. + required: false - name: temporal_corrupt_thresh - dtype: float32 + dtype: float64 doc: Threshold for temporal corruption index. Traces above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. + required: false - name: spatial_corrupt_thresh - dtype: float32 + dtype: float64 doc: Threshold for spatial corruption index. Images above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. + required: false - name: T_min_snr - dtype: float32 + dtype: float64 doc: Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. Cell refinement parameter. + required: false - name: size_lower_limit - dtype: float32 + dtype: float64 doc: Lower size limit for found cells. Cell refinement parameter. + required: false - name: size_upper_limit - dtype: float32 + dtype: float64 doc: Lower size limit for found cells. Cell refinement parameter. + required: false - name: eccent_thresh - dtype: float32 + dtype: float64 doc: Upper limit of eccentricity for found cells. Cell refinement parameter. + required: false - name: low_ST_index_thresh - dtype: float32 + dtype: float64 doc: Lower limit of spatiotemporal activity index. Cell refinement parameter. + required: false + - name: low_cell_area_flag + dtype: bool + doc: Boolean flag indicating lower limit of cell area. + required: false - name: high_ST_index_thresh - dtype: float32 + dtype: float64 doc: Upper limit limit of spatiotemporal activity index. Cell refinement parameter. + required: false - name: low_ST_corr_thresh - dtype: float32 + dtype: float64 doc: Lower limit of spatiotemporal corelation. Cell refinement parameter. + required: false - name: confidence_thresh - dtype: float32 - doc: Confidence threhsold for found cells. Cell refinement parameter. + dtype: float64 + doc: Confidence threshold for found cells. Cell refinement parameter. + required: false - name: downsample_time_by - dtype: text - doc: Time downsampling factor. If set to auto downsampling factor based on avg - cell radius and avg calcium event time constant. + dtype: float64 + doc: Time downsampling factor. + required: false - name: min_tau_after_downsampling - dtype: float32 - doc: Minimum event tau after downsampling. Used when downsample_time_by = auto + dtype: float64 + doc: Minimum event tau after downsampling. + required: false - name: downsample_space_by - dtype: text - doc: Spatial downsampling factor. If set to auto downsampling factor based on - avg cell radius and avg calcium event time constant. + dtype: float64 + doc: Spatial downsampling factor. + required: false - name: min_radius_after_downsampling - dtype: float32 - doc: Minimum avg radius after downsampling. Used when downsample_space_by = auto. + dtype: float64 + doc: Minimum avg radius after downsampling. + required: false - name: reestimate_S_if_downsampled dtype: bool - doc: Boolean flag. When set to true, images are re-estimated from full movie at - the end. When false, images are upsampled by interpolation. + doc: Boolean flag. When set to True, images are re-estimated from full movie at + the end. When False, images are upsampled by interpolation. + required: false - name: reestimate_T_if_downsampled dtype: bool - doc: Boolean flag. When set to true, traces are re-estimated from full movie at - the end. When false, traces are upsampled by interpolation. + doc: Boolean flag. When set to True, traces are re-estimated from full movie at + the end. When False, traces are upsampled by interpolation. + required: false - name: adaptive_kappa dtype: bool - doc: Boolean flag. If true, then during cell finding, the robust esimation loss + doc: Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively set its robustness parameter + required: false + - name: arbitrary_mask + dtype: bool + doc: Indicates whether to use an arbitrary mask on the image. + required: false - name: smoothing_ratio_x2y - dtype: float32 + dtype: float64 doc: If the movie contains mainly objects that are elongated in one dimension (e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension. + required: false - name: spatial_lowpass_cutoff - dtype: float32 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' + dtype: float64 + doc: Cutoff determining the strength of butterworth spatial filtering of the movie. + Values defined relative to the average cell radius. + required: false - name: smooth_T dtype: bool - doc: Boolean flag indicating whether calculated traces are smoothed using median - filtering. + doc: Indicates whether calculated traces are smoothed using median filtering. + required: false - name: smooth_S dtype: bool - doc: Boolean flag indicating whether calculated images are smoothed using a 2-D - Gaussian filter. + doc: Indicates whether calculated images are smoothed using a 2-D Gaussian filter. + required: false - name: l1_penalty_factor - dtype: float32 + dtype: float64 doc: Strength of l1 regularization penalty to be applied when estimating the temporal components. + required: false - name: max_iter_S - dtype: uint - doc: Maximum number of iterations for S estimation steps + dtype: float64 + doc: Maximum number of iterations for S estimation steps. + required: false - name: max_iter_T - dtype: uint - doc: Maximum number of iterations for T estimation steps + dtype: float64 + doc: Maximum number of iterations for T estimation steps. + required: false - name: TOL_sub - dtype: float32 + dtype: float64 doc: If the 1-step relative change in the objective within each T and S optimization is less than this, the respective optimization is terminated. + required: false - name: TOL_main - dtype: float32 - doc: ' If the relative change in the main objective function between 2 consecutive - alternating minimization steps is less than this, cell extraction is terminated.' + dtype: float64 + doc: If the relative change in the main objective function between 2 consecutive + alternating minimization steps is less than this, cell extraction is terminated. + required: false - name: T_lower_snr_threshold - dtype: float32 + dtype: float64 doc: Lower SNR threshold for found traces. + required: false - name: save_all_found dtype: bool - doc: Boolean flag. Save all spatial and temporal components found + doc: Determines whether to save all spatial and temporal components found. + required: false + - name: plot_loss + dtype: bool + doc: Indicates whether empirical risk was plotted against iterations during alternating + estimation. + required: false datasets: + - name: baseline_quantile + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: Baseline quantile for Df/F calculation in preprocessing. + quantity: '?' - name: S_init dtype: double dims: - - heightxwidth - - num_cells + - height + - width shape: - null - null doc: Cell images provided to algorithm as the initial set of cells, skipping its - native initialization + native initialization. quantity: '?' - name: T_init dtype: double @@ -237,7 +413,7 @@ groups: - null - null doc: Cell traces provided to algorithm as the initial set of traces, skipping - its native initialization + its native initialization. quantity: '?' - name: movie_mask dtype: double @@ -247,5 +423,45 @@ groups: shape: - null - null - doc: Movie mask. + doc: The circular mask to apply for microendoscopic movies during preprocessing. + quantity: '?' + - name: is_pixel_valid + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: num_frames + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: num_iter_stop_quality_checks + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: pick_gpu + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available quantity: '?' From 9c8aba2a9896a86369383196cff3399d7525e05b Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:51:53 +0200 Subject: [PATCH 12/18] black --- src/pynwb/ndx_extract/__init__.py | 20 +- src/pynwb/ndx_extract/extract.py | 1252 ++++++++++++++++------------- 2 files changed, 716 insertions(+), 556 deletions(-) diff --git a/src/pynwb/ndx_extract/__init__.py b/src/pynwb/ndx_extract/__init__.py index 274c62f..7de9197 100644 --- a/src/pynwb/ndx_extract/__init__.py +++ b/src/pynwb/ndx_extract/__init__.py @@ -3,20 +3,22 @@ # Set path of the namespace.yaml file to the expected install location ndx_extract_specpath = os.path.join( - os.path.dirname(__file__), - 'spec', - 'ndx-extract.namespace.yaml' + os.path.dirname(__file__), "spec", "ndx-extract.namespace.yaml" ) # If the extension has not been installed yet but we are running directly from # the git repo if not os.path.exists(ndx_extract_specpath): - ndx_extract_specpath = os.path.abspath(os.path.join( - os.path.dirname(__file__), - '..', '..', '..', - 'spec', - 'ndx-extract.namespace.yaml' - )) + ndx_extract_specpath = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "..", + "spec", + "ndx-extract.namespace.yaml", + ) + ) # Load the namespace load_namespaces(ndx_extract_specpath) diff --git a/src/pynwb/ndx_extract/extract.py b/src/pynwb/ndx_extract/extract.py index a82d822..b525f9b 100644 --- a/src/pynwb/ndx_extract/extract.py +++ b/src/pynwb/ndx_extract/extract.py @@ -4,564 +4,722 @@ from pynwb.ophys import ImageSegmentation -@register_class('EXTRACTSegmentation', 'ndx-extract') +@register_class("EXTRACTSegmentation", "ndx-extract") class EXTRACTSegmentation(ImageSegmentation): """EXTRACT configuration parameters added to an ImageSegmentation object.""" - __nwbfields__ = ('version', 'preprocess', 'pre_mask_on', 'pre_mask_radius', - 'fix_zero_FOV_strips', 'medfilt_outlier_pixels', 'skip_dff', - 'second_df', 'skip_highpass', 'spatial_highpass_cutoff', - 'temporal_denoising', 'remove_background', 'use_default_gpu', - 'use_gpu', 'multi_gpu', 'parallel_cpu', 'avg_cell_radius', - 'avg_event_tau', 'avg_yield_threshold', 'num_partitions_x', - 'num_partitions_y', 'trace_output_option', 'trace_quantile', - 'dendrite_aware', 'crop_circular', 'use_sparse_arrays', 'verbose', - 'compact_output', 'hyperparameter_tuning_flag', - 'remove_duplicate_cells', 'max_iter', 'minimal_checks', - 'cellfind_min_snr', 'cellfind_max_steps', 'cellfind_kappa_std_ratio', - 'cellfind_filter_type', 'cellfind_numpix_threshold', - 'cellfind_adaptive_kappa', 'cellfind_spatial_highpass_cutoff', - 'moving_radius', 'moving_radius_spatial', 'moving_radius_temporal', - 'init_with_gaussian', 'high2low_brightness_ratio', - 'visualize_cellfinding', 'visualize_cellfinding_full_range', - 'visualize_cellfinding_max', 'visualize_cellfinding_min', - 'visualize_cellfinding_show_bad_cells', 'kappa_std_ratio', - 'T_dup_corr_thresh', 'T_dup_thresh', 'S_dup_corr_thresh', - 'S_corr_thresh', 'T_corr_thresh', 'temporal_corrupt_thresh', - 'spatial_corrupt_thresh', 'T_min_snr', 'size_lower_limit', - 'size_upper_limit', 'eccent_thresh', 'low_ST_index_thresh', - 'low_cell_area_flag', 'high_ST_index_thresh', - 'low_ST_corr_thresh', 'confidence_thresh', 'downsample_time_by', - 'min_tau_after_downsampling', 'downsample_space_by', - 'min_radius_after_downsampling', 'reestimate_S_if_downsampled', - 'reestimate_T_if_downsampled', 'adaptive_kappa', 'arbitrary_mask', - 'smoothing_ratio_x2y', 'spatial_lowpass_cutoff', 'smooth_T', - 'smooth_S', 'l1_penalty_factor', 'max_iter_S', 'max_iter_T', - 'TOL_sub', 'TOL_main', 'T_lower_snr_threshold', 'save_all_found', - 'plot_loss', 'baseline_quantile', 'S_init', 'T_init', 'movie_mask', - 'is_pixel_valid', 'num_frames', 'num_iter_stop_quality_checks', 'pick_gpu') + __nwbfields__ = ( + "version", + "preprocess", + "pre_mask_on", + "pre_mask_radius", + "fix_zero_FOV_strips", + "medfilt_outlier_pixels", + "skip_dff", + "second_df", + "skip_highpass", + "spatial_highpass_cutoff", + "temporal_denoising", + "remove_background", + "use_default_gpu", + "use_gpu", + "multi_gpu", + "parallel_cpu", + "avg_cell_radius", + "avg_event_tau", + "avg_yield_threshold", + "num_partitions_x", + "num_partitions_y", + "trace_output_option", + "trace_quantile", + "dendrite_aware", + "crop_circular", + "use_sparse_arrays", + "verbose", + "compact_output", + "hyperparameter_tuning_flag", + "remove_duplicate_cells", + "max_iter", + "minimal_checks", + "cellfind_min_snr", + "cellfind_max_steps", + "cellfind_kappa_std_ratio", + "cellfind_filter_type", + "cellfind_numpix_threshold", + "cellfind_adaptive_kappa", + "cellfind_spatial_highpass_cutoff", + "moving_radius", + "moving_radius_spatial", + "moving_radius_temporal", + "init_with_gaussian", + "high2low_brightness_ratio", + "visualize_cellfinding", + "visualize_cellfinding_full_range", + "visualize_cellfinding_max", + "visualize_cellfinding_min", + "visualize_cellfinding_show_bad_cells", + "kappa_std_ratio", + "T_dup_corr_thresh", + "T_dup_thresh", + "S_dup_corr_thresh", + "S_corr_thresh", + "T_corr_thresh", + "temporal_corrupt_thresh", + "spatial_corrupt_thresh", + "T_min_snr", + "size_lower_limit", + "size_upper_limit", + "eccent_thresh", + "low_ST_index_thresh", + "low_cell_area_flag", + "high_ST_index_thresh", + "low_ST_corr_thresh", + "confidence_thresh", + "downsample_time_by", + "min_tau_after_downsampling", + "downsample_space_by", + "min_radius_after_downsampling", + "reestimate_S_if_downsampled", + "reestimate_T_if_downsampled", + "adaptive_kappa", + "arbitrary_mask", + "smoothing_ratio_x2y", + "spatial_lowpass_cutoff", + "smooth_T", + "smooth_S", + "l1_penalty_factor", + "max_iter_S", + "max_iter_T", + "TOL_sub", + "TOL_main", + "T_lower_snr_threshold", + "save_all_found", + "plot_loss", + "baseline_quantile", + "S_init", + "T_init", + "movie_mask", + "is_pixel_valid", + "num_frames", + "num_iter_stop_quality_checks", + "pick_gpu", + ) @docval( *get_docval( ImageSegmentation.__init__, "name", ), # required - {'name': 'version', - 'type': str, - 'doc': 'The version of EXTRACT.', - 'default': None, - }, - {'name': 'preprocess', - 'type': bool, - 'doc': 'Indicates whether or not data preprocessing was applied before main EXTRACT function.', - 'default': None, - }, - {'name': 'pre_mask_on', - 'type': bool, - 'doc': 'Indicates whether or not an image mask was applied for preprocessing.', - 'default': None, - }, - {'name': 'pre_mask_radius', - 'type': float, - 'doc': 'If an image mask was applied for preprocessing, indicates the radius of the image mask.', - 'default': None, - }, - {'name': 'fix_zero_FOV_strips', - 'type': bool, - 'doc': "Indicates whether or not find and fix spatial slices that are occasionally zero due to " - "frame registration during preprocessing.", - 'default': None, - }, - {'name': 'medfilt_outlier_pixels', - 'type': bool, - 'doc': "Determines whether outlier pixels in the movie should be replaced with their " - "neighborhood median.", - 'default': None, - }, - {'name': 'skip_dff', - 'type': bool, - 'doc': "Indicates whether to skip Df/F calculation in preprocessing.", - 'default': None, - }, - {'name': 'second_df', - 'type': float, - 'doc': "The setting for second df calculation.", - 'default': None, - }, - {'name': 'skip_highpass', - 'type': bool, - 'doc': "Indicates whether to skip highpass filtering in preprocessing.", - 'default': None, - }, - {'name': 'spatial_highpass_cutoff', - 'type': float, - 'doc': "Cutoff determining the strength of butterworth spatial filtering of the movie. " - "Values defined relative to the average cell radius.", - 'default': None, - }, - {'name': 'temporal_denoising', - 'type': bool, - 'doc': "Determines whether to apply temporal wavelet denoising.", - 'default': None, - }, - {'name': 'remove_background', - 'type': bool, - 'doc': "Determines whether to subtract the (spatial) background " - "(largest spatiotemporal mode of the movie matrix).", - 'default': None, - }, - {'name': 'use_default_gpu', - 'type': bool, - 'doc': "Indicates whether or not use the default GPU.", - 'default': None, - }, - {'name': 'use_gpu', - 'type': bool, - 'doc': "Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU.", - 'default': None, - }, - {'name': 'multi_gpu', - 'type': bool, - 'doc': "Indicates whether multiple GPUs were used.", - 'default': None, - }, - {'name': 'parallel_cpu', - 'type': bool, - 'doc': "Indicates whether parallel CPUs were used.", - 'default': None, - }, - {'name': 'avg_cell_radius', - 'type': float, - 'doc': "Radius estimate for an average cell in the movie.", - 'default': None, - }, - {'name': 'avg_event_tau', - 'type': float, - 'doc': "Determines the average event tau.", - 'default': None, - }, - {'name': 'avg_yield_threshold', - 'type': float, - 'doc': "Determines the average yield threshold.", - 'default': None, - }, - {'name': 'num_partitions_x', - 'type': float, - 'doc': "Number of movie partitions in x dimension.", - 'default': None, - }, - {'name': 'num_partitions_y', - 'type': float, - 'doc': "Number of movie partitions in y dimension.", - 'default': None, - }, - {'name': 'trace_output_option', - 'type': str, - 'doc': "Raw or non-negative output traces.", - 'default': None, - }, - {'name': 'trace_quantile', - 'type': float, - 'doc': "No description available.", - 'default': None, - }, - {'name': 'dendrite_aware', - 'type': bool, - 'doc': "Determines whether or not dendrites are preserved in the output " - "for movies where dendrites are present.", - 'default': None, - }, - {'name': 'crop_circular', - 'type': bool, - 'doc': "For microendoscopic movies, whether or not automatically cropping out " - "the region outside the circular imaging region.", - 'default': None, - }, - {'name': 'use_sparse_arrays', - 'type': bool, - 'doc': "Determines whether not the output cell images were saved as sparse arrays.", - 'default': None, - }, - {'name': 'verbose', - 'type': float, - 'doc': "Indicates the level of verbosity.", - 'default': None, - }, - {'name': 'compact_output', - 'type': bool, - 'doc': "Indicates whether or not leave out bad components that were found but " - "then eliminated from the output.", - 'default': None, - }, - {'name': 'hyperparameter_tuning_flag', - 'type': bool, - 'doc': "Indicates whether or not use internal hyperparameter tuning.", - 'default': None, - }, - {'name': 'remove_duplicate_cells', - 'type': bool, - 'doc': "For movies processed in multiple partitions, " - "this flag controls duplicate removal in the overlap regions.", - 'default': None, - }, - {'name': 'max_iter', - 'type': float, - 'doc': "Maximum number of alternating estimation iterations.", - 'default': None, - }, - {'name': 'minimal_checks', - 'type': float, - 'doc': "Minimum number of checks that are performed.", - 'default': None, - }, - {'name': 'cellfind_min_snr', - 'type': float, - 'doc': "Minimum peak SNR value for an object to be considered as a cell.", - 'default': None, - }, - {'name': 'cellfind_max_steps', - 'type': float, - 'doc': "Maximum number of cell candidate initialization during cell finding step.", - 'default': None, - }, - {'name': 'cellfind_kappa_std_ratio', - 'type': float, - 'doc': "Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.", - 'default': None, - }, - {'name': 'cellfind_filter_type', - 'type': str, - 'doc': "Type of the spatial smoothing filter used for cell finding.", - 'default': None, - }, - {'name': 'cellfind_numpix_threshold', - 'type': float, - 'doc': "During cell finding, objects with an area < cellfind_numpix_threshold are discarded.", - 'default': None, - }, - {'name': 'cellfind_adaptive_kappa', - 'type': bool, - 'doc': "If True, then during cell finding, the robust esimation loss will adaptively " - "set its robustness parameter", - 'default': None, - }, - {'name': 'cellfind_spatial_highpass_cutoff', - 'type': float, - 'doc': "Cutoff determining the strength of butterworth spatial filtering of the movie. " - "Values defined relative to the average cell radius. ", - 'default': None, - }, - {'name': 'moving_radius', - 'type': float, - 'doc': "Deprecated variable for older EXTRACT file versions. " - "Radius of moving average filter in the case when cellfind_filter_type = moveavg " - "(moving average)", - 'default': None, - }, - {'name': 'moving_radius_spatial', - 'type': float, - 'doc': "Radius of moving average filter in the case when cellfind_filter_type = moveavg " - "(moving average) for the image.", - 'default': None, - }, - {'name': 'moving_radius_temporal', - 'type': float, - 'doc': "Radius of moving average filter in the case when cellfind_filter_type = moveavg " - "(moving average) for the traces.", - 'default': None, - }, - {'name': 'init_with_gaussian', - 'type': bool, - 'doc': "If True, then during cell finding, each cell is initialized with a gaussian shape " - "prior to robust estimation. If False, then initialization is done with a " - "correlation image (preferred for movies with dendrites).", - 'default': None, - }, - {'name': 'high2low_brightness_ratio', - 'type': float, - 'doc': "Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to " - "stop cell finding process.", - 'default': None, - }, - {'name': 'visualize_cellfinding', - 'type': float, - 'doc': "The visualization setting for cell finding.", - 'default': None, - }, - {'name': 'visualize_cellfinding_full_range', - 'type': float, - 'doc': "The visualization setting for cell finding.", - 'default': None, - }, - {'name': 'visualize_cellfinding_max', - 'type': float, - 'doc': "The visualization setting for cell finding.", - 'default': None, - }, - {'name': 'visualize_cellfinding_min', - 'type': float, - 'doc': "The visualization setting for cell finding.", - 'default': None, - }, - {'name': 'visualize_cellfinding_show_bad_cells', - 'type': float, - 'doc': "The visualization setting for cell finding.", - 'default': None, - }, - {'name': 'kappa_std_ratio', - 'type': float, - 'doc': "Kappa will be set to this times the noise std during the cell refinement process. " - "Cell refinement parameter.", - 'default': None, - }, - {'name': 'T_dup_corr_thresh', - 'type': float, - 'doc': "Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh " - "are eliminated. Cell refinement parameter.", - 'default': None, - }, - {'name': 'T_dup_thresh', - 'type': float, - 'doc': "Threshold for traces. Cell refinement parameter.", - 'default': None, - }, - {'name': 'S_dup_corr_thresh', - 'type': float, - 'doc': "Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh " - "are eliminated. Cell refinement parameter.", - 'default': None, - }, - {'name': 'S_corr_thresh', - 'type': float, - 'doc': "Image correlation threshold. Cell refinement parameter.", - 'default': None, - }, - {'name': 'T_corr_thresh', - 'type': float, - 'doc': "Trace correlation threshold. Cell refinement parameter.", - 'default': None, - }, - {'name': 'temporal_corrupt_thresh', - 'type': float, - 'doc': "Threshold for temporal corruption index. Traces above this threshold are eliminated duirng " - "alternating minimization routine. Cell refinement parameter.", - 'default': None, - }, - {'name': 'spatial_corrupt_thresh', - 'type': float, - 'doc': "Threshold for spatial corruption index. Images above this threshold are eliminated duirng " - "alternating minimization routine. Cell refinement parameter.", - 'default': None, - }, - {'name': 'T_min_snr', - 'type': float, - 'doc': "Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. " - "Cell refinement parameter.", - 'default': None, - }, - {'name': 'size_lower_limit', - 'type': float, - 'doc': "Lower size limit for found cells. Cell refinement parameter.", - 'default': None, - }, - {'name': 'size_upper_limit', - 'type': float, - 'doc': "Lower size limit for found cells. Cell refinement parameter.", - 'default': None, - }, - {'name': 'eccent_thresh', - 'type': float, - 'doc': "Upper limit of eccentricity for found cells. Cell refinement parameter.", - 'default': None, - }, - {'name': 'low_ST_index_thresh', - 'type': float, - 'doc': "Lower limit of spatiotemporal activity index. Cell refinement parameter.", - 'default': None, - }, - {'name': 'low_cell_area_flag', - 'type': bool, - 'doc': "Boolean flag indicating lower limit of cell area.", - 'default': None, - }, - {'name': 'high_ST_index_thresh', - 'type': float, - 'doc': "Upper limit limit of spatiotemporal activity index. Cell refinement parameter.", - 'default': None, - }, - {'name': 'low_ST_corr_thresh', - 'type': float, - 'doc': "Lower limit of spatiotemporal corelation. Cell refinement parameter.", - 'default': None, - }, - {'name': 'confidence_thresh', - 'type': float, - 'doc': "Confidence threshold for found cells. Cell refinement parameter.", - 'default': None, - }, - {'name': 'downsample_time_by', - 'type': float, - 'doc': "Time downsampling factor.", - 'default': None, - }, - {'name': 'min_tau_after_downsampling', - 'type': float, - 'doc': "Minimum event tau after downsampling.", - 'default': None, - }, - {'name': 'downsample_space_by', - 'type': float, - 'doc': "Spatial downsampling factor.", - 'default': None, - }, - {'name': 'min_radius_after_downsampling', - 'type': float, - 'doc': "Minimum avg radius after downsampling.", - 'default': None, - }, - {'name': 'reestimate_S_if_downsampled', - 'type': bool, - 'doc': "Boolean flag. When set to True, images are re-estimated from full movie at the end. " - "When False, images are upsampled by interpolation.", - 'default': None, - }, - {'name': 'reestimate_T_if_downsampled', - 'type': bool, - 'doc': "Boolean flag. When set to True, traces are re-estimated from full movie at the end. " - "When False, traces are upsampled by interpolation.", - 'default': None, - }, - {'name': 'adaptive_kappa', - 'type': bool, - 'doc': "Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " - "set its robustness parameter", - 'default': None, - }, - {'name': 'arbitrary_mask', - 'type': bool, - 'doc': "Indicates whether to use an arbitrary mask on the image.", - 'default': None, - }, - {'name': 'smoothing_ratio_x2y', - 'type': float, - 'doc': "If the movie contains mainly objects that are elongated in one dimension " - "(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.", - 'default': None, - }, - {'name': 'spatial_lowpass_cutoff', - 'type': float, - 'doc': "Cutoff determining the strength of butterworth spatial filtering of the movie. " - "Values defined relative to the average cell radius.", - 'default': None, - }, - {'name': 'smooth_T', - 'type': bool, - 'doc': "Indicates whether calculated traces are smoothed using median filtering.", - 'default': None, - }, - {'name': 'smooth_S', - 'type': bool, - 'doc': "Indicates whether calculated images are smoothed using a 2-D Gaussian filter.", - 'default': None, - }, - {'name': 'l1_penalty_factor', - 'type': float, - 'doc': "Strength of l1 regularization penalty to be applied when estimating the temporal components.", - 'default': None, - }, - {'name': 'max_iter_S', - 'type': float, - 'doc': "Maximum number of iterations for S estimation steps.", - 'default': None, - }, - {'name': 'max_iter_T', - 'type': float, - 'doc': "Maximum number of iterations for T estimation steps.", - 'default': None, - }, - {'name': 'TOL_sub', - 'type': float, - 'doc': "If the 1-step relative change in the objective within each T and S optimization is less than this, " - "the respective optimization is terminated.", - 'default': None, - }, - {'name': 'TOL_main', - 'type': float, - 'doc': "If the relative change in the main objective function between 2 consecutive alternating " - "minimization steps is less than this, cell extraction is terminated.", - 'default': None, - }, - {'name': 'T_lower_snr_threshold', - 'type': float, - 'doc': "Lower SNR threshold for found traces.", - 'default': None, - }, - {'name': 'save_all_found', - 'type': bool, - 'doc': "Determines whether to save all spatial and temporal components found.", - 'default': None, - }, - {'name': 'plot_loss', - 'type': bool, - 'doc': "Indicates whether empirical risk was plotted against iterations during alternating estimation.", - 'default': None, - }, - {'name': 'baseline_quantile', - 'type': ('array_data', 'data'), - 'doc': "Baseline quantile for Df/F calculation in preprocessing." - "The first dimension denotes the height of the image, the second dimension " - "corresponds to the width of the image.", - 'default': None, - 'shape': (None, None), - }, - {'name': 'S_init', - 'type': ('array_data', 'data'), - 'doc': "Cell images provided to algorithm as the initial set of cells, skipping its native initialization." - "The first dimension denotes the height of the image, the second dimension " - "corresponds to the width of the image.", - 'default': None, - 'shape': (None, None), - }, - {'name': 'T_init', - 'type': ('array_data', 'data'), - 'doc': "Cell traces provided to algorithm as the initial set of traces, skipping its native initialization." - "The first dimension denotes the number of cells, the second dimension " - "corresponds to time.", - 'default': None, - 'shape': (None, None), - }, - {'name': 'movie_mask', - 'type': ('array_data', 'data'), - 'doc': "The circular mask to apply for microendoscopic movies during preprocessing.", - 'default': None, - 'shape': (None, None), - }, - {'name': 'is_pixel_valid', - 'type': ('array_data', 'data'), - 'doc': "No description available", - 'default': None, - 'shape': (None, None), - }, - {'name': 'num_frames', - 'type': ('array_data', 'data'), - 'doc': "No description available", - 'default': None, - }, - {'name': 'num_iter_stop_quality_checks', - 'type': ('array_data', 'data'), - 'doc': "No description available", - 'default': None, - 'shape': (None, None), - }, - {'name': 'pick_gpu', - 'type': ('array_data', 'data'), - 'doc': "No description available", - 'default': None, - 'shape': (None, None), - }, + { + "name": "version", + "type": str, + "doc": "The version of EXTRACT.", + "default": None, + }, + { + "name": "preprocess", + "type": bool, + "doc": "Indicates whether or not data preprocessing was applied before main EXTRACT function.", + "default": None, + }, + { + "name": "pre_mask_on", + "type": bool, + "doc": "Indicates whether or not an image mask was applied for preprocessing.", + "default": None, + }, + { + "name": "pre_mask_radius", + "type": float, + "doc": "If an image mask was applied for preprocessing, indicates the radius of the image mask.", + "default": None, + }, + { + "name": "fix_zero_FOV_strips", + "type": bool, + "doc": "Indicates whether or not find and fix spatial slices that are occasionally zero due to " + "frame registration during preprocessing.", + "default": None, + }, + { + "name": "medfilt_outlier_pixels", + "type": bool, + "doc": "Determines whether outlier pixels in the movie should be replaced with their " + "neighborhood median.", + "default": None, + }, + { + "name": "skip_dff", + "type": bool, + "doc": "Indicates whether to skip Df/F calculation in preprocessing.", + "default": None, + }, + { + "name": "second_df", + "type": float, + "doc": "The setting for second df calculation.", + "default": None, + }, + { + "name": "skip_highpass", + "type": bool, + "doc": "Indicates whether to skip highpass filtering in preprocessing.", + "default": None, + }, + { + "name": "spatial_highpass_cutoff", + "type": float, + "doc": "Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius.", + "default": None, + }, + { + "name": "temporal_denoising", + "type": bool, + "doc": "Determines whether to apply temporal wavelet denoising.", + "default": None, + }, + { + "name": "remove_background", + "type": bool, + "doc": "Determines whether to subtract the (spatial) background " + "(largest spatiotemporal mode of the movie matrix).", + "default": None, + }, + { + "name": "use_default_gpu", + "type": bool, + "doc": "Indicates whether or not use the default GPU.", + "default": None, + }, + { + "name": "use_gpu", + "type": bool, + "doc": "Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU.", + "default": None, + }, + { + "name": "multi_gpu", + "type": bool, + "doc": "Indicates whether multiple GPUs were used.", + "default": None, + }, + { + "name": "parallel_cpu", + "type": bool, + "doc": "Indicates whether parallel CPUs were used.", + "default": None, + }, + { + "name": "avg_cell_radius", + "type": float, + "doc": "Radius estimate for an average cell in the movie.", + "default": None, + }, + { + "name": "avg_event_tau", + "type": float, + "doc": "Determines the average event tau.", + "default": None, + }, + { + "name": "avg_yield_threshold", + "type": float, + "doc": "Determines the average yield threshold.", + "default": None, + }, + { + "name": "num_partitions_x", + "type": float, + "doc": "Number of movie partitions in x dimension.", + "default": None, + }, + { + "name": "num_partitions_y", + "type": float, + "doc": "Number of movie partitions in y dimension.", + "default": None, + }, + { + "name": "trace_output_option", + "type": str, + "doc": "Raw or non-negative output traces.", + "default": None, + }, + { + "name": "trace_quantile", + "type": float, + "doc": "No description available.", + "default": None, + }, + { + "name": "dendrite_aware", + "type": bool, + "doc": "Determines whether or not dendrites are preserved in the output " + "for movies where dendrites are present.", + "default": None, + }, + { + "name": "crop_circular", + "type": bool, + "doc": "For microendoscopic movies, whether or not automatically cropping out " + "the region outside the circular imaging region.", + "default": None, + }, + { + "name": "use_sparse_arrays", + "type": bool, + "doc": "Determines whether not the output cell images were saved as sparse arrays.", + "default": None, + }, + { + "name": "verbose", + "type": float, + "doc": "Indicates the level of verbosity.", + "default": None, + }, + { + "name": "compact_output", + "type": bool, + "doc": "Indicates whether or not leave out bad components that were found but " + "then eliminated from the output.", + "default": None, + }, + { + "name": "hyperparameter_tuning_flag", + "type": bool, + "doc": "Indicates whether or not use internal hyperparameter tuning.", + "default": None, + }, + { + "name": "remove_duplicate_cells", + "type": bool, + "doc": "For movies processed in multiple partitions, " + "this flag controls duplicate removal in the overlap regions.", + "default": None, + }, + { + "name": "max_iter", + "type": float, + "doc": "Maximum number of alternating estimation iterations.", + "default": None, + }, + { + "name": "minimal_checks", + "type": float, + "doc": "Minimum number of checks that are performed.", + "default": None, + }, + { + "name": "cellfind_min_snr", + "type": float, + "doc": "Minimum peak SNR value for an object to be considered as a cell.", + "default": None, + }, + { + "name": "cellfind_max_steps", + "type": float, + "doc": "Maximum number of cell candidate initialization during cell finding step.", + "default": None, + }, + { + "name": "cellfind_kappa_std_ratio", + "type": float, + "doc": "Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization.", + "default": None, + }, + { + "name": "cellfind_filter_type", + "type": str, + "doc": "Type of the spatial smoothing filter used for cell finding.", + "default": None, + }, + { + "name": "cellfind_numpix_threshold", + "type": float, + "doc": "During cell finding, objects with an area < cellfind_numpix_threshold are discarded.", + "default": None, + }, + { + "name": "cellfind_adaptive_kappa", + "type": bool, + "doc": "If True, then during cell finding, the robust esimation loss will adaptively " + "set its robustness parameter", + "default": None, + }, + { + "name": "cellfind_spatial_highpass_cutoff", + "type": float, + "doc": "Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius. ", + "default": None, + }, + { + "name": "moving_radius", + "type": float, + "doc": "Deprecated variable for older EXTRACT file versions. " + "Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average)", + "default": None, + }, + { + "name": "moving_radius_spatial", + "type": float, + "doc": "Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average) for the image.", + "default": None, + }, + { + "name": "moving_radius_temporal", + "type": float, + "doc": "Radius of moving average filter in the case when cellfind_filter_type = moveavg " + "(moving average) for the traces.", + "default": None, + }, + { + "name": "init_with_gaussian", + "type": bool, + "doc": "If True, then during cell finding, each cell is initialized with a gaussian shape " + "prior to robust estimation. If False, then initialization is done with a " + "correlation image (preferred for movies with dendrites).", + "default": None, + }, + { + "name": "high2low_brightness_ratio", + "type": float, + "doc": "Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to " + "stop cell finding process.", + "default": None, + }, + { + "name": "visualize_cellfinding", + "type": float, + "doc": "The visualization setting for cell finding.", + "default": None, + }, + { + "name": "visualize_cellfinding_full_range", + "type": float, + "doc": "The visualization setting for cell finding.", + "default": None, + }, + { + "name": "visualize_cellfinding_max", + "type": float, + "doc": "The visualization setting for cell finding.", + "default": None, + }, + { + "name": "visualize_cellfinding_min", + "type": float, + "doc": "The visualization setting for cell finding.", + "default": None, + }, + { + "name": "visualize_cellfinding_show_bad_cells", + "type": float, + "doc": "The visualization setting for cell finding.", + "default": None, + }, + { + "name": "kappa_std_ratio", + "type": float, + "doc": "Kappa will be set to this times the noise std during the cell refinement process. " + "Cell refinement parameter.", + "default": None, + }, + { + "name": "T_dup_corr_thresh", + "type": float, + "doc": "Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh " + "are eliminated. Cell refinement parameter.", + "default": None, + }, + { + "name": "T_dup_thresh", + "type": float, + "doc": "Threshold for traces. Cell refinement parameter.", + "default": None, + }, + { + "name": "S_dup_corr_thresh", + "type": float, + "doc": "Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh " + "are eliminated. Cell refinement parameter.", + "default": None, + }, + { + "name": "S_corr_thresh", + "type": float, + "doc": "Image correlation threshold. Cell refinement parameter.", + "default": None, + }, + { + "name": "T_corr_thresh", + "type": float, + "doc": "Trace correlation threshold. Cell refinement parameter.", + "default": None, + }, + { + "name": "temporal_corrupt_thresh", + "type": float, + "doc": "Threshold for temporal corruption index. Traces above this threshold are eliminated duirng " + "alternating minimization routine. Cell refinement parameter.", + "default": None, + }, + { + "name": "spatial_corrupt_thresh", + "type": float, + "doc": "Threshold for spatial corruption index. Images above this threshold are eliminated duirng " + "alternating minimization routine. Cell refinement parameter.", + "default": None, + }, + { + "name": "T_min_snr", + "type": float, + "doc": "Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. " + "Cell refinement parameter.", + "default": None, + }, + { + "name": "size_lower_limit", + "type": float, + "doc": "Lower size limit for found cells. Cell refinement parameter.", + "default": None, + }, + { + "name": "size_upper_limit", + "type": float, + "doc": "Lower size limit for found cells. Cell refinement parameter.", + "default": None, + }, + { + "name": "eccent_thresh", + "type": float, + "doc": "Upper limit of eccentricity for found cells. Cell refinement parameter.", + "default": None, + }, + { + "name": "low_ST_index_thresh", + "type": float, + "doc": "Lower limit of spatiotemporal activity index. Cell refinement parameter.", + "default": None, + }, + { + "name": "low_cell_area_flag", + "type": bool, + "doc": "Boolean flag indicating lower limit of cell area.", + "default": None, + }, + { + "name": "high_ST_index_thresh", + "type": float, + "doc": "Upper limit limit of spatiotemporal activity index. Cell refinement parameter.", + "default": None, + }, + { + "name": "low_ST_corr_thresh", + "type": float, + "doc": "Lower limit of spatiotemporal corelation. Cell refinement parameter.", + "default": None, + }, + { + "name": "confidence_thresh", + "type": float, + "doc": "Confidence threshold for found cells. Cell refinement parameter.", + "default": None, + }, + { + "name": "downsample_time_by", + "type": float, + "doc": "Time downsampling factor.", + "default": None, + }, + { + "name": "min_tau_after_downsampling", + "type": float, + "doc": "Minimum event tau after downsampling.", + "default": None, + }, + { + "name": "downsample_space_by", + "type": float, + "doc": "Spatial downsampling factor.", + "default": None, + }, + { + "name": "min_radius_after_downsampling", + "type": float, + "doc": "Minimum avg radius after downsampling.", + "default": None, + }, + { + "name": "reestimate_S_if_downsampled", + "type": bool, + "doc": "Boolean flag. When set to True, images are re-estimated from full movie at the end. " + "When False, images are upsampled by interpolation.", + "default": None, + }, + { + "name": "reestimate_T_if_downsampled", + "type": bool, + "doc": "Boolean flag. When set to True, traces are re-estimated from full movie at the end. " + "When False, traces are upsampled by interpolation.", + "default": None, + }, + { + "name": "adaptive_kappa", + "type": bool, + "doc": "Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively " + "set its robustness parameter", + "default": None, + }, + { + "name": "arbitrary_mask", + "type": bool, + "doc": "Indicates whether to use an arbitrary mask on the image.", + "default": None, + }, + { + "name": "smoothing_ratio_x2y", + "type": float, + "doc": "If the movie contains mainly objects that are elongated in one dimension " + "(e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension.", + "default": None, + }, + { + "name": "spatial_lowpass_cutoff", + "type": float, + "doc": "Cutoff determining the strength of butterworth spatial filtering of the movie. " + "Values defined relative to the average cell radius.", + "default": None, + }, + { + "name": "smooth_T", + "type": bool, + "doc": "Indicates whether calculated traces are smoothed using median filtering.", + "default": None, + }, + { + "name": "smooth_S", + "type": bool, + "doc": "Indicates whether calculated images are smoothed using a 2-D Gaussian filter.", + "default": None, + }, + { + "name": "l1_penalty_factor", + "type": float, + "doc": "Strength of l1 regularization penalty to be applied when estimating the temporal components.", + "default": None, + }, + { + "name": "max_iter_S", + "type": float, + "doc": "Maximum number of iterations for S estimation steps.", + "default": None, + }, + { + "name": "max_iter_T", + "type": float, + "doc": "Maximum number of iterations for T estimation steps.", + "default": None, + }, + { + "name": "TOL_sub", + "type": float, + "doc": "If the 1-step relative change in the objective within each T and S optimization is less than this, " + "the respective optimization is terminated.", + "default": None, + }, + { + "name": "TOL_main", + "type": float, + "doc": "If the relative change in the main objective function between 2 consecutive alternating " + "minimization steps is less than this, cell extraction is terminated.", + "default": None, + }, + { + "name": "T_lower_snr_threshold", + "type": float, + "doc": "Lower SNR threshold for found traces.", + "default": None, + }, + { + "name": "save_all_found", + "type": bool, + "doc": "Determines whether to save all spatial and temporal components found.", + "default": None, + }, + { + "name": "plot_loss", + "type": bool, + "doc": "Indicates whether empirical risk was plotted against iterations during alternating estimation.", + "default": None, + }, + { + "name": "baseline_quantile", + "type": ("array_data", "data"), + "doc": "Baseline quantile for Df/F calculation in preprocessing." + "The first dimension denotes the height of the image, the second dimension " + "corresponds to the width of the image.", + "default": None, + "shape": (None, None), + }, + { + "name": "S_init", + "type": ("array_data", "data"), + "doc": "Cell images provided to algorithm as the initial set of cells, skipping its native initialization." + "The first dimension denotes the height of the image, the second dimension " + "corresponds to the width of the image.", + "default": None, + "shape": (None, None), + }, + { + "name": "T_init", + "type": ("array_data", "data"), + "doc": "Cell traces provided to algorithm as the initial set of traces, skipping its native initialization." + "The first dimension denotes the number of cells, the second dimension " + "corresponds to time.", + "default": None, + "shape": (None, None), + }, + { + "name": "movie_mask", + "type": ("array_data", "data"), + "doc": "The circular mask to apply for microendoscopic movies during preprocessing.", + "default": None, + "shape": (None, None), + }, + { + "name": "is_pixel_valid", + "type": ("array_data", "data"), + "doc": "No description available", + "default": None, + "shape": (None, None), + }, + { + "name": "num_frames", + "type": ("array_data", "data"), + "doc": "No description available", + "default": None, + }, + { + "name": "num_iter_stop_quality_checks", + "type": ("array_data", "data"), + "doc": "No description available", + "default": None, + "shape": (None, None), + }, + { + "name": "pick_gpu", + "type": ("array_data", "data"), + "doc": "No description available", + "default": None, + "shape": (None, None), + }, ) def __init__(self, **kwargs): - name = popargs('name', kwargs) + name = popargs("name", kwargs) super().__init__(name=name) for extract_config_name, extract_config_value in kwargs.items(): setattr(self, extract_config_name, extract_config_value) From 30acb165cec7b61d866a82e51c0e480b920f2b9d Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 29 Sep 2022 13:54:16 +0200 Subject: [PATCH 13/18] remove unused attributes from test --- src/pynwb/tests/test_extract.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pynwb/tests/test_extract.py b/src/pynwb/tests/test_extract.py index 5d75594..a139594 100644 --- a/src/pynwb/tests/test_extract.py +++ b/src/pynwb/tests/test_extract.py @@ -14,9 +14,6 @@ class TestExtractSegmentation(TestCase): @classmethod def setUpClass(cls): - cls.file_path = "extract_public_output.mat" - cls.output_struct_name = "output" - cls.session_start_time = datetime.now().astimezone() cls.image_segmentation_name = "ExtractSegmentation" From 46600f1b78b6d1121533576ae8707d5362ac8f72 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 5 Oct 2022 11:19:00 +0200 Subject: [PATCH 14/18] Revert "update yaml files" This reverts commit 8f7d0646d9ebbc37f165b50b30ba9197bdbb0b33. --- spec/ndx-EXTRACT.namespace.yaml | 6 ++---- src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/spec/ndx-EXTRACT.namespace.yaml b/spec/ndx-EXTRACT.namespace.yaml index bb0d2de..8306bf8 100644 --- a/spec/ndx-EXTRACT.namespace.yaml +++ b/spec/ndx-EXTRACT.namespace.yaml @@ -1,15 +1,13 @@ namespaces: - author: - Cesar Echavarria - - Szonja Weigl contact: - cesar.echavarria@catalystneuro.com - - szonja.weigl@catalystneuro.com - doc: NWB Extension for storage of EXTRACT parameters and output + doc: NWB:N Extension for storage of EXTRACT parameters and output name: ndx-extract schema: - namespace: core neurodata_types: - ImageSegmentation - source: ndx-extract.extensions.yaml - version: 0.2.0 + version: 0.1.1 diff --git a/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml b/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml index bb0d2de..8306bf8 100644 --- a/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml +++ b/src/pynwb/ndx_extract/spec/ndx-extract.namespace.yaml @@ -1,15 +1,13 @@ namespaces: - author: - Cesar Echavarria - - Szonja Weigl contact: - cesar.echavarria@catalystneuro.com - - szonja.weigl@catalystneuro.com - doc: NWB Extension for storage of EXTRACT parameters and output + doc: NWB:N Extension for storage of EXTRACT parameters and output name: ndx-extract schema: - namespace: core neurodata_types: - ImageSegmentation - source: ndx-extract.extensions.yaml - version: 0.2.0 + version: 0.1.1 From 23928cfb93553fac8ad2257fac271f11613889f1 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 5 Oct 2022 11:19:08 +0200 Subject: [PATCH 15/18] Revert "update yaml files" This reverts commit ae29798909d78cf6bc4705126843d355155ca7bc. --- spec/ndx-EXTRACT.extensions.yaml | 438 +++++------------- .../spec/ndx-extract.extensions.yaml | 400 ++++------------ 2 files changed, 204 insertions(+), 634 deletions(-) diff --git a/spec/ndx-EXTRACT.extensions.yaml b/spec/ndx-EXTRACT.extensions.yaml index 86c8cab..91099cd 100644 --- a/spec/ndx-EXTRACT.extensions.yaml +++ b/spec/ndx-EXTRACT.extensions.yaml @@ -3,406 +3,232 @@ groups: neurodata_type_inc: ImageSegmentation doc: EXTRACT configuration parameters attributes: - - name: version - dtype: text - doc: The version of EXTRACT. - required: false - name: preprocess - dtype: bool - doc: Indicates whether or not data preprocessing was applied before main EXTRACT - function. - required: false - - name: pre_mask_on - dtype: bool - doc: Indicates whether or not an image mask was applied for preprocessing. - required: false - - name: pre_mask_radius - dtype: float64 - doc: The radius of the image mask. - required: false + dtype: int32 + doc: int32ean flag indicating data preprocessing before main EXTRACT function - name: fix_zero_FOV_strips - dtype: bool - doc: Indicates whether or not find and fix spatial slices that are occasionally - zero due to frame registration during preprocessing. - required: false + dtype: int32 + doc: int32ean flag. Find and fix spatial slices that are occasionally zero due + to frame registration during preprocessing. - name: medfilt_outlier_pixels - dtype: bool - doc: Determines whether outlier pixels in the movie should be replaced with their - neighborhood median. - required: false + dtype: int32 + doc: int32ean flag determining whether outlier pixels in the movie should be replaced + with their neighborhood median. - name: skip_dff - dtype: bool - doc: Indicates whether to skip Df/F calculation in preprocessing. - required: false - - name: second_df - dtype: float64 - doc: The setting for second df calculation. - required: false + dtype: int32 + doc: int32ean flag. Skip Df/F calculation in preprocessing. + - name: baseline_quantile + dtype: float32 + doc: Baseline quantile for Df/F calculation in preprocessing. - name: skip_highpass - dtype: bool - doc: Indicates whether to skip highpass filtering in preprocessing. - required: false + dtype: int32 + doc: int32ean flag. Skip highpass filtering in preprocessing. - name: spatial_highpass_cutoff - dtype: float64 - doc: Cutoff determining the strength of butterworth spatial filtering of the movie. - Values defined relative to the average cell radius. - required: false + dtype: float32 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' - name: temporal_denoising - dtype: bool - doc: Determines whether to apply temporal wavelet denoising. - required: false + dtype: int32 + doc: int32ean flag that determines whether to apply temporal wavelet denoising. - name: remove_background - dtype: bool - doc: Determines whether to subtract the (spatial) background (largest spatiotemporal - mode of the movie matrix). - required: false - - name: use_default_gpu - dtype: bool - doc: Indicates whether or not use the default GPU. - required: false - - name: use_gpu - dtype: bool - doc: Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU. - required: false - - name: multi_gpu - dtype: bool - doc: Indicates whether multiple GPUs were used. - required: false - - name: parallel_cpu - dtype: bool - doc: Indicates whether parallel CPUs were used. - required: false + dtype: int32 + doc: int32ean flag that determines whether to subtract the (spatial) background + (largest spatiotemporal mode of the movie matrix). - name: avg_cell_radius - dtype: float64 + dtype: float32 doc: Radius estimate for an average cell in the movie. - required: false - - name: avg_event_tau - dtype: float64 - doc: Determines the average event tau. - required: false - - name: avg_yield_threshold - dtype: float64 - doc: Determines the average yield threshold. - required: false - name: num_partitions_x - dtype: float64 + dtype: uint32 doc: Number of movie partitions in x dimension. required: false - name: num_partitions_y - dtype: float64 + dtype: uint32 doc: Number of movie partitions in y dimension. required: false - name: trace_output_option dtype: text - doc: Raw or non-negative output traces. - required: false - - name: trace_quantile - dtype: float64 - doc: No description available. - required: false + doc: Raw or non-negative output traces - name: dendrite_aware - dtype: bool - doc: Determines whether or not dendrites are preserved in the output for movies - where dendrites are present. - required: false + dtype: int32 + doc: int32ean flag, set it to true if dendrites exist in the movie & are desired + in the output. - name: crop_circular - dtype: bool - doc: For microendoscopic movies, whether or not automatically cropping out the - region outside the circular imaging region. - required: false + dtype: int32 + doc: int32ean flag. For microendoscopic movies, set it to true for automatically + cropping out the region outside the circular imaging region. - name: use_sparse_arrays - dtype: bool - doc: Determines whether not the output cell images were saved as sparse arrays. - required: false - - name: verbose - dtype: float64 - doc: Indicates the level of verbosity. - required: false + dtype: int32 + doc: int32ean flag. If set to true, then the output cell images will be saved + as sparse arrays - name: compact_output - dtype: bool - doc: Indicates whether or not leave out bad components that were found but then - eliminated from the output. - required: false + dtype: int32 + doc: int32ean flag. If set to true, then the output will not include bad components + that were found but then eliminated - name: hyperparameter_tuning_flag - dtype: bool - doc: Indicates whether or not use internal hyperparameter tuning. - required: false + dtype: int32 + doc: int32ean flag indicating internal hyperparameter tuning. - name: remove_duplicate_cells - dtype: bool + dtype: int32 doc: For movies processed in multiple partitions, this flag controls duplicate removal in the overlap regions. - required: false - name: max_iter - dtype: float64 + dtype: uint32 doc: Maximum number of alternating estimation iterations. - required: false - - name: minimal_checks - dtype: float64 - doc: Minimum number of checks that are performed. - required: false - name: cellfind_min_snr - dtype: float64 + dtype: float32 doc: Minimum peak SNR value for an object to be considered as a cell. - required: false - name: cellfind_max_steps - dtype: float64 + dtype: int doc: Maximum number of cell candidate initialization during cell finding step. - required: false - name: cellfind_kappa_std_ratio - dtype: float64 + dtype: float32 doc: Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization. - required: false - name: cellfind_filter_type dtype: text doc: Type of the spatial smoothing filter used for cell finding. - required: false - name: cellfind_numpix_threshold - dtype: float64 + dtype: float32 doc: During cell finding, objects with an area < cellfind_numpix_threshold are discarded. - required: false - - name: cellfind_adaptive_kappa - dtype: bool - doc: If True, then during cell finding, the robust esimation loss will adaptively - set its robustness parameter - required: false - - name: cellfind_spatial_highpass_cutoff - dtype: float64 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' - required: false - name: moving_radius dtype: float32 - doc: Deprecated variable for older EXTRACT file versions. Radius of moving average - filter in the case when cellfind_filter_type = moveavg (moving average) - required: false - - name: moving_radius_spatial - dtype: float64 - doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) for the image. - required: false - - name: moving_radius_temporal - dtype: float64 doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) for the traces. - required: false + (moving average) - name: init_with_gaussian - dtype: bool - doc: If True, then during cell finding, each cell is initialized with a gaussian - shape prior to robust estimation. If False, then initialization is done with - a correlation image (preferred for movies with dendrites). - required: false + dtype: int32 + doc: int32ean flag. If true, then during cell finding, each cell is initialized + with a gaussian shape prior to robust estimation. If false, then initialization + is done with a correlation image (preferred for movies with dendrites). - name: high2low_brightness_ratio - dtype: float64 + dtype: float32 doc: Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to stop cell finding process. - required: false - - name: visualize_cellfinding - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_full_range - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_max - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_min - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_show_bad_cells - dtype: float64 - doc: The visualization setting for cell finding. - required: false - name: kappa_std_ratio - dtype: float64 + dtype: float32 doc: Kappa will be set to this times the noise std during the cell refinement process. Cell refinement parameter. - required: false - name: T_dup_corr_thresh - dtype: float64 + dtype: float32 doc: Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh are eliminated. Cell refinement parameter. - required: false - - name: T_dup_thresh - dtype: float64 - doc: Threshold for traces. Cell refinement parameter. - required: false - name: S_dup_corr_thresh - dtype: float64 + dtype: float32 doc: Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh are eliminated. Cell refinement parameter. - required: false - - name: S_corr_thresh - dtype: float64 - doc: Image correlation threshold. Cell refinement parameter. - required: false - - name: T_corr_thresh - dtype: float64 - doc: Trace correlation threshold. Cell refinement parameter. - required: false - name: temporal_corrupt_thresh - dtype: float64 + dtype: float32 doc: Threshold for temporal corruption index. Traces above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. - required: false - name: spatial_corrupt_thresh - dtype: float64 + dtype: float32 doc: Threshold for spatial corruption index. Images above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. - required: false - name: T_min_snr - dtype: float64 + dtype: float32 doc: Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. Cell refinement parameter. - required: false - name: size_lower_limit - dtype: float64 + dtype: float32 doc: Lower size limit for found cells. Cell refinement parameter. - required: false - name: size_upper_limit - dtype: float64 + dtype: float32 doc: Lower size limit for found cells. Cell refinement parameter. - required: false - name: eccent_thresh - dtype: float64 + dtype: float32 doc: Upper limit of eccentricity for found cells. Cell refinement parameter. - required: false - name: low_ST_index_thresh - dtype: float64 + dtype: float32 doc: Lower limit of spatiotemporal activity index. Cell refinement parameter. - required: false - - name: low_cell_area_flag - dtype: bool - doc: Boolean flag indicating lower limit of cell area. - required: false - name: high_ST_index_thresh - dtype: float64 + dtype: float32 doc: Upper limit limit of spatiotemporal activity index. Cell refinement parameter. - required: false - name: low_ST_corr_thresh - dtype: float64 + dtype: float32 doc: Lower limit of spatiotemporal corelation. Cell refinement parameter. - required: false - name: confidence_thresh - dtype: float64 - doc: Confidence threshold for found cells. Cell refinement parameter. - required: false + dtype: float32 + doc: Confidence threhsold for found cells. Cell refinement parameter. - name: downsample_time_by - dtype: float64 - doc: Time downsampling factor. + dtype: float32 + doc: Time downsampling factor. If set to auto downsampling factor based on avg + cell radius and avg calcium event time constant. required: false - name: min_tau_after_downsampling - dtype: float64 - doc: Minimum event tau after downsampling. - required: false + dtype: float32 + doc: Minimum event tau after downsampling. Used when downsample_time_by = auto - name: downsample_space_by - dtype: float64 - doc: Spatial downsampling factor. + dtype: float32 + doc: Spatial downsampling factor. If set to auto downsampling factor based on + avg cell radius and avg calcium event time constant. required: false - name: min_radius_after_downsampling - dtype: float64 - doc: Minimum avg radius after downsampling. - required: false + dtype: float32 + doc: Minimum avg radius after downsampling. Used when downsample_space_by = auto. - name: reestimate_S_if_downsampled - dtype: bool - doc: Boolean flag. When set to True, images are re-estimated from full movie at - the end. When False, images are upsampled by interpolation. - required: false + dtype: int32 + doc: int32ean flag. When set to true, images are re-estimated from full movie + at the end. When false, images are upsampled by interpolation. - name: reestimate_T_if_downsampled - dtype: bool - doc: Boolean flag. When set to True, traces are re-estimated from full movie at - the end. When False, traces are upsampled by interpolation. - required: false + dtype: int32 + doc: int32ean flag. When set to true, traces are re-estimated from full movie + at the end. When false, traces are upsampled by interpolation. - name: adaptive_kappa - dtype: bool - doc: Boolean flag. If True, then during cell finding, the robust esimation loss + dtype: int32 + doc: int32ean flag. If true, then during cell finding, the robust esimation loss will adaptively set its robustness parameter - required: false - - name: arbitrary_mask - dtype: bool - doc: Indicates whether to use an arbitrary mask on the image. - required: false - name: smoothing_ratio_x2y - dtype: float64 + dtype: float32 doc: If the movie contains mainly objects that are elongated in one dimension (e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension. - required: false - name: spatial_lowpass_cutoff - dtype: float64 - doc: Cutoff determining the strength of butterworth spatial filtering of the movie. - Values defined relative to the average cell radius. - required: false + dtype: float32 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' - name: smooth_T - dtype: bool - doc: Indicates whether calculated traces are smoothed using median filtering. - required: false + dtype: int32 + doc: int32ean flag indicating whether calculated traces are smoothed using median + filtering. - name: smooth_S - dtype: bool - doc: Indicates whether calculated images are smoothed using a 2-D Gaussian filter. - required: false + dtype: int32 + doc: int32ean flag indicating whether calculated images are smoothed using a 2-D + Gaussian filter. - name: l1_penalty_factor - dtype: float64 + dtype: float32 doc: Strength of l1 regularization penalty to be applied when estimating the temporal components. - required: false - name: max_iter_S - dtype: float64 - doc: Maximum number of iterations for S estimation steps. - required: false + dtype: uint32 + doc: Maximum number of iterations for S estimation steps - name: max_iter_T - dtype: float64 - doc: Maximum number of iterations for T estimation steps. - required: false + dtype: uint32 + doc: Maximum number of iterations for T estimation steps - name: TOL_sub - dtype: float64 + dtype: float32 doc: If the 1-step relative change in the objective within each T and S optimization is less than this, the respective optimization is terminated. - required: false - name: TOL_main - dtype: float64 - doc: If the relative change in the main objective function between 2 consecutive - alternating minimization steps is less than this, cell extraction is terminated. - required: false + dtype: float32 + doc: ' If the relative change in the main objective function between 2 consecutive + alternating minimization steps is less than this, cell extraction is terminated.' - name: T_lower_snr_threshold - dtype: float64 + dtype: float32 doc: Lower SNR threshold for found traces. - required: false - name: save_all_found - dtype: bool - doc: Determines whether to save all spatial and temporal components found. - required: false - - name: plot_loss - dtype: bool - doc: Indicates whether empirical risk was plotted against iterations during alternating - estimation. - required: false + dtype: int32 + doc: int32ean flag. Save all spatial and temporal components found datasets: - - name: baseline_quantile - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: Baseline quantile for Df/F calculation in preprocessing. - quantity: '?' - name: S_init dtype: double dims: - - height - - width + - heightxwidth + - num_cells shape: - null - null doc: Cell images provided to algorithm as the initial set of cells, skipping its - native initialization. + native initialization quantity: '?' - name: T_init dtype: double @@ -413,7 +239,7 @@ groups: - null - null doc: Cell traces provided to algorithm as the initial set of traces, skipping - its native initialization. + its native initialization quantity: '?' - name: movie_mask dtype: double @@ -423,45 +249,5 @@ groups: shape: - null - null - doc: The circular mask to apply for microendoscopic movies during preprocessing. - quantity: '?' - - name: is_pixel_valid - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available - quantity: '?' - - name: num_frames - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available - quantity: '?' - - name: num_iter_stop_quality_checks - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available - quantity: '?' - - name: pick_gpu - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available + doc: Movie mask. quantity: '?' diff --git a/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml b/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml index 86c8cab..b6f0a1c 100644 --- a/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml +++ b/src/pynwb/ndx_extract/spec/ndx-extract.extensions.yaml @@ -3,406 +3,230 @@ groups: neurodata_type_inc: ImageSegmentation doc: EXTRACT configuration parameters attributes: - - name: version - dtype: text - doc: The version of EXTRACT. - required: false - name: preprocess dtype: bool - doc: Indicates whether or not data preprocessing was applied before main EXTRACT - function. - required: false - - name: pre_mask_on - dtype: bool - doc: Indicates whether or not an image mask was applied for preprocessing. - required: false - - name: pre_mask_radius - dtype: float64 - doc: The radius of the image mask. - required: false + doc: Boolean flag indicating data preprocessing before main EXTRACT function - name: fix_zero_FOV_strips dtype: bool - doc: Indicates whether or not find and fix spatial slices that are occasionally - zero due to frame registration during preprocessing. - required: false + doc: Boolean flag. Find and fix spatial slices that are occasionally zero due + to frame registration during preprocessing. - name: medfilt_outlier_pixels dtype: bool - doc: Determines whether outlier pixels in the movie should be replaced with their - neighborhood median. - required: false + doc: Boolean flag determining whether outlier pixels in the movie should be replaced + with their neighborhood median. - name: skip_dff dtype: bool - doc: Indicates whether to skip Df/F calculation in preprocessing. - required: false - - name: second_df - dtype: float64 - doc: The setting for second df calculation. - required: false + doc: Boolean flag. Skip Df/F calculation in preprocessing. + - name: baseline_quantile + dtype: float32 + doc: Baseline quantile for Df/F calculation in preprocessing. - name: skip_highpass dtype: bool - doc: Indicates whether to skip highpass filtering in preprocessing. - required: false + doc: Boolean flag. Skip highpass filtering in preprocessing. - name: spatial_highpass_cutoff - dtype: float64 - doc: Cutoff determining the strength of butterworth spatial filtering of the movie. - Values defined relative to the average cell radius. - required: false + dtype: float32 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' - name: temporal_denoising dtype: bool - doc: Determines whether to apply temporal wavelet denoising. - required: false + doc: Boolean flag that determines whether to apply temporal wavelet denoising. - name: remove_background dtype: bool - doc: Determines whether to subtract the (spatial) background (largest spatiotemporal - mode of the movie matrix). - required: false - - name: use_default_gpu - dtype: bool - doc: Indicates whether or not use the default GPU. - required: false - - name: use_gpu - dtype: bool - doc: Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU. - required: false - - name: multi_gpu - dtype: bool - doc: Indicates whether multiple GPUs were used. - required: false - - name: parallel_cpu - dtype: bool - doc: Indicates whether parallel CPUs were used. - required: false + doc: Boolean flag that determines whether to subtract the (spatial) background + (largest spatiotemporal mode of the movie matrix). - name: avg_cell_radius - dtype: float64 + dtype: float32 doc: Radius estimate for an average cell in the movie. - required: false - - name: avg_event_tau - dtype: float64 - doc: Determines the average event tau. - required: false - - name: avg_yield_threshold - dtype: float64 - doc: Determines the average yield threshold. - required: false - name: num_partitions_x - dtype: float64 + dtype: uint doc: Number of movie partitions in x dimension. required: false - name: num_partitions_y - dtype: float64 + dtype: uint doc: Number of movie partitions in y dimension. required: false - name: trace_output_option dtype: text - doc: Raw or non-negative output traces. - required: false - - name: trace_quantile - dtype: float64 - doc: No description available. - required: false + doc: Raw or non-negative output traces - name: dendrite_aware dtype: bool - doc: Determines whether or not dendrites are preserved in the output for movies - where dendrites are present. - required: false + doc: Boolean flag, set it to true if dendrites exist in the movie & are desired + in the output. - name: crop_circular dtype: bool - doc: For microendoscopic movies, whether or not automatically cropping out the - region outside the circular imaging region. - required: false + doc: Boolean flag. For microendoscopic movies, set it to true for automatically + cropping out the region outside the circular imaging region. - name: use_sparse_arrays dtype: bool - doc: Determines whether not the output cell images were saved as sparse arrays. - required: false - - name: verbose - dtype: float64 - doc: Indicates the level of verbosity. - required: false + doc: Boolean flag. If set to true, then the output cell images will be saved as + sparse arrays - name: compact_output dtype: bool - doc: Indicates whether or not leave out bad components that were found but then - eliminated from the output. - required: false + doc: Boolean flag. If set to true, then the output will not include bad components + that were found but then eliminated - name: hyperparameter_tuning_flag dtype: bool - doc: Indicates whether or not use internal hyperparameter tuning. - required: false + doc: Boolean flag indicating internal hyperparameter tuning. - name: remove_duplicate_cells dtype: bool doc: For movies processed in multiple partitions, this flag controls duplicate removal in the overlap regions. - required: false - name: max_iter - dtype: float64 + dtype: uint doc: Maximum number of alternating estimation iterations. - required: false - - name: minimal_checks - dtype: float64 - doc: Minimum number of checks that are performed. - required: false - name: cellfind_min_snr - dtype: float64 + dtype: float32 doc: Minimum peak SNR value for an object to be considered as a cell. - required: false - name: cellfind_max_steps - dtype: float64 + dtype: int doc: Maximum number of cell candidate initialization during cell finding step. - required: false - name: cellfind_kappa_std_ratio - dtype: float64 + dtype: float32 doc: Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization. - required: false - name: cellfind_filter_type dtype: text doc: Type of the spatial smoothing filter used for cell finding. - required: false - name: cellfind_numpix_threshold - dtype: float64 + dtype: float32 doc: During cell finding, objects with an area < cellfind_numpix_threshold are discarded. - required: false - - name: cellfind_adaptive_kappa - dtype: bool - doc: If True, then during cell finding, the robust esimation loss will adaptively - set its robustness parameter - required: false - - name: cellfind_spatial_highpass_cutoff - dtype: float64 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' - required: false - name: moving_radius dtype: float32 - doc: Deprecated variable for older EXTRACT file versions. Radius of moving average - filter in the case when cellfind_filter_type = moveavg (moving average) - required: false - - name: moving_radius_spatial - dtype: float64 - doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) for the image. - required: false - - name: moving_radius_temporal - dtype: float64 doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) for the traces. - required: false + (moving average) - name: init_with_gaussian dtype: bool - doc: If True, then during cell finding, each cell is initialized with a gaussian - shape prior to robust estimation. If False, then initialization is done with - a correlation image (preferred for movies with dendrites). - required: false + doc: Boolean flag. If true, then during cell finding, each cell is initialized + with a gaussian shape prior to robust estimation. If false, then initialization + is done with a correlation image (preferred for movies with dendrites). - name: high2low_brightness_ratio - dtype: float64 + dtype: float32 doc: Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to stop cell finding process. - required: false - - name: visualize_cellfinding - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_full_range - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_max - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_min - dtype: float64 - doc: The visualization setting for cell finding. - required: false - - name: visualize_cellfinding_show_bad_cells - dtype: float64 - doc: The visualization setting for cell finding. - required: false - name: kappa_std_ratio - dtype: float64 + dtype: float32 doc: Kappa will be set to this times the noise std during the cell refinement process. Cell refinement parameter. - required: false - name: T_dup_corr_thresh - dtype: float64 + dtype: float32 doc: Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh are eliminated. Cell refinement parameter. - required: false - - name: T_dup_thresh - dtype: float64 - doc: Threshold for traces. Cell refinement parameter. - required: false - name: S_dup_corr_thresh - dtype: float64 + dtype: float32 doc: Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh are eliminated. Cell refinement parameter. - required: false - - name: S_corr_thresh - dtype: float64 - doc: Image correlation threshold. Cell refinement parameter. - required: false - - name: T_corr_thresh - dtype: float64 - doc: Trace correlation threshold. Cell refinement parameter. - required: false - name: temporal_corrupt_thresh - dtype: float64 + dtype: float32 doc: Threshold for temporal corruption index. Traces above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. - required: false - name: spatial_corrupt_thresh - dtype: float64 + dtype: float32 doc: Threshold for spatial corruption index. Images above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. - required: false - name: T_min_snr - dtype: float64 + dtype: float32 doc: Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. Cell refinement parameter. - required: false - name: size_lower_limit - dtype: float64 + dtype: float32 doc: Lower size limit for found cells. Cell refinement parameter. - required: false - name: size_upper_limit - dtype: float64 + dtype: float32 doc: Lower size limit for found cells. Cell refinement parameter. - required: false - name: eccent_thresh - dtype: float64 + dtype: float32 doc: Upper limit of eccentricity for found cells. Cell refinement parameter. - required: false - name: low_ST_index_thresh - dtype: float64 + dtype: float32 doc: Lower limit of spatiotemporal activity index. Cell refinement parameter. - required: false - - name: low_cell_area_flag - dtype: bool - doc: Boolean flag indicating lower limit of cell area. - required: false - name: high_ST_index_thresh - dtype: float64 + dtype: float32 doc: Upper limit limit of spatiotemporal activity index. Cell refinement parameter. - required: false - name: low_ST_corr_thresh - dtype: float64 + dtype: float32 doc: Lower limit of spatiotemporal corelation. Cell refinement parameter. - required: false - name: confidence_thresh - dtype: float64 - doc: Confidence threshold for found cells. Cell refinement parameter. - required: false + dtype: float32 + doc: Confidence threhsold for found cells. Cell refinement parameter. - name: downsample_time_by - dtype: float64 - doc: Time downsampling factor. - required: false + dtype: text + doc: Time downsampling factor. If set to auto downsampling factor based on avg + cell radius and avg calcium event time constant. - name: min_tau_after_downsampling - dtype: float64 - doc: Minimum event tau after downsampling. - required: false + dtype: float32 + doc: Minimum event tau after downsampling. Used when downsample_time_by = auto - name: downsample_space_by - dtype: float64 - doc: Spatial downsampling factor. - required: false + dtype: text + doc: Spatial downsampling factor. If set to auto downsampling factor based on + avg cell radius and avg calcium event time constant. - name: min_radius_after_downsampling - dtype: float64 - doc: Minimum avg radius after downsampling. - required: false + dtype: float32 + doc: Minimum avg radius after downsampling. Used when downsample_space_by = auto. - name: reestimate_S_if_downsampled dtype: bool - doc: Boolean flag. When set to True, images are re-estimated from full movie at - the end. When False, images are upsampled by interpolation. - required: false + doc: Boolean flag. When set to true, images are re-estimated from full movie at + the end. When false, images are upsampled by interpolation. - name: reestimate_T_if_downsampled dtype: bool - doc: Boolean flag. When set to True, traces are re-estimated from full movie at - the end. When False, traces are upsampled by interpolation. - required: false + doc: Boolean flag. When set to true, traces are re-estimated from full movie at + the end. When false, traces are upsampled by interpolation. - name: adaptive_kappa dtype: bool - doc: Boolean flag. If True, then during cell finding, the robust esimation loss + doc: Boolean flag. If true, then during cell finding, the robust esimation loss will adaptively set its robustness parameter - required: false - - name: arbitrary_mask - dtype: bool - doc: Indicates whether to use an arbitrary mask on the image. - required: false - name: smoothing_ratio_x2y - dtype: float64 + dtype: float32 doc: If the movie contains mainly objects that are elongated in one dimension (e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension. - required: false - name: spatial_lowpass_cutoff - dtype: float64 - doc: Cutoff determining the strength of butterworth spatial filtering of the movie. - Values defined relative to the average cell radius. - required: false + dtype: float32 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' - name: smooth_T dtype: bool - doc: Indicates whether calculated traces are smoothed using median filtering. - required: false + doc: Boolean flag indicating whether calculated traces are smoothed using median + filtering. - name: smooth_S dtype: bool - doc: Indicates whether calculated images are smoothed using a 2-D Gaussian filter. - required: false + doc: Boolean flag indicating whether calculated images are smoothed using a 2-D + Gaussian filter. - name: l1_penalty_factor - dtype: float64 + dtype: float32 doc: Strength of l1 regularization penalty to be applied when estimating the temporal components. - required: false - name: max_iter_S - dtype: float64 - doc: Maximum number of iterations for S estimation steps. - required: false + dtype: uint + doc: Maximum number of iterations for S estimation steps - name: max_iter_T - dtype: float64 - doc: Maximum number of iterations for T estimation steps. - required: false + dtype: uint + doc: Maximum number of iterations for T estimation steps - name: TOL_sub - dtype: float64 + dtype: float32 doc: If the 1-step relative change in the objective within each T and S optimization is less than this, the respective optimization is terminated. - required: false - name: TOL_main - dtype: float64 - doc: If the relative change in the main objective function between 2 consecutive - alternating minimization steps is less than this, cell extraction is terminated. - required: false + dtype: float32 + doc: ' If the relative change in the main objective function between 2 consecutive + alternating minimization steps is less than this, cell extraction is terminated.' - name: T_lower_snr_threshold - dtype: float64 + dtype: float32 doc: Lower SNR threshold for found traces. - required: false - name: save_all_found dtype: bool - doc: Determines whether to save all spatial and temporal components found. - required: false - - name: plot_loss - dtype: bool - doc: Indicates whether empirical risk was plotted against iterations during alternating - estimation. - required: false + doc: Boolean flag. Save all spatial and temporal components found datasets: - - name: baseline_quantile - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: Baseline quantile for Df/F calculation in preprocessing. - quantity: '?' - name: S_init dtype: double dims: - - height - - width + - heightxwidth + - num_cells shape: - null - null doc: Cell images provided to algorithm as the initial set of cells, skipping its - native initialization. + native initialization quantity: '?' - name: T_init dtype: double @@ -413,7 +237,7 @@ groups: - null - null doc: Cell traces provided to algorithm as the initial set of traces, skipping - its native initialization. + its native initialization quantity: '?' - name: movie_mask dtype: double @@ -423,45 +247,5 @@ groups: shape: - null - null - doc: The circular mask to apply for microendoscopic movies during preprocessing. - quantity: '?' - - name: is_pixel_valid - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available - quantity: '?' - - name: num_frames - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available - quantity: '?' - - name: num_iter_stop_quality_checks - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available - quantity: '?' - - name: pick_gpu - dtype: double - dims: - - height - - width - shape: - - null - - null - doc: No description available + doc: Movie mask. quantity: '?' From d4ea5d3ea642e0fc1270d1f357e52ed69ea54ee7 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 5 Oct 2022 11:22:55 +0200 Subject: [PATCH 16/18] update yaml files in spec folder --- spec/ndx-EXTRACT.extensions.yaml | 438 +++++++++++++++++++++++-------- spec/ndx-EXTRACT.namespace.yaml | 6 +- 2 files changed, 330 insertions(+), 114 deletions(-) diff --git a/spec/ndx-EXTRACT.extensions.yaml b/spec/ndx-EXTRACT.extensions.yaml index 91099cd..86c8cab 100644 --- a/spec/ndx-EXTRACT.extensions.yaml +++ b/spec/ndx-EXTRACT.extensions.yaml @@ -3,232 +3,406 @@ groups: neurodata_type_inc: ImageSegmentation doc: EXTRACT configuration parameters attributes: + - name: version + dtype: text + doc: The version of EXTRACT. + required: false - name: preprocess - dtype: int32 - doc: int32ean flag indicating data preprocessing before main EXTRACT function + dtype: bool + doc: Indicates whether or not data preprocessing was applied before main EXTRACT + function. + required: false + - name: pre_mask_on + dtype: bool + doc: Indicates whether or not an image mask was applied for preprocessing. + required: false + - name: pre_mask_radius + dtype: float64 + doc: The radius of the image mask. + required: false - name: fix_zero_FOV_strips - dtype: int32 - doc: int32ean flag. Find and fix spatial slices that are occasionally zero due - to frame registration during preprocessing. + dtype: bool + doc: Indicates whether or not find and fix spatial slices that are occasionally + zero due to frame registration during preprocessing. + required: false - name: medfilt_outlier_pixels - dtype: int32 - doc: int32ean flag determining whether outlier pixels in the movie should be replaced - with their neighborhood median. + dtype: bool + doc: Determines whether outlier pixels in the movie should be replaced with their + neighborhood median. + required: false - name: skip_dff - dtype: int32 - doc: int32ean flag. Skip Df/F calculation in preprocessing. - - name: baseline_quantile - dtype: float32 - doc: Baseline quantile for Df/F calculation in preprocessing. + dtype: bool + doc: Indicates whether to skip Df/F calculation in preprocessing. + required: false + - name: second_df + dtype: float64 + doc: The setting for second df calculation. + required: false - name: skip_highpass - dtype: int32 - doc: int32ean flag. Skip highpass filtering in preprocessing. + dtype: bool + doc: Indicates whether to skip highpass filtering in preprocessing. + required: false - name: spatial_highpass_cutoff - dtype: float32 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' + dtype: float64 + doc: Cutoff determining the strength of butterworth spatial filtering of the movie. + Values defined relative to the average cell radius. + required: false - name: temporal_denoising - dtype: int32 - doc: int32ean flag that determines whether to apply temporal wavelet denoising. + dtype: bool + doc: Determines whether to apply temporal wavelet denoising. + required: false - name: remove_background - dtype: int32 - doc: int32ean flag that determines whether to subtract the (spatial) background - (largest spatiotemporal mode of the movie matrix). + dtype: bool + doc: Determines whether to subtract the (spatial) background (largest spatiotemporal + mode of the movie matrix). + required: false + - name: use_default_gpu + dtype: bool + doc: Indicates whether or not use the default GPU. + required: false + - name: use_gpu + dtype: bool + doc: Indicates whether to run EXTRACT on GPU. If False, EXTRACT was run on CPU. + required: false + - name: multi_gpu + dtype: bool + doc: Indicates whether multiple GPUs were used. + required: false + - name: parallel_cpu + dtype: bool + doc: Indicates whether parallel CPUs were used. + required: false - name: avg_cell_radius - dtype: float32 + dtype: float64 doc: Radius estimate for an average cell in the movie. + required: false + - name: avg_event_tau + dtype: float64 + doc: Determines the average event tau. + required: false + - name: avg_yield_threshold + dtype: float64 + doc: Determines the average yield threshold. + required: false - name: num_partitions_x - dtype: uint32 + dtype: float64 doc: Number of movie partitions in x dimension. required: false - name: num_partitions_y - dtype: uint32 + dtype: float64 doc: Number of movie partitions in y dimension. required: false - name: trace_output_option dtype: text - doc: Raw or non-negative output traces + doc: Raw or non-negative output traces. + required: false + - name: trace_quantile + dtype: float64 + doc: No description available. + required: false - name: dendrite_aware - dtype: int32 - doc: int32ean flag, set it to true if dendrites exist in the movie & are desired - in the output. + dtype: bool + doc: Determines whether or not dendrites are preserved in the output for movies + where dendrites are present. + required: false - name: crop_circular - dtype: int32 - doc: int32ean flag. For microendoscopic movies, set it to true for automatically - cropping out the region outside the circular imaging region. + dtype: bool + doc: For microendoscopic movies, whether or not automatically cropping out the + region outside the circular imaging region. + required: false - name: use_sparse_arrays - dtype: int32 - doc: int32ean flag. If set to true, then the output cell images will be saved - as sparse arrays + dtype: bool + doc: Determines whether not the output cell images were saved as sparse arrays. + required: false + - name: verbose + dtype: float64 + doc: Indicates the level of verbosity. + required: false - name: compact_output - dtype: int32 - doc: int32ean flag. If set to true, then the output will not include bad components - that were found but then eliminated + dtype: bool + doc: Indicates whether or not leave out bad components that were found but then + eliminated from the output. + required: false - name: hyperparameter_tuning_flag - dtype: int32 - doc: int32ean flag indicating internal hyperparameter tuning. + dtype: bool + doc: Indicates whether or not use internal hyperparameter tuning. + required: false - name: remove_duplicate_cells - dtype: int32 + dtype: bool doc: For movies processed in multiple partitions, this flag controls duplicate removal in the overlap regions. + required: false - name: max_iter - dtype: uint32 + dtype: float64 doc: Maximum number of alternating estimation iterations. + required: false + - name: minimal_checks + dtype: float64 + doc: Minimum number of checks that are performed. + required: false - name: cellfind_min_snr - dtype: float32 + dtype: float64 doc: Minimum peak SNR value for an object to be considered as a cell. + required: false - name: cellfind_max_steps - dtype: int + dtype: float64 doc: Maximum number of cell candidate initialization during cell finding step. + required: false - name: cellfind_kappa_std_ratio - dtype: float32 + dtype: float64 doc: Kappa will be set to this times the noise std for the component-wise EXTRACT during initialization. + required: false - name: cellfind_filter_type dtype: text doc: Type of the spatial smoothing filter used for cell finding. + required: false - name: cellfind_numpix_threshold - dtype: float32 + dtype: float64 doc: During cell finding, objects with an area < cellfind_numpix_threshold are discarded. + required: false + - name: cellfind_adaptive_kappa + dtype: bool + doc: If True, then during cell finding, the robust esimation loss will adaptively + set its robustness parameter + required: false + - name: cellfind_spatial_highpass_cutoff + dtype: float64 + doc: 'Cutoff determining the strength of butterworth spatial filtering of the + movie. Values defined relative to the average cell radius. ' + required: false - name: moving_radius dtype: float32 + doc: Deprecated variable for older EXTRACT file versions. Radius of moving average + filter in the case when cellfind_filter_type = moveavg (moving average) + required: false + - name: moving_radius_spatial + dtype: float64 + doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg + (moving average) for the image. + required: false + - name: moving_radius_temporal + dtype: float64 doc: Radius of moving average filter in the case when cellfind_filter_type = moveavg - (moving average) + (moving average) for the traces. + required: false - name: init_with_gaussian - dtype: int32 - doc: int32ean flag. If true, then during cell finding, each cell is initialized - with a gaussian shape prior to robust estimation. If false, then initialization - is done with a correlation image (preferred for movies with dendrites). + dtype: bool + doc: If True, then during cell finding, each cell is initialized with a gaussian + shape prior to robust estimation. If False, then initialization is done with + a correlation image (preferred for movies with dendrites). + required: false - name: high2low_brightness_ratio - dtype: float32 + dtype: float64 doc: Threshold for ratio of pixel compared to most bright region in FOV. Used to determine when to stop cell finding process. + required: false + - name: visualize_cellfinding + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_full_range + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_max + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_min + dtype: float64 + doc: The visualization setting for cell finding. + required: false + - name: visualize_cellfinding_show_bad_cells + dtype: float64 + doc: The visualization setting for cell finding. + required: false - name: kappa_std_ratio - dtype: float32 + dtype: float64 doc: Kappa will be set to this times the noise std during the cell refinement process. Cell refinement parameter. + required: false - name: T_dup_corr_thresh - dtype: float32 + dtype: float64 doc: Through alternating estimation, cells that have higher trace correlation than T_dup_corr_thresh are eliminated. Cell refinement parameter. + required: false + - name: T_dup_thresh + dtype: float64 + doc: Threshold for traces. Cell refinement parameter. + required: false - name: S_dup_corr_thresh - dtype: float32 + dtype: float64 doc: Through alternating estimation, cells that have higher image correlation than S_dup_corr_thresh are eliminated. Cell refinement parameter. + required: false + - name: S_corr_thresh + dtype: float64 + doc: Image correlation threshold. Cell refinement parameter. + required: false + - name: T_corr_thresh + dtype: float64 + doc: Trace correlation threshold. Cell refinement parameter. + required: false - name: temporal_corrupt_thresh - dtype: float32 + dtype: float64 doc: Threshold for temporal corruption index. Traces above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. + required: false - name: spatial_corrupt_thresh - dtype: float32 + dtype: float64 doc: Threshold for spatial corruption index. Images above this threshold are eliminated duirng alternating minimization routine. Cell refinement parameter. + required: false - name: T_min_snr - dtype: float32 + dtype: float64 doc: Threshold for temporal SNR. Cells with temporal SNR below this value will be eliminated. Cell refinement parameter. + required: false - name: size_lower_limit - dtype: float32 + dtype: float64 doc: Lower size limit for found cells. Cell refinement parameter. + required: false - name: size_upper_limit - dtype: float32 + dtype: float64 doc: Lower size limit for found cells. Cell refinement parameter. + required: false - name: eccent_thresh - dtype: float32 + dtype: float64 doc: Upper limit of eccentricity for found cells. Cell refinement parameter. + required: false - name: low_ST_index_thresh - dtype: float32 + dtype: float64 doc: Lower limit of spatiotemporal activity index. Cell refinement parameter. + required: false + - name: low_cell_area_flag + dtype: bool + doc: Boolean flag indicating lower limit of cell area. + required: false - name: high_ST_index_thresh - dtype: float32 + dtype: float64 doc: Upper limit limit of spatiotemporal activity index. Cell refinement parameter. + required: false - name: low_ST_corr_thresh - dtype: float32 + dtype: float64 doc: Lower limit of spatiotemporal corelation. Cell refinement parameter. + required: false - name: confidence_thresh - dtype: float32 - doc: Confidence threhsold for found cells. Cell refinement parameter. + dtype: float64 + doc: Confidence threshold for found cells. Cell refinement parameter. + required: false - name: downsample_time_by - dtype: float32 - doc: Time downsampling factor. If set to auto downsampling factor based on avg - cell radius and avg calcium event time constant. + dtype: float64 + doc: Time downsampling factor. required: false - name: min_tau_after_downsampling - dtype: float32 - doc: Minimum event tau after downsampling. Used when downsample_time_by = auto + dtype: float64 + doc: Minimum event tau after downsampling. + required: false - name: downsample_space_by - dtype: float32 - doc: Spatial downsampling factor. If set to auto downsampling factor based on - avg cell radius and avg calcium event time constant. + dtype: float64 + doc: Spatial downsampling factor. required: false - name: min_radius_after_downsampling - dtype: float32 - doc: Minimum avg radius after downsampling. Used when downsample_space_by = auto. + dtype: float64 + doc: Minimum avg radius after downsampling. + required: false - name: reestimate_S_if_downsampled - dtype: int32 - doc: int32ean flag. When set to true, images are re-estimated from full movie - at the end. When false, images are upsampled by interpolation. + dtype: bool + doc: Boolean flag. When set to True, images are re-estimated from full movie at + the end. When False, images are upsampled by interpolation. + required: false - name: reestimate_T_if_downsampled - dtype: int32 - doc: int32ean flag. When set to true, traces are re-estimated from full movie - at the end. When false, traces are upsampled by interpolation. + dtype: bool + doc: Boolean flag. When set to True, traces are re-estimated from full movie at + the end. When False, traces are upsampled by interpolation. + required: false - name: adaptive_kappa - dtype: int32 - doc: int32ean flag. If true, then during cell finding, the robust esimation loss + dtype: bool + doc: Boolean flag. If True, then during cell finding, the robust esimation loss will adaptively set its robustness parameter + required: false + - name: arbitrary_mask + dtype: bool + doc: Indicates whether to use an arbitrary mask on the image. + required: false - name: smoothing_ratio_x2y - dtype: float32 + dtype: float64 doc: If the movie contains mainly objects that are elongated in one dimension (e.g. dendrites), this parameter is useful for more smoothing in either x or y dimension. + required: false - name: spatial_lowpass_cutoff - dtype: float32 - doc: 'Cutoff determining the strength of butterworth spatial filtering of the - movie. Values defined relative to the average cell radius. ' + dtype: float64 + doc: Cutoff determining the strength of butterworth spatial filtering of the movie. + Values defined relative to the average cell radius. + required: false - name: smooth_T - dtype: int32 - doc: int32ean flag indicating whether calculated traces are smoothed using median - filtering. + dtype: bool + doc: Indicates whether calculated traces are smoothed using median filtering. + required: false - name: smooth_S - dtype: int32 - doc: int32ean flag indicating whether calculated images are smoothed using a 2-D - Gaussian filter. + dtype: bool + doc: Indicates whether calculated images are smoothed using a 2-D Gaussian filter. + required: false - name: l1_penalty_factor - dtype: float32 + dtype: float64 doc: Strength of l1 regularization penalty to be applied when estimating the temporal components. + required: false - name: max_iter_S - dtype: uint32 - doc: Maximum number of iterations for S estimation steps + dtype: float64 + doc: Maximum number of iterations for S estimation steps. + required: false - name: max_iter_T - dtype: uint32 - doc: Maximum number of iterations for T estimation steps + dtype: float64 + doc: Maximum number of iterations for T estimation steps. + required: false - name: TOL_sub - dtype: float32 + dtype: float64 doc: If the 1-step relative change in the objective within each T and S optimization is less than this, the respective optimization is terminated. + required: false - name: TOL_main - dtype: float32 - doc: ' If the relative change in the main objective function between 2 consecutive - alternating minimization steps is less than this, cell extraction is terminated.' + dtype: float64 + doc: If the relative change in the main objective function between 2 consecutive + alternating minimization steps is less than this, cell extraction is terminated. + required: false - name: T_lower_snr_threshold - dtype: float32 + dtype: float64 doc: Lower SNR threshold for found traces. + required: false - name: save_all_found - dtype: int32 - doc: int32ean flag. Save all spatial and temporal components found + dtype: bool + doc: Determines whether to save all spatial and temporal components found. + required: false + - name: plot_loss + dtype: bool + doc: Indicates whether empirical risk was plotted against iterations during alternating + estimation. + required: false datasets: + - name: baseline_quantile + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: Baseline quantile for Df/F calculation in preprocessing. + quantity: '?' - name: S_init dtype: double dims: - - heightxwidth - - num_cells + - height + - width shape: - null - null doc: Cell images provided to algorithm as the initial set of cells, skipping its - native initialization + native initialization. quantity: '?' - name: T_init dtype: double @@ -239,7 +413,7 @@ groups: - null - null doc: Cell traces provided to algorithm as the initial set of traces, skipping - its native initialization + its native initialization. quantity: '?' - name: movie_mask dtype: double @@ -249,5 +423,45 @@ groups: shape: - null - null - doc: Movie mask. + doc: The circular mask to apply for microendoscopic movies during preprocessing. + quantity: '?' + - name: is_pixel_valid + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: num_frames + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: num_iter_stop_quality_checks + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available + quantity: '?' + - name: pick_gpu + dtype: double + dims: + - height + - width + shape: + - null + - null + doc: No description available quantity: '?' diff --git a/spec/ndx-EXTRACT.namespace.yaml b/spec/ndx-EXTRACT.namespace.yaml index 8306bf8..bb0d2de 100644 --- a/spec/ndx-EXTRACT.namespace.yaml +++ b/spec/ndx-EXTRACT.namespace.yaml @@ -1,13 +1,15 @@ namespaces: - author: - Cesar Echavarria + - Szonja Weigl contact: - cesar.echavarria@catalystneuro.com - doc: NWB:N Extension for storage of EXTRACT parameters and output + - szonja.weigl@catalystneuro.com + doc: NWB Extension for storage of EXTRACT parameters and output name: ndx-extract schema: - namespace: core neurodata_types: - ImageSegmentation - source: ndx-extract.extensions.yaml - version: 0.1.1 + version: 0.2.0 From a8fb1fffc416c795d79823a63cc45c03626fb27d Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 19 Oct 2022 15:14:07 +0200 Subject: [PATCH 17/18] fix missing required attribute warning for plane segmentation in the unittest --- src/pynwb/tests/test_extract.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/pynwb/tests/test_extract.py b/src/pynwb/tests/test_extract.py index a139594..85fa9c8 100644 --- a/src/pynwb/tests/test_extract.py +++ b/src/pynwb/tests/test_extract.py @@ -7,6 +7,7 @@ from hdmf.testing import TestCase from numpy.testing import assert_array_equal from pynwb import NWBFile, NWBHDF5IO +from pynwb.ophys import OpticalChannel from ndx_extract import EXTRACTSegmentation @@ -24,7 +25,28 @@ def setUp(self): identifier="file_id", session_start_time=self.session_start_time, ) - + # Create the device and optical channel for the imaging plane object. + device = self.nwbfile.create_device( + name="Microscope", + description="My two-photon microscope", + manufacturer="The best microscope manufacturer" + ) + optical_channel = OpticalChannel( + name="OpticalChannel", + description="My optical channel", + emission_lambda=500. + ) + # The imaging plane is required for creating a plane segmentation which + # in turn is a required attribute for an image segmentation. + self.imaging_plane = self.nwbfile.create_imaging_plane( + name="ImagingPlane", + description='A very interesting part of the brain.', + optical_channel=optical_channel, + device=device, + excitation_lambda=600., + indicator="GFP", + location="V1", + ) self.tmpdir = Path(mkdtemp()) self.nwbfile_path = self.tmpdir / "test_nwb.nwb" @@ -47,6 +69,12 @@ def test_extract_roundtrip(self): movie_mask=np.ones((15, 15)), ) + # Create a plane segmentation and add it to the image segmentation object. + image_segmentation.create_plane_segmentation( + imaging_plane=self.imaging_plane, + description="The description about this plane segmentation." + ) + ophys_module = self.nwbfile.create_processing_module( name="ophys", description="optical physiology processed data", From 78be9bcc954e7533c66e0fddeabe41796b1c936e Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 19 Oct 2022 20:51:54 +0200 Subject: [PATCH 18/18] update README.md --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/README.md b/README.md index 202584c..6dd35ed 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,57 @@ This extension allows for the storage of configuration options used by the [EXTR ## Usage + +### Python +Install the extension from [PyPI](https://pypi.org/project/ndx-extract/) +```shell +pip install ndx-extract +``` +Usage: +```python +from datetime import datetime +from ndx_extract import EXTRACTSegmentation +from pynwb import NWBFile, NWBHDF5IO + +# Create the NWBfile +nwbfile = NWBFile( + session_description="The mouse in open exploration.", + identifier="Mouse5_Day3", + session_start_time=datetime.now().astimezone(), +) +# Create the processing module +ophys_module = nwbfile.create_processing_module( + name="ophys", + description="optical physiology processed data", +) +# Create the segmentation object and define the configuration properties +# The properties that can be defined are listed at spec/ndx-EXTRACT.extensions.yaml +image_segmentation = EXTRACTSegmentation( + name="ImageSegmentation", + version="1.1.0", + preprocess=True, + trace_output_option="nonneg", +) +# Add this image segmentation to the processing module +ophys_module.add(image_segmentation) + +# Writing the NWB file +with NWBHDF5IO("image_segmentation.nwb", mode="w") as io: + io.write(nwbfile) + +# Reading the NWB file and accessing the segmentation parameters +with NWBHDF5IO("image_segmentation.nwb", mode="r") as io: + nwbfile_in = io.read() + nwbfile_in.processing["ophys"].data_interfaces["ImageSegmentation"].version + nwbfile_in.processing["ophys"].data_interfaces["ImageSegmentation"].preprocess + nwbfile_in.processing["ophys"].data_interfaces["ImageSegmentation"].trace_output_option +``` + +Running the tests: +```shell + python -m unittest src/pynwb/tests/test_extract.py +``` + ### MATLAB install: ```matlab