From 85956d586b7d28c462f14d210aa7cc596938bd71 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 30 May 2023 15:11:17 +0200 Subject: [PATCH 01/18] modify add_two_photon_series to add_photon_series to be able to add OnePhotonSeries to nwbfile --- .../tools/roiextractors/roiextractors.py | 77 +++++++++++++------ 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index a98d9e8bd6..366e0ed4e3 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -1,6 +1,6 @@ from collections import defaultdict from copy import deepcopy -from typing import Optional +from typing import Literal, Optional from warnings import warn import numpy as np @@ -18,6 +18,7 @@ Fluorescence, ImageSegmentation, ImagingPlane, + OnePhotonSeries, OpticalChannel, PlaneSegmentation, RoiResponseSeries, @@ -83,6 +84,12 @@ def get_default_ophys_metadata() -> dict: unit="n.a.", ) + default_one_photon_series = dict( + name="OnePhotonSeries", + description="Imaging data from one-photon excitation microscopy.", + unit="n.a.", + ) + default_image_segmentation = dict( name="ImageSegmentation", plane_segmentations=[ @@ -101,13 +108,17 @@ def get_default_ophys_metadata() -> dict: ImageSegmentation=default_image_segmentation, ImagingPlane=[default_imaging_plane], TwoPhotonSeries=[default_two_photon_series], + OnePhotonSeries=[default_one_photon_series], ), ) return metadata -def get_nwb_imaging_metadata(imgextractor: ImagingExtractor) -> dict: +def get_nwb_imaging_metadata( + imgextractor: ImagingExtractor, + photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", +) -> dict: """ Convert metadata from the ImagingExtractor into nwb specific metadata. @@ -135,10 +146,16 @@ def get_nwb_imaging_metadata(imgextractor: ImagingExtractor) -> dict: ) # TwoPhotonSeries update: - metadata["Ophys"]["TwoPhotonSeries"][0].update(dimension=list(imgextractor.get_image_size())) + metadata["Ophys"][photon_series_type][0].update(dimension=list(imgextractor.get_image_size())) plane_name = metadata["Ophys"]["ImagingPlane"][0]["name"] - metadata["Ophys"]["TwoPhotonSeries"][0]["imaging_plane"] = plane_name + metadata["Ophys"][photon_series_type][0]["imaging_plane"] = plane_name + + if photon_series_type == "TwoPhotonSeries": + photon_series_to_pop = "OnePhotonSeries" + else: + photon_series_to_pop = "TwoPhotonSeries" + _ = metadata["Ophys"].pop(photon_series_to_pop) # remove what Segmentation extractor will input: _ = metadata["Ophys"].pop("ImageSegmentation") @@ -269,11 +286,12 @@ def add_image_segmentation(nwbfile: NWBFile, metadata: dict) -> NWBFile: return nwbfile -def add_two_photon_series( +def add_photon_series( imaging: ImagingExtractor, nwbfile: NWBFile, metadata: dict, - two_photon_series_index: int = 0, + photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", + photon_series_index: int = 0, iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, use_times=False, # TODO: to be removed @@ -295,50 +313,61 @@ def add_two_photon_series( iterator_options = iterator_options or dict() metadata_copy = deepcopy(metadata) - metadata_copy = dict_deep_update(get_nwb_imaging_metadata(imaging), metadata_copy, append_list=False) + assert photon_series_type in [ + "OnePhotonSeries", + "TwoPhotonSeries", + ], "'photon_series_type' must be either 'OnePhotonSeries' or 'TwoPhotonSeries'." + metadata_copy = dict_deep_update( + get_nwb_imaging_metadata(imaging, photon_series_type=photon_series_type), metadata_copy, append_list=False + ) - # Tests if TwoPhotonSeries already exists in acquisition - two_photon_series_metadata = metadata_copy["Ophys"]["TwoPhotonSeries"][two_photon_series_index] - two_photon_series_name = two_photon_series_metadata["name"] + # Tests if TwoPhotonSeries//OnePhotonSeries already exists in acquisition + photon_series_metadata = metadata_copy["Ophys"][photon_series_type][photon_series_index] + photon_series_name = photon_series_metadata["name"] - if two_photon_series_name in nwbfile.acquisition: - warn(f"{two_photon_series_name} already on nwbfile") + if photon_series_name in nwbfile.acquisition: + warn(f"{photon_series_name} already on nwbfile") return nwbfile # Add the image plane to nwb nwbfile = add_imaging_plane(nwbfile=nwbfile, metadata=metadata_copy) - imaging_plane_name = two_photon_series_metadata["imaging_plane"] + imaging_plane_name = photon_series_metadata["imaging_plane"] imaging_plane = nwbfile.get_imaging_plane(name=imaging_plane_name) - two_photon_series_metadata.update(imaging_plane=imaging_plane) + photon_series_metadata.update(imaging_plane=imaging_plane) # Add the data - two_p_series_kwargs = two_photon_series_metadata + photon_series_kwargs = photon_series_metadata frames_to_iterator = _imaging_frames_to_hdmf_iterator( imaging=imaging, iterator_type=iterator_type, iterator_options=iterator_options, ) data = H5DataIO(data=frames_to_iterator, compression=True) - two_p_series_kwargs.update(data=data) + photon_series_kwargs.update(data=data) # Add dimension - two_p_series_kwargs.update(dimension=imaging.get_image_size()) + photon_series_kwargs.update(dimension=imaging.get_image_size()) # Add timestamps or rate if imaging.has_time_vector(): timestamps = imaging.frame_to_time(np.arange(imaging.get_num_frames())) estimated_rate = calculate_regular_series_rate(series=timestamps) if estimated_rate: - two_p_series_kwargs.update(starting_time=timestamps[0], rate=estimated_rate) + photon_series_kwargs.update(starting_time=timestamps[0], rate=estimated_rate) else: - two_p_series_kwargs.update(timestamps=H5DataIO(data=timestamps, compression="gzip"), rate=None) + photon_series_kwargs.update(timestamps=H5DataIO(data=timestamps, compression="gzip"), rate=None) else: rate = float(imaging.get_sampling_frequency()) - two_p_series_kwargs.update(starting_time=0.0, rate=rate) + photon_series_kwargs.update(starting_time=0.0, rate=rate) # Add the TwoPhotonSeries to the nwbfile - two_photon_series = TwoPhotonSeries(**two_p_series_kwargs) - nwbfile.add_acquisition(two_photon_series) + photon_series = dict( + OnePhotonSeries=OnePhotonSeries, + TwoPhotonSeries=TwoPhotonSeries, + )[ + photon_series_type + ](**photon_series_kwargs) + nwbfile.add_acquisition(photon_series) return nwbfile @@ -429,6 +458,7 @@ def write_imaging( verbose: bool = True, iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, + photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", buffer_size: Optional[int] = None, # TODO: to be removed ): """ @@ -492,10 +522,11 @@ def write_imaging( nwbfile_path=nwbfile_path, nwbfile=nwbfile, metadata=metadata, overwrite=overwrite, verbose=verbose ) as nwbfile_out: add_devices(nwbfile=nwbfile_out, metadata=metadata) - add_two_photon_series( + add_photon_series( imaging=imaging, nwbfile=nwbfile_out, metadata=metadata, + photon_series_type=photon_series_type, iterator_type=iterator_type, iterator_options=iterator_options, ) From 573e27173c156df084defd587a7b352b7345b76b Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 30 May 2023 15:11:40 +0200 Subject: [PATCH 02/18] add tests --- tests/test_ophys/test_tools_roiextractors.py | 60 +++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index d575a89f57..ffc6725d14 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -16,6 +16,7 @@ from parameterized import param, parameterized from pynwb import NWBHDF5IO, H5DataIO, NWBFile from pynwb.device import Device +from pynwb.ophys import OnePhotonSeries from roiextractors.testing import ( generate_dummy_imaging_extractor, generate_dummy_segmentation_extractor, @@ -27,9 +28,9 @@ add_fluorescence_traces, add_image_segmentation, add_imaging_plane, + add_photon_series, add_plane_segmentation, add_summary_images, - add_two_photon_series, check_if_imaging_fits_into_memory, ) from neuroconv.tools.roiextractors.imagingextractordatachunkiterator import ( @@ -1121,7 +1122,7 @@ def test_add_fluorescence_traces_irregular_timestamps_with_metadata(self): assert_array_equal(roi_response_series[series_name].timestamps.data, times) -class TestAddTwoPhotonSeries(TestCase): +class TestAddPhotonSeries(TestCase): @classmethod def setUpClass(cls): cls.session_start_time = datetime.now().astimezone() @@ -1172,7 +1173,7 @@ def setUp(self): def test_default_values(self): """Test adding two photon series with default values.""" - add_two_photon_series(imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata) + add_photon_series(imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata) # Check data acquisition_modules = self.nwbfile.acquisition @@ -1204,7 +1205,7 @@ def test_invalid_iterator_type_raises_error(self): AssertionError, "'iterator_type' must be either 'v1', 'v2' (recommended), or None.", ): - add_two_photon_series( + add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1237,7 +1238,7 @@ def test_non_iterative_write_assertion(self): def test_non_iterative_two_photon(self): """Test adding two photon series with using DataChunkIterator as iterator type.""" - add_two_photon_series( + add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1257,7 +1258,7 @@ def test_non_iterative_two_photon(self): def test_v1_iterator(self): """Test adding two photon series with using DataChunkIterator as iterator type.""" - add_two_photon_series( + add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1283,7 +1284,7 @@ def test_iterator_options_propagation(self): """Test that iterator options are propagated to the data chunk iterator.""" buffer_shape = (20, 5, 5) chunk_shape = (10, 5, 5) - add_two_photon_series( + add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata, @@ -1301,7 +1302,7 @@ def test_iterator_options_propagation(self): def test_add_two_photon_series_roundtrip(self): metadata = self.metadata - add_two_photon_series(imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=metadata) + add_photon_series(imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=metadata) # Write the data to disk nwbfile_path = Path(mkdtemp()) / "two_photon_roundtrip.nwb" @@ -1330,6 +1331,49 @@ def test_add_two_photon_series_roundtrip(self): assert self.imaging_plane_name in imaging_planes_in_file assert len(imaging_planes_in_file) == 1 + def test_add_invalid_photon_series_type(self): + """Test error is raised when adding photon series with invalid 'photon_series_type'.""" + with self.assertRaisesWith( + AssertionError, + "'photon_series_type' must be either 'OnePhotonSeries' or 'TwoPhotonSeries'.", + ): + add_photon_series( + imaging=self.imaging_extractor, + nwbfile=self.nwbfile, + metadata=self.metadata, + photon_series_type="invalid", + ) + + def test_add_one_photon_series(self): + """Test adding one photon series with metadata.""" + + one_photon_series_name = "one_photon_series_name" + one_photon_series_metadata = dict( + name=one_photon_series_name, + pmt_gain=60.0, + binning=2, + power=500.0, + imaging_plane=self.imaging_plane_name, + unit="n.a.", + ) + metadata = deepcopy(self.metadata) + _ = metadata["Ophys"].pop("TwoPhotonSeries") + metadata["Ophys"].update(OnePhotonSeries=[one_photon_series_metadata]) + + add_photon_series( + imaging=self.imaging_extractor, + nwbfile=self.nwbfile, + metadata=metadata, + photon_series_type="OnePhotonSeries", + ) + self.assertIn(one_photon_series_name, self.nwbfile.acquisition) + one_photon_series = self.nwbfile.acquisition[one_photon_series_name] + self.assertIsInstance(one_photon_series, OnePhotonSeries) + self.assertEqual(one_photon_series.pmt_gain, 60.0) + self.assertEqual(one_photon_series.binning, 2) + self.assertEqual(one_photon_series.power, 500.0) + self.assertEqual(one_photon_series.unit, "n.a.") + class TestAddSummaryImages(unittest.TestCase): def setUp(self): From dbc62b199e72488c4c2aa7a3190469d391df5c7a Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 30 May 2023 15:28:24 +0200 Subject: [PATCH 03/18] add add_photon_series to init --- src/neuroconv/tools/roiextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuroconv/tools/roiextractors/__init__.py b/src/neuroconv/tools/roiextractors/__init__.py index d51930d310..a04b9442aa 100644 --- a/src/neuroconv/tools/roiextractors/__init__.py +++ b/src/neuroconv/tools/roiextractors/__init__.py @@ -3,9 +3,9 @@ add_fluorescence_traces, add_image_segmentation, add_imaging_plane, + add_photon_series, add_plane_segmentation, add_summary_images, - add_two_photon_series, check_if_imaging_fits_into_memory, get_nwb_imaging_metadata, get_nwb_segmentation_metadata, From ccfc814861a7ccdfca18c385ce8cf091b70298ff Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Wed, 31 May 2023 15:42:29 +0200 Subject: [PATCH 04/18] add OnePhotonSeries to metadata_schema.json --- src/neuroconv/schemas/metadata_schema.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/neuroconv/schemas/metadata_schema.json b/src/neuroconv/schemas/metadata_schema.json index f9e1932437..eee17c38ef 100644 --- a/src/neuroconv/schemas/metadata_schema.json +++ b/src/neuroconv/schemas/metadata_schema.json @@ -248,6 +248,19 @@ "imaging_plane": {"type": "string"} } } + }, + "OnePhotonSeries": { + "type": "array", + "items": { + "title": "OnePhotonSeries", + "type": "object", + "required": ["name", "description"], + "properties": { + "name": {"type": "string", "default": "OnePhotonSeries"}, + "description": {"type": "string"}, + "imaging_plane": {"type": "string"} + } + } } } } From b516756d435a0b310185e90d9399c634cd73a4fd Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 1 Jun 2023 12:15:22 +0200 Subject: [PATCH 05/18] refactor default ophys metadata functions --- .../tools/roiextractors/roiextractors.py | 100 ++++++++++-------- 1 file changed, 57 insertions(+), 43 deletions(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 366e0ed4e3..4d6d015956 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -34,14 +34,15 @@ from ..hdmf import SliceableDataChunkIterator from ..nwb_helpers import get_default_nwbfile_metadata, get_module, make_or_load_nwbfile from ...utils import ( + DeepDict, OptionalFilePathType, calculate_regular_series_rate, dict_deep_update, ) -def get_default_ophys_metadata() -> dict: - """Fill default metadata for optical physiology.""" +def get_default_ophys_metadata() -> DeepDict: + """Fill default metadata for Device and ImagingPlane.""" metadata = get_default_nwbfile_metadata() default_device = dict(name="Microscope") @@ -62,6 +63,45 @@ def get_default_ophys_metadata() -> dict: optical_channel=[default_optical_channel], ) + metadata.update( + Ophys=dict( + Device=[default_device], + ImagingPlane=[default_imaging_plane], + ), + ) + + return metadata + + +def get_default_imaging_metadata( + photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", +) -> DeepDict: + """Fill default metadata for imaging. + + Parameters + photon_series_type : {'OnePhotonSeries', 'TwoPhotonSeries'}, optional + The type of photon series metadata to add. The default is to fill metadata for TwoPhotonSeries. + """ + metadata = get_default_ophys_metadata() + imaging_plane_name = metadata["Ophys"]["ImagingPlane"][0]["name"] + + one_photon_description = "Imaging data from one-photon excitation microscopy." + two_photon_description = "Imaging data from two-photon excitation microscopy." + default_photon_series = dict( + name=photon_series_type, + description=two_photon_description if photon_series_type == "TwoPhotonSeries" else one_photon_description, + unit="n.a.", + imaging_plane=imaging_plane_name, + ) + metadata["Ophys"].update({photon_series_type: [default_photon_series]}) + + return metadata + + +def get_default_segmentation_metadata() -> DeepDict: + """Fill default metadata for segmentation.""" + metadata = get_default_ophys_metadata() + default_fluorescence_roi_response_series = dict( name="RoiResponseSeries", description="Array of raw fluorescence traces.", unit="n.a." ) @@ -71,25 +111,17 @@ def get_default_ophys_metadata() -> dict: roi_response_series=[default_fluorescence_roi_response_series], ) - default_dff_roi_response_series = dict(name="RoiResponseSeries", description="Array of df/F traces.", unit="n.a.") + default_dff_roi_response_series = dict( + name="RoiResponseSeries", + description="Array of df/F traces.", + unit="n.a.", + ) default_df_over_f = dict( name="DfOverF", roi_response_series=[default_dff_roi_response_series], ) - default_two_photon_series = dict( - name="TwoPhotonSeries", - description="Imaging data from two-photon excitation microscopy.", - unit="n.a.", - ) - - default_one_photon_series = dict( - name="OnePhotonSeries", - description="Imaging data from one-photon excitation microscopy.", - unit="n.a.", - ) - default_image_segmentation = dict( name="ImageSegmentation", plane_segmentations=[ @@ -100,15 +132,11 @@ def get_default_ophys_metadata() -> dict: ], ) - metadata.update( - Ophys=dict( - Device=[default_device], + metadata["Ophys"].update( + dict( Fluorescence=default_fluorescence, DfOverF=default_df_over_f, ImageSegmentation=default_image_segmentation, - ImagingPlane=[default_imaging_plane], - TwoPhotonSeries=[default_two_photon_series], - OnePhotonSeries=[default_one_photon_series], ), ) @@ -124,9 +152,10 @@ def get_nwb_imaging_metadata( Parameters ---------- - imgextractor: ImagingExtractor + imgextractor : ImagingExtractor + photon_series_type : {'OnePhotonSeries', 'TwoPhotonSeries'}, optional """ - metadata = get_default_ophys_metadata() + metadata = get_default_imaging_metadata(photon_series_type=photon_series_type) channel_name_list = imgextractor.get_channel_names() or ( ["OpticalChannel"] @@ -145,22 +174,8 @@ def get_nwb_imaging_metadata( ) ) - # TwoPhotonSeries update: metadata["Ophys"][photon_series_type][0].update(dimension=list(imgextractor.get_image_size())) - plane_name = metadata["Ophys"]["ImagingPlane"][0]["name"] - metadata["Ophys"][photon_series_type][0]["imaging_plane"] = plane_name - - if photon_series_type == "TwoPhotonSeries": - photon_series_to_pop = "OnePhotonSeries" - else: - photon_series_to_pop = "TwoPhotonSeries" - _ = metadata["Ophys"].pop(photon_series_to_pop) - - # remove what Segmentation extractor will input: - _ = metadata["Ophys"].pop("ImageSegmentation") - _ = metadata["Ophys"].pop("Fluorescence") - _ = metadata["Ophys"].pop("DfOverF") return metadata @@ -271,7 +286,7 @@ def add_image_segmentation(nwbfile: NWBFile, metadata: dict) -> NWBFile: """ # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_ophys_metadata() + default_metadata = get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) image_segmentation_metadata = metadata_copy["Ophys"]["ImageSegmentation"] @@ -541,7 +556,7 @@ def get_nwb_segmentation_metadata(sgmextractor: SegmentationExtractor) -> dict: ---------- sgmextractor: SegmentationExtractor """ - metadata = get_default_ophys_metadata() + metadata = get_default_segmentation_metadata() # Optical Channel name: for i in range(sgmextractor.get_num_channels()): ch_name = sgmextractor.get_channel_names()[i] @@ -563,8 +578,7 @@ def get_nwb_segmentation_metadata(sgmextractor: SegmentationExtractor) -> dict: description=f"description of {trace_name} traces", ) ) - # remove what imaging extractor will input: - _ = metadata["Ophys"].pop("TwoPhotonSeries") + return metadata @@ -633,7 +647,7 @@ def add_plane_segmentation( # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_ophys_metadata() + default_metadata = get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) image_segmentation_metadata = metadata_copy["Ophys"]["ImageSegmentation"] @@ -758,7 +772,7 @@ def add_fluorescence_traces( # Set the defaults and required infrastructure metadata_copy = deepcopy(metadata) - default_metadata = get_default_ophys_metadata() + default_metadata = get_default_segmentation_metadata() metadata_copy = dict_deep_update(default_metadata, metadata_copy, append_list=False) # df/F metadata From edf47e9bf6c1566f6b9af2ad504e9bc831b8717d Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 1 Jun 2023 12:59:58 +0200 Subject: [PATCH 06/18] update docsstring --- .../tools/roiextractors/roiextractors.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 4d6d015956..bd43e4e970 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -315,7 +315,28 @@ def add_photon_series( """ Auxiliary static method for nwbextractor. - Adds two photon series from imaging object as TwoPhotonSeries to nwbfile object. + Adds photon series from ImagingExtractor to NWB file object. + The photon series can be added to the NWB file either as a TwoPhotonSeries + or OnePhotonSeries object. + + Parameters + ---------- + imaging : ImagingExtractor + The imaging extractor to get the data from. + nwbfile : NWBFile + The nwbfile to add the photon series to. + metadata: dict + The metadata for the photon series. + photon_series_type: {'OnePhotonSeries', 'TwoPhotonSeries'}, optional + The type of photon series to add, default is TwoPhotonSeries. + photon_series_index: int, default: 0 + The metadata for the photon series is a list of the different photon series to add. + Specify which element of the list with this parameter. + + Returns + ------- + NWBFile + The NWBFile passed as an input with the photon series added. """ if use_times: warn("Keyword argument 'use_times' is deprecated and will be removed on or after August 1st, 2022.") From b88b314d9053fdde73abc12727400c446e4b7c19 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 1 Jun 2023 14:17:14 +0200 Subject: [PATCH 07/18] refactor --- src/neuroconv/tools/roiextractors/roiextractors.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index bd43e4e970..178176f209 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -358,8 +358,8 @@ def add_photon_series( ) # Tests if TwoPhotonSeries//OnePhotonSeries already exists in acquisition - photon_series_metadata = metadata_copy["Ophys"][photon_series_type][photon_series_index] - photon_series_name = photon_series_metadata["name"] + photon_series_kwargs = metadata_copy["Ophys"][photon_series_type][photon_series_index] + photon_series_name = photon_series_kwargs["name"] if photon_series_name in nwbfile.acquisition: warn(f"{photon_series_name} already on nwbfile") @@ -367,12 +367,11 @@ def add_photon_series( # Add the image plane to nwb nwbfile = add_imaging_plane(nwbfile=nwbfile, metadata=metadata_copy) - imaging_plane_name = photon_series_metadata["imaging_plane"] + imaging_plane_name = photon_series_kwargs["imaging_plane"] imaging_plane = nwbfile.get_imaging_plane(name=imaging_plane_name) - photon_series_metadata.update(imaging_plane=imaging_plane) + photon_series_kwargs.update(imaging_plane=imaging_plane) # Add the data - photon_series_kwargs = photon_series_metadata frames_to_iterator = _imaging_frames_to_hdmf_iterator( imaging=imaging, iterator_type=iterator_type, @@ -396,7 +395,7 @@ def add_photon_series( rate = float(imaging.get_sampling_frequency()) photon_series_kwargs.update(starting_time=0.0, rate=rate) - # Add the TwoPhotonSeries to the nwbfile + # Add the photon series to the nwbfile (either as OnePhotonSeries or TwoPhotonSeries) photon_series = dict( OnePhotonSeries=OnePhotonSeries, TwoPhotonSeries=TwoPhotonSeries, From 3fb0fef08d589b5d62eb535020857dc80e6f736c Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 1 Jun 2023 15:17:52 +0200 Subject: [PATCH 08/18] conditionally require `photon_series_type` when metadata contains 'OnePhotonSeries' --- src/neuroconv/tools/roiextractors/roiextractors.py | 3 +++ tests/test_ophys/test_tools_roiextractors.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 178176f209..5aa78ec0a1 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -356,6 +356,9 @@ def add_photon_series( metadata_copy = dict_deep_update( get_nwb_imaging_metadata(imaging, photon_series_type=photon_series_type), metadata_copy, append_list=False ) + assert ( + "OnePhotonSeries" in metadata_copy["Ophys"] and photon_series_type == "OnePhotonSeries" + ), "Received metadata for 'OnePhotonSeries' but `photon_series_type` was not explicitly specified." # Tests if TwoPhotonSeries//OnePhotonSeries already exists in acquisition photon_series_kwargs = metadata_copy["Ophys"][photon_series_type][photon_series_index] diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index ffc6725d14..6cafbaff02 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -1344,6 +1344,18 @@ def test_add_invalid_photon_series_type(self): photon_series_type="invalid", ) + def test_add_photon_series_inconclusive_metadata(self): + """Test error is raised when `photon_series_type` specifies 'TwoPhotonSeries' but metadata contains 'OnePhotonSeries'.""" + with self.assertRaisesWith( + AssertionError, + "Received metadata for 'OnePhotonSeries' but `photon_series_type` was not explicitly specified.", + ): + add_photon_series( + imaging=self.imaging_extractor, + nwbfile=self.nwbfile, + metadata=self.one_photon_series_metadata, + ) + def test_add_one_photon_series(self): """Test adding one photon series with metadata.""" From e3b6169ee33d249a37c28176385f3c887a993950 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 1 Jun 2023 15:19:19 +0200 Subject: [PATCH 09/18] refactor metadata in tests --- tests/test_ophys/test_tools_roiextractors.py | 114 ++++++++++++------- 1 file changed, 73 insertions(+), 41 deletions(-) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index 6cafbaff02..f885fa0c49 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -1126,46 +1126,57 @@ class TestAddPhotonSeries(TestCase): @classmethod def setUpClass(cls): cls.session_start_time = datetime.now().astimezone() - cls.device_name = "optical_device" cls.num_frames = 30 cls.num_rows = 10 cls.num_columns = 15 - def setUp(self): - self.nwbfile = NWBFile( - session_description="session_description", - identifier="file_id", - session_start_time=self.session_start_time, - ) - self.metadata = dict(Ophys=dict()) + metadata = dict(Ophys=dict()) - self.device_metadata = dict(name=self.device_name) - self.metadata["Ophys"].update(Device=[self.device_metadata]) + cls.device_name = "optical_device" + device_metadata = dict(name=cls.device_name) - self.optical_channel_metadata = dict( + optical_channel_metadata = dict( name="optical_channel", emission_lambda=np.nan, description="description", ) - self.imaging_plane_name = "imaging_plane_name" - self.imaging_plane_metadata = dict( - name=self.imaging_plane_name, - optical_channel=[self.optical_channel_metadata], + cls.imaging_plane_name = "imaging_plane_name" + imaging_plane_metadata = dict( + name=cls.imaging_plane_name, + optical_channel=[optical_channel_metadata], description="image_plane_description", - device=self.device_name, + device=cls.device_name, excitation_lambda=np.nan, indicator="unknown", location="unknown", ) - self.metadata["Ophys"].update(ImagingPlane=[self.imaging_plane_metadata]) + metadata["Ophys"].update( + Device=[device_metadata], + ImagingPlane=[imaging_plane_metadata], + ) - self.two_photon_series_name = "two_photon_series_name" - self.two_photon_series_metadata = dict( - name=self.two_photon_series_name, imaging_plane=self.imaging_plane_name, unit="n.a." + photon_series_metadata = dict(imaging_plane=cls.imaging_plane_name, unit="n.a.") + + cls.two_photon_series_metadata = deepcopy(metadata) + cls.two_photon_series_name = "two_photon_series_name" + cls.two_photon_series_metadata["Ophys"].update( + dict(TwoPhotonSeries=[dict(name=cls.two_photon_series_name, **photon_series_metadata)]) + ) + + cls.one_photon_series_metadata = deepcopy(metadata) + cls.one_photon_series_name = "one_photon_series_name" + cls.one_photon_series_metadata["Ophys"].update( + dict(OnePhotonSeries=[dict(name=cls.one_photon_series_name, **photon_series_metadata)]) + ) + + def setUp(self): + self.nwbfile = NWBFile( + session_description="session_description", + identifier="file_id", + session_start_time=self.session_start_time, ) - self.metadata["Ophys"].update(TwoPhotonSeries=[self.two_photon_series_metadata]) self.imaging_extractor = generate_dummy_imaging_extractor( self.num_frames, num_rows=self.num_rows, num_columns=self.num_columns @@ -1173,7 +1184,9 @@ def setUp(self): def test_default_values(self): """Test adding two photon series with default values.""" - add_photon_series(imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.metadata) + add_photon_series( + imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata + ) # Check data acquisition_modules = self.nwbfile.acquisition @@ -1208,7 +1221,7 @@ def test_invalid_iterator_type_raises_error(self): add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, - metadata=self.metadata, + metadata=self.two_photon_series_metadata, iterator_type="invalid", ) @@ -1241,7 +1254,7 @@ def test_non_iterative_two_photon(self): add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, - metadata=self.metadata, + metadata=self.two_photon_series_metadata, iterator_type=None, ) @@ -1261,7 +1274,7 @@ def test_v1_iterator(self): add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, - metadata=self.metadata, + metadata=self.two_photon_series_metadata, iterator_type="v1", ) @@ -1287,7 +1300,7 @@ def test_iterator_options_propagation(self): add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, - metadata=self.metadata, + metadata=self.two_photon_series_metadata, iterator_type="v2", iterator_options=dict(buffer_shape=buffer_shape, chunk_shape=chunk_shape), ) @@ -1300,9 +1313,9 @@ def test_iterator_options_propagation(self): self.assertEqual(data_chunk_iterator.chunk_shape, chunk_shape) def test_add_two_photon_series_roundtrip(self): - metadata = self.metadata - - add_photon_series(imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=metadata) + add_photon_series( + imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=self.two_photon_series_metadata + ) # Write the data to disk nwbfile_path = Path(mkdtemp()) / "two_photon_roundtrip.nwb" @@ -1340,7 +1353,7 @@ def test_add_invalid_photon_series_type(self): add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, - metadata=self.metadata, + metadata=self.two_photon_series_metadata, photon_series_type="invalid", ) @@ -1359,33 +1372,52 @@ def test_add_photon_series_inconclusive_metadata(self): def test_add_one_photon_series(self): """Test adding one photon series with metadata.""" - one_photon_series_name = "one_photon_series_name" - one_photon_series_metadata = dict( - name=one_photon_series_name, + metadata = deepcopy(self.one_photon_series_metadata) + one_photon_series_metadata = metadata["Ophys"]["OnePhotonSeries"][0] + one_photon_series_metadata.update( pmt_gain=60.0, binning=2, power=500.0, - imaging_plane=self.imaging_plane_name, - unit="n.a.", ) - metadata = deepcopy(self.metadata) - _ = metadata["Ophys"].pop("TwoPhotonSeries") - metadata["Ophys"].update(OnePhotonSeries=[one_photon_series_metadata]) - add_photon_series( imaging=self.imaging_extractor, nwbfile=self.nwbfile, metadata=metadata, photon_series_type="OnePhotonSeries", ) - self.assertIn(one_photon_series_name, self.nwbfile.acquisition) - one_photon_series = self.nwbfile.acquisition[one_photon_series_name] + self.assertIn(self.one_photon_series_name, self.nwbfile.acquisition) + one_photon_series = self.nwbfile.acquisition[self.one_photon_series_name] self.assertIsInstance(one_photon_series, OnePhotonSeries) self.assertEqual(one_photon_series.pmt_gain, 60.0) self.assertEqual(one_photon_series.binning, 2) self.assertEqual(one_photon_series.power, 500.0) self.assertEqual(one_photon_series.unit, "n.a.") + def test_add_one_photon_series_roundtrip(self): + add_photon_series( + imaging=self.imaging_extractor, + nwbfile=self.nwbfile, + metadata=self.one_photon_series_metadata, + photon_series_type="OnePhotonSeries", + ) + + # Write the data to disk + nwbfile_path = Path(mkdtemp()) / "one_photon_roundtrip.nwb" + with NWBHDF5IO(nwbfile_path, "w") as io: + io.write(self.nwbfile) + + with NWBHDF5IO(nwbfile_path, "r") as io: + read_nwbfile = io.read() + + # Check data + acquisition_modules = read_nwbfile.acquisition + assert self.one_photon_series_name in acquisition_modules + one_photon_series = acquisition_modules[self.one_photon_series_name].data + + # NWB stores images as num_columns x num_rows + expected_one_photon_series_shape = (self.num_frames, self.num_columns, self.num_rows) + assert one_photon_series.shape == expected_one_photon_series_shape + class TestAddSummaryImages(unittest.TestCase): def setUp(self): From 6071c08815c2ea2e505740559aa0c27770272d0a Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 1 Jun 2023 15:28:49 +0200 Subject: [PATCH 10/18] fix assert --- src/neuroconv/tools/roiextractors/roiextractors.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 5aa78ec0a1..3c21daeb1d 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -356,9 +356,10 @@ def add_photon_series( metadata_copy = dict_deep_update( get_nwb_imaging_metadata(imaging, photon_series_type=photon_series_type), metadata_copy, append_list=False ) - assert ( - "OnePhotonSeries" in metadata_copy["Ophys"] and photon_series_type == "OnePhotonSeries" - ), "Received metadata for 'OnePhotonSeries' but `photon_series_type` was not explicitly specified." + if photon_series_type == "TwoPhotonSeries": + assert ( + "OnePhotonSeries" not in metadata_copy["Ophys"] + ), "Received metadata for 'OnePhotonSeries' but `photon_series_type` was not explicitly specified." # Tests if TwoPhotonSeries//OnePhotonSeries already exists in acquisition photon_series_kwargs = metadata_copy["Ophys"][photon_series_type][photon_series_index] From c8199001034721c0c1c866497385cd37c37fc8b7 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Mon, 5 Jun 2023 14:37:20 +0200 Subject: [PATCH 11/18] merge get_default_imaging_metadata into get_nwb_imaging_metadata --- .../tools/roiextractors/roiextractors.py | 44 +++++++------------ 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 3c21daeb1d..25a185a87d 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -73,31 +73,6 @@ def get_default_ophys_metadata() -> DeepDict: return metadata -def get_default_imaging_metadata( - photon_series_type: Literal["OnePhotonSeries", "TwoPhotonSeries"] = "TwoPhotonSeries", -) -> DeepDict: - """Fill default metadata for imaging. - - Parameters - photon_series_type : {'OnePhotonSeries', 'TwoPhotonSeries'}, optional - The type of photon series metadata to add. The default is to fill metadata for TwoPhotonSeries. - """ - metadata = get_default_ophys_metadata() - imaging_plane_name = metadata["Ophys"]["ImagingPlane"][0]["name"] - - one_photon_description = "Imaging data from one-photon excitation microscopy." - two_photon_description = "Imaging data from two-photon excitation microscopy." - default_photon_series = dict( - name=photon_series_type, - description=two_photon_description if photon_series_type == "TwoPhotonSeries" else one_photon_description, - unit="n.a.", - imaging_plane=imaging_plane_name, - ) - metadata["Ophys"].update({photon_series_type: [default_photon_series]}) - - return metadata - - def get_default_segmentation_metadata() -> DeepDict: """Fill default metadata for segmentation.""" metadata = get_default_ophys_metadata() @@ -155,18 +130,20 @@ def get_nwb_imaging_metadata( imgextractor : ImagingExtractor photon_series_type : {'OnePhotonSeries', 'TwoPhotonSeries'}, optional """ - metadata = get_default_imaging_metadata(photon_series_type=photon_series_type) + metadata = get_default_ophys_metadata() channel_name_list = imgextractor.get_channel_names() or ( ["OpticalChannel"] if imgextractor.get_num_channels() == 1 else [f"OpticalChannel{idx}" for idx in range(imgextractor.get_num_channels())] ) + + imaging_plane = metadata["Ophys"]["ImagingPlane"][0] for index, channel_name in enumerate(channel_name_list): if index == 0: - metadata["Ophys"]["ImagingPlane"][0]["optical_channel"][index]["name"] = channel_name + imaging_plane["optical_channel"][index]["name"] = channel_name else: - metadata["Ophys"]["ImagingPlane"][0]["optical_channel"].append( + imaging_plane["optical_channel"].append( dict( name=channel_name, emission_lambda=np.nan, @@ -174,7 +151,16 @@ def get_nwb_imaging_metadata( ) ) - metadata["Ophys"][photon_series_type][0].update(dimension=list(imgextractor.get_image_size())) + one_photon_description = "Imaging data from one-photon excitation microscopy." + two_photon_description = "Imaging data from two-photon excitation microscopy." + photon_series_metadata = dict( + name=photon_series_type, + description=two_photon_description if photon_series_type == "TwoPhotonSeries" else one_photon_description, + unit="n.a.", + imaging_plane=imaging_plane["name"], + dimension=list(imgextractor.get_image_size()), + ) + metadata["Ophys"].update({photon_series_type: [photon_series_metadata]}) return metadata From ff7488baadcc5c8cfe150a09ef5953a833a448c4 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 11:55:26 +0200 Subject: [PATCH 12/18] remove pynwb constraint for python 3.11 --- requirements-minimal.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 7f060dea38..b1cc49b805 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -6,7 +6,6 @@ scipy>=1.4.1 h5py>=2.10.0 hdmf>=3.4.7 pynwb>=1.4.0;python_version>='3.8' -pynwb>=1.4.0,<2.3.2;python_version>='3.11' psutil>=5.8.0 tqdm>=4.60.0 dandi>=0.46.2 From 9d5a758ef8296825f7a7c43bed89feb89e7ab15c Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 13:45:11 +0200 Subject: [PATCH 13/18] fix mask tests for numpy 1.24.2 --- tests/test_ophys/test_tools_roiextractors.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_ophys/test_tools_roiextractors.py b/tests/test_ophys/test_tools_roiextractors.py index f885fa0c49..d5ca5eb126 100644 --- a/tests/test_ophys/test_tools_roiextractors.py +++ b/tests/test_ophys/test_tools_roiextractors.py @@ -4,7 +4,7 @@ from pathlib import Path from tempfile import mkdtemp from types import MethodType -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Tuple from unittest.mock import Mock import numpy as np @@ -293,6 +293,15 @@ def _generate_casted_test_masks(num_rois: int, mask_type: Literal["pixel", "voxe return casted_masks +def assert_masks_equal(mask: List[List[Tuple[int, int, int]]], expected_mask: List[List[Tuple[int, int, int]]]): + """ + Asserts that two lists of pixel masks of inhomogeneous shape are equal. + """ + assert len(mask) == len(expected_mask) + for mask_ind in range(len(mask)): + assert_array_equal(mask[mask_ind], expected_mask[mask_ind]) + + class TestAddPlaneSegmentation(unittest.TestCase): @classmethod def setUpClass(cls): @@ -488,7 +497,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd plane_segmentation = plane_segmentations[self.plane_segmentation_name] true_pixel_masks = _generate_casted_test_masks(num_rois=self.num_rois, mask_type="pixel") - assert_array_equal(plane_segmentation["pixel_mask"], true_pixel_masks) + assert_masks_equal(plane_segmentation["pixel_mask"][:], true_pixel_masks) def test_voxel_masks(self): """Test the voxel mask option for writing a plane segmentation table.""" @@ -519,7 +528,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd plane_segmentation = plane_segmentations[self.plane_segmentation_name] true_voxel_masks = _generate_casted_test_masks(num_rois=self.num_rois, mask_type="voxel") - assert_array_equal(plane_segmentation["voxel_mask"], true_voxel_masks) + assert_masks_equal(plane_segmentation["voxel_mask"][:], true_voxel_masks) def test_none_masks(self): """Test the None mask_type option for writing a plane segmentation table.""" @@ -577,7 +586,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd plane_segmentation = plane_segmentations[self.plane_segmentation_name] true_voxel_masks = _generate_casted_test_masks(num_rois=self.num_rois, mask_type="pixel") - assert_array_equal(plane_segmentation["pixel_mask"], true_voxel_masks) + assert_masks_equal(plane_segmentation["pixel_mask"][:], true_voxel_masks) def test_voxel_masks_auto_switch(self): segmentation_extractor = generate_dummy_segmentation_extractor( @@ -614,7 +623,7 @@ def get_roi_pixel_masks(self, roi_ids: Optional[ArrayLike] = None) -> List[np.nd plane_segmentation = plane_segmentations[self.plane_segmentation_name] true_voxel_masks = _generate_casted_test_masks(num_rois=self.num_rois, mask_type="voxel") - assert_array_equal(plane_segmentation["voxel_mask"], true_voxel_masks) + assert_masks_equal(plane_segmentation["voxel_mask"][:], true_voxel_masks) def test_not_overwriting_plane_segmentation_if_same_name(self): """Test that adding a plane segmentation with the same name will not overwrite From bcdbcdd6cecf97226edddbfad515545a54ab2988 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 14:52:10 +0200 Subject: [PATCH 14/18] update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aece9bffaf..11618876c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,9 @@ * Added basic temporal alignment methods to the AudioInterface. `align_starting_time` is split into `align_starting_times` (list of times, one per audio file) and `align_global_starting_time` (shift all by a scalar amount). `align_by_interpolation` and other timestamp-based approaches is not yet implemented for this interface. [PR #402](https://github.com/catalystneuro/neuroconv/pull/402) * Changed the order of recording properties extraction in `NeuroscopeRecordingInterface` and `NeuroScopeLFPInterface` to make them consistent with each other [PR #466](https://github.com/catalystneuro/neuroconv/pull/466) * The `ScanImageImagingInterface` has been updated to read metadata from more recent versions of ScanImage [PR #457](https://github.com/catalystneuro/neuroconv/pull/457) +* Refactored `add_two_photon_series()` to `add_photon_series()` and added `photon_series_type` optional argument which can be either `"OnePhotonSeries"` or `"TwoPhotonSeries"`. + Changed `get_default_ophys_metadata()` to add `Device` and `ImagingPlane` metadata which are both used by imaging and segmentation. + Added `photon_series_type` to `get_nwb_imaging_metadata()` to fill metadata for `OnePhotonSeries` or `TwoPhotonSeries`. [PR #462](https://github.com/catalystneuro/neuroconv/pull/462) ### Testing * The tests for `automatic_dandi_upload` now follow up-to-date DANDI validation rules for file name conventions. [PR #310](https://github.com/catalystneuro/neuroconv/pull/310) From 6817990c78c6b109ab49a113e3feac9e7db0f68c Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 15:20:40 +0200 Subject: [PATCH 15/18] deprecate two_photon_series_index --- src/neuroconv/tools/roiextractors/roiextractors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index 25a185a87d..a6df231749 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -293,6 +293,7 @@ def add_photon_series( metadata: dict, photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", photon_series_index: int = 0, + two_photon_series_index: int = 0, # TODO: to be removed iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, use_times=False, # TODO: to be removed @@ -331,6 +332,8 @@ def add_photon_series( "Keyword argument 'buffer_size' is deprecated and will be removed on or after September 1st, 2022." "Specify as a key in the new 'iterator_options' dictionary instead." ) + if two_photon_series_index: + warn("Keyword argument 'two_photon_series_index' is deprecated. Use 'photon_series_index' instead.") iterator_options = iterator_options or dict() From 011e6184a86cb1997f4a9a47556d106f3a323a38 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 15:24:34 +0200 Subject: [PATCH 16/18] pin pynwb to 2.3.2 --- requirements-minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-minimal.txt b/requirements-minimal.txt index b1cc49b805..7432435884 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -5,7 +5,7 @@ PyYAML>=5.4 scipy>=1.4.1 h5py>=2.10.0 hdmf>=3.4.7 -pynwb>=1.4.0;python_version>='3.8' +pynwb==2.3.2;python_version>='3.8' psutil>=5.8.0 tqdm>=4.60.0 dandi>=0.46.2 From d7ce2ff9e6ce65af1a4b578ee0501a0ae4577eee Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 15:35:12 +0200 Subject: [PATCH 17/18] change default two_photon_series_index to None --- src/neuroconv/tools/roiextractors/roiextractors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/neuroconv/tools/roiextractors/roiextractors.py b/src/neuroconv/tools/roiextractors/roiextractors.py index a6df231749..b0184d6655 100644 --- a/src/neuroconv/tools/roiextractors/roiextractors.py +++ b/src/neuroconv/tools/roiextractors/roiextractors.py @@ -293,7 +293,7 @@ def add_photon_series( metadata: dict, photon_series_type: Literal["TwoPhotonSeries", "OnePhotonSeries"] = "TwoPhotonSeries", photon_series_index: int = 0, - two_photon_series_index: int = 0, # TODO: to be removed + two_photon_series_index: Optional[int] = None, # TODO: to be removed iterator_type: Optional[str] = "v2", iterator_options: Optional[dict] = None, use_times=False, # TODO: to be removed @@ -334,6 +334,7 @@ def add_photon_series( ) if two_photon_series_index: warn("Keyword argument 'two_photon_series_index' is deprecated. Use 'photon_series_index' instead.") + photon_series_index = two_photon_series_index iterator_options = iterator_options or dict() From 18e66ee1cc3f0f4dd61a72da2f3a7c8615cc97ec Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Tue, 6 Jun 2023 15:37:27 +0200 Subject: [PATCH 18/18] remove constraint for numpy --- requirements-minimal.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 7432435884..ce7b31675e 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -1,5 +1,4 @@ -numpy>=1.22.0,<1.24;python_version<'3.11' -numpy>=1.22.0;python_version>='3.11' +numpy>=1.22.0 jsonschema>=3.2.0 PyYAML>=5.4 scipy>=1.4.1