From 57f7ffb6bb3f93e58bfade6bee3beba13b1723e5 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 28 Mar 2024 10:43:22 +0100 Subject: [PATCH 1/7] keep properties order in add_electrode --- src/neuroconv/tools/spikeinterface/spikeinterface.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 579b902e1..5413bcb73 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -319,6 +319,7 @@ def add_electrodes(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: d schema_properties = required_schema_properties | optional_schema_properties electrode_table_previous_properties = set(nwbfile.electrodes.colnames) if nwbfile.electrodes else set() + order_of_properties = list(data_to_add.keys()) extracted_properties = set(data_to_add) properties_to_add_by_rows = required_schema_properties | electrode_table_previous_properties properties_to_add_by_columns = extracted_properties - properties_to_add_by_rows @@ -387,7 +388,9 @@ def add_electrodes(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: d indexes_for_default_values = electrodes_df.index.difference(indexes_for_new_data).values # Add properties as columns - for property in properties_to_add_by_columns - {"channel_name"}: + unordered_properties_to_add_by_columns = properties_to_add_by_columns - {"channel_name"} + ordered_properties_to_add_by_columns = sorted(unordered_properties_to_add_by_columns, key=order_of_properties.index) + for property in ordered_properties_to_add_by_columns: cols_args = data_to_add[property] data = cols_args["data"] if np.issubdtype(data.dtype, np.integer): From f810a71a20229ffc2960abe1617de029d89aa4fc Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 28 Mar 2024 13:30:07 +0100 Subject: [PATCH 2/7] add test --- .../test_ecephys/test_tools_spikeinterface.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_ecephys/test_tools_spikeinterface.py b/tests/test_ecephys/test_tools_spikeinterface.py index 0540db9d6..9da5cc70e 100644 --- a/tests/test_ecephys/test_tools_spikeinterface.py +++ b/tests/test_ecephys/test_tools_spikeinterface.py @@ -881,6 +881,32 @@ def test_property_metadata_mismatch(self): expected_property_2_values = ["", "", "value_1", "value_2", "value_3", "value_4"] self.assertListEqual(actual_property_2_values, expected_property_2_values) + def test_custom_property_order(self): + """ + Test that custom properties are be added to the electrodes table in the order they are set. + """ + self.recording_1.set_property(key="custom_property_x", values=[0.1] * self.num_channels) + self.recording_1.set_property(key="custom_property_y", values=[0.2] * self.num_channels) + self.recording_1.set_property(key="custom_property_z", values=[0.3] * self.num_channels) + self.recording_1.set_property(key="custom_property_1", values=["value_1"] * self.num_channels) + + add_electrodes(recording=self.recording_1, nwbfile=self.nwbfile) + + expected_columns_in_order = [ + "location", + "group", + "group_name", + "channel_name", + "custom_property_x", + "custom_property_y", + "custom_property_z", + "custom_property_1", + "rel_x", + "rel_y", + ] + electrodes_columns = list(self.nwbfile.electrodes.colnames) + self.assertListEqual(expected_columns_in_order, electrodes_columns) + class TestAddUnitsTable(TestCase): @classmethod From c521d760a845651aa0a18f8452f0a30e2dba8804 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Thu, 28 Mar 2024 13:38:31 +0100 Subject: [PATCH 3/7] update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1720eb930..b53276b27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Upcoming +### Improvements + +* In `add_electrodes()` custom properties are added to the electrodes table in the order they are set in the recording extractor. [PR #793](https://github.com/catalystneuro/neuroconv/issues/793) + # v0.4.8 (March 20, 2024) ### Bug fixes From e1f2c6c8be81d5d24c857f8c621cac555de929d2 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Fri, 29 Mar 2024 13:12:15 +0100 Subject: [PATCH 4/7] add default electrodes metadata --- .../tools/spikeinterface/spikeinterface.py | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 5413bcb73..02b05e572 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -49,6 +49,42 @@ def get_nwb_metadata(recording: BaseRecording, metadata: dict = None): return metadata +def get_default_electrodes_metadata(recording: BaseRecording, exclude: tuple = ()): + """ + Return default metadata for all electrode fields. + + Parameters + ---------- + recording: spikeinterface.BaseRecording + exclude: tuple + An iterable containing the string names of channel properties in the RecordingExtractor + object to ignore. + """ + recording_properties_si_mapping = dict(brain_area="location", location=["rel_x", "rel_y", "rel_z"]) + metadata = get_nwb_metadata(recording=recording) + metadata["Ecephys"].update(Electrodes=list()) + electrodes_metadata = metadata["Ecephys"]["Electrodes"] + electrodes_metadata.append(dict(name="group_name", description="no description")) + electrodes_metadata.append(dict(name="channel_name", description="no description")) + + recorder_properties = recording.get_property_keys() + excluded_properties = list(exclude) + ["group", "contact_vector"] + properties_to_add_to_metadata = [ + property for property in recorder_properties if property not in excluded_properties + ] + + for property in properties_to_add_to_metadata: + if property == "brain_area": + electrodes_metadata.append(dict(name="location", description="location")) + elif property == "location": + for location_property in recording_properties_si_mapping[property]: + electrodes_metadata.append(dict(name=location_property, description="no description")) + else: + electrodes_metadata.append(dict(name=property, description="no description")) + + return metadata + + def add_devices(nwbfile: pynwb.NWBFile, metadata: Optional[DeepDict] = None): """ Add device information to nwbfile object. @@ -211,10 +247,11 @@ def add_electrodes(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: d """ assert isinstance(nwbfile, pynwb.NWBFile), "'nwbfile' should be of type pynwb.NWBFile" + default_metadata = get_default_electrodes_metadata(recording=recording, exclude=exclude) + metadata = dict_deep_update(default_metadata, metadata) + # Test that metadata has the expected structure - electrodes_metadata = list() - if metadata is not None: - electrodes_metadata = metadata.get("Ecephys", dict()).get("Electrodes", list()) + electrodes_metadata = metadata["Ecephys"]["Electrodes"] required_keys = {"name", "description"} assert all( @@ -319,7 +356,6 @@ def add_electrodes(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: d schema_properties = required_schema_properties | optional_schema_properties electrode_table_previous_properties = set(nwbfile.electrodes.colnames) if nwbfile.electrodes else set() - order_of_properties = list(data_to_add.keys()) extracted_properties = set(data_to_add) properties_to_add_by_rows = required_schema_properties | electrode_table_previous_properties properties_to_add_by_columns = extracted_properties - properties_to_add_by_rows From 6e9da7da5d6084824396a1688a851a2fb80c99db Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Fri, 29 Mar 2024 13:12:53 +0100 Subject: [PATCH 5/7] order properties by metadata --- src/neuroconv/tools/spikeinterface/spikeinterface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 02b05e572..116b9edd5 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -425,6 +425,7 @@ def add_electrodes(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: d # Add properties as columns unordered_properties_to_add_by_columns = properties_to_add_by_columns - {"channel_name"} + order_of_properties = [property["name"] for property in electrodes_metadata] ordered_properties_to_add_by_columns = sorted(unordered_properties_to_add_by_columns, key=order_of_properties.index) for property in ordered_properties_to_add_by_columns: cols_args = data_to_add[property] From 403b63af9b922105a12083b2de35d4b14e71be0e Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Fri, 29 Mar 2024 13:13:14 +0100 Subject: [PATCH 6/7] modify test --- .../test_ecephys/test_tools_spikeinterface.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/test_ecephys/test_tools_spikeinterface.py b/tests/test_ecephys/test_tools_spikeinterface.py index 9da5cc70e..04f9724a5 100644 --- a/tests/test_ecephys/test_tools_spikeinterface.py +++ b/tests/test_ecephys/test_tools_spikeinterface.py @@ -888,22 +888,44 @@ def test_custom_property_order(self): self.recording_1.set_property(key="custom_property_x", values=[0.1] * self.num_channels) self.recording_1.set_property(key="custom_property_y", values=[0.2] * self.num_channels) self.recording_1.set_property(key="custom_property_z", values=[0.3] * self.num_channels) - self.recording_1.set_property(key="custom_property_1", values=["value_1"] * self.num_channels) - add_electrodes(recording=self.recording_1, nwbfile=self.nwbfile) + self.recording_2.set_property(key="custom_property_2", values=["value_1"] * self.num_channels) + + metadata = dict( + Ecephys=dict( + Electrodes=[ + dict(name="custom_property_x", description="custom description."), + dict(name="custom_property_y", description="custom description."), + dict(name="custom_property_z", description="custom description."), + dict(name="custom_property_2", description="custom description."), + ] + ) + ) expected_columns_in_order = [ "location", "group", "group_name", "channel_name", + "rel_x", + "rel_y", "custom_property_x", "custom_property_y", "custom_property_z", - "custom_property_1", - "rel_x", - "rel_y", + "custom_property_2", ] + + add_electrodes( + recording=self.recording_1, + nwbfile=self.nwbfile, + metadata=metadata, + ) + add_electrodes( + recording=self.recording_2, + nwbfile=self.nwbfile, + metadata=metadata, + ) + electrodes_columns = list(self.nwbfile.electrodes.colnames) self.assertListEqual(expected_columns_in_order, electrodes_columns) From c01ef147f06ae9884702fd8f7e74f545d1acd846 Mon Sep 17 00:00:00 2001 From: weiglszonja Date: Fri, 29 Mar 2024 13:23:40 +0100 Subject: [PATCH 7/7] fix metadata default --- src/neuroconv/tools/spikeinterface/spikeinterface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/neuroconv/tools/spikeinterface/spikeinterface.py b/src/neuroconv/tools/spikeinterface/spikeinterface.py index 116b9edd5..4c185857c 100644 --- a/src/neuroconv/tools/spikeinterface/spikeinterface.py +++ b/src/neuroconv/tools/spikeinterface/spikeinterface.py @@ -248,6 +248,7 @@ def add_electrodes(recording: BaseRecording, nwbfile: pynwb.NWBFile, metadata: d assert isinstance(nwbfile, pynwb.NWBFile), "'nwbfile' should be of type pynwb.NWBFile" default_metadata = get_default_electrodes_metadata(recording=recording, exclude=exclude) + metadata = metadata or dict() metadata = dict_deep_update(default_metadata, metadata) # Test that metadata has the expected structure