From 8e35dd35596dd4d779d78598ccd071ee916003d6 Mon Sep 17 00:00:00 2001 From: Sam Bray Date: Thu, 15 Jun 2023 14:30:26 -0700 Subject: [PATCH 1/9] save multiple LED position data as separate spatialSeries --- .../originators/position_originator.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index 4c283a3f..f4f28b25 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -53,14 +53,24 @@ def make(self, nwb_content: NWBFile): position_df = self.get_position_with_corrected_timestamps( position_tracking_path[0] ) - position.create_spatial_series( - name=f"series_{dataset_ind}", - description=", ".join(position_df.columns.tolist()), - data=np.asarray(position_df), - conversion=conversion, - reference_frame="Upper left corner of video frame", - timestamps=np.asarray(position_df.index), - ) + #Multi-position split. + #TODO: generalize key names? + key_lists = [['xloc','yloc','zloc'], + ['xloc2','yloc2','zloc2'], + ['xloc3','yloc3','zloc3']] + led_number = 0 + for valid_keys in key_lists: + key_set = [key for key in position_df.columns.tolist() if key in valid_keys] + if len(key_set)==0: continue + position.create_spatial_series( + name=f"led_{led_number}_series_{dataset_ind}", + description=", ".join(key_set), + data=np.asarray(position_df[key_set]), + conversion=conversion, + reference_frame="Upper left corner of video frame", + timestamps=np.asarray(position_df.index), + ) + led_number += 1 first_timestamps.append(position_df.index[0]) except IndexError: video_file_path = glob.glob( From 119fccac337756a62b146e3564febc76d80e87fb Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 16 Jun 2023 09:43:59 -0700 Subject: [PATCH 2/9] reduce potential position variable names Co-authored-by: Eric Denovellis --- .../processing/builder/originators/position_originator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index f4f28b25..ef782c68 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -59,9 +59,9 @@ def make(self, nwb_content: NWBFile): ['xloc2','yloc2','zloc2'], ['xloc3','yloc3','zloc3']] led_number = 0 - for valid_keys in key_lists: + for led_number, valid_keys in enumerate(key_lists): key_set = [key for key in position_df.columns.tolist() if key in valid_keys] - if len(key_set)==0: continue + if len(key_set) > 0: position.create_spatial_series( name=f"led_{led_number}_series_{dataset_ind}", description=", ".join(key_set), @@ -70,7 +70,6 @@ def make(self, nwb_content: NWBFile): reference_frame="Upper left corner of video frame", timestamps=np.asarray(position_df.index), ) - led_number += 1 first_timestamps.append(position_df.index[0]) except IndexError: video_file_path = glob.glob( From 61320164d7ccbd9100404a10f32fc38e09760d0d Mon Sep 17 00:00:00 2001 From: Sam Bray Date: Fri, 16 Jun 2023 09:47:37 -0700 Subject: [PATCH 3/9] correct contditional indentation --- .../builder/originators/position_originator.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index ef782c68..7a8d7ed0 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -62,15 +62,15 @@ def make(self, nwb_content: NWBFile): for led_number, valid_keys in enumerate(key_lists): key_set = [key for key in position_df.columns.tolist() if key in valid_keys] if len(key_set) > 0: - position.create_spatial_series( - name=f"led_{led_number}_series_{dataset_ind}", - description=", ".join(key_set), - data=np.asarray(position_df[key_set]), - conversion=conversion, - reference_frame="Upper left corner of video frame", - timestamps=np.asarray(position_df.index), - ) - first_timestamps.append(position_df.index[0]) + position.create_spatial_series( + name=f"led_{led_number}_series_{dataset_ind}", + description=", ".join(key_set), + data=np.asarray(position_df[key_set]), + conversion=conversion, + reference_frame="Upper left corner of video frame", + timestamps=np.asarray(position_df.index), + ) + first_timestamps.append(position_df.index[0]) except IndexError: video_file_path = glob.glob( os.path.join(pos_path, "*.pos_cameraHWFrameCount.dat") From 3dafd50bcdf725c9f738cc4718bc03a0e672bad1 Mon Sep 17 00:00:00 2001 From: Sam Bray Date: Fri, 16 Jun 2023 11:36:01 -0700 Subject: [PATCH 4/9] only single first timestamp per epoch --- .../processing/builder/originators/position_originator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index 7a8d7ed0..9581fea6 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -70,7 +70,7 @@ def make(self, nwb_content: NWBFile): reference_frame="Upper left corner of video frame", timestamps=np.asarray(position_df.index), ) - first_timestamps.append(position_df.index[0]) + first_timestamps.append(position_df.index[0]) except IndexError: video_file_path = glob.glob( os.path.join(pos_path, "*.pos_cameraHWFrameCount.dat") From 8603821ef3c8e9f16116503b8d4f444d112b5f95 Mon Sep 17 00:00:00 2001 From: Sam Bray Date: Fri, 16 Jun 2023 14:23:55 -0700 Subject: [PATCH 5/9] reduce valid position name options. Ensure both LED's us xloc, yloc as final column names --- .../processing/builder/originators/position_originator.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index 9581fea6..dff79014 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -55,16 +55,15 @@ def make(self, nwb_content: NWBFile): ) #Multi-position split. #TODO: generalize key names? - key_lists = [['xloc','yloc','zloc'], - ['xloc2','yloc2','zloc2'], - ['xloc3','yloc3','zloc3']] + key_lists = [['xloc','yloc',], #led 0 + ['xloc2','yloc2',],] #led 1 led_number = 0 for led_number, valid_keys in enumerate(key_lists): key_set = [key for key in position_df.columns.tolist() if key in valid_keys] if len(key_set) > 0: position.create_spatial_series( name=f"led_{led_number}_series_{dataset_ind}", - description=", ".join(key_set), + description=", ".join(['xloc','yloc']), data=np.asarray(position_df[key_set]), conversion=conversion, reference_frame="Upper left corner of video frame", From 4d4c25daba82e8dac3c8e086745260511a02e501 Mon Sep 17 00:00:00 2001 From: Samuel Bray Date: Fri, 16 Jun 2023 14:48:20 -0700 Subject: [PATCH 6/9] simple cleanup Co-authored-by: Eric Denovellis --- rec_to_nwb/processing/builder/originators/position_originator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index dff79014..187608e2 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -57,7 +57,6 @@ def make(self, nwb_content: NWBFile): #TODO: generalize key names? key_lists = [['xloc','yloc',], #led 0 ['xloc2','yloc2',],] #led 1 - led_number = 0 for led_number, valid_keys in enumerate(key_lists): key_set = [key for key in position_df.columns.tolist() if key in valid_keys] if len(key_set) > 0: From 91f1a52f128387c2f00055e00249286d3036a3a3 Mon Sep 17 00:00:00 2001 From: Sam Bray Date: Fri, 16 Jun 2023 15:05:58 -0700 Subject: [PATCH 7/9] resolve redundant first_timestamps addon --- .../processing/builder/originators/position_originator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index 187608e2..d7a5580d 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -68,7 +68,7 @@ def make(self, nwb_content: NWBFile): reference_frame="Upper left corner of video frame", timestamps=np.asarray(position_df.index), ) - first_timestamps.append(position_df.index[0]) + first_timestamps.append(position_df.index[0]) except IndexError: video_file_path = glob.glob( os.path.join(pos_path, "*.pos_cameraHWFrameCount.dat") From 0396a66b97eac1b68bed890e50b371190bd6ed62 Mon Sep 17 00:00:00 2001 From: Sam Bray Date: Thu, 6 Jul 2023 09:47:52 -0700 Subject: [PATCH 8/9] Add position frame indexes and non-repeating timestamps as new processing modules --- .../originators/position_originator.py | 57 ++++++++++++++++--- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index d7a5580d..fdb27abc 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd -from pynwb import NWBFile, ProcessingModule +from pynwb import NWBFile, ProcessingModule, TimeSeries from pynwb.behavior import Position from rec_to_binaries.read_binaries import readTrodesExtractedDataFile from rec_to_nwb.processing.exceptions.invalid_metadata_exception import ( @@ -53,22 +53,63 @@ def make(self, nwb_content: NWBFile): position_df = self.get_position_with_corrected_timestamps( position_tracking_path[0] ) - #Multi-position split. - #TODO: generalize key names? - key_lists = [['xloc','yloc',], #led 0 - ['xloc2','yloc2',],] #led 1 + # Multi-position split. + # TODO: generalize key names? + key_lists = [ + [ + "xloc", + "yloc", + ], # led 0 + [ + "xloc2", + "yloc2", + ], + ] # led 1 for led_number, valid_keys in enumerate(key_lists): - key_set = [key for key in position_df.columns.tolist() if key in valid_keys] + key_set = [ + key for key in position_df.columns.tolist() if key in valid_keys + ] if len(key_set) > 0: position.create_spatial_series( name=f"led_{led_number}_series_{dataset_ind}", - description=", ".join(['xloc','yloc']), + description=", ".join(["xloc", "yloc"]), data=np.asarray(position_df[key_set]), conversion=conversion, reference_frame="Upper left corner of video frame", timestamps=np.asarray(position_df.index), ) first_timestamps.append(position_df.index[0]) + # add the video frame index as a new processing module + if "position_frame_index" not in nwb_content.processing: + nwb_content.create_processing_module( + name="position_frame_index", + description="stores video frame index for each position timestep", + ) + # add timeseries for each frame index set (once per series because led's share timestamps) + nwb_content.processing["position_frame_index"].add( + TimeSeries( + name=f"series_{dataset_ind}", + data=np.asarray(position_df["video_frame_ind"]), + unit="N/A", + timestamps=np.asarray(position_df.index), + ) + ) + # add the video non-repeat timestamp labels as a new processing module + if "non_repeat_timestamp_labels" not in nwb_content.processing: + nwb_content.create_processing_module( + name="non_repeat_timestamp_labels", + description="stores non_repeat_labels for each position timestep", + ) + # add timeseries for each non-repeat timestamp labels set (once per series because led's share timestamps) + nwb_content.processing["non_repeat_timestamp_labels"].add( + TimeSeries( + name=f"series_{dataset_ind}", + data=np.asarray(position_df["non_repeat_timestamp_labels"]), + unit="N/A", + timestamps=np.asarray(position_df.index), + ) + ) + except IndexError: video_file_path = glob.glob( os.path.join(pos_path, "*.pos_cameraHWFrameCount.dat") @@ -233,7 +274,6 @@ def get_position_with_corrected_timestamps(position_tracking_path): @staticmethod def get_corrected_timestamps_without_position(hw_frame_count_path): - video_info = readTrodesExtractedDataFile(hw_frame_count_path) video_info = pd.DataFrame(video_info["data"]).set_index("PosTimestamp") # On AVT cameras, HWFrame counts wraps to 0 above this value. @@ -516,7 +556,6 @@ def remove_acquisition_timing_pause_non_ptp( def correct_timestamps_for_camera_to_mcu_lag( frame_count, camera_systime, camera_to_mcu_lag ): - regression_result = linregress(frame_count, camera_systime - camera_to_mcu_lag) corrected_camera_systime = ( regression_result.intercept + frame_count * regression_result.slope From 73c83f7d11d0b7225ef28811a975e52fd261d224 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Tue, 11 Jul 2023 08:40:49 -0700 Subject: [PATCH 9/9] Update position_originator.py --- rec_to_nwb/processing/builder/originators/position_originator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rec_to_nwb/processing/builder/originators/position_originator.py b/rec_to_nwb/processing/builder/originators/position_originator.py index cd94a252..bed43ca6 100644 --- a/rec_to_nwb/processing/builder/originators/position_originator.py +++ b/rec_to_nwb/processing/builder/originators/position_originator.py @@ -273,7 +273,6 @@ def get_position_with_corrected_timestamps(position_tracking_path, ptp_enabled): ) @staticmethod - def get_corrected_timestamps_without_position(hw_frame_count_path, ptp_enabled): video_info = readTrodesExtractedDataFile(hw_frame_count_path) video_info = pd.DataFrame(video_info["data"]).set_index("PosTimestamp")