From c3962f72abef8cc29007862c12ea556a50f66466 Mon Sep 17 00:00:00 2001 From: owinter Date: Sat, 21 Dec 2024 09:30:51 +0000 Subject: [PATCH 01/16] allow none labels in plot insertions --- brainbox/ephys_plots.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/brainbox/ephys_plots.py b/brainbox/ephys_plots.py index 66075cd26..9057cf7fd 100644 --- a/brainbox/ephys_plots.py +++ b/brainbox/ephys_plots.py @@ -439,20 +439,22 @@ def plot_brain_regions(channel_ids, channel_depths=None, brain_regions=None, dis bar_kwargs.update(**kwargs) color = col / 255 ax.bar(x=0.5, height=height, color=color, bottom=reg[0], **kwargs) - if label == 'right': - ax.yaxis.tick_right() - ax.set_yticks(region_labels[:, 0].astype(int)) - ax.yaxis.set_tick_params(labelsize=8) - ax.set_ylim(np.nanmin(channel_depths), np.nanmax(channel_depths)) - ax.get_xaxis().set_visible(False) - ax.set_yticklabels(region_labels[:, 1]) - if label == 'right': - ax.yaxis.tick_right() - ax.spines['left'].set_visible(False) - else: - ax.spines['right'].set_visible(False) ax.spines['top'].set_visible(False) ax.spines['bottom'].set_visible(False) + if not label is None: + if label == 'right': + ax.yaxis.tick_right() + ax.set_yticks(region_labels[:, 0].astype(int)) + ax.yaxis.set_tick_params(labelsize=8) + ax.set_ylim(np.nanmin(channel_depths), np.nanmax(channel_depths)) + ax.get_xaxis().set_visible(False) + ax.set_yticklabels(region_labels[:, 1]) + if label == 'right': + ax.yaxis.tick_right() + ax.spines['left'].set_visible(False) + else: + ax.spines['right'].set_visible(False) + if title: ax.set_title(title) From e81480b6af8faa2bb319302333e736213c6b8524 Mon Sep 17 00:00:00 2001 From: owinter Date: Sat, 21 Dec 2024 09:31:10 +0000 Subject: [PATCH 02/16] rename spike sorting log --- ibllib/pipes/ephys_tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index 8596a1619..e876a8b1d 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -602,14 +602,14 @@ def signature(self): ('*sync.npy', f'{self.device_collection}/{self.pname}', True) ], 'output_files': [ - # ./raw_ephys_data/probe00/ + # ./raw_ephys_data/{self.pname}/ ('_iblqc_ephysTimeRmsAP.rms.npy', f'{self.device_collection}/{self.pname}/', True), ('_iblqc_ephysTimeRmsAP.timestamps.npy', f'{self.device_collection}/{self.pname}/', True), ('_iblqc_ephysSaturation.samples.npy', f'{self.device_collection}/{self.pname}/', True), - # ./spike_sorters/iblsorter/probe00 - ('spike_sorting_iblsorter.log', f'spike_sorters/{self._sortername}/{self.pname}', True), + # ./spike_sorters/iblsorter/{self.pname} + (f'_ibl_log.info_{self.SPIKE_SORTER_NAME}.log', f'spike_sorters/{self._sortername}/{self.pname}', True), ('_kilosort_raw.output.tar', f'spike_sorters/{self._sortername}/{self.pname}/', True), - # ./alf/probe00/iblsorter + # ./alf/{self.pname}/iblsorter ('_kilosort_whitening.matrix.npy', f'alf/{self.pname}/{self._sortername}/', True), ('_phy_spikes_subset.channels.npy', f'alf/{self.pname}/{self._sortername}/', True), ('_phy_spikes_subset.spikes.npy', f'alf/{self.pname}/{self._sortername}/', True), @@ -713,11 +713,11 @@ def _run_iblsort(self, ap_file): sorter_dir = self.session_path.joinpath("spike_sorters", self.SPIKE_SORTER_NAME, self.pname) self.FORCE_RERUN = False if not self.FORCE_RERUN: - log_file = sorter_dir.joinpath(f"spike_sorting_{self.SPIKE_SORTER_NAME}.log") + log_file = sorter_dir.joinpath(f"_ibl_log.info_{self.SPIKE_SORTER_NAME}.log") if log_file.exists(): run_version = self._fetch_iblsorter_run_version(log_file) if packaging.version.parse(run_version) >= packaging.version.parse('1.7.0'): - _logger.info(f"Already ran: spike_sorting_{self.SPIKE_SORTER_NAME}.log" + _logger.info(f"Already ran: {log_file}" f" found in {sorter_dir}, skipping.") return sorter_dir else: From 558bde4b7446b9758fcb206528ebfbb4133c81ee Mon Sep 17 00:00:00 2001 From: owinter Date: Sat, 21 Dec 2024 20:35:09 +0000 Subject: [PATCH 03/16] spike sorting log in alf collection for task signature --- ibllib/pipes/ephys_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index e876a8b1d..e1fd7f731 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -607,9 +607,9 @@ def signature(self): ('_iblqc_ephysTimeRmsAP.timestamps.npy', f'{self.device_collection}/{self.pname}/', True), ('_iblqc_ephysSaturation.samples.npy', f'{self.device_collection}/{self.pname}/', True), # ./spike_sorters/iblsorter/{self.pname} - (f'_ibl_log.info_{self.SPIKE_SORTER_NAME}.log', f'spike_sorters/{self._sortername}/{self.pname}', True), ('_kilosort_raw.output.tar', f'spike_sorters/{self._sortername}/{self.pname}/', True), # ./alf/{self.pname}/iblsorter + (f'_ibl_log.info_{self.SPIKE_SORTER_NAME}.log', f'alf/{self.pname}/{self._sortername}', True), ('_kilosort_whitening.matrix.npy', f'alf/{self.pname}/{self._sortername}/', True), ('_phy_spikes_subset.channels.npy', f'alf/{self.pname}/{self._sortername}/', True), ('_phy_spikes_subset.spikes.npy', f'alf/{self.pname}/{self._sortername}/', True), From 6c372636bedb174f10063daa2e7c42fc7bb471ed Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 31 Dec 2024 09:07:12 +0000 Subject: [PATCH 04/16] add waveform datasets to the spike sorting task signature --- ibllib/pipes/ephys_tasks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index e1fd7f731..e154d89d9 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -638,6 +638,10 @@ def signature(self): ('templates.amps.npy', f'alf/{self.pname}/{self._sortername}/', True), ('templates.waveforms.npy', f'alf/{self.pname}/{self._sortername}/', True), ('templates.waveformsChannels.npy', f'alf/{self.pname}/{self._sortername}/', True), + ('waveforms.channels.npz', f'alf/{self.pname}/{self._sortername}/', True), + ('waveforms.table.pqt', f'alf/{self.pname}/{self._sortername}/', True), + ('waveforms.templates.npy', f'alf/{self.pname}/{self._sortername}/', True), + ('waveforms.traces.npy', f'alf/{self.pname}/{self._sortername}/', True), ], } return signature From ea32cae772e97118ccf7fe27a300609ffd70fc9b Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 31 Dec 2024 13:54:11 +0000 Subject: [PATCH 05/16] spike sorting loader default switched to iblsorter --- brainbox/io/one.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/brainbox/io/one.py b/brainbox/io/one.py index fc621d98c..8e491c88d 100644 --- a/brainbox/io/one.py +++ b/brainbox/io/one.py @@ -899,12 +899,13 @@ def load_spike_sorting_object(self, obj, *args, **kwargs): self.download_spike_sorting_object(obj, *args, **kwargs) return self._load_object(self.files[obj]) - def get_version(self, spike_sorter='pykilosort'): + def get_version(self, spike_sorter=None): + spike_sorter = (spike_sorter or self.spike_sorter) or 'iblsorter' collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter) dset = self.one.alyx.rest('datasets', 'list', session=self.eid, collection=collection, name='spikes.times.npy') return dset[0]['version'] if len(dset) else 'unknown' - def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_types=None, collection=None, + def download_spike_sorting_object(self, obj, spike_sorter=None, dataset_types=None, collection=None, attribute=None, missing='raise', **kwargs): """ Downloads an ALF object @@ -917,6 +918,7 @@ def download_spike_sorting_object(self, obj, spike_sorter='pykilosort', dataset_ :param missing: 'raise' (default) or 'ignore' :return: """ + spike_sorter = (spike_sorter or self.spike_sorter) or 'iblsorter' if len(self.collections) == 0: return {}, {}, {} self.collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter) From f6b54c41418f805cfe0a0053e58d1de2493ec3cf Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 31 Dec 2024 13:54:31 +0000 Subject: [PATCH 06/16] remove temporary folder before waveform extraction to clear some space --- ibllib/pipes/ephys_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index e154d89d9..1cbdd149d 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -806,6 +806,8 @@ def _run(self): out = ibllib.ephys.spikes.ks2_to_tar(sorter_dir, tar_dir, force=self.FORCE_RERUN) out_files.extend(out) # run waveform extraction + _logger.info(f"Cleaning up temporary folder {self.scratch_folder_run}") + shutil.rmtree(self.scratch_folder_run, ignore_errors=True) _logger.info("Running waveform extraction") spikes = alfio.load_object(probe_out_path, 'spikes', attribute=['samples', 'clusters']) clusters = alfio.load_object(probe_out_path, 'clusters', attribute=['channels']) From 1d0107ce1931ea0d21a25ba14ec5fb8f3cce3f0e Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Tue, 31 Dec 2024 14:23:22 +0000 Subject: [PATCH 07/16] add iblsorter log to the alf folder --- ibllib/pipes/ephys_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index 1cbdd149d..8454d78a8 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -785,7 +785,7 @@ def _run(self): bin_file=ap_file, ampfactor=self._sample2v(ap_file), ) - logfile = sorter_dir.joinpath(f"spike_sorting_{self.SPIKE_SORTER_NAME}.log") + logfile = sorter_dir.joinpath(f"_ibl_log.info_{self.SPIKE_SORTER_NAME}.log") if logfile.exists(): shutil.copyfile(logfile, probe_out_path.joinpath(f"_ibl_log.info_{self.SPIKE_SORTER_NAME}.log")) # recover the QC files from the spike sorting output and copy them From b9b074d15e437f5fc5063fbac1684e1edec1be5b Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Wed, 1 Jan 2025 18:30:28 +0000 Subject: [PATCH 08/16] on GPU lock found, raise if task mode is set to raise --- ibllib/pipes/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ibllib/pipes/tasks.py b/ibllib/pipes/tasks.py index 660c6502d..755875398 100644 --- a/ibllib/pipes/tasks.py +++ b/ibllib/pipes/tasks.py @@ -255,6 +255,8 @@ def run(self, **kwargs): self.log = new_log if self.clobber else self.log + new_log _logger.removeHandler(ch) ch.close() + if self.on_error == 'raise': + raise FileExistsError(f'Job {self.__class__} exited as a lock was found at {self._lock_file_path()}') return self.status outputs = self._run(**kwargs) _logger.info(f'Job {self.__class__} complete') From 18348917232b89e941a4d6a0cfdefe1a94421d2f Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Wed, 1 Jan 2025 18:30:54 +0000 Subject: [PATCH 09/16] patcher: add context to registration exception --- ibllib/oneibl/patcher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ibllib/oneibl/patcher.py b/ibllib/oneibl/patcher.py index 987c26bf6..10b70a729 100644 --- a/ibllib/oneibl/patcher.py +++ b/ibllib/oneibl/patcher.py @@ -189,7 +189,10 @@ def patch_dataset(self, file_list, dry=False, ftp=False, **kwargs): return # from the dataset info, set flatIron flag to exists=True for p, d in zip(file_list, response): - self._patch_dataset(p, dset_id=d['id'], revision=d['revision'], dry=dry, ftp=ftp) + try: + self._patch_dataset(p, dset_id=d['id'], revision=d['revision'], dry=dry, ftp=ftp) + except Exception as e: + raise Exception(f'Error registering file {p}') from e return response def patch_datasets(self, file_list, **kwargs): From 375c7c5ec8f140a43d21fe505c15c79db1c5c64f Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Wed, 1 Jan 2025 18:31:02 +0000 Subject: [PATCH 10/16] flake --- brainbox/behavior/training.py | 20 ++++++++++---------- brainbox/ephys_plots.py | 2 +- ibllib/pipes/scan_fix_passive_files.py | 2 +- ibllib/plots/figures.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/brainbox/behavior/training.py b/brainbox/behavior/training.py index a8f2f383d..b17329784 100644 --- a/brainbox/behavior/training.py +++ b/brainbox/behavior/training.py @@ -430,24 +430,24 @@ def display_status(subj, sess_dates, status, perf_easy=None, n_trials=None, psyc f"{sess_dates[2]}]") elif psych_20 is None: print(f"\n{subj} : {status} \nSession dates={[x for x in sess_dates]}, " - f"Perf easy={[np.around(pe,2) for pe in perf_easy]}, " + f"Perf easy={[np.around(pe, 2) for pe in perf_easy]}, " f"N trials={[nt for nt in n_trials]} " f"\nPsych fit over last 3 sessions: " - f"bias={np.around(psych[0],2)}, thres={np.around(psych[1],2)}, " - f"lapse_low={np.around(psych[2],2)}, lapse_high={np.around(psych[3],2)} " + f"bias={np.around(psych[0], 2)}, thres={np.around(psych[1], 2)}, " + f"lapse_low={np.around(psych[2], 2)}, lapse_high={np.around(psych[3], 2)} " f"\nMedian reaction time at 0 contrast over last 3 sessions = " - f"{np.around(rt,2)}") + f"{np.around(rt, 2)}") else: print(f"\n{subj} : {status} \nSession dates={[x for x in sess_dates]}, " - f"Perf easy={[np.around(pe,2) for pe in perf_easy]}, " + f"Perf easy={[np.around(pe, 2) for pe in perf_easy]}, " f"N trials={[nt for nt in n_trials]} " f"\nPsych fit over last 3 sessions (20): " - f"bias={np.around(psych_20[0],2)}, thres={np.around(psych_20[1],2)}, " - f"lapse_low={np.around(psych_20[2],2)}, lapse_high={np.around(psych_20[3],2)} " - f"\nPsych fit over last 3 sessions (80): bias={np.around(psych_80[0],2)}, " - f"thres={np.around(psych_80[1],2)}, lapse_low={np.around(psych_80[2],2)}, " - f"lapse_high={np.around(psych_80[3],2)} " + f"bias={np.around(psych_20[0], 2)}, thres={np.around(psych_20[1], 2)}, " + f"lapse_low={np.around(psych_20[2], 2)}, lapse_high={np.around(psych_20[3], 2)} " + f"\nPsych fit over last 3 sessions (80): bias={np.around(psych_80[0], 2)}, " + f"thres={np.around(psych_80[1], 2)}, lapse_low={np.around(psych_80[2], 2)}, " + f"lapse_high={np.around(psych_80[3], 2)} " f"\nMedian reaction time at 0 contrast over last 3 sessions = " f"{np.around(rt, 2)}") diff --git a/brainbox/ephys_plots.py b/brainbox/ephys_plots.py index 9057cf7fd..d87d81eb9 100644 --- a/brainbox/ephys_plots.py +++ b/brainbox/ephys_plots.py @@ -441,7 +441,7 @@ def plot_brain_regions(channel_ids, channel_depths=None, brain_regions=None, dis ax.bar(x=0.5, height=height, color=color, bottom=reg[0], **kwargs) ax.spines['top'].set_visible(False) ax.spines['bottom'].set_visible(False) - if not label is None: + if label is not None: if label == 'right': ax.yaxis.tick_right() ax.set_yticks(region_labels[:, 0].astype(int)) diff --git a/ibllib/pipes/scan_fix_passive_files.py b/ibllib/pipes/scan_fix_passive_files.py index fab675c5f..bdc2430c6 100644 --- a/ibllib/pipes/scan_fix_passive_files.py +++ b/ibllib/pipes/scan_fix_passive_files.py @@ -68,7 +68,7 @@ def move_rename_pairs(from_to_pairs): for i, (src, dst) in enumerate(from_to_pairs): src = Path(src) dst = Path(dst) - log.info(f"Moving {i+1} of {len(from_to_pairs)}: \n{src}\n--> {dst}") + log.info(f"Moving {i + 1} of {len(from_to_pairs)}: \n{src}\n--> {dst}") try: shutil.move(str(src / "raw_behavior_data"), str(dst / "raw_passive_data")) ffile = src.joinpath("passive_data_for_ephys.flag") diff --git a/ibllib/plots/figures.py b/ibllib/plots/figures.py index 14a6bb554..38cb60195 100644 --- a/ibllib/plots/figures.py +++ b/ibllib/plots/figures.py @@ -644,7 +644,7 @@ def raw_destripe(raw, fs, t0, i_plt, n_plt, fig, axs = plt.subplots(nrows=1, ncols=n_plt, figsize=(14, 5), gridspec_kw={'width_ratios': 4 * n_plt}) if i_plt > len(axs) - 1: # Error - raise ValueError(f'The given increment of subplot ({i_plt+1}) ' + raise ValueError(f'The given increment of subplot ({i_plt + 1}) ' f'is larger than the total number of subplots ({len(axs)})') [nc, ns] = raw.shape From fed7b2e5b2d7515a3e019c19b5a6fce16c9da091 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Fri, 3 Jan 2025 09:53:54 +0000 Subject: [PATCH 11/16] check only size not hash for S3 re-runs --- ibllib/oneibl/data_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibllib/oneibl/data_handlers.py b/ibllib/oneibl/data_handlers.py index a0b085150..c66fde278 100644 --- a/ibllib/oneibl/data_handlers.py +++ b/ibllib/oneibl/data_handlers.py @@ -766,7 +766,7 @@ def setUp(self, **_): :return: """ df = super().getData() - self.one._check_filesystem(df) + self.one._check_filesystem(df, check_hash=False) def uploadData(self, outputs, version, **kwargs): """ From d4b3bbad4a70d1be0fd5c6a641c980f5e3f98550 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Fri, 3 Jan 2025 10:35:30 +0000 Subject: [PATCH 12/16] setup logger for iblsorter --- ibllib/pipes/ephys_tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index 8454d78a8..1d29d4cec 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -1,9 +1,9 @@ import logging -import traceback from pathlib import Path -import subprocess import re import shutil +import subprocess +import traceback import packaging.version import numpy as np @@ -13,6 +13,7 @@ from ibldsp.utils import rms from ibldsp.waveform_extraction import extract_wfs_cbin import one.alf.io as alfio +import iblutil.util from ibllib.misc import check_nvidia_driver from ibllib.pipes import base_tasks @@ -714,6 +715,7 @@ def _run_iblsort(self, ap_file): (discontinued support for old spike sortings in the probe folder <1.5.5) :return: path of the folder containing ks2 spike sorting output """ + iblutil.util.setup_logger('iblsorter', level='INFO') sorter_dir = self.session_path.joinpath("spike_sorters", self.SPIKE_SORTER_NAME, self.pname) self.FORCE_RERUN = False if not self.FORCE_RERUN: @@ -726,7 +728,6 @@ def _run_iblsort(self, ap_file): return sorter_dir else: self.FORCE_RERUN = True - _logger.info(f"job progress command: tail -f {self.scratch_folder_run} *.log") self.scratch_folder_run.mkdir(parents=True, exist_ok=True) check_nvidia_driver() try: From b520d884538ece542970838c23647652faa2ad43 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Fri, 3 Jan 2025 17:16:41 +0000 Subject: [PATCH 13/16] update ibl-neuropixels requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b890b3e5e..1c6f40fdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ seaborn>=0.9.0 tqdm>=4.32.1 # ibl libraries iblatlas>=0.5.3 -ibl-neuropixel>=1.5.0 +ibl-neuropixel>=1.6.1 iblutil>=1.13.0 iblqt>=0.3.2 mtscomp>=1.0.1 From 1d2401e6e94881069b9e49abc8e8e5a207d82be7 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Fri, 3 Jan 2025 17:30:42 +0000 Subject: [PATCH 14/16] add waveform files to the output list --- ibllib/pipes/ephys_tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index 1d29d4cec..c396fc5e0 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -813,7 +813,7 @@ def _run(self): spikes = alfio.load_object(probe_out_path, 'spikes', attribute=['samples', 'clusters']) clusters = alfio.load_object(probe_out_path, 'clusters', attribute=['channels']) channels = alfio.load_object(probe_out_path, 'channels') - extract_wfs_cbin( + _output_waveform_files = extract_wfs_cbin( bin_file=ap_file, output_dir=probe_out_path, spike_samples=spikes['samples'], @@ -829,6 +829,7 @@ def _run(self): preprocess_steps=["phase_shift", "bad_channel_interpolation", "butterworth", "car"], scratch_dir=self.scratch_folder_run, ) + out_files.extend(_output_waveform_files) _logger.info(f"Cleaning up temporary folder {self.scratch_folder_run}") shutil.rmtree(self.scratch_folder_run, ignore_errors=True) if self.one: @@ -852,5 +853,5 @@ def _run(self): chns = np.load(probe_out_path.joinpath('channels.localCoordinates.npy')) out = get_aligned_channels(ins[0], chns, one=self.one, save_dir=probe_out_path) out_files.extend(out) - + self.assert_expected_outputs() return out_files From 1d5bd9a6bef3c074113214e7182b451eebbc8523 Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Sat, 4 Jan 2025 08:56:04 +0000 Subject: [PATCH 15/16] remove duplicate files in output task --- ibllib/pipes/ephys_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibllib/pipes/ephys_tasks.py b/ibllib/pipes/ephys_tasks.py index c396fc5e0..9df0053bb 100644 --- a/ibllib/pipes/ephys_tasks.py +++ b/ibllib/pipes/ephys_tasks.py @@ -853,5 +853,5 @@ def _run(self): chns = np.load(probe_out_path.joinpath('channels.localCoordinates.npy')) out = get_aligned_channels(ins[0], chns, one=self.one, save_dir=probe_out_path) out_files.extend(out) - self.assert_expected_outputs() - return out_files + self.assert_expected_outputs() + return sorted(list(set(out_files))) From f3f44adca53684ca07c5dc00449277fcbc4ef97a Mon Sep 17 00:00:00 2001 From: Olivier Winter Date: Mon, 6 Jan 2025 11:03:36 +0000 Subject: [PATCH 16/16] replace ricker function that has been removed in scipy --- ibllib/tests/test_ephys.py | 3 ++- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ibllib/tests/test_ephys.py b/ibllib/tests/test_ephys.py index d852b34f5..3b9870afe 100644 --- a/ibllib/tests/test_ephys.py +++ b/ibllib/tests/test_ephys.py @@ -8,6 +8,7 @@ from one.api import ONE import neuropixel from ibldsp import voltage +import ibldsp.utils from ibllib.ephys import ephysqc, spikes from ibllib.tests import TEST_DB @@ -59,7 +60,7 @@ def synthetic_with_bad_channels(): st = st[st < ns].astype(np.int32) stripes = np.zeros(ns) stripes[st] = 1 - stripes = scipy.signal.convolve(stripes, scipy.signal.ricker(1200, 40), 'same') * 1e-6 * 2500 + stripes = scipy.signal.convolve(stripes, ibldsp.utils.ricker(1200, 40), 'same') * 1e-6 * 2500 data = data + stripes[:, np.newaxis] noise = np.random.randn(*data.shape) * 1e-6 * 10 diff --git a/requirements.txt b/requirements.txt index 1c6f40fdf..815c73b6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ seaborn>=0.9.0 tqdm>=4.32.1 # ibl libraries iblatlas>=0.5.3 -ibl-neuropixel>=1.6.1 +ibl-neuropixel>=1.6.2 iblutil>=1.13.0 iblqt>=0.3.2 mtscomp>=1.0.1