Skip to content

Commit

Permalink
Merge pull request #840 from int-brain-lab/remove_old_pipeline
Browse files Browse the repository at this point in the history 
Remove old pipeline
  • Loading branch information
@k1o0
k1o0 authored Sep 19, 2024
2 parents cd81887 + 0d8fa11 commit 75511b1
Show file tree
Hide file tree
Showing 43 changed files with 1,042 additions and 5,384 deletions.
156 changes: 2 additions & 154 deletions ibllib/io/extractors/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,39 +208,6 @@ def run_extractor_classes(classes, session_path=None, **kwargs):
return outputs, files


def _get_task_types_json_config():
"""
Return the extractor types map.
This function is only used for legacy sessions, i.e. those without an experiment description
file and will be removed in favor of :func:`_get_task_extractor_map`, which directly returns
the Bpod extractor class name. The experiment description file cuts out the need for pipeline
name identifiers.
Returns
-------
Dict[str, str]
A map of task protocol to task extractor identifier, e.g. 'ephys', 'habituation', etc.
See Also
--------
_get_task_extractor_map - returns a map of task protocol to Bpod trials extractor class name.
"""
with open(Path(__file__).parent.joinpath('extractor_types.json')) as fp:
task_types = json.load(fp)
try:
# look if there are custom extractor types in the personal projects repo
import projects.base
custom_extractors = Path(projects.base.__file__).parent.joinpath('extractor_types.json')
_logger.debug('Loading extractor types from %s', custom_extractors)
with open(custom_extractors) as fp:
custom_task_types = json.load(fp)
task_types.update(custom_task_types)
except (ModuleNotFoundError, FileNotFoundError):
pass
return task_types


def get_task_protocol(session_path, task_collection='raw_behavior_data'):
"""
Return the task protocol name from task settings.
Expand Down Expand Up @@ -273,125 +240,6 @@ def get_task_protocol(session_path, task_collection='raw_behavior_data'):
return


def get_task_extractor_type(task_name):
"""
(DEPRECATED) Returns the task type string from the full pybpod task name.
Parameters
----------
task_name : str
The complete task protocol name from the PYBPOD_PROTOCOL field of the task settings.
Returns
-------
str
The extractor type identifier. Examples include 'biased', 'habituation', 'training',
'ephys', 'mock_ephys' and 'sync_ephys'.
Examples
--------
>>> get_task_extractor_type('_iblrig_tasks_biasedChoiceWorld3.7.0')
'biased'
>>> get_task_extractor_type('_iblrig_tasks_trainingChoiceWorld3.6.0')
'training'
"""
if isinstance(task_name, Path):
task_name = get_task_protocol(task_name)
if task_name is None:
return
task_types = _get_task_types_json_config()

task_type = task_types.get(task_name, None)
if task_type is None: # Try lazy matching of name
task_type = next((task_types[tt] for tt in task_types if tt in task_name), None)
if task_type is None:
_logger.warning(f'No extractor type found for {task_name}')
return task_type


def get_session_extractor_type(session_path, task_collection='raw_behavior_data'):
"""
(DEPRECATED) Infer trials extractor type from task settings.
From a session path, loads the settings file, finds the task and checks if extractors exist.
Examples include 'biased', 'habituation', 'training', 'ephys', 'mock_ephys', and 'sync_ephys'.
Note this should only be used for legacy sessions, i.e. those without an experiment description
file.
Parameters
----------
session_path : str, pathlib.Path
The session path for which to determine the pipeline.
task_collection : str
The session path directory containing the raw task data.
Returns
-------
str or False
The task extractor type, e.g. 'biased', 'habituation', 'ephys', or False if unknown.
"""
task_protocol = get_task_protocol(session_path, task_collection=task_collection)
if task_protocol is None:
_logger.error(f'ABORT: No task protocol found in "{task_collection}" folder {session_path}')
return False
extractor_type = get_task_extractor_type(task_protocol)
if extractor_type:
return extractor_type
else:
return False


def get_pipeline(session_path, task_collection='raw_behavior_data'):
"""
(DEPRECATED) Get the pre-processing pipeline name from a session path.
Note this is only suitable for legacy sessions, i.e. those without an experiment description
file. This function will be removed in the future.
Parameters
----------
session_path : str, pathlib.Path
The session path for which to determine the pipeline.
task_collection : str
The session path directory containing the raw task data.
Returns
-------
str
The pipeline name inferred from the extractor type, e.g. 'ephys', 'training', 'widefield'.
"""
stype = get_session_extractor_type(session_path, task_collection=task_collection)
return _get_pipeline_from_task_type(stype)


def _get_pipeline_from_task_type(stype):
"""
(DEPRECATED) Return the pipeline from the task type.
Some task types directly define the pipeline. Note this is only suitable for legacy sessions,
i.e. those without an experiment description file. This function will be removed in the future.
Parameters
----------
stype : str
The session type or task extractor type, e.g. 'habituation', 'ephys', etc.
Returns
-------
str
A task pipeline identifier.
"""
if stype in ['ephys_biased_opto', 'ephys', 'ephys_training', 'mock_ephys', 'sync_ephys']:
return 'ephys'
elif stype in ['habituation', 'training', 'biased', 'biased_opto']:
return 'training'
elif isinstance(stype, str) and 'widefield' in stype:
return 'widefield'
else:
return stype or ''


def _get_task_extractor_map():
"""
Load the task protocol extractor map.
Expand All @@ -406,8 +254,8 @@ def _get_task_extractor_map():
task_extractors = json.load(fp)
try:
# look if there are custom extractor types in the personal projects repo
import projects.base
custom_extractors = Path(projects.base.__file__).parent.joinpath(FILENAME)
import projects
custom_extractors = Path(projects.__file__).parent.joinpath(FILENAME)
with open(custom_extractors, 'r') as fp:
custom_task_types = json.load(fp)
task_extractors.update(custom_task_types)
Expand Down
49 changes: 3 additions & 46 deletions ibllib/io/extractors/biased_trials.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
from ibllib.io.extractors.training_trials import (
Choice, FeedbackTimes, FeedbackType, GoCueTimes, GoCueTriggerTimes,
IncludedTrials, Intervals, ProbabilityLeft, ResponseTimes, RewardVolume,
StimOnTimes_deprecated, StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes,
StimOnTriggerTimes, StimOnOffFreezeTimes, ItiInTimes,
StimOffTriggerTimes, StimFreezeTriggerTimes, ErrorCueTriggerTimes, PhasePosQuiescence)
from ibllib.io.extractors.training_wheel import Wheel

__all__ = ['extract_all', 'BiasedTrials', 'EphysTrials']
__all__ = ['BiasedTrials', 'EphysTrials']


class ContrastLR(BaseBpodTrialsExtractor):
Expand Down Expand Up @@ -176,7 +176,7 @@ class EphysTrials(BaseBpodTrialsExtractor):
- Contrast, phase, position, probabilityLeft and quiescence is extracted differently
"""
save_names = ('_ibl_trials.goCueTrigger_times.npy', '_ibl_trials.stimOnTrigger_times.npy', None,
'_ibl_trials.stimOffTrigger_times', None, None,
'_ibl_trials.stimOffTrigger_times.npy', None, None,
'_ibl_trials.table.pqt', '_ibl_trials.stimOff_times.npy', None, '_ibl_wheel.timestamps.npy',
'_ibl_wheel.position.npy', '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None,
'_ibl_trials.included.npy', None, None, '_ibl_trials.quiescencePeriod.npy')
Expand Down Expand Up @@ -205,46 +205,3 @@ def _extract(self, extractor_classes=None, **kwargs) -> dict:
base + extractor_classes, session_path=self.session_path, bpod_trials=self.bpod_trials,
settings=self.settings, save=False, task_collection=self.task_collection)
return {k: out[k] for k in self.var_names}


def extract_all(session_path, save=False, bpod_trials=False, settings=False, extra_classes=None,
task_collection='raw_behavior_data', save_path=None):
"""
Same as training_trials.extract_all except...
- there is no RepNum
- ContrastLR is extracted differently
- IncludedTrials is only extracted for 5.0.0 or greater
:param session_path:
:param save:
:param bpod_trials:
:param settings:
:param extra_classes: additional BaseBpodTrialsExtractor subclasses for custom extractions
:return:
"""
if not bpod_trials:
bpod_trials = raw.load_data(session_path, task_collection=task_collection)
if not settings:
settings = raw.load_settings(session_path, task_collection=task_collection)
if settings is None:
settings = {'IBLRIG_VERSION': '100.0.0'}

if settings['IBLRIG_VERSION'] == '':
settings['IBLRIG_VERSION'] = '100.0.0'

# Version check
if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'):
# We now extract a single trials table
base = [BiasedTrials]
else:
base = [
GoCueTriggerTimes, Intervals, Wheel, FeedbackType, ContrastLR, ProbabilityLeft, Choice,
StimOnTimes_deprecated, RewardVolume, FeedbackTimes, ResponseTimes, GoCueTimes, PhasePosQuiescence
]

if extra_classes:
base.extend(extra_classes)

out, fil = run_extractor_classes(base, save=save, session_path=session_path, bpod_trials=bpod_trials, settings=settings,
task_collection=task_collection, path_out=save_path)
return out, fil
90 changes: 1 addition & 89 deletions ibllib/io/extractors/bpod_trials.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,104 +5,16 @@
"""
import logging
import importlib
from collections import OrderedDict
import warnings

from packaging import version
from ibllib.io.extractors import habituation_trials, training_trials, biased_trials, opto_trials
from ibllib.io.extractors.base import get_bpod_extractor_class, protocol2extractor
from ibllib.io.extractors.habituation_trials import HabituationTrials
from ibllib.io.extractors.training_trials import TrainingTrials
from ibllib.io.extractors.biased_trials import BiasedTrials, EphysTrials
from ibllib.io.extractors.base import get_session_extractor_type, BaseBpodTrialsExtractor
import ibllib.io.raw_data_loaders as rawio
from ibllib.io.extractors.base import BaseBpodTrialsExtractor

_logger = logging.getLogger(__name__)


def extract_all(session_path, save=True, bpod_trials=None, settings=None,
task_collection='raw_behavior_data', extractor_type=None, save_path=None):
"""
Extracts a training session from its path. NB: Wheel must be extracted first in order to
extract trials.firstMovement_times.
Parameters
----------
session_path : str, pathlib.Path
The path to the session to be extracted.
task_collection : str
The subfolder containing the raw Bpod data files.
save : bool
If true, save the output files to save_path.
bpod_trials : list of dict
The loaded Bpod trial data. If None, attempts to load _iblrig_taskData.raw from
raw_task_collection.
settings : dict
The loaded Bpod settings. If None, attempts to load _iblrig_taskSettings.raw from
raw_task_collection.
extractor_type : str
The type of extraction. Supported types are {'ephys', 'biased', 'biased_opto',
'ephys_biased_opto', 'training', 'ephys_training', 'habituation'}. If None, extractor type
determined from settings.
save_path : str, pathlib.Path
The location of the output files if save is true. Defaults to <session_path>/alf.
Returns
-------
dict
The extracted trials data.
dict
The extracted wheel data.
list of pathlib.Path
The output files if save is true.
"""
warnings.warn('`extract_all` functions soon to be removed, use `bpod_trials.get_bpod_extractor` instead', FutureWarning)
if not extractor_type:
extractor_type = get_session_extractor_type(session_path, task_collection=task_collection)
_logger.info(f'Extracting {session_path} as {extractor_type}')
bpod_trials = bpod_trials or rawio.load_data(session_path, task_collection=task_collection)
settings = settings or rawio.load_settings(session_path, task_collection=task_collection)
_logger.info(f'{extractor_type} session on {settings["PYBPOD_BOARD"]}')

# Determine which additional extractors are required
extra = []
if extractor_type == 'ephys': # Should exclude 'ephys_biased'
_logger.debug('Engaging biased TrialsTableEphys')
extra.append(biased_trials.TrialsTableEphys)
if extractor_type in ['biased_opto', 'ephys_biased_opto']:
_logger.debug('Engaging opto_trials LaserBool')
extra.append(opto_trials.LaserBool)

# Determine base extraction
if extractor_type in ['training', 'ephys_training']:
trials, files_trials = training_trials.extract_all(session_path, bpod_trials=bpod_trials, settings=settings, save=save,
task_collection=task_collection, save_path=save_path)
# This is hacky but avoids extracting the wheel twice.
# files_trials should contain wheel files at the end.
files_wheel = []
wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k})
elif 'biased' in extractor_type or 'ephys' in extractor_type:
trials, files_trials = biased_trials.extract_all(
session_path, bpod_trials=bpod_trials, settings=settings, save=save, extra_classes=extra,
task_collection=task_collection, save_path=save_path)

files_wheel = []
wheel = OrderedDict({k: trials.pop(k) for k in tuple(trials.keys()) if 'wheel' in k})
elif extractor_type == 'habituation':
if settings['IBLRIG_VERSION'] and \
version.parse(settings['IBLRIG_VERSION']) <= version.parse('5.0.0'):
_logger.warning('No extraction of legacy habituation sessions')
return None, None, None
trials, files_trials = habituation_trials.extract_all(session_path, bpod_trials=bpod_trials, settings=settings, save=save,
task_collection=task_collection, save_path=save_path)
wheel = None
files_wheel = []
else:
raise ValueError(f'No extractor for task {extractor_type}')
_logger.info('session extracted \n') # timing info in log
return trials, wheel, (files_trials + files_wheel) if save else None


def get_bpod_extractor(session_path, protocol=None, task_collection='raw_behavior_data') -> BaseBpodTrialsExtractor:
"""
Returns an extractor for a given session.
Expand Down
Loading

0 comments on commit 75511b1

Please sign in to comment.