One3 (#926)

* set ONE on the v3 branch

* auto -> remote mode; fix unit tests and imports

* fix the version reading from log bug

* Fix brainbox.io tests - flake

* set ONE on the latest v3.0.0 branch

* bump ONE requirement after release

---------

Co-authored-by: Miles Wells <k1o0@5tk.co>

Author: oliche

brainbox/io/one.py

@@ -918,7 +918,8 @@ def download_spike_sorting_object(self, obj, spike_sorter=None, dataset_types=No
         :param missing: 'raise' (default) or 'ignore'
         :return:
         """
-        spike_sorter = (spike_sorter or self.spike_sorter) or 'iblsorter'
+        if spike_sorter is None:
+            spike_sorter = self.spike_sorter if self.spike_sorter is not None else 'iblsorter'
         if len(self.collections) == 0:
             return {}, {}, {}
         self.collection = self._get_spike_sorting_collection(spike_sorter=spike_sorter)

brainbox/tests/test_behavior.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 import unittest
 from unittest import mock
+from functools import partial
 import numpy as np
 import pickle
 import copy
@@ -250,7 +251,10 @@ def test_query_criterion(self):
             'ready4ephysrig': ['2019-04-10', 'abf5109c-d780-44c8-9561-83e857c7bc01'],
             'ready4recording': ['2019-04-11', '7dc3c44b-225f-4083-be3d-07b8562885f4']
         }
-        with mock.patch.object(one.alyx, 'rest', return_value={'json': {'trained_criteria': status_map}}):
+
+        # Mock output of subjects read endpoint only
+        side_effect = partial(self._rest_mock, one.alyx.rest, {'json': {'trained_criteria': status_map}})
+        with mock.patch.object(one.alyx, 'rest', side_effect=side_effect):
             eid, n_sessions, n_days = train.query_criterion(subject, 'in_training', one=one)
             self.assertEqual('01390fcc-4f86-4707-8a3b-4d9309feb0a1', eid)
             self.assertEqual(1, n_sessions)
@@ -267,3 +271,24 @@ def test_query_criterion(self):
             self.assertIsNone(n_sessions)
             self.assertIsNone(n_days)
             self.assertRaises(ValueError, train.query_criterion, subject, 'foobar', one=one)
+
+    def _rest_mock(self, alyx_rest, return_value, *args, **kwargs):
+        """Mock return value of AlyxClient.rest function depending on input.
+
+        If using the subjects endpoint, return `return_value`. Otherwise, calls the original method.
+
+        Parameters
+        ----------
+        alyx_rest : function
+            one.webclient.AlyxClient.rest method.
+        return_value : any
+            The mock data to return.
+
+        Returns
+        -------
+        dict, list
+            Either `return_value` or the original method output.
+        """
+        if args[0] == 'subjects':
+            return return_value
+        return alyx_rest(*args, **kwargs)

ibllib/io/video.py

@@ -151,12 +151,26 @@ def get_video_meta(video_path, one=None):
 
 
 def url_from_eid(eid, label=None, one=None):
-    """Return the video URL(s) for a given eid
-
-    :param eid: The session id
-    :param label: The video label (e.g. 'body') or a tuple thereof
-    :param one: An instance of ONE
-    :return: The URL string if the label is a string, otherwise a dict of urls with labels as keys
+    """Return the video URL(s) for a given eid.
+
+    Parameters
+    ----------
+    eid : UUID, str
+        The session ID.
+    label : str, tuple of str
+        The video label (e.g. 'body') or a tuple thereof.
+    one : one.api.One
+        An instance of ONE.
+
+    Returns
+    -------
+    str, dict of str
+        The URL string if the label is a string, otherwise a dict of urls with labels as keys.
+
+    Raises
+    ------
+    ValueError
+        Video label is unreckognized.  See `VIDEO_LABELS` for valid labels.
     """
     valid_labels = VIDEO_LABELS
     if not (label is None or np.isin(label, valid_labels).all()):

ibllib/pipes/ephys_tasks.py

@@ -702,11 +702,16 @@ def _fetch_iblsorter_run_version(log_file):
         '\x1b[0m15:39:37.919 [I] ibl:90               Starting Pykilosort version ibl_1.3.0^[[0m\n'
         """
         with open(log_file) as fid:
-            line = fid.readline()
-        version = re.search('version (.*), output', line)
-        version = version or re.search('version (.*)', line)  # old versions have output, new have a version line
-        version = re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', '', version.group(1))
-        return version
+            for m in range(50):
+                line = fid.readline()
+                print(line.strip())
+                version = re.search('version (.*)', line)
+                if not line or version:
+                    break
+        if version is not None:
+            version = re.sub(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])', '', version.group(1))
+            version = version.replace(',', ' ').split(' ')[0]  # breaks the string after the first space
+            return version
 
     def _run_iblsort(self, ap_file):
         """

ibllib/pipes/mesoscope_tasks.py

@@ -1135,7 +1135,7 @@ def register_fov(self, meta: dict, suffix: str = None) -> (list, list):
             assert set(fov.keys()) >= {'MLAPDV', 'nXnYnZ', 'roiUUID'}
             # Field of view
             alyx_FOV = {
-                'session': self.session_path.as_posix() if dry else self.path2eid(),
+                'session': self.session_path.as_posix() if dry else str(self.path2eid()),
                 'imaging_type': 'mesoscope', 'name': f'FOV_{i:02}',
                 'stack': stack_ids.get(fov['roiUUID'])
             }

ibllib/pipes/tasks.py

@@ -91,6 +91,7 @@
 from one.api import ONE
 from one import webclient
 import one.alf.io as alfio
+from one.alf.path import ALFPath
 
 _logger = logging.getLogger(__name__)
 TASK_STATUS_SET = {'Waiting', 'Held', 'Started', 'Errored', 'Empty', 'Complete', 'Incomplete', 'Abandoned'}
@@ -493,10 +494,13 @@ def assert_expected_inputs(self, raise_error=True, raise_ambiguous=False):
         # Some sessions may contain revisions and without ONE it's difficult to determine which
         # are the default datasets. Likewise SDSC may contain multiple datasets with different
         # UUIDs in the name after patching data.
-        variant_datasets = alfio.find_variants(files, extra=False)
+        valid_alf_files = filter(ALFPath.is_valid_alf, files)
+        variant_datasets = alfio.find_variants(valid_alf_files, extra=False)
+        if len(variant_datasets) < len(files):
+            _logger.warning('Some files are not ALF datasets and will not be checked for ambiguity')
         if any(map(len, variant_datasets.values())):
             # Keep those with variants and make paths relative to session for logging purposes
-            to_frag = lambda x: x.relative_to(self.session_path).as_posix()  # noqa
+            to_frag = lambda x: x.relative_to_session().as_posix()  # noqa
             ambiguous = {
                 to_frag(k): [to_frag(x) for x in v]
                 for k, v in variant_datasets.items() if any(v)}

ibllib/qc/task_qc_viewer/task_qc.py

@@ -244,7 +244,7 @@ def show_session_task_qc(qc_or_session=None, bpod_only=False, local=False, one=N
         task_qc = qc_or_session
         qc = QcFrame(task_qc)
     else:  # assumed to be eid or session path
-        one = one or ONE(mode='local' if local else 'auto')
+        one = one or ONE(mode='local' if local else 'remote')
         if not is_session_path(Path(qc_or_session)):
             eid = one.to_eid(qc_or_session)
             session_path = one.eid2path(eid)

ibllib/tests/qc/test_base_qc.py

@@ -16,12 +16,11 @@ class TestQC(unittest.TestCase):
     """Test base QC class."""
 
     eid = None
-    """str: An experiment UUID to use for updating QC fields."""
+    """UUID: An experiment UUID to use for updating QC fields."""
 
     @classmethod
     def setUpClass(cls):
-        _, eid = register_new_session(one, subject='ZM_1150')
-        cls.eid = str(eid)
+        _, cls.eid = register_new_session(one, subject='ZM_1150')
 
     def setUp(self) -> None:
         ses = one.alyx.rest('sessions', 'partial_update', id=self.eid, data={'qc': 'NOT_SET'})
@@ -67,7 +66,7 @@ def test_update(self) -> None:
         current = self.qc.update(outcome)
         self.assertIs(spec.QC.PASS, current, 'Failed to update QC field')
         # Check that extended QC field was updated
-        extended = one.alyx.get('/sessions/' + self.eid, clobber=True)['extended_qc']
+        extended = one.alyx.get('/sessions/' + str(self.eid), clobber=True)['extended_qc']
         updated = 'experimenter' in extended and extended['experimenter'] == outcome
         self.assertTrue(updated, 'failed to update extended_qc field')
         # Check that outcome property is set
@@ -78,7 +77,7 @@ def test_update(self) -> None:
         namespace = 'task'
         current = self.qc.update(outcome, namespace=namespace)
         self.assertIs(spec.QC.FAIL, current, 'Failed to update QC field')
-        extended = one.alyx.get('/sessions/' + self.eid, clobber=True)['extended_qc']
+        extended = one.alyx.get('/sessions/' + str(self.eid), clobber=True)['extended_qc']
         updated = namespace in extended and extended[namespace] == outcome.upper()
         self.assertTrue(updated, 'failed to update extended_qc field')
 
@@ -87,7 +86,7 @@ def test_update(self) -> None:
         namespace = 'task'
         current = self.qc.update(outcome)
         self.assertNotEqual(spec.QC.PASS, current, 'QC field updated with less severe outcome')
-        extended = one.alyx.get('/sessions/' + self.eid, clobber=True)['extended_qc']
+        extended = one.alyx.get('/sessions/' + str(self.eid), clobber=True)['extended_qc']
         updated = namespace in extended and extended[namespace] != outcome
         self.assertTrue(updated, 'failed to update extended_qc field')
 
@@ -96,7 +95,7 @@ def test_update(self) -> None:
         namespace = 'task'
         current = self.qc.update(outcome, override=True, namespace=namespace)
         self.assertEqual(spec.QC.NOT_SET, current, 'QC field updated with less severe outcome')
-        extended = one.alyx.get('/sessions/' + self.eid, clobber=True)['extended_qc']
+        extended = one.alyx.get('/sessions/' + str(self.eid), clobber=True)['extended_qc']
         updated = namespace in extended and extended[namespace] == outcome
         self.assertTrue(updated, 'failed to update extended_qc field')
 

ibllib/tests/test_io.py

@@ -370,7 +370,9 @@ def test_delete_empty_folders(self):
 class TestVideo(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
-        cls.one = ONE(**TEST_DB)
+        cls.one = ONE(**TEST_DB, mode='local')
+        if cls.one._cache.sessions.empty:
+            cls.one.load_cache()
         if 'public' in cls.one.alyx._par.HTTP_DATA_SERVER:
             cls.one.alyx._par = cls.one.alyx._par.set(
                 'HTTP_DATA_SERVER', cls.one.alyx._par.HTTP_DATA_SERVER.rsplit('/', 1)[0])
@@ -398,7 +400,7 @@ def test_label_from_path(self):
         self.assertIsNone(label)
 
     def test_url_from_eid(self):
-        assert self.one.mode != 'remote'
+        self.one.mode = 'local'
         actual = video.url_from_eid(self.eid, 'left', self.one)
         self.assertEqual(self.url, actual)
         actual = video.url_from_eid(self.eid, one=self.one)
@@ -410,10 +412,12 @@ def test_url_from_eid(self):
 
         # Test remote mode
         old_mode = self.one.mode
-        self.one.mode = 'remote'
-        actual = video.url_from_eid(self.eid, label='left', one=self.one)
-        self.assertEqual(self.url, actual)
-        self.one.mode = old_mode
+        try:
+            self.one.mode = 'remote'
+            actual = video.url_from_eid(self.eid, label='left', one=self.one)
+            self.assertEqual(self.url, actual)
+        finally:
+            self.one.mode = old_mode
 
         # Test arg checks
         with self.assertRaises(ValueError):

ibllib/tests/test_mesoscope.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 import subprocess
 from copy import deepcopy
+import uuid
 
 from one.api import ONE
 import numpy as np
@@ -286,7 +287,7 @@ def test_update_surgery_json(self):
                 task.update_surgery_json(meta, normal_vector)
         finally:
             # ONE function is cached so we must reset the mode for other tests
-            one.mode = 'auto'
+            one.mode = 'remote'
 
 
 class TestRegisterFOV(unittest.TestCase):
@@ -311,17 +312,19 @@ def test_register_fov(self):
                   'bottomLeft': [2317.3, -2181.4, -466.3], 'bottomRight': [2862.7, -2206.9, -679.4],
                   'center': [2596.1, -1900.5, -588.6]}
         meta = {'FOV': [{'MLAPDV': mlapdv, 'nXnYnZ': [512, 512, 1], 'roiUUID': 0}]}
-        with unittest.mock.patch.object(task.one.alyx, 'rest') as mock_rest:
+        eid = uuid.uuid4()
+        with unittest.mock.patch.object(task.one.alyx, 'rest') as mock_rest, \
+                unittest.mock.patch.object(task.one, 'path2eid', return_value=eid):
             task.register_fov(meta, 'estimate')
         calls = mock_rest.call_args_list
-        self.assertEqual(3, len(calls))
+        self.assertEqual(2, len(calls))
 
-        args, kwargs = calls[1]
+        args, kwargs = calls[0]
         self.assertEqual(('fields-of-view', 'create'), args)
-        expected = {'data': {'session': None, 'imaging_type': 'mesoscope', 'name': 'FOV_00', 'stack': None}}
+        expected = {'data': {'session': str(eid), 'imaging_type': 'mesoscope', 'name': 'FOV_00', 'stack': None}}
         self.assertEqual(expected, kwargs)
 
-        args, kwargs = calls[2]
+        args, kwargs = calls[1]
         self.assertEqual(('fov-location', 'create'), args)
         expected = ['field_of_view', 'default_provenance', 'coordinate_system', 'n_xyz', 'provenance', 'x', 'y', 'z',
                     'brain_region']
@@ -350,7 +353,7 @@ def tearDown(self) -> None:
         The ONE function is cached and therefore the One object persists beyond this test.
         Here we return the mode back to the default after testing behaviour in offline mode.
         """
-        self.one.mode = 'auto'
+        self.one.mode = 'remote'
 
 
 class TestImagingMeta(unittest.TestCase):

ibllib/tests/test_oneibl.py

@@ -17,7 +17,7 @@
 from one.api import ONE
 from one.webclient import AlyxClient
 import one.alf.exceptions as alferr
-from one.util import QC_TYPE
+from one.alf.cache import QC_TYPE
 import iblutil.io.params as iopar
 
 from ibllib.oneibl import patcher, registration, data_handlers as handlers
@@ -612,7 +612,7 @@ def test_server_upload_data(self, register_dataset_mock):
 
     def test_getData(self):
         """Test for DataHandler.getData method."""
-        one = ONE(**TEST_DB, mode='auto')
+        one = ONE(**TEST_DB, mode='remote')
         session_path = Path('KS005/2019-04-01/001')
         task = ChoiceWorldTrialsBpod(session_path, one=one, collection='raw_behavior_data')
         task.get_signatures()
@@ -698,7 +698,7 @@ class TestSDSCDataHandler(unittest.TestCase):
     def setUp(self):
         tmp = tempfile.TemporaryDirectory()
         self.addCleanup(tmp.cleanup)
-        self.one = ONE(**TEST_DB, mode='auto')
+        self.one = ONE(**TEST_DB, mode='remote')
         self.patch_path = Path(tmp.name, 'patch')
         self.root_path = Path(tmp.name, 'root')
         self.root_path.mkdir(), self.patch_path.mkdir()

requirements.txt

@@ -27,7 +27,7 @@ ibl-neuropixel>=1.6.2
 iblutil>=1.13.0
 iblqt>=0.4.2
 mtscomp>=1.0.1
-ONE-api>=2.11
+ONE-api==3.0b3
 phylib>=2.6.0
 psychofit
 slidingRP>=1.1.1  # steinmetz lab refractory period metrics