Add microsaccades: interface and default model helper (#492)

* integrate brain-score/model-tools#75 to brain-score/vision

* add exception for when temp file write fails

* fix error with tf temp file management

* add bandaid to a DataAssembly index problem

* move microsaccades to the model side

* update comments

* remove indexing bug

* fix bug with activations.shape

* Apply suggestions from code review

Co-authored-by: Martin Schrimpf <mschrimpf@users.noreply.github.com>

* Delete brainscore_vision/data/scialom2024/__init__.py

* address review changes

* remove needless import

* fix bug with temporary file handling test

* assume number_of_trials=1 and require_variance=False when getting stored activations

* fix bug with access to ActivationsExtractorHelper.set_visual_degrees

* move extractor calls to ModelCommitment generic

* add check for whether activations_model exists

* fix bug with TestVisualDegrees

* add link to BrainModel issue

* remove shifts from stimulus set packaging

* change link signatures

* change microsaccade call signature

* microsaccades are now computed on both a pixel and degree basis

* Apply suggestions from code review

Co-authored-by: Martin Schrimpf <mschrimpf@users.noreply.github.com>

* fix outdated comments, type hints, etc.

* change function call to reduce repetition

* add kwargs to microsaccade helpers

* refactor microsaccade usage to their own class to improve readability

* refactor microsaccade coords into MicrosaccadeHelper

* refactor microsaccade building

* change the way MultiIndex is set

* fix tf/pytorch/keras bug with image shape calculation

* cv2 reshaping in translate

* add test for exact microsaccades

* fix microsaccade indexing

* rename test to be in line with current naming

* add require_variance to _from_paths_stored

* reduce unnecessarily long test times by reducing the number of trials tests run for, while keeping the test the same

---------

Co-authored-by: Martin Schrimpf <mschrimpf@users.noreply.github.com>

Author: benlonnqvist

brainscore_vision/model_helpers/activations/core.py

@@ -24,6 +24,10 @@ def __init__(self, identifier,
                  visual_degrees=8):
         self.layers = layers
         self.activations_model = activations_model
+        # We set the visual degrees of the ActivationsExtractorHelper here to avoid changing its signature.
+        #  The ideal solution would be to not expose the _extractor of the activations_model here, but to change
+        #  the signature of the ActivationsExtractorHelper. See https://github.com/brain-score/vision/issues/554
+        self.activations_model._extractor.set_visual_degrees(visual_degrees)  # for microsaccades
         self._visual_degrees = visual_degrees
         # region-layer mapping
         if region_layer_map is None:

brainscore_vision/model_helpers/brain_transformation/__init__.py

@@ -23,21 +23,30 @@ def __init__(self, identifier, activations_model, region_layer_map, visual_degre
     def identifier(self):
         return self._identifier
 
-    def look_at(self, stimuli, number_of_trials=1):
+    def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False):
+        """
+        :param number_of_trials: An integer that determines how many repetitions of the same image the model performs.
+        :param require_variance: Whether to require models to return different activations for the same stimuli or not.
+                                  For detailed information, see
+                                  :meth:`~brainscore_vision.model_helpers.activations.ActivationsExtractorHelper.__call__`,
+        """
         layer_regions = {}
         for region in self.recorded_regions:
             layers = self.region_layer_map[region]
             layers = make_list(layers)
             for layer in layers:
                 assert layer not in layer_regions, f"layer {layer} has already been assigned for {layer_regions[layer]}"
                 layer_regions[layer] = region
-        activations = self.run_activations(
-            stimuli, layers=list(layer_regions.keys()), number_of_trials=number_of_trials)
+        activations = self.run_activations(stimuli,
+                                           layers=list(layer_regions.keys()),
+                                           number_of_trials=number_of_trials,
+                                           require_variance=require_variance)
         activations['region'] = 'neuroid', [layer_regions[layer] for layer in activations['layer'].values]
         return activations
 
-    def run_activations(self, stimuli, layers, number_of_trials=1):
-        activations = self.activations_model(stimuli, layers=layers)
+    def run_activations(self, stimuli, layers, number_of_trials=1, require_variance=None):
+        activations = self.activations_model(stimuli, layers=layers, number_of_trials=number_of_trials,
+                                             require_variance=require_variance)
         return activations
 
     def start_task(self, task):

brainscore_vision/model_helpers/brain_transformation/neural.py

@@ -18,6 +18,7 @@ dependencies = [
     "importlib-metadata<5", # workaround to https://github.com/brain-score/brainio/issues/28
     "scikit-learn", # for metric_helpers/transformations.py cross-validation
     "scipy", # for benchmark_helpers/properties_common.py
+    "opencv-python",  # for microsaccades
     "h5py",
     "tqdm",
     "gitpython",

pyproject.toml

@@ -163,6 +163,20 @@ def tfslim_vgg16():
     pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=pytest.mark.memory_intense),
 ]
 
+# exact microsaccades for pytorch_alexnet, grayscale.png, for 1 and 10 number_of_trials
+exact_microsaccades = {"x_degrees": {1: np.array([0.]),
+                                     10: np.array([0., -0.00639121, -0.02114204, -0.02616418, -0.02128906,
+                                                   -0.00941355, 0.00596172, 0.02166913, 0.03523793, 0.04498976])},
+                       "y_degrees": {1: np.array([0.]),
+                                     10: np.array([0., 0.0144621, 0.00728107, -0.00808922, -0.02338324, -0.0340791,
+                                                   -0.03826824, -0.03578336, -0.02753704, -0.01503068])},
+                       "x_pixels": {1: np.array([0.]),
+                                    10: np.array([0., -0.17895397, -0.59197722, -0.73259714, -0.59609364, -0.26357934,
+                                                  0.16692818, 0.60673569, 0.98666196, 1.25971335])},
+                       "y_pixels": {1: np.array([0.]),
+                                    10: np.array([0., 0.40493885, 0.20386999, -0.22649819, -0.65473077, -0.95421482,
+                                                  -1.07151061, -1.00193403, -0.77103707, -0.42085896])}}
+
 
 @pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
                                         'palletized.png'])
@@ -189,6 +203,68 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits):
     return activations
 
 
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+@pytest.mark.parametrize("number_of_trials", [1, 3, 10])
+def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number_of_trials):
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=number_of_trials,
+                                        require_variance=True)
+
+    assert activations is not None
+    assert len(activations['microsaccade_shift_x_pixels']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_y_pixels']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_x_degrees']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_y_degrees']) == number_of_trials * len(stimulus_paths)
+
+
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+@pytest.mark.parametrize("require_variance", [False, True])
+@pytest.mark.parametrize("number_of_trials", [1, 3, 10])
+def test_require_variance_presentation_length(model_ctr, layers, image_name, require_variance, number_of_trials):
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers,
+                                        number_of_trials=number_of_trials, require_variance=require_variance)
+
+    assert activations is not None
+    if require_variance:
+        assert len(activations['presentation']) == number_of_trials
+    else:
+        assert len(activations['presentation']) == 1
+
+
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_temporary_file_handling(model_ctr, layers, image_name):
+    import tempfile
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=2,
+                                        require_variance=True)
+    temp_files = [f for f in os.listdir(tempfile.gettempdir()) if f.startswith('temp') and f.endswith('.png')]
+
+    assert activations is not None
+    assert len(temp_files) == 0
+
+
 def _build_stimulus_set(image_names):
     stimulus_set = StimulusSet([{'stimulus_id': image_name, 'some_meta': image_name[::-1]}
                                 for image_name in image_names])
@@ -223,9 +299,51 @@ def test_exact_activations(pca_components):
                                        image_name='rgb.jpg', pca_components=pca_components, logits=False)
     path_to_expected = Path(__file__).parent / f'alexnet-rgb-{pca_components}.nc'
     expected = xr.load_dataarray(path_to_expected)
+
+    # Originally, the `stimulus_path` Index was used to index into xarrays in Brain-Score, but this was changed
+    #  as a part of PR #492 to a MultiIndex to allow metadata to be attached to multiple repetitions of the same
+    #  `stimulus_path`. Old .nc files need to be updated to use the `presentation` index instead of `stimulus_path`,
+    #  and instead of changing the extant activations, this test was simply modified to simulate that.
+    expected = expected.rename({'stimulus_path': 'presentation'})
+
     assert (activations == expected).all()
 
 
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("number_of_trials", [1, 10])
+def test_exact_microsaccades(number_of_trials):
+    image_name = 'grayscale.png'
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = pytorch_alexnet()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+    # the exact microsaccades were computed at this extent
+    assert activations_extractor._extractor._microsaccade_helper.microsaccade_extent_degrees == 0.05
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=['features.12'],
+                                        number_of_trials=number_of_trials, require_variance=True)
+
+    assert activations is not None
+    # test with np.isclose instead of == since while the arrays are visually equal, == often fails due to float errors
+    assert np.isclose(activations['microsaccade_shift_x_degrees'].values,
+                      exact_microsaccades['x_degrees'][number_of_trials],
+                      rtol=1e-05,
+                      atol=1e-08).all()
+    assert np.isclose(activations['microsaccade_shift_y_degrees'].values,
+                       exact_microsaccades['y_degrees'][number_of_trials],
+                       rtol=1e-05,
+                       atol=1e-08).all()
+    assert np.isclose(activations['microsaccade_shift_x_pixels'].values,
+                      exact_microsaccades['x_pixels'][number_of_trials],
+                      rtol=1e-05,
+                      atol=1e-08).all()
+    assert np.isclose(activations['microsaccade_shift_y_pixels'].values,
+                      exact_microsaccades['y_pixels'][number_of_trials],
+                      rtol=1e-05,
+                      atol=1e-08).all()
+
+
 @pytest.mark.memory_intense
 @pytest.mark.parametrize(["model_ctr", "internal_layers"], [
     (pytorch_alexnet, ['features.12', 'classifier.5']),

tests/test_model_helpers/activations/test___init__.py

@@ -1,9 +1,18 @@
+from unittest.mock import Mock
+
 from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
 from brainscore_vision.model_helpers.utils import fullname
 
 
 class TestVisualDegrees:
     def test_standard_commitment(self):
-        brain_model = ModelCommitment(identifier=fullname(self), activations_model=None,
+        # create mock ActivationsExtractorHelper with a mock set_visual_degrees to avoid failing set_visual_degrees()
+        mock_extractor = Mock()
+        mock_extractor.set_visual_degrees = Mock()
+        mock_activations_model = Mock()
+        mock_activations_model._extractor = mock_extractor
+
+        # Initialize ModelCommitment with the mock activations_model
+        brain_model = ModelCommitment(identifier=fullname(self), activations_model=mock_activations_model,
                                       layers=['dummy'])
         assert brain_model.visual_degrees() == 8