Add callable microsaccades as intended by #492

brainscore_vision/model_helpers/activations/core.py

@@ -73,7 +73,8 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, requi
         for hook in self._stimulus_set_hooks.copy().values():  # copy to avoid stale handles
             stimulus_set = hook(stimulus_set)
         stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']]
-        activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier)
+        activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier,
+                                      require_variance=require_variance)
         activations = attach_stimulus_set_meta(activations,
                                                stimulus_set,
                                                number_of_trials=self._microsaccade_helper.number_of_trials,
@@ -578,15 +579,10 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, requ
     assert (np.array(assembly_paths) == np.array(repeated_stimulus_paths)).all()
     repeated_stimulus_ids = np.repeat(stimulus_set['stimulus_id'].values, replication_factor)
 
-    if replication_factor > 1:
-        # repeat over the presentation dimension to accommodate multiple runs per stimulus
-        assembly = xr.concat([assembly for _ in range(replication_factor)], dim='presentation')
     assembly = assembly.reset_index('presentation')
     assembly['stimulus_path'] = ('presentation', repeated_stimulus_ids)
     assembly = assembly.rename({'stimulus_path': 'stimulus_id'})
 
-    assert (np.array(assembly_paths) == np.array(stimulus_paths)).all()
-
     all_columns = []
     for column in stimulus_set.columns:
         repeated_values = np.repeat(stimulus_set[column].values, replication_factor)

brainscore_vision/model_helpers/brain_transformation/__init__.py

@@ -62,9 +62,9 @@ def start_task(self, task: BrainModel.Task, *args, **kwargs):
         else:
             self.do_behavior = False
 
-    def look_at(self, stimuli, number_of_trials=1):
+    def look_at(self, stimuli, number_of_trials: int = 1, require_variance: bool = False):
         if self.do_behavior:
-            return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials)
+            return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials, require_variance=require_variance)
         else:
             return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials)
 

brainscore_vision/model_helpers/brain_transformation/behavior.py

@@ -43,9 +43,10 @@ def start_task(self, task: BrainModel.Task, choice_labels):
         self.current_task = task
         self.choice_labels = choice_labels
 
-    def look_at(self, stimuli, number_of_trials=1):
+    def look_at(self, stimuli, number_of_trials: int = 1, require_variance: bool = False):
         assert self.current_task == BrainModel.Task.label
-        logits = self.activations_model(stimuli, layers=['logits'])
+        logits = self.activations_model(stimuli, layers=['logits'], number_of_trials=number_of_trials,
+                                        require_variance=require_variance)
         choices = self.logits_to_choice(logits)
         return choices
 
@@ -183,20 +184,23 @@ def __init__(self, identifier, activations_model, layer):
     def identifier(self):
         return self._identifier
 
-    def start_task(self, task: BrainModel.Task, fitting_stimuli):
+    def start_task(self, task: BrainModel.Task, fitting_stimuli, number_of_trials=1, require_variance=False):
         assert task in [BrainModel.Task.passive, BrainModel.Task.probabilities]
         self.current_task = task
 
-        fitting_features = self.activations_model(fitting_stimuli, layers=self.readout)
+        fitting_features = self.activations_model(fitting_stimuli, layers=self.readout,
+                                                  number_of_trials=number_of_trials,
+                                                  require_variance=require_variance)
         fitting_features = fitting_features.transpose('presentation', 'neuroid')
         assert all(fitting_features['stimulus_id'].values == fitting_stimuli['stimulus_id'].values), \
             "stimulus_id ordering is incorrect"
-        self.classifier.fit(fitting_features, fitting_stimuli['image_label'])
+        self.classifier.fit(fitting_features, fitting_features['image_label'])
 
-    def look_at(self, stimuli, number_of_trials=1):
+    def look_at(self, stimuli, number_of_trials=1, require_variance=False):
         if self.current_task is BrainModel.Task.passive:
             return
-        features = self.activations_model(stimuli, layers=self.readout)
+        features = self.activations_model(stimuli, layers=self.readout, number_of_trials=number_of_trials,
+                                          require_variance=require_variance)
         features = features.transpose('presentation', 'neuroid')
         prediction = self.classifier.predict_proba(features)
         return prediction
@@ -259,13 +263,13 @@ def start_task(self, task: BrainModel.Task):
         assert task == BrainModel.Task.odd_one_out
         self.current_task = task
 
-    def look_at(self, triplets, number_of_trials=1):
+    def look_at(self, triplets, number_of_trials: int = 1, require_variance: bool = False):
         # Compute unique features and image_pathst
         stimuli = triplets.drop_duplicates(subset=['stimulus_id'])
         stimuli = stimuli.sort_values(by='stimulus_id')
 
         # Get features
-        features = self.activations_model(stimuli, layers=self.readout)
+        features = self.activations_model(stimuli, layers=self.readout, require_variance=require_variance)
         features = features.transpose('presentation', 'neuroid')
 
         # Compute similarity matrix

tests/test_model_helpers/activations/test___init__.py

@@ -292,6 +292,25 @@ def test_from_stimulus_set(model_ctr, layers, pca_components):
         assert len(activations['neuroid']) == pca_components * len(layers)
 
 
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("number_of_trials", [3, 10])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_microsaccades_from_stimulus_set(model_ctr, layers, number_of_trials):
+    image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']
+    stimulus_set = _build_stimulus_set(image_names)
+
+    activations_extractor = model_ctr()
+    activations_extractor._extractor.set_visual_degrees(8.)
+    activations_extractor._extractor._microsaccade_helper.number_of_trials = number_of_trials
+    activations = activations_extractor.from_stimulus_set(stimulus_set, layers=layers, stimuli_identifier=False,
+                                                          require_variance=True)
+
+    assert activations is not None
+    assert len(activations['presentation']) == len(image_names) * number_of_trials
+    assert set(activations['stimulus_id'].values) == set(image_names)
+    assert len(np.unique(activations['layer'])) == len(layers)
+
+
 @pytest.mark.memory_intense
 @pytest.mark.parametrize("pca_components", [None, 1000])
 def test_exact_activations(pca_components):