diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index f37631dbb..a9f537250 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -73,7 +73,8 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, requi for hook in self._stimulus_set_hooks.copy().values(): # copy to avoid stale handles stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] - activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) + activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier, + require_variance=require_variance) activations = attach_stimulus_set_meta(activations, stimulus_set, number_of_trials=self._microsaccade_helper.number_of_trials, @@ -578,15 +579,10 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, requ assert (np.array(assembly_paths) == np.array(repeated_stimulus_paths)).all() repeated_stimulus_ids = np.repeat(stimulus_set['stimulus_id'].values, replication_factor) - if replication_factor > 1: - # repeat over the presentation dimension to accommodate multiple runs per stimulus - assembly = xr.concat([assembly for _ in range(replication_factor)], dim='presentation') assembly = assembly.reset_index('presentation') assembly['stimulus_path'] = ('presentation', repeated_stimulus_ids) assembly = assembly.rename({'stimulus_path': 'stimulus_id'}) - assert (np.array(assembly_paths) == np.array(stimulus_paths)).all() - all_columns = [] for column in stimulus_set.columns: repeated_values = np.repeat(stimulus_set[column].values, replication_factor) diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index c41188615..a1a3c1dc7 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -62,9 +62,9 @@ def start_task(self, task: BrainModel.Task, *args, **kwargs): else: self.do_behavior = False - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials: int = 1, require_variance: bool = False): if self.do_behavior: - return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials) + return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials, require_variance=require_variance) else: return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials) diff --git a/brainscore_vision/model_helpers/brain_transformation/behavior.py b/brainscore_vision/model_helpers/brain_transformation/behavior.py index 17f02453d..28aeb9e6a 100644 --- a/brainscore_vision/model_helpers/brain_transformation/behavior.py +++ b/brainscore_vision/model_helpers/brain_transformation/behavior.py @@ -43,9 +43,10 @@ def start_task(self, task: BrainModel.Task, choice_labels): self.current_task = task self.choice_labels = choice_labels - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials: int = 1, require_variance: bool = False): assert self.current_task == BrainModel.Task.label - logits = self.activations_model(stimuli, layers=['logits']) + logits = self.activations_model(stimuli, layers=['logits'], number_of_trials=number_of_trials, + require_variance=require_variance) choices = self.logits_to_choice(logits) return choices @@ -183,20 +184,23 @@ def __init__(self, identifier, activations_model, layer): def identifier(self): return self._identifier - def start_task(self, task: BrainModel.Task, fitting_stimuli): + def start_task(self, task: BrainModel.Task, fitting_stimuli, number_of_trials=1, require_variance=False): assert task in [BrainModel.Task.passive, BrainModel.Task.probabilities] self.current_task = task - fitting_features = self.activations_model(fitting_stimuli, layers=self.readout) + fitting_features = self.activations_model(fitting_stimuli, layers=self.readout, + number_of_trials=number_of_trials, + require_variance=require_variance) fitting_features = fitting_features.transpose('presentation', 'neuroid') assert all(fitting_features['stimulus_id'].values == fitting_stimuli['stimulus_id'].values), \ "stimulus_id ordering is incorrect" - self.classifier.fit(fitting_features, fitting_stimuli['image_label']) + self.classifier.fit(fitting_features, fitting_features['image_label']) - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, require_variance=False): if self.current_task is BrainModel.Task.passive: return - features = self.activations_model(stimuli, layers=self.readout) + features = self.activations_model(stimuli, layers=self.readout, number_of_trials=number_of_trials, + require_variance=require_variance) features = features.transpose('presentation', 'neuroid') prediction = self.classifier.predict_proba(features) return prediction @@ -259,13 +263,13 @@ def start_task(self, task: BrainModel.Task): assert task == BrainModel.Task.odd_one_out self.current_task = task - def look_at(self, triplets, number_of_trials=1): + def look_at(self, triplets, number_of_trials: int = 1, require_variance: bool = False): # Compute unique features and image_pathst stimuli = triplets.drop_duplicates(subset=['stimulus_id']) stimuli = stimuli.sort_values(by='stimulus_id') # Get features - features = self.activations_model(stimuli, layers=self.readout) + features = self.activations_model(stimuli, layers=self.readout, require_variance=require_variance) features = features.transpose('presentation', 'neuroid') # Compute similarity matrix diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 108666982..99b36cb98 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -292,6 +292,25 @@ def test_from_stimulus_set(model_ctr, layers, pca_components): assert len(activations['neuroid']) == pca_components * len(layers) +@pytest.mark.memory_intense +@pytest.mark.parametrize("number_of_trials", [3, 10]) +@pytest.mark.parametrize(["model_ctr", "layers"], models_layers) +def test_microsaccades_from_stimulus_set(model_ctr, layers, number_of_trials): + image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png'] + stimulus_set = _build_stimulus_set(image_names) + + activations_extractor = model_ctr() + activations_extractor._extractor.set_visual_degrees(8.) + activations_extractor._extractor._microsaccade_helper.number_of_trials = number_of_trials + activations = activations_extractor.from_stimulus_set(stimulus_set, layers=layers, stimuli_identifier=False, + require_variance=True) + + assert activations is not None + assert len(activations['presentation']) == len(image_names) * number_of_trials + assert set(activations['stimulus_id'].values) == set(image_names) + assert len(np.unique(activations['layer'])) == len(layers) + + @pytest.mark.memory_intense @pytest.mark.parametrize("pca_components", [None, 1000]) def test_exact_activations(pca_components):