From ce7eda737318b28bb9b6a10f3d7a64d09d74a4ae Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 14 Jun 2024 15:34:01 +0200 Subject: [PATCH 1/5] add callable microsaccades --- .../model_helpers/activations/core.py | 8 ++----- .../brain_transformation/__init__.py | 4 ++-- .../brain_transformation/behavior.py | 22 +++++++++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index f37631dbb..a9f537250 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -73,7 +73,8 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, requi for hook in self._stimulus_set_hooks.copy().values(): # copy to avoid stale handles stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] - activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) + activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier, + require_variance=require_variance) activations = attach_stimulus_set_meta(activations, stimulus_set, number_of_trials=self._microsaccade_helper.number_of_trials, @@ -578,15 +579,10 @@ def attach_stimulus_set_meta(assembly, stimulus_set, number_of_trials: int, requ assert (np.array(assembly_paths) == np.array(repeated_stimulus_paths)).all() repeated_stimulus_ids = np.repeat(stimulus_set['stimulus_id'].values, replication_factor) - if replication_factor > 1: - # repeat over the presentation dimension to accommodate multiple runs per stimulus - assembly = xr.concat([assembly for _ in range(replication_factor)], dim='presentation') assembly = assembly.reset_index('presentation') assembly['stimulus_path'] = ('presentation', repeated_stimulus_ids) assembly = assembly.rename({'stimulus_path': 'stimulus_id'}) - assert (np.array(assembly_paths) == np.array(stimulus_paths)).all() - all_columns = [] for column in stimulus_set.columns: repeated_values = np.repeat(stimulus_set[column].values, replication_factor) diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index c41188615..5f49f9284 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -62,9 +62,9 @@ def start_task(self, task: BrainModel.Task, *args, **kwargs): else: self.do_behavior = False - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, **kwargs): if self.do_behavior: - return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials) + return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials, **kwargs) else: return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials) diff --git a/brainscore_vision/model_helpers/brain_transformation/behavior.py b/brainscore_vision/model_helpers/brain_transformation/behavior.py index 17f02453d..82ce23764 100644 --- a/brainscore_vision/model_helpers/brain_transformation/behavior.py +++ b/brainscore_vision/model_helpers/brain_transformation/behavior.py @@ -183,20 +183,28 @@ def __init__(self, identifier, activations_model, layer): def identifier(self): return self._identifier - def start_task(self, task: BrainModel.Task, fitting_stimuli): + def start_task(self, task: BrainModel.Task, fitting_stimuli, number_of_trials=1, require_variance=False): assert task in [BrainModel.Task.passive, BrainModel.Task.probabilities] self.current_task = task - fitting_features = self.activations_model(fitting_stimuli, layers=self.readout) + fitting_features = self.activations_model(fitting_stimuli, layers=self.readout, + number_of_trials=number_of_trials, + require_variance=require_variance) fitting_features = fitting_features.transpose('presentation', 'neuroid') - assert all(fitting_features['stimulus_id'].values == fitting_stimuli['stimulus_id'].values), \ - "stimulus_id ordering is incorrect" - self.classifier.fit(fitting_features, fitting_stimuli['image_label']) + if require_variance and number_of_trials > 1: + # if microsaccades were collected for fitting trials, we use the image labels that + # ActivationsExtractorHelper collected for us due to the dimension mismatch with the fitting_stimuli + self.classifier.fit(fitting_features, fitting_features['image_label']) + else: + assert all(fitting_features['stimulus_id'].values == fitting_stimuli['stimulus_id'].values), \ + "stimulus_id ordering is incorrect" + self.classifier.fit(fitting_features, fitting_stimuli['image_label']) - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, require_variance=False): if self.current_task is BrainModel.Task.passive: return - features = self.activations_model(stimuli, layers=self.readout) + features = self.activations_model(stimuli, layers=self.readout, number_of_trials=number_of_trials, + require_variance=require_variance) features = features.transpose('presentation', 'neuroid') prediction = self.classifier.predict_proba(features) return prediction From 49a2d9cba2a0f6eac878f0e9523788761a0d68f4 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 17 Jun 2024 10:53:50 +0200 Subject: [PATCH 2/5] add test to catch errors in stimulus set meta attachment when using microsaccades --- .../activations/test___init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 108666982..58cbe8fbd 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -292,6 +292,24 @@ def test_from_stimulus_set(model_ctr, layers, pca_components): assert len(activations['neuroid']) == pca_components * len(layers) +@pytest.mark.parametrize("number_of_trials", [3, 10]) +@pytest.mark.parametrize(["model_ctr", "layers"], models_layers) +def test_microsaccades_from_stimulus_set(model_ctr, layers, number_of_trials): + image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png'] + stimulus_set = _build_stimulus_set(image_names) + + activations_extractor = model_ctr() + activations_extractor._extractor.set_visual_degrees(8.) + activations_extractor._extractor._microsaccade_helper.number_of_trials = number_of_trials + activations = activations_extractor.from_stimulus_set(stimulus_set, layers=layers, stimuli_identifier=False, + require_variance=True) + + assert activations is not None + assert len(activations['presentation']) == len(image_names) * number_of_trials + assert set(activations['stimulus_id'].values) == set(image_names) + assert len(np.unique(activations['layer'])) == len(layers) + + @pytest.mark.memory_intense @pytest.mark.parametrize("pca_components", [None, 1000]) def test_exact_activations(pca_components): From df6731b93ae6f1aa8731703be0d580c41ffc6ef5 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 17 Jun 2024 11:09:38 +0200 Subject: [PATCH 3/5] mark new test as memory intense --- tests/test_model_helpers/activations/test___init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_model_helpers/activations/test___init__.py b/tests/test_model_helpers/activations/test___init__.py index 58cbe8fbd..99b36cb98 100644 --- a/tests/test_model_helpers/activations/test___init__.py +++ b/tests/test_model_helpers/activations/test___init__.py @@ -292,6 +292,7 @@ def test_from_stimulus_set(model_ctr, layers, pca_components): assert len(activations['neuroid']) == pca_components * len(layers) +@pytest.mark.memory_intense @pytest.mark.parametrize("number_of_trials", [3, 10]) @pytest.mark.parametrize(["model_ctr", "layers"], models_layers) def test_microsaccades_from_stimulus_set(model_ctr, layers, number_of_trials): From 1664f5b9752b1c4a22f90a9856dca1ad242b8f08 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 21 Jun 2024 08:15:00 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Martin Schrimpf --- .../model_helpers/brain_transformation/__init__.py | 4 ++-- .../model_helpers/brain_transformation/behavior.py | 11 +++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index 5f49f9284..a1a3c1dc7 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -62,9 +62,9 @@ def start_task(self, task: BrainModel.Task, *args, **kwargs): else: self.do_behavior = False - def look_at(self, stimuli, number_of_trials=1, **kwargs): + def look_at(self, stimuli, number_of_trials: int = 1, require_variance: bool = False): if self.do_behavior: - return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials, **kwargs) + return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials, require_variance=require_variance) else: return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials) diff --git a/brainscore_vision/model_helpers/brain_transformation/behavior.py b/brainscore_vision/model_helpers/brain_transformation/behavior.py index 82ce23764..58679bf7c 100644 --- a/brainscore_vision/model_helpers/brain_transformation/behavior.py +++ b/brainscore_vision/model_helpers/brain_transformation/behavior.py @@ -191,14 +191,9 @@ def start_task(self, task: BrainModel.Task, fitting_stimuli, number_of_trials=1, number_of_trials=number_of_trials, require_variance=require_variance) fitting_features = fitting_features.transpose('presentation', 'neuroid') - if require_variance and number_of_trials > 1: - # if microsaccades were collected for fitting trials, we use the image labels that - # ActivationsExtractorHelper collected for us due to the dimension mismatch with the fitting_stimuli - self.classifier.fit(fitting_features, fitting_features['image_label']) - else: - assert all(fitting_features['stimulus_id'].values == fitting_stimuli['stimulus_id'].values), \ - "stimulus_id ordering is incorrect" - self.classifier.fit(fitting_features, fitting_stimuli['image_label']) + assert all(fitting_features['stimulus_id'].values == fitting_stimuli['stimulus_id'].values), \ + "stimulus_id ordering is incorrect" + self.classifier.fit(fitting_features, fitting_features['image_label']) def look_at(self, stimuli, number_of_trials=1, require_variance=False): if self.current_task is BrainModel.Task.passive: From 4d1a7737cc50a5d6158e68f37b15b12fe4ce1dcb Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Fri, 21 Jun 2024 09:52:38 +0200 Subject: [PATCH 5/5] add require_variance to the signature of look_at for all behavioral arbiters (required since the generic look_at no longer takes kwargs but specifically require_variance) --- .../model_helpers/brain_transformation/behavior.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/brainscore_vision/model_helpers/brain_transformation/behavior.py b/brainscore_vision/model_helpers/brain_transformation/behavior.py index 58679bf7c..28aeb9e6a 100644 --- a/brainscore_vision/model_helpers/brain_transformation/behavior.py +++ b/brainscore_vision/model_helpers/brain_transformation/behavior.py @@ -43,9 +43,10 @@ def start_task(self, task: BrainModel.Task, choice_labels): self.current_task = task self.choice_labels = choice_labels - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials: int = 1, require_variance: bool = False): assert self.current_task == BrainModel.Task.label - logits = self.activations_model(stimuli, layers=['logits']) + logits = self.activations_model(stimuli, layers=['logits'], number_of_trials=number_of_trials, + require_variance=require_variance) choices = self.logits_to_choice(logits) return choices @@ -262,13 +263,13 @@ def start_task(self, task: BrainModel.Task): assert task == BrainModel.Task.odd_one_out self.current_task = task - def look_at(self, triplets, number_of_trials=1): + def look_at(self, triplets, number_of_trials: int = 1, require_variance: bool = False): # Compute unique features and image_pathst stimuli = triplets.drop_duplicates(subset=['stimulus_id']) stimuli = stimuli.sort_values(by='stimulus_id') # Get features - features = self.activations_model(stimuli, layers=self.readout) + features = self.activations_model(stimuli, layers=self.readout, require_variance=require_variance) features = features.transpose('presentation', 'neuroid') # Compute similarity matrix