From 0e9a89dc9af5e97759f8b681c961fabaaa537289 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 18 Mar 2024 11:28:31 +0100 Subject: [PATCH 1/3] add require_variance to the signature of look_at, and update a missing docstring ref --- brainscore_vision/model_helpers/activations/core.py | 3 ++- brainscore_vision/model_interface.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index f37631dbb..24b3640d1 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -45,7 +45,8 @@ def __call__(self, stimuli, layers, stimuli_identifier=None, number_of_trials: i :param number_of_trials: An integer that determines how many repetitions of the same model performs. :param require_variance: A bool that asks models to output different responses to the same stimuli (i.e., allows stochastic responses to identical stimuli, even in otherwise deterministic base models). - We here implement this using microsaccades. For more, see ... + We here implement this using microsaccades. For more, see + :cls:`~brainscore_vision.model_helpers.activations.core.MicrosaccadeHelper`. """ if require_variance: diff --git a/brainscore_vision/model_interface.py b/brainscore_vision/model_interface.py index ea9f67313..95b3a922c 100644 --- a/brainscore_vision/model_interface.py +++ b/brainscore_vision/model_interface.py @@ -196,8 +196,8 @@ def start_recording(self, recording_target: RecordingTarget, time_bins: List[Tup """ raise NotImplementedError() - def look_at(self, stimuli: Union[StimulusSet, List[str]], number_of_trials=1) \ - -> Union[BehavioralAssembly, NeuroidAssembly]: + def look_at(self, stimuli: Union[StimulusSet, List[str]], number_of_trials: int = 1, + require_variance: bool = False) -> Union[BehavioralAssembly, NeuroidAssembly]: """ Digest a set of stimuli and return requested outputs. Which outputs to return is instructed by the :meth:`~brainscore_vision.model_interface.BrainMode.start_task` and @@ -207,6 +207,9 @@ def look_at(self, stimuli: Union[StimulusSet, List[str]], number_of_trials=1) \ or a list of image file paths :param number_of_trials: The number of repeated trials of the stimuli that the model should average over. E.g. 10 or 35. Non-stochastic models can likely ignore this parameter. + :param require_variance: Allows all models access to different activations to the same stimuli, currently + implemented using microsaccades. For more information, see the class + :cls:`~brainscore_vision.model_helpers.activations.core.MicrosaccadeHelper`. :return: task behaviors or recordings as instructed """ raise NotImplementedError() From d64281e150d6857b32b8c6b6d198ed2c5273fd72 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 18 Mar 2024 13:39:18 +0100 Subject: [PATCH 2/3] add traversal paths to require_variance for non-neural models --- brainscore_vision/model_helpers/activations/core.py | 3 ++- .../model_helpers/brain_transformation/__init__.py | 8 +++++--- .../model_helpers/brain_transformation/behavior.py | 12 ++++++------ .../model_helpers/brain_transformation/temporal.py | 5 +++-- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/brainscore_vision/model_helpers/activations/core.py b/brainscore_vision/model_helpers/activations/core.py index 24b3640d1..71100c00b 100644 --- a/brainscore_vision/model_helpers/activations/core.py +++ b/brainscore_vision/model_helpers/activations/core.py @@ -74,7 +74,8 @@ def from_stimulus_set(self, stimulus_set, layers, stimuli_identifier=None, requi for hook in self._stimulus_set_hooks.copy().values(): # copy to avoid stale handles stimulus_set = hook(stimulus_set) stimuli_paths = [str(stimulus_set.get_stimulus(stimulus_id)) for stimulus_id in stimulus_set['stimulus_id']] - activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier) + activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier, + require_variance=require_variance) activations = attach_stimulus_set_meta(activations, stimulus_set, number_of_trials=self._microsaccade_helper.number_of_trials, diff --git a/brainscore_vision/model_helpers/brain_transformation/__init__.py b/brainscore_vision/model_helpers/brain_transformation/__init__.py index e378f87ff..96b26971b 100644 --- a/brainscore_vision/model_helpers/brain_transformation/__init__.py +++ b/brainscore_vision/model_helpers/brain_transformation/__init__.py @@ -62,11 +62,13 @@ def start_task(self, task: BrainModel.Task, *args, **kwargs): else: self.do_behavior = False - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): if self.do_behavior: - return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials) + return self.behavior_model.look_at(stimuli, number_of_trials=number_of_trials, + require_variance=require_variance) else: - return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials) + return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials, + require_variance=require_variance) def start_recording(self, recording_target, time_bins): return self.layer_model.start_recording(recording_target, time_bins) diff --git a/brainscore_vision/model_helpers/brain_transformation/behavior.py b/brainscore_vision/model_helpers/brain_transformation/behavior.py index 72c70fb69..967c2a34e 100644 --- a/brainscore_vision/model_helpers/brain_transformation/behavior.py +++ b/brainscore_vision/model_helpers/brain_transformation/behavior.py @@ -43,9 +43,9 @@ def start_task(self, task: BrainModel.Task, choice_labels): self.current_task = task self.choice_labels = choice_labels - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): assert self.current_task == BrainModel.Task.label - logits = self.activations_model(stimuli, layers=['logits']) + logits = self.activations_model(stimuli, layers=['logits'], require_variance=require_variance) choices = self.logits_to_choice(logits) return choices @@ -176,10 +176,10 @@ def start_task(self, task: BrainModel.Task, fitting_stimuli): "stimulus_id ordering is incorrect" self.classifier.fit(fitting_features, fitting_stimuli['image_label']) - def look_at(self, stimuli, number_of_trials=1): + def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): if self.current_task is BrainModel.Task.passive: return - features = self.activations_model(stimuli, layers=self.readout) + features = self.activations_model(stimuli, layers=self.readout, require_variance=require_variance) features = features.transpose('presentation', 'neuroid') prediction = self.classifier.predict_proba(features) return prediction @@ -242,13 +242,13 @@ def start_task(self, task: BrainModel.Task): assert task == BrainModel.Task.odd_one_out self.current_task = task - def look_at(self, triplets, number_of_trials=1): + def look_at(self, triplets, number_of_trials=1, require_variance: bool = False): # Compute unique features and image_pathst stimuli = triplets.drop_duplicates(subset=['stimulus_id']) stimuli = stimuli.sort_values(by='stimulus_id') # Get features - features = self.activations_model(stimuli, layers=self.readout) + features = self.activations_model(stimuli, layers=self.readout, require_variance=require_variance) features = features.transpose('presentation', 'neuroid') # Compute similarity matrix diff --git a/brainscore_vision/model_helpers/brain_transformation/temporal.py b/brainscore_vision/model_helpers/brain_transformation/temporal.py index c9030156f..3413bc3f6 100644 --- a/brainscore_vision/model_helpers/brain_transformation/temporal.py +++ b/brainscore_vision/model_helpers/brain_transformation/temporal.py @@ -28,8 +28,9 @@ def start_recording(self, recording_target: BrainModel.RecordingTarget, time_bin def visual_degrees(self) -> int: return self._layer_model.visual_degrees() - def look_at(self, stimuli, number_of_trials=1): - responses = self._layer_model.look_at(stimuli, number_of_trials=number_of_trials) + def look_at(self, stimuli, number_of_trials=1, require_variance: bool = False): + responses = self._layer_model.look_at(stimuli, number_of_trials=number_of_trials, + require_variance=require_variance) time_responses = [] self._logger.debug(f'Repeating single assembly across time bins {self._time_bins}') for time_bin in self._time_bins: From b256d93605e0f951c840554164940d9525cd16f7 Mon Sep 17 00:00:00 2001 From: Ben Lonnqvist Date: Mon, 18 Mar 2024 13:50:03 +0100 Subject: [PATCH 3/3] Update brainscore_vision/model_interface.py Co-authored-by: Martin Schrimpf --- brainscore_vision/model_interface.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/brainscore_vision/model_interface.py b/brainscore_vision/model_interface.py index 95b3a922c..59571c39c 100644 --- a/brainscore_vision/model_interface.py +++ b/brainscore_vision/model_interface.py @@ -207,9 +207,7 @@ def look_at(self, stimuli: Union[StimulusSet, List[str]], number_of_trials: int or a list of image file paths :param number_of_trials: The number of repeated trials of the stimuli that the model should average over. E.g. 10 or 35. Non-stochastic models can likely ignore this parameter. - :param require_variance: Allows all models access to different activations to the same stimuli, currently - implemented using microsaccades. For more information, see the class - :cls:`~brainscore_vision.model_helpers.activations.core.MicrosaccadeHelper`. + :param require_variance: Whether to require models to return different activations for the same stimuli or not. This is typically true in biological measurements due to e.g. micro-saccades and noise. Ideally this would always be the same in the models, but for compute reasons this flag allows specifying the need more directly. :return: task behaviors or recordings as instructed """ raise NotImplementedError()