diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4f3d518b9..3162d6773 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -28,4 +28,13 @@ Do NOT leave comments about: Aim for fewer, higher-signal comments. A review with 2-3 important comments is better than 15 trivial ones. -Follow `.github/instructions/style-guide.instructions.md` for style guidelines. And look in `.github/instructions/` for specific instructions on the different components. +## Instruction Files + +BEFORE editing or code-reviewing any file, you MUST read the `.github/instructions/` files whose `applyTo` patterns match the files you are about to edit. For example: +- Editing/code-reviewing `pyrit/**/*.py` → read `style-guide.instructions.md` and `user-custom.instructions.md` +- Editing/code-reviewing `pyrit/scenario/**` → also read `scenarios.instructions.md` +- Editing/code-reviewing `pyrit/prompt_converter/**` → also read `converters.instructions.md` +- Editing/code-reviewing `tests/**` → also read `test.instructions.md` +- Editing/code-reviewing `doc/**/*.py` or `doc/**/*.ipynb` → also read `docs.instructions.md` + +Follow every rule in the applicable instruction files. Do not skip this step. diff --git a/.pyrit_conf_example b/.pyrit_conf_example index 1d800ead8..92ec37c4a 100644 --- a/.pyrit_conf_example +++ b/.pyrit_conf_example @@ -24,6 +24,8 @@ memory_db_type: sqlite # Available initializers: # - simple: Basic OpenAI configuration (requires OPENAI_CHAT_* env vars) # - airt: AI Red Team setup with Azure OpenAI (requires AZURE_OPENAI_* env vars) +# - targets: Registers available prompt targets into the TargetRegistry +# - scorers: Registers pre-configured scorers into the ScorerRegistry # - load_default_datasets: Loads default datasets for all registered scenarios # - objective_list: Sets default objectives for scenarios # - openai_objective_target: Sets up OpenAI target for scenarios @@ -38,13 +40,14 @@ memory_db_type: sqlite # Example: # initializers: # - simple -# - name: target +# - name: targets # args: # tags: # - default # - scorer initializers: - name: simple + - name: load_default_datasets - name: scorers - name: targets args: diff --git a/doc/code/scenarios/1_configuring_scenarios.ipynb b/doc/code/scenarios/1_configuring_scenarios.ipynb index b28777532..277e6f487 100644 --- a/doc/code/scenarios/1_configuring_scenarios.ipynb +++ b/doc/code/scenarios/1_configuring_scenarios.ipynb @@ -36,8 +36,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['./.pyrit/.env']\n", - "Loaded environment file: ./.pyrit/.env\n" + "Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n", + "Loaded environment file: ./.pyrit/.env\n", + "Loaded environment file: ./.pyrit/.env.local\n" ] } ], @@ -74,47 +75,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\r", - "Loading datasets - this can take a few minutes: 0%| | 0/46 [00:00 None: + """ + Add tags to an existing registry entry. + + Args: + name: The registry name of the entry to tag. + tags: Tags to add. Accepts a ``dict[str, str]`` + or a ``list[str]`` (each string becomes a key with value ``""``). + + Raises: + KeyError: If no entry with the given name exists. + """ + entry = self._registry_items.get(name) + if entry is None: + raise KeyError(f"No entry named '{name}' in registry.") + entry.tags.update(self._normalize_tags(tags)) + self._metadata_cache = None + def list_metadata( self, *, diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index 443dd6c43..7329b6f72 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -24,14 +24,15 @@ from pyrit.memory.memory_models import ScenarioResultEntry from pyrit.models import AttackResult from pyrit.models.scenario_result import ScenarioIdentifier, ScenarioResult -from pyrit.prompt_target import PromptTarget +from pyrit.prompt_target import OpenAIChatTarget, PromptTarget +from pyrit.registry import ScorerRegistry from pyrit.scenario.core.atomic_attack import AtomicAttack from pyrit.scenario.core.dataset_configuration import DatasetConfiguration from pyrit.scenario.core.scenario_strategy import ( ScenarioCompositeStrategy, ScenarioStrategy, ) -from pyrit.score import Scorer, TrueFalseScorer +from pyrit.score import Scorer, SelfAskRefusalScorer, TrueFalseInverterScorer, TrueFalseScorer if TYPE_CHECKING: from pyrit.executor.attack.core.attack_config import AttackScoringConfig @@ -171,6 +172,19 @@ def default_dataset_config(cls) -> DatasetConfiguration: DatasetConfiguration: The default dataset configuration. """ + def _get_default_objective_scorer(self) -> TrueFalseScorer: + # Deferred import to avoid circular dependency: + from pyrit.setup.initializers.components.scorers import ScorerInitializerTags + + entries = ScorerRegistry.get_registry_singleton().get_by_tag(tag=ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER) + if entries and isinstance(entries[0].instance, TrueFalseScorer): + scorer = entries[0].instance + logger.info(f"Using registered default objective scorer: {type(scorer).__name__}") + return scorer + scorer = TrueFalseInverterScorer(scorer=SelfAskRefusalScorer(chat_target=OpenAIChatTarget())) + logger.info(f"No registered default objective scorer found, using fallback: {type(scorer).__name__}") + return scorer + @apply_defaults async def initialize_async( self, diff --git a/pyrit/scenario/scenarios/airt/content_harms.py b/pyrit/scenario/scenarios/airt/content_harms.py index 0fcc816ad..c92f702c2 100644 --- a/pyrit/scenario/scenarios/airt/content_harms.py +++ b/pyrit/scenario/scenarios/airt/content_harms.py @@ -28,7 +28,7 @@ ScenarioCompositeStrategy, ScenarioStrategy, ) -from pyrit.score import SelfAskRefusalScorer, TrueFalseInverterScorer, TrueFalseScorer +from pyrit.score import TrueFalseScorer logger = logging.getLogger(__name__) @@ -185,8 +185,9 @@ def __init__( removed_in="0.13.0", ) - self._objective_scorer: TrueFalseScorer = objective_scorer if objective_scorer else self._get_default_scorer() - self._scorer_config = AttackScoringConfig(objective_scorer=self._objective_scorer) + self._objective_scorer: TrueFalseScorer = ( + objective_scorer if objective_scorer else self._get_default_objective_scorer() + ) self._adversarial_chat = adversarial_chat if adversarial_chat else self._get_default_adversarial_target() super().__init__( @@ -206,19 +207,6 @@ def _get_default_adversarial_target(self) -> OpenAIChatTarget: temperature=1.2, ) - def _get_default_scorer(self) -> TrueFalseInverterScorer: - endpoint = os.environ.get("AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT") - return TrueFalseInverterScorer( - scorer=SelfAskRefusalScorer( - chat_target=OpenAIChatTarget( - endpoint=endpoint, - api_key=get_azure_openai_auth(endpoint), - model_name=os.environ.get("AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL"), - temperature=0.9, - ) - ), - ) - def _resolve_seed_groups_by_harm(self) -> dict[str, list[SeedAttackGroup]]: """ Resolve seed groups from deprecated objectives_by_harm or dataset configuration. @@ -310,7 +298,7 @@ def _get_single_turn_attacks( """ prompt_sending_attack = PromptSendingAttack( objective_target=self._objective_target, - attack_scoring_config=self._scorer_config, + attack_scoring_config=AttackScoringConfig(objective_scorer=self._objective_scorer), ) role_play_attack = RolePlayAttack( @@ -356,7 +344,7 @@ def _get_multi_turn_attacks( """ many_shot_jailbreak_attack = ManyShotJailbreakAttack( objective_target=self._objective_target, - attack_scoring_config=self._scorer_config, + attack_scoring_config=AttackScoringConfig(objective_scorer=self._objective_scorer), ) tap_attack = TreeOfAttacksWithPruningAttack( diff --git a/pyrit/scenario/scenarios/airt/jailbreak.py b/pyrit/scenario/scenarios/airt/jailbreak.py index c3e1e72db..826622b97 100644 --- a/pyrit/scenario/scenarios/airt/jailbreak.py +++ b/pyrit/scenario/scenarios/airt/jailbreak.py @@ -25,8 +25,6 @@ from pyrit.scenario.core.scenario import Scenario from pyrit.scenario.core.scenario_strategy import ScenarioCompositeStrategy, ScenarioStrategy from pyrit.score import ( - SelfAskRefusalScorer, - TrueFalseInverterScorer, TrueFalseScorer, ) @@ -156,9 +154,9 @@ def __init__( " or `jailbreak_names` (specific selection)." ) - if not objective_scorer: - objective_scorer = self._get_default_objective_scorer() - self._scorer_config = AttackScoringConfig(objective_scorer=objective_scorer) + self._objective_scorer: TrueFalseScorer = ( + objective_scorer if objective_scorer else self._get_default_objective_scorer() + ) self._num_templates = num_templates self._num_attempts = num_attempts @@ -183,7 +181,7 @@ def __init__( super().__init__( version=self.VERSION, strategy_class=JailbreakStrategy, - objective_scorer=objective_scorer, + objective_scorer=self._objective_scorer, include_default_baseline=include_baseline, scenario_result_id=scenario_result_id, ) @@ -191,28 +189,6 @@ def __init__( # Will be resolved in _get_atomic_attacks_async self._seed_groups: Optional[list[SeedAttackGroup]] = None - def _get_default_objective_scorer(self) -> TrueFalseScorer: - """ - Retrieve the default objective scorer. - - The scoring mechanism detects non-refusal: if the target model refuses to - comply with the jailbreak attempt, we want that to register as a negative score. - A successful jailbreak (non-refusal) scores as True. - - Returns: - TrueFalseScorer: A scorer that returns True when the model does NOT refuse. - """ - endpoint = os.getenv("AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT") - return TrueFalseInverterScorer( - scorer=SelfAskRefusalScorer( - chat_target=OpenAIChatTarget( - endpoint=endpoint, - api_key=get_azure_openai_auth(endpoint), - model_name=os.environ.get("AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL"), - ) - ) - ) - def _create_adversarial_target(self) -> OpenAIChatTarget: """ Create a new adversarial target instance. @@ -292,7 +268,7 @@ async def _get_atomic_attack_from_strategy_async( attack: Optional[Union[ManyShotJailbreakAttack, PromptSendingAttack, RolePlayAttack, SkeletonKeyAttack]] = None args = { "objective_target": self._objective_target, - "attack_scoring_config": self._scorer_config, + "attack_scoring_config": AttackScoringConfig(objective_scorer=self._objective_scorer), "attack_converter_config": converter_config, } match strategy: diff --git a/pyrit/scenario/scenarios/foundry/red_team_agent.py b/pyrit/scenario/scenarios/foundry/red_team_agent.py index afbbfabd2..6809b18d1 100644 --- a/pyrit/scenario/scenarios/foundry/red_team_agent.py +++ b/pyrit/scenario/scenarios/foundry/red_team_agent.py @@ -69,14 +69,6 @@ ScenarioCompositeStrategy, ScenarioStrategy, ) -from pyrit.score import ( - AzureContentFilterScorer, - FloatScaleThresholdScorer, - SelfAskRefusalScorer, - TrueFalseCompositeScorer, - TrueFalseInverterScorer, - TrueFalseScoreAggregator, -) if TYPE_CHECKING: from pyrit.executor.attack.core.attack_strategy import AttackStrategy @@ -287,9 +279,9 @@ def __init__( self._objectives = objectives # Store for backward compatibility self._adversarial_chat = adversarial_chat if adversarial_chat else self._get_default_adversarial_target() - self._attack_scoring_config = ( - attack_scoring_config if attack_scoring_config else self._get_default_scoring_config() - ) + if not attack_scoring_config: + attack_scoring_config = AttackScoringConfig(objective_scorer=self._get_default_objective_scorer()) + self._attack_scoring_config = attack_scoring_config objective_scorer = self._attack_scoring_config.objective_scorer if not objective_scorer: @@ -356,26 +348,6 @@ def _get_default_adversarial_target(self) -> OpenAIChatTarget: temperature=1.2, ) - def _get_default_scoring_config(self) -> AttackScoringConfig: - endpoint = os.environ.get("AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT") - objective_scorer = TrueFalseCompositeScorer( - aggregator=TrueFalseScoreAggregator.AND, - scorers=[ - FloatScaleThresholdScorer(scorer=AzureContentFilterScorer(), threshold=0.5), - TrueFalseInverterScorer( - scorer=SelfAskRefusalScorer( - chat_target=OpenAIChatTarget( - endpoint=endpoint, - api_key=get_azure_openai_auth(endpoint), - model_name=os.environ.get("AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL"), - temperature=0.9, - ) - ), - ), - ], - ) - return AttackScoringConfig(objective_scorer=objective_scorer) - def _get_attack_from_strategy(self, composite_strategy: ScenarioCompositeStrategy) -> AtomicAttack: """ Get an atomic attack for the specified strategy composition. diff --git a/pyrit/score/__init__.py b/pyrit/score/__init__.py index 39706be1d..5ff485883 100644 --- a/pyrit/score/__init__.py +++ b/pyrit/score/__init__.py @@ -45,6 +45,7 @@ ScorerMetricsWithIdentity, ) from pyrit.score.scorer_evaluation.scorer_metrics_io import ( + find_objective_metrics_by_eval_hash, get_all_harm_metrics, get_all_objective_metrics, ) @@ -114,6 +115,7 @@ "ScorerMetricsWithIdentity", "get_all_harm_metrics", "get_all_objective_metrics", + "find_objective_metrics_by_eval_hash", "ScorerPromptValidator", "SelfAskCategoryScorer", "SelfAskGeneralFloatScaleScorer", diff --git a/pyrit/score/audio_transcript_scorer.py b/pyrit/score/audio_transcript_scorer.py index 1395e3b96..9c7e7e3f4 100644 --- a/pyrit/score/audio_transcript_scorer.py +++ b/pyrit/score/audio_transcript_scorer.py @@ -39,7 +39,7 @@ def _is_compliant_wav(input_path: str, *, sample_rate: int, channels: int) -> bo is_pcm_s16 = codec_name == "pcm_s16le" is_correct_rate = stream.rate == sample_rate is_correct_channels = stream.channels == channels - return is_pcm_s16 and is_correct_rate and is_correct_channels + return bool(is_pcm_s16 and is_correct_rate and is_correct_channels) except Exception: return False diff --git a/pyrit/setup/initializers/components/__init__.py b/pyrit/setup/initializers/components/__init__.py index 0049708f4..f38b2032e 100644 --- a/pyrit/setup/initializers/components/__init__.py +++ b/pyrit/setup/initializers/components/__init__.py @@ -3,11 +3,13 @@ """Component initializers for targets, scorers, and other components.""" -from pyrit.setup.initializers.components.scorers import ScorerInitializer -from pyrit.setup.initializers.components.targets import TargetConfig, TargetInitializer +from pyrit.setup.initializers.components.scorers import ScorerInitializer, ScorerInitializerTags +from pyrit.setup.initializers.components.targets import TargetConfig, TargetInitializer, TargetInitializerTags __all__ = [ "ScorerInitializer", + "ScorerInitializerTags", "TargetConfig", "TargetInitializer", + "TargetInitializerTags", ] diff --git a/pyrit/setup/initializers/components/scorers.py b/pyrit/setup/initializers/components/scorers.py index 06b304ebc..b845994da 100644 --- a/pyrit/setup/initializers/components/scorers.py +++ b/pyrit/setup/initializers/components/scorers.py @@ -12,7 +12,8 @@ import logging from collections.abc import Callable -from typing import TYPE_CHECKING, Literal, Optional +from enum import Enum +from typing import TYPE_CHECKING, Optional from azure.ai.contentsafety.models import TextCategory @@ -30,6 +31,7 @@ TrueFalseInverterScorer, TrueFalseQuestionPaths, TrueFalseScoreAggregator, + find_objective_metrics_by_eval_hash, ) from pyrit.setup.initializers.pyrit_initializer import InitializerParameter, PyRITInitializer @@ -38,8 +40,14 @@ logger = logging.getLogger(__name__) -# Shared tag type with TargetInitializer -ScorerTag = Literal["default"] + +class ScorerInitializerTags(str, Enum): + """Tags applied to scorer registry entries by ScorerInitializer.""" + + DEFAULT = "default" + BEST_OBJECTIVE_F1 = "best_objective_f1" + DEFAULT_OBJECTIVE_SCORER = "default_objective_scorer" + # Target registry names used by scorer configurations. GPT4O_TARGET: str = "azure_openai_gpt4o" @@ -284,6 +292,32 @@ async def initialize_async(self) -> None: gpt4o, ) + # Register the scorer with the best objective F1 score + self._register_best_objective_f1(scorer_registry) + + def _register_best_objective_f1(self, scorer_registry: ScorerRegistry) -> None: + """Find the registered scorer with the highest objective F1 and tag it as best_objective_f1.""" + best_name: str | None = None + best_f1: float = -1.0 + + for entry in scorer_registry.get_all_instances(): + eval_hash = entry.instance.get_identifier().eval_hash + if not eval_hash: + continue + metrics = find_objective_metrics_by_eval_hash(eval_hash=eval_hash) + if metrics is not None and metrics.f1_score > best_f1: + best_f1 = metrics.f1_score + best_name = entry.name + + if best_name is not None: + scorer_registry.add_tags( + name=best_name, + tags=[ScorerInitializerTags.BEST_OBJECTIVE_F1, ScorerInitializerTags.DEFAULT_OBJECTIVE_SCORER], + ) + logger.info(f"Tagged {best_name} as {ScorerInitializerTags.BEST_OBJECTIVE_F1} with F1={best_f1:.4f}") + else: + logger.warning("No registered scorer with objective metrics; skipping best_objective_f1 tagging.") + def _try_register( self, scorer_registry: ScorerRegistry, diff --git a/pyrit/setup/initializers/components/targets.py b/pyrit/setup/initializers/components/targets.py index 87715c884..178e5d290 100644 --- a/pyrit/setup/initializers/components/targets.py +++ b/pyrit/setup/initializers/components/targets.py @@ -15,7 +15,8 @@ import logging import os from dataclasses import dataclass, field -from typing import Any, Literal, Optional +from enum import Enum +from typing import Any, Optional from pyrit.auth import get_azure_openai_auth, get_azure_token_provider from pyrit.prompt_target import ( @@ -36,10 +37,12 @@ logger = logging.getLogger(__name__) -# Literal type for target tags -TargetTag = Literal["default", "scorer", "all"] +class TargetInitializerTags(str, Enum): + """Tags used by TargetInitializer for filtering which targets to register.""" -ALL_TARGET_TAGS: list[str] = ["default", "scorer"] + DEFAULT = "default" + SCORER = "scorer" + ALL = "all" @dataclass @@ -66,7 +69,7 @@ class TargetConfig: underlying_model_var: Optional[str] = None temperature: Optional[float] = None extra_kwargs: dict[str, Any] = field(default_factory=dict) - tags: list[TargetTag] = field(default_factory=lambda: ["default"]) + tags: list[TargetInitializerTags] = field(default_factory=lambda: [TargetInitializerTags.DEFAULT]) # Define all supported target configurations. @@ -322,7 +325,7 @@ class TargetConfig: model_var="AZURE_OPENAI_GPT4O_MODEL", underlying_model_var="AZURE_OPENAI_GPT4O_UNDERLYING_MODEL", temperature=0.0, - tags=["scorer"], + tags=[TargetInitializerTags.SCORER], ), TargetConfig( registry_name="azure_openai_gpt4o_temp9", @@ -332,7 +335,7 @@ class TargetConfig: model_var="AZURE_OPENAI_GPT4O_MODEL", underlying_model_var="AZURE_OPENAI_GPT4O_UNDERLYING_MODEL", temperature=0.9, - tags=["scorer"], + tags=[TargetInitializerTags.SCORER], ), TargetConfig( registry_name="azure_gpt4o_unsafe_chat_temp0", @@ -342,7 +345,7 @@ class TargetConfig: model_var="AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL", underlying_model_var="AZURE_OPENAI_GPT4O_UNSAFE_CHAT_UNDERLYING_MODEL", temperature=0.0, - tags=["scorer"], + tags=[TargetInitializerTags.SCORER], ), TargetConfig( registry_name="azure_gpt4o_unsafe_chat_temp9", @@ -352,7 +355,7 @@ class TargetConfig: model_var="AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL", underlying_model_var="AZURE_OPENAI_GPT4O_UNSAFE_CHAT_UNDERLYING_MODEL", temperature=0.9, - tags=["scorer"], + tags=[TargetInitializerTags.SCORER], ), ] @@ -480,8 +483,8 @@ async def initialize_async(self) -> None: tags matching the configured tags are registered. """ tags = self.params.get("tags", ["default"]) - if "all" in tags: - tags = ALL_TARGET_TAGS + if TargetInitializerTags.ALL in tags: + tags = [tag for tag in TargetInitializerTags if tag != TargetInitializerTags.ALL] for config in TARGET_CONFIGS: if not any(tag in tags for tag in config.tags): diff --git a/tests/unit/registry/test_base_instance_registry.py b/tests/unit/registry/test_base_instance_registry.py index 91ef0b686..08ea2c132 100644 --- a/tests/unit/registry/test_base_instance_registry.py +++ b/tests/unit/registry/test_base_instance_registry.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import pytest + from pyrit.identifiers import ComponentIdentifier from pyrit.registry.instance_registries.base_instance_registry import BaseInstanceRegistry, RegistryEntry @@ -453,3 +455,67 @@ def test_iter_allows_for_loop(self): """Test that the registry can be used in a for loop.""" collected = list(self.registry) assert collected == ["name1", "name2"] + + +class TestBaseInstanceRegistryAddTags: + """Tests for add_tags functionality in BaseInstanceRegistry.""" + + def setup_method(self): + """Reset and get a fresh registry for each test.""" + ConcreteTestRegistry.reset_instance() + self.registry = ConcreteTestRegistry.get_registry_singleton() + + def teardown_method(self): + """Reset the singleton after each test.""" + ConcreteTestRegistry.reset_instance() + + def test_add_tags_with_list(self): + """Test adding list-style tags to an existing entry.""" + self.registry.register("value", name="entry1") + self.registry.add_tags(name="entry1", tags=["fast", "default"]) + + entry = self.registry.get_entry("entry1") + assert entry is not None + assert entry.tags == {"fast": "", "default": ""} + + def test_add_tags_with_dict(self): + """Test adding dict-style tags to an existing entry.""" + self.registry.register("value", name="entry1") + self.registry.add_tags(name="entry1", tags={"role": "scorer"}) + + entry = self.registry.get_entry("entry1") + assert entry is not None + assert entry.tags == {"role": "scorer"} + + def test_add_tags_merges_with_existing(self): + """Test that add_tags merges new tags with existing ones.""" + self.registry.register("value", name="entry1", tags={"existing": "yes"}) + self.registry.add_tags(name="entry1", tags=["new_tag"]) + + entry = self.registry.get_entry("entry1") + assert entry is not None + assert entry.tags == {"existing": "yes", "new_tag": ""} + + def test_add_tags_raises_for_missing_entry(self): + """Test that add_tags raises KeyError for a non-existent entry.""" + with pytest.raises(KeyError, match="No entry named 'missing'"): + self.registry.add_tags(name="missing", tags=["tag"]) + + def test_add_tags_invalidates_metadata_cache(self): + """Test that add_tags invalidates the metadata cache.""" + self.registry.register("value", name="entry1") + self.registry.list_metadata() # Build cache + + self.registry.add_tags(name="entry1", tags=["new"]) + + # Cache should be invalidated (None), next call rebuilds + assert self.registry._metadata_cache is None + + def test_add_tags_entries_findable_by_get_by_tag(self): + """Test that entries are findable via get_by_tag after add_tags.""" + self.registry.register("value", name="entry1") + self.registry.add_tags(name="entry1", tags=["best_scorer"]) + + results = self.registry.get_by_tag(tag="best_scorer") + assert len(results) == 1 + assert results[0].name == "entry1" diff --git a/tests/unit/scenarios/test_content_harms.py b/tests/unit/scenarios/test_content_harms.py index ef81b03ba..33a20c6df 100644 --- a/tests/unit/scenarios/test_content_harms.py +++ b/tests/unit/scenarios/test_content_harms.py @@ -223,7 +223,7 @@ class TestContentHarmsBasic: """Basic tests for ContentHarms initialization and properties.""" @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_initialization_with_minimal_parameters( self, @@ -251,7 +251,7 @@ async def test_initialization_with_minimal_parameters( assert scenario._objective_target == mock_objective_target @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_initialization_with_custom_strategies( self, @@ -287,11 +287,11 @@ def test_initialization_with_custom_scorer( objective_scorer=mock_objective_scorer, ) - # The scorer is stored in _scorer_config.objective_scorer - assert scenario._scorer_config.objective_scorer == mock_objective_scorer + # The scorer is stored in _objective_scorer + assert scenario._objective_scorer == mock_objective_scorer @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_initialization_with_custom_max_concurrency( self, @@ -313,7 +313,7 @@ async def test_initialization_with_custom_max_concurrency( assert scenario._max_concurrency == 10 @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_initialization_with_custom_dataset_path( self, @@ -336,7 +336,7 @@ async def test_initialization_with_custom_dataset_path( assert scenario is not None @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_initialization_defaults_to_all_strategy( self, @@ -362,6 +362,7 @@ def test_get_default_strategy_returns_all(self): """Test that get_default_strategy returns ALL strategy.""" assert ContentHarms.get_default_strategy() == ContentHarmsStrategy.ALL + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch.dict( "os.environ", { @@ -370,12 +371,14 @@ def test_get_default_strategy_returns_all(self): "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL": "gpt-4", }, ) - def test_get_default_adversarial_target(self, mock_objective_target): + def test_get_default_adversarial_target(self, mock_get_scorer, mock_objective_target, mock_objective_scorer): """Test default adversarial target creation.""" + mock_get_scorer.return_value = mock_objective_scorer scenario = ContentHarms() assert scenario._adversarial_chat is not None + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch.dict( "os.environ", { @@ -384,16 +387,18 @@ def test_get_default_adversarial_target(self, mock_objective_target): "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL": "gpt-4", }, ) - def test_get_default_scorer(self, mock_objective_target): - """Test default scorer creation.""" + def test_get_default_objective_scorer(self, mock_get_scorer, mock_objective_target, mock_objective_scorer): + """Test default objective scorer is set from base class.""" + mock_get_scorer.return_value = mock_objective_scorer scenario = ContentHarms() - assert scenario._objective_scorer is not None + assert scenario._objective_scorer == mock_objective_scorer def test_scenario_version(self): """Test that scenario has correct version.""" assert ContentHarms.VERSION == 1 + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch.dict( "os.environ", { @@ -404,9 +409,10 @@ def test_scenario_version(self): ) @pytest.mark.asyncio async def test_initialize_raises_exception_when_no_datasets_available( - self, mock_objective_target, mock_adversarial_target + self, mock_get_scorer, mock_objective_target, mock_adversarial_target, mock_objective_scorer ): """Test that initialization raises ValueError when datasets are not available in memory.""" + mock_get_scorer.return_value = mock_objective_scorer # Don't mock _get_objectives_by_harm, let it try to load from empty memory scenario = ContentHarms(adversarial_chat=mock_adversarial_target) @@ -414,7 +420,7 @@ async def test_initialize_raises_exception_when_no_datasets_available( await scenario.initialize_async(objective_target=mock_objective_target) @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_initialization_with_max_retries( self, @@ -436,7 +442,7 @@ async def test_initialization_with_max_retries( assert scenario._max_retries == 3 @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_memory_labels_are_stored( self, @@ -493,12 +499,13 @@ async def test_initialization_with_all_parameters( assert scenario._objective_target == mock_objective_target assert scenario._adversarial_chat == mock_adversarial_target - assert scenario._scorer_config.objective_scorer == mock_objective_scorer + assert scenario._objective_scorer == mock_objective_scorer assert scenario._memory_labels == memory_labels assert scenario._max_concurrency == 5 assert scenario._max_retries == 2 @pytest.mark.asyncio + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") @patch.dict( "os.environ", @@ -509,7 +516,13 @@ async def test_initialization_with_all_parameters( }, ) async def test_initialization_with_objectives_by_harm( - self, mock_get_seed_attack_groups, mock_objective_target, mock_adversarial_target, mock_seed_groups + self, + mock_get_seed_attack_groups, + mock_get_scorer, + mock_objective_target, + mock_adversarial_target, + mock_objective_scorer, + mock_seed_groups, ): """Test initialization with custom objectives_by_harm parameter.""" # Setup custom objectives by harm @@ -518,6 +531,7 @@ async def test_initialization_with_objectives_by_harm( "violence": mock_seed_groups("violence"), } + mock_get_scorer.return_value = mock_objective_scorer mock_get_seed_attack_groups.return_value = custom_objectives scenario = ContentHarms( @@ -705,7 +719,7 @@ class TestContentHarmsAttackGroups: """Tests for the single-turn and multi-turn attack generation.""" @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_get_single_turn_attacks_returns_prompt_sending_and_role_play( self, @@ -737,7 +751,7 @@ async def test_get_single_turn_attacks_returns_prompt_sending_and_role_play( assert RolePlayAttack in attack_types @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_get_multi_turn_attacks_returns_many_shot_and_tap( self, @@ -769,7 +783,7 @@ async def test_get_multi_turn_attacks_returns_many_shot_and_tap( assert TreeOfAttacksWithPruningAttack in attack_types @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_get_strategy_attacks_includes_all_groups( self, @@ -809,7 +823,7 @@ async def test_get_strategy_attacks_includes_all_groups( assert TreeOfAttacksWithPruningAttack in attack_types @pytest.mark.asyncio - @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarms._get_default_scorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch("pyrit.scenario.scenarios.airt.content_harms.ContentHarmsDatasetConfiguration.get_seed_attack_groups") async def test_get_strategy_attacks_raises_when_not_initialized( self, diff --git a/tests/unit/scenarios/test_foundry.py b/tests/unit/scenarios/test_foundry.py index dc41d8b7a..6558ac8c4 100644 --- a/tests/unit/scenarios/test_foundry.py +++ b/tests/unit/scenarios/test_foundry.py @@ -241,30 +241,28 @@ async def test_init_with_memory_labels( assert scenario._memory_labels == memory_labels - @patch("pyrit.scenario.scenarios.foundry.red_team_agent.TrueFalseCompositeScorer") + @patch("pyrit.scenario.core.scenario.Scenario._get_default_objective_scorer") @patch.dict( "os.environ", { "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT": "https://test.openai.azure.com/", "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY": "test-key", "AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL": "gpt-4", - "AZURE_CONTENT_SAFETY_API_ENDPOINT": "https://test.content.azure.com/", - "AZURE_CONTENT_SAFETY_API_KEY": "test-content-key", }, ) def test_init_creates_default_scorer_when_not_provided( - self, mock_composite, mock_objective_target, mock_memory_seed_groups + self, mock_get_scorer, mock_objective_target, mock_memory_seed_groups ): """Test that initialization creates default scorer when not provided.""" - # Mock the composite scorer - mock_composite_instance = MagicMock(spec=TrueFalseScorer) - mock_composite.return_value = mock_composite_instance + mock_scorer_instance = MagicMock(spec=TrueFalseScorer) + mock_get_scorer.return_value = mock_scorer_instance with patch.object(RedTeamAgent, "_resolve_seed_groups", return_value=mock_memory_seed_groups): scenario = RedTeamAgent() - # Verify default scorer was created - mock_composite.assert_called_once() + # Verify default scorer was used + mock_get_scorer.assert_called_once() + assert scenario._attack_scoring_config.objective_scorer == mock_scorer_instance # seed_groups are resolved lazily during _get_atomic_attacks_async assert scenario._objectives is None diff --git a/tests/unit/scenarios/test_jailbreak.py b/tests/unit/scenarios/test_jailbreak.py index 666dc5cff..6ee4d4bd0 100644 --- a/tests/unit/scenarios/test_jailbreak.py +++ b/tests/unit/scenarios/test_jailbreak.py @@ -7,7 +7,6 @@ import pytest -from pyrit.executor.attack.core.attack_config import AttackScoringConfig from pyrit.executor.attack.single_turn.many_shot_jailbreak import ManyShotJailbreakAttack from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack from pyrit.executor.attack.single_turn.role_play import RolePlayAttack @@ -159,7 +158,7 @@ def test_init_with_custom_scorer(self, mock_objective_scorer, mock_memory_seed_g """Test initialization with custom scorer.""" with patch.object(Jailbreak, "_resolve_seed_groups", return_value=mock_memory_seed_groups): scenario = Jailbreak(objective_scorer=mock_objective_scorer) - assert isinstance(scenario._scorer_config, AttackScoringConfig) + assert scenario._objective_scorer == mock_objective_scorer def test_init_with_num_templates(self, mock_random_num_templates): """Test initialization with num_templates provided.""" @@ -438,7 +437,7 @@ async def test_no_target_duplication_async( await scenario.initialize_async(objective_target=mock_objective_target) objective_target = scenario._objective_target - scorer_target = scenario._scorer_config.objective_scorer # type: ignore[arg-type] + scorer_target = scenario._objective_scorer assert objective_target != scorer_target diff --git a/tests/unit/scenarios/test_scenario.py b/tests/unit/scenarios/test_scenario.py index 50192add6..7f0298201 100644 --- a/tests/unit/scenarios/test_scenario.py +++ b/tests/unit/scenarios/test_scenario.py @@ -3,7 +3,7 @@ """Tests for the scenarios.Scenario class.""" -from unittest.mock import AsyncMock, MagicMock, PropertyMock +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pytest @@ -806,3 +806,38 @@ async def test_standalone_baseline_uses_dataset_config_seeds(self, mock_objectiv baseline_attack = scenario._atomic_attacks[0] assert baseline_attack.atomic_attack_name == "baseline" assert baseline_attack.seed_groups == expected_seeds + + +class TestGetDefaultObjectiveScorer: + """Tests for Scenario._get_default_objective_scorer method.""" + + @patch("pyrit.scenario.core.scenario.ScorerRegistry") + def test_returns_registry_scorer_when_tagged(self, mock_registry_cls) -> None: + """Test that a tagged scorer from the registry is returned.""" + from pyrit.score import TrueFalseScorer + + mock_scorer = MagicMock(spec=TrueFalseScorer) + mock_scorer.__class__ = TrueFalseScorer + + mock_entry = MagicMock() + mock_entry.instance = mock_scorer + + mock_registry = MagicMock() + mock_registry.get_by_tag.return_value = [mock_entry] + mock_registry_cls.get_registry_singleton.return_value = mock_registry + + result = Scenario._get_default_objective_scorer(MagicMock()) + assert result is mock_scorer + + @patch("pyrit.scenario.core.scenario.OpenAIChatTarget") + @patch("pyrit.scenario.core.scenario.ScorerRegistry") + def test_returns_fallback_when_registry_empty(self, mock_registry_cls, mock_oai_target) -> None: + """Test fallback to TrueFalseInverterScorer when no tagged scorer exists.""" + from pyrit.score import TrueFalseInverterScorer + + mock_registry = MagicMock() + mock_registry.get_by_tag.return_value = [] + mock_registry_cls.get_registry_singleton.return_value = mock_registry + + result = Scenario._get_default_objective_scorer(MagicMock()) + assert isinstance(result, TrueFalseInverterScorer) diff --git a/tests/unit/setup/test_scorer_initializer.py b/tests/unit/setup/test_scorer_initializer.py index a59057e32..f3449a244 100644 --- a/tests/unit/setup/test_scorer_initializer.py +++ b/tests/unit/setup/test_scorer_initializer.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. import os -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest @@ -228,3 +228,118 @@ async def test_get_info_execution_order_is_two(self) -> None: """Test that get_info reports execution_order of 2.""" info = await ScorerInitializer.get_info_async() assert info["execution_order"] == 2 + + +@pytest.mark.usefixtures("patch_central_database") +class TestScorerInitializerBestObjectiveF1: + """Tests for _register_best_objective_f1 tagging behavior.""" + + def setup_method(self) -> None: + """Reset registries before each test.""" + ScorerRegistry.reset_instance() + TargetRegistry.reset_instance() + + def teardown_method(self) -> None: + """Clean up after each test.""" + ScorerRegistry.reset_instance() + TargetRegistry.reset_instance() + + def _register_mock_target(self, *, name: str) -> OpenAIChatTarget: + """Register a mock OpenAIChatTarget in the TargetRegistry.""" + target = MagicMock(spec=OpenAIChatTarget) + target._temperature = None + target._endpoint = f"https://test-{name}.openai.azure.com" + target._api_key = "test_key" + target._model_name = "test-model" + target._underlying_model = "gpt-4o" + registry = TargetRegistry.get_registry_singleton() + registry.register_instance(target, name=name) + return target + + @pytest.mark.asyncio + @patch("pyrit.setup.initializers.components.scorers.find_objective_metrics_by_eval_hash") + async def test_best_objective_f1_tags_best_scorer(self, mock_find_metrics) -> None: + """Test that _register_best_objective_f1 tags the scorer with highest F1.""" + self._register_mock_target(name=GPT4O_TARGET) + + mock_metrics = MagicMock() + mock_metrics.f1_score = 0.85 + mock_find_metrics.return_value = mock_metrics + + init = ScorerInitializer() + await init.initialize_async() + + registry = ScorerRegistry.get_registry_singleton() + results = registry.get_by_tag(tag="best_objective_f1") + assert len(results) >= 1 + + @pytest.mark.asyncio + @patch("pyrit.setup.initializers.components.scorers.find_objective_metrics_by_eval_hash") + async def test_best_objective_f1_no_metrics_skips_tagging(self, mock_find_metrics) -> None: + """Test that no scorer is tagged when no metrics are available.""" + self._register_mock_target(name=GPT4O_TARGET) + mock_find_metrics.return_value = None + + init = ScorerInitializer() + await init.initialize_async() + + registry = ScorerRegistry.get_registry_singleton() + results = registry.get_by_tag(tag="best_objective_f1") + assert len(results) == 0 + + @pytest.mark.asyncio + @patch("pyrit.setup.initializers.components.scorers.find_objective_metrics_by_eval_hash") + async def test_best_objective_f1_picks_highest_f1(self, mock_find_metrics) -> None: + """Test that the scorer with the highest F1 score gets tagged.""" + self._register_mock_target(name=GPT4O_TARGET) + self._register_mock_target(name=GPT4O_TEMP9_TARGET) + + def mock_metrics_by_hash(*, eval_hash: str) -> MagicMock | None: + metrics = MagicMock() + if "refusal" in eval_hash.lower() if eval_hash else False: + metrics.f1_score = 0.5 + return metrics + # Default: return higher F1 for all scorers that have a hash + metrics.f1_score = 0.9 + return metrics + + mock_find_metrics.side_effect = mock_metrics_by_hash + + init = ScorerInitializer() + await init.initialize_async() + + registry = ScorerRegistry.get_registry_singleton() + results = registry.get_by_tag(tag="best_objective_f1") + assert len(results) == 1 + # Should also have the default_objective_scorer tag + assert "default_objective_scorer" in results[0].tags + + @pytest.mark.asyncio + @patch("pyrit.setup.initializers.components.scorers.find_objective_metrics_by_eval_hash") + async def test_best_objective_f1_does_not_add_extra_entry(self, mock_find_metrics) -> None: + """Test that tagging best_objective_f1 doesn't increase registry count.""" + self._register_mock_target(name=GPT4O_TARGET) + + mock_metrics = MagicMock() + mock_metrics.f1_score = 0.85 + mock_find_metrics.return_value = mock_metrics + + init = ScorerInitializer() + await init.initialize_async() + + registry = ScorerRegistry.get_registry_singleton() + count_with_tag = len(registry) + + # Reset and run without metrics to get baseline count + ScorerRegistry.reset_instance() + TargetRegistry.reset_instance() + self._register_mock_target(name=GPT4O_TARGET) + mock_find_metrics.return_value = None + + init2 = ScorerInitializer() + await init2.initialize_async() + + registry2 = ScorerRegistry.get_registry_singleton() + count_without_tag = len(registry2) + + assert count_with_tag == count_without_tag diff --git a/uv.lock b/uv.lock index c27a32692..968200513 100644 --- a/uv.lock +++ b/uv.lock @@ -396,6 +396,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/23/b65f568ed0c22f1efacb744d2db1a33c8068f384b8c9b482b52ebdbc3ef6/authlib-1.6.9-py2.py3-none-any.whl", hash = "sha256:f08b4c14e08f0861dc18a32357b33fbcfd2ea86cfe3fe149484b4d764c4a0ac3", size = 244197, upload-time = "2026-03-02T07:44:00.307Z" }, ] +[[package]] +name = "av" +version = "17.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/eb/abca886df3a091bc406feb5ff71b4c4f426beaae6b71b9697264ce8c7211/av-17.0.0.tar.gz", hash = "sha256:c53685df73775a8763c375c7b2d62a6cb149d992a26a4b098204da42ade8c3df", size = 4410769, upload-time = "2026-03-14T14:38:45.868Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/4d/ea1ac272eeea83014daca1783679a9e9f894e1e68e5eb4f717dd8813da2a/av-17.0.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:4b21bcff4144acae658c0efb011fa8668c7a9638384f3ae7f5add33f35b907c6", size = 23407827, upload-time = "2026-03-14T14:37:47.337Z" }, + { url = "https://files.pythonhosted.org/packages/54/1a/e433766470c57c9c1c8558021de4d2466b3403ed629e48722d39d12baa6c/av-17.0.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:17cd518fc88dc449ce9dcfd0b40e9b3530266927375a743efc80d510adfb188b", size = 18829899, upload-time = "2026-03-14T14:37:50.493Z" }, + { url = "https://files.pythonhosted.org/packages/5f/25/95ad714f950c188495ffbfef235d06a332123d6f266026a534801ffc2171/av-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9a8b7b63a92d8dc7cbe5000546e4684176124ddd49fdd9c12570e3aa6dadf11a", size = 35348062, upload-time = "2026-03-14T14:37:52.964Z" }, + { url = "https://files.pythonhosted.org/packages/7a/db/7f3f9e92f2ac8dba639ab01d69a33b723aa16b5e3e612dbfe667fbc02dcd/av-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:8706ce9b5d8d087d093b46a9781e7532c4a9e13874bca1da468be78efc56cecc", size = 37684503, upload-time = "2026-03-14T14:37:55.628Z" }, + { url = "https://files.pythonhosted.org/packages/c1/53/3b356b14ba72354688c8d9777cf67b707769b6e14b63aaeb0cddeeac8d32/av-17.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3a074835ce807434451086993fedfb3b223dacedb2119ab9d7a72480f2d77f32", size = 36547601, upload-time = "2026-03-14T14:37:58.465Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8d/f489cd6f9fe9c8b38dca00ecb39dc38836761767a4ec07dd95e62e124ac3/av-17.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f8ef8e8f1a0cbb2e0ad49266015e2277801a916e2186ac9451b493ff6dfdec27", size = 38815129, upload-time = "2026-03-14T14:38:01.277Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bd/e42536234e37caffd1a054de1a0e6abca226c5686e9672726a8d95511422/av-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a795e153ff31a6430e974b4e6ad0d0fab695b78e3f17812293a0a34cd03ee6a9", size = 28984602, upload-time = "2026-03-14T14:38:03.632Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fb/55e3b5b5d1fc61466292f26fbcbabafa2642f378dc48875f8f554591e1a4/av-17.0.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ed4013fac77c309a4a68141dcf6148f1821bb1073a36d4289379762a6372f711", size = 23238424, upload-time = "2026-03-14T14:38:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/52/03/9ace1acc08bc9ae38c14bf3a4b1360e995e4d999d1d33c2cbd7c9e77582a/av-17.0.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:e44b6c83e9f3be9f79ee87d0b77a27cea9a9cd67bd630362c86b7e56a748dfbb", size = 18709043, upload-time = "2026-03-14T14:38:08.288Z" }, + { url = "https://files.pythonhosted.org/packages/00/c0/637721f3cd5bb8bd16105a1a08efd781fc12f449931bdb3a4d0cfd63fa55/av-17.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b440da6ac47da0629d509316f24bcd858f33158dbdd0f1b7293d71e99beb26de", size = 34018780, upload-time = "2026-03-14T14:38:10.45Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/d19bc3257dd985d55337d7f0414c019414b97e16cd3690ebf9941a847543/av-17.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1060cba85f97f4a337311169d92c0b5e143452cfa5ca0e65fa499d7955e8592e", size = 36358757, upload-time = "2026-03-14T14:38:13.092Z" }, + { url = "https://files.pythonhosted.org/packages/52/6c/a1f4f2677bae6f2ade7a8a18e90ebdcf70690c9b1c4e40e118aa30fa313f/av-17.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:deda202e6021cfc7ba3e816897760ec5431309d59a4da1f75df3c0e9413d71e7", size = 35195281, upload-time = "2026-03-14T14:38:15.789Z" }, + { url = "https://files.pythonhosted.org/packages/90/ea/52b0fc6f69432c7bf3f5fbe6f707113650aa40a1a05b9096ffc2bba4f77d/av-17.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffaf266a1a9c2148072de0a4b5ae98061465178d2cfaa69ee089761149342974", size = 37444817, upload-time = "2026-03-14T14:38:18.563Z" }, + { url = "https://files.pythonhosted.org/packages/34/ad/d2172966282cb8f146c13b6be7416efefde74186460c5e1708ddfc13dba6/av-17.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:45a35a40b2875bf2f98de7c952d74d960f92f319734e6d28e03b4c62a49e6f49", size = 28888553, upload-time = "2026-03-14T14:38:21.223Z" }, + { url = "https://files.pythonhosted.org/packages/b0/bb/c5a4c4172c514d631fb506e6366b503576b8c7f29809cf42aca73e28ff01/av-17.0.0-cp311-abi3-win_arm64.whl", hash = "sha256:3d32e9b5c5bbcb872a0b6917b352a1db8a42142237826c9b49a36d5dbd9e9c26", size = 21916910, upload-time = "2026-03-14T14:38:23.706Z" }, +] + [[package]] name = "azure-ai-contentsafety" version = "1.0.0" @@ -5752,6 +5775,7 @@ dependencies = [ { name = "aiofiles" }, { name = "appdirs" }, { name = "art" }, + { name = "av" }, { name = "azure-ai-contentsafety" }, { name = "azure-core" }, { name = "azure-identity" }, @@ -5792,6 +5816,7 @@ dependencies = [ [package.optional-dependencies] all = [ { name = "accelerate" }, + { name = "av" }, { name = "azure-ai-ml" }, { name = "azure-cognitiveservices-speech" }, { name = "azureml-mlflow" }, @@ -5884,6 +5909,8 @@ requires-dist = [ { name = "aiofiles", specifier = ">=24,<25" }, { name = "appdirs", specifier = ">=1.4.0" }, { name = "art", specifier = ">=6.5.0" }, + { name = "av", specifier = ">=14.0.0" }, + { name = "av", marker = "extra == 'all'", specifier = ">=14.0.0" }, { name = "azure-ai-contentsafety", specifier = ">=1.0.0" }, { name = "azure-ai-ml", marker = "extra == 'all'", specifier = ">=1.27.1" }, { name = "azure-ai-ml", marker = "extra == 'gcg'", specifier = ">=1.27.1" },