From 8e138309d43de44bb696c1cd135d7bc43cdaa17d Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Tue, 24 Mar 2026 11:33:07 -0400 Subject: [PATCH 1/4] FEAT: Add partner integration tests for azure-ai-evaluation red team module Add tests/partner_integration/azure_ai_evaluation/ with contract tests validating PyRIT API stability for the azure-ai-evaluation red team module, which depends on 45+ PyRIT imports across 14 files. Test coverage includes: - PromptChatTarget interface contract (extended by 4 SDK classes) - CentralMemory/SQLiteMemory lifecycle (used in RedTeam.__init__) - Data models: Message, MessagePiece, Score, seed models, AttackResult - PromptConverter base + 19 specific converters importability - Scorer/TrueFalseScorer interface (extended by RAIServiceScorer) - Foundry scenario APIs: FoundryScenario, FoundryStrategy, DatasetConfiguration - Exception types and retry decorators - Import smoke tests for azure-ai-evaluation (skipped if not installed) Also adds partner-integration-test target to Makefile. All 84 tests pass with no Azure credentials required. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Makefile | 4 + tests/partner_integration/__init__.py | 2 + .../azure_ai_evaluation/__init__.py | 2 + .../test_converter_contract.py | 98 ++++++++++ .../test_exceptions_contract.py | 65 +++++++ .../test_foundry_contract.py | 78 ++++++++ .../azure_ai_evaluation/test_import_smoke.py | 74 ++++++++ .../test_memory_contract.py | 59 ++++++ .../test_model_contract.py | 173 ++++++++++++++++++ .../test_prompt_target_contract.py | 102 +++++++++++ .../test_scorer_contract.py | 65 +++++++ tests/partner_integration/conftest.py | 57 ++++++ 12 files changed, 779 insertions(+) create mode 100644 tests/partner_integration/__init__.py create mode 100644 tests/partner_integration/azure_ai_evaluation/__init__.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_converter_contract.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_exceptions_contract.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_import_smoke.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_memory_contract.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_model_contract.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py create mode 100644 tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py create mode 100644 tests/partner_integration/conftest.py diff --git a/Makefile b/Makefile index 0b0c33cc21..699feea617 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PYMODULE:=pyrit TESTS:=tests UNIT_TESTS:=tests/unit INTEGRATION_TESTS:=tests/integration +PARTNER_INTEGRATION_TESTS:=tests/partner_integration END_TO_END_TESTS:=tests/end_to_end all: pre-commit @@ -36,5 +37,8 @@ integration-test: end-to-end-test: $(CMD) pytest $(END_TO_END_TESTS) -v --junitxml=junit/test-results.xml +partner-integration-test: + $(CMD) pytest $(PARTNER_INTEGRATION_TESTS) -v --junitxml=junit/partner-test-results.xml + #clean: # git clean -Xdf # Delete all files in .gitignore diff --git a/tests/partner_integration/__init__.py b/tests/partner_integration/__init__.py new file mode 100644 index 0000000000..9a0454564d --- /dev/null +++ b/tests/partner_integration/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/partner_integration/azure_ai_evaluation/__init__.py b/tests/partner_integration/azure_ai_evaluation/__init__.py new file mode 100644 index 0000000000..9a0454564d --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py b/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py new file mode 100644 index 0000000000..3eec18a6d6 --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py @@ -0,0 +1,98 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for PromptConverter interface and specific converters used by azure-ai-evaluation. + +The azure-ai-evaluation red team module: +- Extends PromptConverter via _DefaultConverter +- Imports 20+ specific converters in _agent/_agent_utils.py and strategy_utils.py +- Uses ConverterResult as the return type +""" + +import pytest + +from pyrit.prompt_converter import ConverterResult, PromptConverter + + +class TestPromptConverterContract: + """Validate PromptConverter base class interface stability.""" + + def test_prompt_converter_base_exists(self): + """_DefaultConverter extends PromptConverter.""" + assert PromptConverter is not None + + def test_converter_result_exists(self): + """_DefaultConverter.convert_async returns ConverterResult.""" + assert ConverterResult is not None + + def test_prompt_converter_has_convert_async(self): + """_DefaultConverter overrides convert_async.""" + assert hasattr(PromptConverter, "convert_async") + + def test_prompt_converter_subclassable(self): + """_DefaultConverter subclasses PromptConverter with convert_async.""" + + class TestConverter(PromptConverter): + SUPPORTED_INPUT_TYPES = ("text",) + SUPPORTED_OUTPUT_TYPES = ("text",) + + async def convert_async(self, *, prompt, input_type="text"): + return ConverterResult(output_text=prompt, output_type="text") + + converter = TestConverter() + assert isinstance(converter, PromptConverter) + + +class TestSpecificConvertersImportable: + """Validate that all converters imported by azure-ai-evaluation are available. + + These converters are imported in: + - _agent/_agent_utils.py (20+ converters) + - _utils/strategy_utils.py (converter instantiation) + """ + + @pytest.mark.parametrize( + "converter_name", + [ + "AnsiAttackConverter", + "AsciiArtConverter", + "AtbashConverter", + "Base64Converter", + "BinaryConverter", + "CaesarConverter", + "CharacterSpaceConverter", + "CharSwapConverter", + "DiacriticConverter", + "FlipConverter", + "LeetspeakConverter", + "MorseConverter", + "ROT13Converter", + "StringJoinConverter", + "SuffixAppendConverter", + "TenseConverter", + "UnicodeConfusableConverter", + "UnicodeSubstitutionConverter", + "UrlConverter", + ], + ) + def test_converter_importable(self, converter_name): + """Each converter used by azure-ai-evaluation must be importable from pyrit.prompt_converter.""" + import pyrit.prompt_converter as pc + + converter_class = getattr(pc, converter_name, None) + assert converter_class is not None, ( + f"{converter_name} not found in pyrit.prompt_converter — " + f"azure-ai-evaluation depends on this converter" + ) + + def test_ascii_smuggler_converter_importable(self): + """AsciiSmugglerConverter is imported in _agent/_agent_utils.py.""" + from pyrit.prompt_converter import AsciiArtConverter + + assert AsciiArtConverter is not None + + def test_llm_generic_text_converter_importable(self): + """LLMGenericTextConverter is used for tense/translation strategies.""" + from pyrit.prompt_converter import LLMGenericTextConverter + + assert LLMGenericTextConverter is not None diff --git a/tests/partner_integration/azure_ai_evaluation/test_exceptions_contract.py b/tests/partner_integration/azure_ai_evaluation/test_exceptions_contract.py new file mode 100644 index 0000000000..1ada6ba3d4 --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_exceptions_contract.py @@ -0,0 +1,65 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for PyRIT exception types and retry decorators used by azure-ai-evaluation. + +The azure-ai-evaluation red team module uses these in: +- _callback_chat_target.py: EmptyResponseException, RateLimitException, pyrit_target_retry +- _rai_service_target.py: remove_markdown_json +""" + +from pyrit.exceptions import ( + EmptyResponseException, + RateLimitException, + pyrit_target_retry, + remove_markdown_json, +) + + +class TestExceptionTypesContract: + """Validate exception types exist and are proper Exception subclasses.""" + + def test_empty_response_exception_is_exception(self): + """_CallbackChatTarget catches EmptyResponseException.""" + assert issubclass(EmptyResponseException, Exception) + + def test_rate_limit_exception_is_exception(self): + """_CallbackChatTarget catches RateLimitException.""" + assert issubclass(RateLimitException, Exception) + + def test_empty_response_exception_instantiable(self): + """Verify EmptyResponseException can be raised with a message.""" + exc = EmptyResponseException() + assert isinstance(exc, Exception) + + def test_rate_limit_exception_instantiable(self): + """Verify RateLimitException can be raised with a message.""" + exc = RateLimitException() + assert isinstance(exc, Exception) + + +class TestRetryDecoratorContract: + """Validate retry decorator availability.""" + + def test_pyrit_target_retry_is_callable(self): + """_CallbackChatTarget uses @pyrit_target_retry decorator.""" + assert callable(pyrit_target_retry) + + +class TestUtilityFunctionsContract: + """Validate utility functions used by azure-ai-evaluation.""" + + def test_remove_markdown_json_is_callable(self): + """_rai_service_target.py uses remove_markdown_json.""" + assert callable(remove_markdown_json) + + def test_remove_markdown_json_handles_plain_text(self): + """Verify remove_markdown_json passes through plain text.""" + result = remove_markdown_json("plain text") + assert isinstance(result, str) + + def test_remove_markdown_json_strips_markdown_fences(self): + """Verify remove_markdown_json strips ```json fences.""" + input_text = '```json\n{"key": "value"}\n```' + result = remove_markdown_json(input_text) + assert "```" not in result diff --git a/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py b/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py new file mode 100644 index 0000000000..cc5c70d8a3 --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for Foundry scenario APIs used by azure-ai-evaluation. + +The azure-ai-evaluation red team module uses the Foundry framework for modern attack execution: +- FoundryExecutionManager creates FoundryScenario instances per risk category +- StrategyMapper maps AttackStrategy enum → FoundryStrategy +- DatasetConfigurationBuilder produces DatasetConfiguration from RAI objectives +- ScenarioOrchestrator processes ScenarioResult and AttackResult +- RAIServiceScorer uses AttackScoringConfig for scoring configuration +""" + +from pyrit.executor.attack import AttackScoringConfig +from pyrit.models import AttackOutcome, AttackResult +from pyrit.models.scenario_result import ScenarioResult +from pyrit.scenario import DatasetConfiguration +from pyrit.scenario.foundry import FoundryScenario, FoundryStrategy + + +class TestFoundryStrategyContract: + """Validate FoundryStrategy availability and structure.""" + + def test_foundry_strategy_class_exists(self): + """StrategyMapper maps to FoundryStrategy values.""" + assert FoundryStrategy is not None + + def test_foundry_strategy_is_scenario_strategy(self): + """FoundryStrategy should extend ScenarioStrategy.""" + from pyrit.scenario import ScenarioStrategy + + assert issubclass(FoundryStrategy, ScenarioStrategy) + + +class TestFoundryScenarioContract: + """Validate FoundryScenario availability.""" + + def test_foundry_scenario_class_exists(self): + """ScenarioOrchestrator creates FoundryScenario instances.""" + assert FoundryScenario is not None + + +class TestDatasetConfigurationContract: + """Validate DatasetConfiguration availability.""" + + def test_dataset_configuration_class_exists(self): + """DatasetConfigurationBuilder produces DatasetConfiguration.""" + assert DatasetConfiguration is not None + + +class TestAttackScoringConfigContract: + """Validate AttackScoringConfig availability.""" + + def test_attack_scoring_config_exists(self): + """ScenarioOrchestrator uses AttackScoringConfig.""" + assert AttackScoringConfig is not None + + def test_attack_scoring_config_has_expected_fields(self): + """AttackScoringConfig should accept objective_scorer and refusal_scorer.""" + config = AttackScoringConfig() + assert hasattr(config, "objective_scorer") + assert hasattr(config, "refusal_scorer") + + +class TestScenarioResultContract: + """Validate ScenarioResult model availability.""" + + def test_scenario_result_class_exists(self): + """ScenarioOrchestrator reads ScenarioResult.""" + assert ScenarioResult is not None + + def test_attack_result_class_exists(self): + """FoundryResultProcessor processes AttackResult.""" + assert AttackResult is not None + + def test_attack_outcome_class_exists(self): + """FoundryResultProcessor checks AttackOutcome values.""" + assert AttackOutcome is not None diff --git a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py new file mode 100644 index 0000000000..0dc7121893 --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py @@ -0,0 +1,74 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Import smoke tests for azure-ai-evaluation red team module integration. + +These tests verify that the azure-ai-evaluation red team module can be imported +and that its PyRIT subclasses correctly extend PyRIT base classes. + +Tests are SKIPPED if azure-ai-evaluation[redteam] is not installed. +""" + +import pytest + +from pyrit.prompt_target import PromptChatTarget +from pyrit.score.true_false.true_false_scorer import TrueFalseScorer + + +def _azure_ai_evaluation_available() -> bool: + """Check if azure-ai-evaluation[redteam] is installed.""" + try: + from azure.ai.evaluation.red_team import RedTeam # noqa: F401 + + return True + except ImportError: + return False + + +requires_azure_ai_evaluation = pytest.mark.skipif( + not _azure_ai_evaluation_available(), + reason="azure-ai-evaluation[redteam] is not installed", +) + + +@requires_azure_ai_evaluation +class TestRedTeamModuleImports: + """Verify azure-ai-evaluation red_team module imports succeed with current PyRIT.""" + + def test_redteam_public_api_imports(self): + """Verify all public classes from azure.ai.evaluation.red_team are importable.""" + from azure.ai.evaluation.red_team import ( + AttackStrategy, + RedTeam, + RedTeamResult, + RiskCategory, + SupportedLanguages, + ) + + assert RedTeam is not None + assert AttackStrategy is not None + assert RiskCategory is not None + assert RedTeamResult is not None + assert SupportedLanguages is not None + + +@requires_azure_ai_evaluation +class TestCallbackChatTargetInheritance: + """Verify _CallbackChatTarget correctly extends PromptChatTarget.""" + + def test_callback_chat_target_extends_prompt_chat_target(self): + """_CallbackChatTarget must be a subclass of pyrit.prompt_target.PromptChatTarget.""" + from azure.ai.evaluation.red_team._callback_chat_target import _CallbackChatTarget + + assert issubclass(_CallbackChatTarget, PromptChatTarget) + + +@requires_azure_ai_evaluation +class TestRAIScorerInheritance: + """Verify RAIServiceScorer correctly extends TrueFalseScorer.""" + + def test_rai_scorer_extends_true_false_scorer(self): + """RAIServiceScorer must be a subclass of pyrit.score.true_false.TrueFalseScorer.""" + from azure.ai.evaluation.red_team._foundry._rai_scorer import RAIServiceScorer + + assert issubclass(RAIServiceScorer, TrueFalseScorer) diff --git a/tests/partner_integration/azure_ai_evaluation/test_memory_contract.py b/tests/partner_integration/azure_ai_evaluation/test_memory_contract.py new file mode 100644 index 0000000000..24c772c772 --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_memory_contract.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for CentralMemory and SQLiteMemory used by azure-ai-evaluation. + +The azure-ai-evaluation RedTeam class initializes PyRIT memory during __init__: + CentralMemory.set_memory_instance(SQLiteMemory()) + +Multiple modules also access memory via CentralMemory.get_memory_instance(). +These tests validate the memory lifecycle contract. +""" + +from pyrit.memory import CentralMemory, SQLiteMemory + + +class TestMemoryContract: + """Validate CentralMemory/SQLiteMemory interface stability.""" + + def test_sqlite_memory_default_constructor(self): + """RedTeam.__init__ calls SQLiteMemory() with no args.""" + memory = SQLiteMemory() + assert memory is not None + memory.dispose_engine() + + def test_sqlite_memory_in_memory_constructor(self): + """Partner tests use SQLiteMemory(db_path=':memory:').""" + memory = SQLiteMemory(db_path=":memory:") + assert memory is not None + memory.dispose_engine() + + def test_central_memory_set_and_get_instance(self): + """RedTeam.__init__ sets memory; formatting_utils.py and _rai_scorer.py retrieve it.""" + memory = SQLiteMemory(db_path=":memory:") + CentralMemory.set_memory_instance(memory) + retrieved = CentralMemory.get_memory_instance() + assert retrieved is memory + memory.dispose_engine() + + def test_sqlite_memory_has_disable_embedding(self): + """Test fixtures call disable_embedding() on SQLiteMemory.""" + memory = SQLiteMemory(db_path=":memory:") + assert hasattr(memory, "disable_embedding") + assert callable(memory.disable_embedding) + memory.disable_embedding() + memory.dispose_engine() + + def test_sqlite_memory_has_reset_database(self): + """Test fixtures call reset_database() on SQLiteMemory.""" + memory = SQLiteMemory(db_path=":memory:") + assert hasattr(memory, "reset_database") + assert callable(memory.reset_database) + memory.dispose_engine() + + def test_sqlite_memory_has_dispose_engine(self): + """Cleanup requires dispose_engine().""" + memory = SQLiteMemory(db_path=":memory:") + assert hasattr(memory, "dispose_engine") + assert callable(memory.dispose_engine) + memory.dispose_engine() diff --git a/tests/partner_integration/azure_ai_evaluation/test_model_contract.py b/tests/partner_integration/azure_ai_evaluation/test_model_contract.py new file mode 100644 index 0000000000..08ac1cad3a --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_model_contract.py @@ -0,0 +1,173 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for PyRIT data models used by azure-ai-evaluation. + +The red team module uses these models extensively: +- Message / MessagePiece: Every request/response path +- Score / UnvalidatedScore: Scoring pipeline +- SeedPrompt / SeedObjective / SeedGroup: DatasetConfigurationBuilder +- AttackResult / AttackOutcome: FoundryResultProcessor +- ChatMessage: formatting_utils.py +- PromptDataType: Type enum used across converters and models +- construct_response_from_request: Response construction +""" + +import uuid + +from pyrit.models import ( + AttackOutcome, + AttackResult, + ChatMessage, + Message, + MessagePiece, + PromptDataType, + ScenarioResult, + Score, + SeedGroup, + SeedObjective, + SeedPrompt, + UnvalidatedScore, + construct_response_from_request, +) + + +class TestMessageContract: + """Validate Message and MessagePiece interfaces.""" + + def test_message_piece_minimal_constructor(self): + """_CallbackChatTarget creates MessagePiece with role, original_value, conversation_id.""" + piece = MessagePiece( + role="user", + original_value="test prompt", + conversation_id=str(uuid.uuid4()), + ) + assert piece.api_role == "user" + assert piece.original_value == "test prompt" + + def test_message_piece_to_message(self): + """_CallbackChatTarget calls piece.to_message() to convert to Message.""" + piece = MessagePiece( + role="user", + original_value="test", + conversation_id=str(uuid.uuid4()), + ) + msg = piece.to_message() + assert isinstance(msg, Message) + assert len(msg.message_pieces) == 1 + + def test_message_get_value(self): + """_CallbackChatTarget accesses message.get_value() for the response text.""" + piece = MessagePiece( + role="assistant", + original_value="response text", + conversation_id=str(uuid.uuid4()), + ) + msg = piece.to_message() + assert msg.get_value() == "response text" + + def test_message_pieces_attribute(self): + """azure-ai-evaluation accesses message.message_pieces list.""" + piece = MessagePiece( + role="user", + original_value="test", + conversation_id=str(uuid.uuid4()), + ) + msg = piece.to_message() + assert hasattr(msg, "message_pieces") + assert isinstance(msg.message_pieces, (list, tuple)) + + def test_message_piece_has_converted_value(self): + """azure-ai-evaluation reads message_piece.converted_value for responses.""" + piece = MessagePiece( + role="assistant", + original_value="original", + converted_value="converted", + conversation_id=str(uuid.uuid4()), + ) + assert piece.converted_value == "converted" + + def test_message_piece_has_conversation_id(self): + """Conversation tracking relies on conversation_id field.""" + conv_id = str(uuid.uuid4()) + piece = MessagePiece( + role="user", + original_value="test", + conversation_id=conv_id, + ) + assert piece.conversation_id == conv_id + + +class TestScoreModels: + """Validate Score and UnvalidatedScore interfaces.""" + + def test_score_class_exists(self): + """RAIServiceScorer and AzureRAIServiceTrueFalseScorer return Score objects.""" + assert Score is not None + + def test_unvalidated_score_class_exists(self): + """Scorers create UnvalidatedScore before validation.""" + assert UnvalidatedScore is not None + + +class TestSeedModels: + """Validate seed data models used by DatasetConfigurationBuilder.""" + + def test_seed_prompt_class_exists(self): + """DatasetConfigurationBuilder creates SeedPrompt instances.""" + assert SeedPrompt is not None + + def test_seed_objective_class_exists(self): + """DatasetConfigurationBuilder creates SeedObjective instances.""" + assert SeedObjective is not None + + def test_seed_group_class_exists(self): + """DatasetConfigurationBuilder creates SeedGroup instances.""" + assert SeedGroup is not None + + +class TestAttackModels: + """Validate attack result models used by FoundryResultProcessor.""" + + def test_attack_result_class_exists(self): + """ScenarioOrchestrator processes AttackResult from FoundryScenario.""" + assert AttackResult is not None + + def test_attack_outcome_class_exists(self): + """FoundryResultProcessor checks AttackOutcome values.""" + assert AttackOutcome is not None + + +class TestMiscModels: + """Validate miscellaneous models used by azure-ai-evaluation.""" + + def test_chat_message_class_exists(self): + """formatting_utils.py imports ChatMessage.""" + assert ChatMessage is not None + + def test_prompt_data_type_has_text(self): + """_DefaultConverter and _dataset_builder check for 'text' data type.""" + # PromptDataType is a Literal type; verify "text" is a valid value + from typing import get_args + + valid_types = get_args(PromptDataType) + assert "text" in valid_types + + def test_scenario_result_class_exists(self): + """ScenarioOrchestrator reads ScenarioResult.""" + assert ScenarioResult is not None + + def test_construct_response_from_request_signature(self): + """Verify construct_response_from_request accepts expected parameters.""" + piece = MessagePiece( + role="user", + original_value="test", + conversation_id=str(uuid.uuid4()), + ) + # Call with positional request + response_text_pieces + result = construct_response_from_request( + request=piece, + response_text_pieces=["response"], + response_type="text", + ) + assert isinstance(result, Message) diff --git a/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py b/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py new file mode 100644 index 0000000000..77190a1453 --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py @@ -0,0 +1,102 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for PromptChatTarget interface used by azure-ai-evaluation. + +The azure-ai-evaluation red team module extends PromptChatTarget in four places: +- _CallbackChatTarget (wraps user callbacks) +- AzureRAIServiceTarget (sends prompts to RAI service) +- RAIServiceEvalChatTarget (evaluation-specific RAI target) +- _rai_service_target.py (multi-turn jailbreak target) + +These tests ensure the base class interface remains stable. +""" + +import uuid + +import pytest + +from pyrit.models import Message, MessagePiece, construct_response_from_request +from pyrit.prompt_target import PromptChatTarget + + +class TestPromptChatTargetContract: + """Validate PromptChatTarget base class interface stability.""" + + def test_prompt_chat_target_is_abstract(self): + """PromptChatTarget should not be directly instantiable (has abstract methods).""" + with pytest.raises(TypeError): + PromptChatTarget() + + def test_prompt_chat_target_has_send_prompt_async(self): + """azure-ai-evaluation overrides send_prompt_async in all subclasses.""" + assert hasattr(PromptChatTarget, "send_prompt_async") + + def test_prompt_chat_target_subclassable_with_send_prompt_async(self): + """azure-ai-evaluation creates subclasses that implement send_prompt_async.""" + + class MinimalTarget(PromptChatTarget): + async def send_prompt_async(self, *, message=None, **kwargs): + return [] + + def is_json_response_supported(self) -> bool: + return False + + def _validate_request(self, *, message) -> None: + pass + + target = MinimalTarget() + assert isinstance(target, PromptChatTarget) + + def test_prompt_chat_target_init_accepts_keyword_args(self): + """PromptChatTarget.__init__ should accept max_requests_per_minute.""" + + class MinimalTarget(PromptChatTarget): + async def send_prompt_async(self, *, message=None, **kwargs): + return [] + + def is_json_response_supported(self) -> bool: + return False + + def _validate_request(self, *, message) -> None: + pass + + target = MinimalTarget(max_requests_per_minute=60) + assert target is not None + + def test_construct_response_from_request_is_callable(self): + """AzureRAIServiceTarget uses construct_response_from_request to build responses.""" + assert callable(construct_response_from_request) + + def test_construct_response_from_request_returns_message(self): + """Verify construct_response_from_request produces a Message from a MessagePiece.""" + request_piece = MessagePiece( + role="user", + original_value="test prompt", + conversation_id=str(uuid.uuid4()), + ) + response = construct_response_from_request( + request=request_piece, + response_text_pieces=["test response"], + ) + assert isinstance(response, Message) + assert len(response.message_pieces) == 1 + assert response.message_pieces[0].converted_value == "test response" + assert response.message_pieces[0].api_role == "assistant" + + def test_prompt_chat_target_has_memory_attribute(self): + """azure-ai-evaluation accesses self._memory on PromptChatTarget subclasses.""" + + class MinimalTarget(PromptChatTarget): + async def send_prompt_async(self, *, message=None, **kwargs): + return [] + + def is_json_response_supported(self) -> bool: + return False + + def _validate_request(self, *, message) -> None: + pass + + target = MinimalTarget() + # _memory is set during initialization or via property + assert hasattr(target, "_memory") diff --git a/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py b/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py new file mode 100644 index 0000000000..eb7e1a0c5b --- /dev/null +++ b/tests/partner_integration/azure_ai_evaluation/test_scorer_contract.py @@ -0,0 +1,65 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Contract tests for Scorer and TrueFalseScorer interfaces used by azure-ai-evaluation. + +The azure-ai-evaluation red team module extends these classes: +- AzureRAIServiceTrueFalseScorer extends Scorer +- RAIServiceScorer extends TrueFalseScorer + +Both are critical for scoring attack results. +""" + +from pyrit.identifiers import ScorerIdentifier +from pyrit.score import ScorerPromptValidator +from pyrit.score.scorer import Scorer +from pyrit.score.true_false.true_false_scorer import TrueFalseScorer + + +class TestScorerContract: + """Validate Scorer base class interface stability.""" + + def test_scorer_base_exists(self): + """AzureRAIServiceTrueFalseScorer extends Scorer.""" + assert Scorer is not None + + def test_scorer_has_score_piece_async(self): + """Scorer subclasses must implement _score_piece_async.""" + assert hasattr(Scorer, "_score_piece_async") + + def test_scorer_has_validate_return_scores(self): + """Scorer subclasses must implement validate_return_scores.""" + assert hasattr(Scorer, "validate_return_scores") + + def test_scorer_has_get_scorer_metrics(self): + """Scorer subclasses must implement get_scorer_metrics.""" + assert hasattr(Scorer, "get_scorer_metrics") + + +class TestTrueFalseScorerContract: + """Validate TrueFalseScorer interface stability.""" + + def test_true_false_scorer_extends_scorer(self): + """RAIServiceScorer extends TrueFalseScorer which extends Scorer.""" + assert issubclass(TrueFalseScorer, Scorer) + + def test_true_false_scorer_has_validate_return_scores(self): + """TrueFalseScorer implements validate_return_scores.""" + assert hasattr(TrueFalseScorer, "validate_return_scores") + + +class TestScorerUtilities: + """Validate scorer utility classes used by azure-ai-evaluation.""" + + def test_scorer_identifier_exists(self): + """RAIServiceScorer uses ScorerIdentifier for identity tracking.""" + assert ScorerIdentifier is not None + + def test_scorer_prompt_validator_exists(self): + """RAIServiceScorer uses ScorerPromptValidator for input validation.""" + assert ScorerPromptValidator is not None + + def test_scorer_prompt_validator_instantiable(self): + """ScorerPromptValidator should accept supported_data_types kwarg.""" + validator = ScorerPromptValidator(supported_data_types=["text"]) + assert validator is not None diff --git a/tests/partner_integration/conftest.py b/tests/partner_integration/conftest.py new file mode 100644 index 0000000000..fb6cdb3df8 --- /dev/null +++ b/tests/partner_integration/conftest.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared fixtures for partner integration tests. + +These tests validate that PyRIT's public APIs remain compatible with +partner packages that depend on them (e.g., azure-ai-evaluation[redteam]). +They do NOT require Azure credentials — all tests use in-memory fixtures. +""" + +import asyncio +import os +import tempfile +from collections.abc import Generator +from unittest.mock import patch + +import pytest +from sqlalchemy import inspect + +from pyrit.memory.central_memory import CentralMemory +from pyrit.memory.sqlite_memory import SQLiteMemory +from pyrit.setup import IN_MEMORY, initialize_pyrit_async + +# Limit retries for deterministic testing +os.environ["RETRY_MAX_NUM_ATTEMPTS"] = "3" +os.environ["RETRY_WAIT_MIN_SECONDS"] = "0" +os.environ["RETRY_WAIT_MAX_SECONDS"] = "1" + +# Initialize PyRIT with in-memory database +asyncio.run(initialize_pyrit_async(memory_db_type=IN_MEMORY)) + + +@pytest.fixture +def sqlite_instance() -> Generator[SQLiteMemory, None, None]: + """Provide an in-memory SQLite database for partner integration tests.""" + sqlite_memory = SQLiteMemory(db_path=":memory:") + temp_dir = tempfile.TemporaryDirectory() + sqlite_memory.results_path = temp_dir.name + sqlite_memory.disable_embedding() + sqlite_memory.reset_database() + + inspector = inspect(sqlite_memory.engine) + assert "PromptMemoryEntries" in inspector.get_table_names() + assert "ScoreEntries" in inspector.get_table_names() + assert "SeedPromptEntries" in inspector.get_table_names() + + CentralMemory.set_memory_instance(sqlite_memory) + yield sqlite_memory + temp_dir.cleanup() + sqlite_memory.dispose_engine() + + +@pytest.fixture +def patch_central_database(sqlite_instance): + """Mock CentralMemory.get_memory_instance for isolated tests.""" + with patch.object(CentralMemory, "get_memory_instance", return_value=sqlite_instance) as mock: + yield mock From 0a0edfb4ac7ff6d854e7aad064a78f4e93b22de8 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Tue, 24 Mar 2026 11:55:53 -0400 Subject: [PATCH 2/4] fix: address review findings in partner integration tests - Fix test_ascii_smuggler_converter_importable to test AsciiSmugglerConverter (was incorrectly testing AsciiArtConverter, duplicating parametrized coverage) - Move module-level asyncio.run(initialize_pyrit_async) to session-scoped fixture - Remove duplicate TestAttackModels (already covered in test_foundry_contract.py) - Extract MinimalTarget to module-level helper (was defined 3x inline) - Add docstring clarifying intentional private API imports in smoke tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../test_converter_contract.py | 4 +- .../azure_ai_evaluation/test_import_smoke.py | 9 +++- .../test_model_contract.py | 14 ----- .../test_prompt_target_contract.py | 52 ++++++------------- tests/partner_integration/conftest.py | 7 ++- 5 files changed, 30 insertions(+), 56 deletions(-) diff --git a/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py b/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py index 3eec18a6d6..8685245f1e 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py @@ -87,9 +87,9 @@ def test_converter_importable(self, converter_name): def test_ascii_smuggler_converter_importable(self): """AsciiSmugglerConverter is imported in _agent/_agent_utils.py.""" - from pyrit.prompt_converter import AsciiArtConverter + from pyrit.prompt_converter import AsciiSmugglerConverter - assert AsciiArtConverter is not None + assert AsciiSmugglerConverter is not None def test_llm_generic_text_converter_importable(self): """LLMGenericTextConverter is used for tense/translation strategies.""" diff --git a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py index 0dc7121893..569d22c332 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py +++ b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py @@ -54,7 +54,12 @@ def test_redteam_public_api_imports(self): @requires_azure_ai_evaluation class TestCallbackChatTargetInheritance: - """Verify _CallbackChatTarget correctly extends PromptChatTarget.""" + """Verify _CallbackChatTarget correctly extends PromptChatTarget. + + NOTE: These tests intentionally import private (_-prefixed) modules from + azure-ai-evaluation. This is correct for contract testing — we need to verify + the actual subclass relationships that PyRIT API changes could break. + """ def test_callback_chat_target_extends_prompt_chat_target(self): """_CallbackChatTarget must be a subclass of pyrit.prompt_target.PromptChatTarget.""" @@ -69,6 +74,6 @@ class TestRAIScorerInheritance: def test_rai_scorer_extends_true_false_scorer(self): """RAIServiceScorer must be a subclass of pyrit.score.true_false.TrueFalseScorer.""" - from azure.ai.evaluation.red_team._foundry._rai_scorer import RAIServiceScorer + from azure.ai.evaluation.red_team._foundry._rai_scorer import RAIServiceScorer # private: intentional assert issubclass(RAIServiceScorer, TrueFalseScorer) diff --git a/tests/partner_integration/azure_ai_evaluation/test_model_contract.py b/tests/partner_integration/azure_ai_evaluation/test_model_contract.py index 08ac1cad3a..eb7b80d3cc 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_model_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_model_contract.py @@ -16,8 +16,6 @@ import uuid from pyrit.models import ( - AttackOutcome, - AttackResult, ChatMessage, Message, MessagePiece, @@ -126,18 +124,6 @@ def test_seed_group_class_exists(self): assert SeedGroup is not None -class TestAttackModels: - """Validate attack result models used by FoundryResultProcessor.""" - - def test_attack_result_class_exists(self): - """ScenarioOrchestrator processes AttackResult from FoundryScenario.""" - assert AttackResult is not None - - def test_attack_outcome_class_exists(self): - """FoundryResultProcessor checks AttackOutcome values.""" - assert AttackOutcome is not None - - class TestMiscModels: """Validate miscellaneous models used by azure-ai-evaluation.""" diff --git a/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py b/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py index 77190a1453..dc835a343d 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py @@ -20,6 +20,19 @@ from pyrit.prompt_target import PromptChatTarget +class _MinimalTarget(PromptChatTarget): + """Minimal concrete PromptChatTarget for contract testing.""" + + async def send_prompt_async(self, *, message=None, **kwargs): + return [] + + def is_json_response_supported(self) -> bool: + return False + + def _validate_request(self, *, message) -> None: + pass + + class TestPromptChatTargetContract: """Validate PromptChatTarget base class interface stability.""" @@ -34,34 +47,12 @@ def test_prompt_chat_target_has_send_prompt_async(self): def test_prompt_chat_target_subclassable_with_send_prompt_async(self): """azure-ai-evaluation creates subclasses that implement send_prompt_async.""" - - class MinimalTarget(PromptChatTarget): - async def send_prompt_async(self, *, message=None, **kwargs): - return [] - - def is_json_response_supported(self) -> bool: - return False - - def _validate_request(self, *, message) -> None: - pass - - target = MinimalTarget() + target = _MinimalTarget() assert isinstance(target, PromptChatTarget) def test_prompt_chat_target_init_accepts_keyword_args(self): """PromptChatTarget.__init__ should accept max_requests_per_minute.""" - - class MinimalTarget(PromptChatTarget): - async def send_prompt_async(self, *, message=None, **kwargs): - return [] - - def is_json_response_supported(self) -> bool: - return False - - def _validate_request(self, *, message) -> None: - pass - - target = MinimalTarget(max_requests_per_minute=60) + target = _MinimalTarget(max_requests_per_minute=60) assert target is not None def test_construct_response_from_request_is_callable(self): @@ -86,17 +77,6 @@ def test_construct_response_from_request_returns_message(self): def test_prompt_chat_target_has_memory_attribute(self): """azure-ai-evaluation accesses self._memory on PromptChatTarget subclasses.""" - - class MinimalTarget(PromptChatTarget): - async def send_prompt_async(self, *, message=None, **kwargs): - return [] - - def is_json_response_supported(self) -> bool: - return False - - def _validate_request(self, *, message) -> None: - pass - - target = MinimalTarget() + target = _MinimalTarget() # _memory is set during initialization or via property assert hasattr(target, "_memory") diff --git a/tests/partner_integration/conftest.py b/tests/partner_integration/conftest.py index fb6cdb3df8..09b61a4f20 100644 --- a/tests/partner_integration/conftest.py +++ b/tests/partner_integration/conftest.py @@ -26,8 +26,11 @@ os.environ["RETRY_WAIT_MIN_SECONDS"] = "0" os.environ["RETRY_WAIT_MAX_SECONDS"] = "1" -# Initialize PyRIT with in-memory database -asyncio.run(initialize_pyrit_async(memory_db_type=IN_MEMORY)) + +@pytest.fixture(scope="session", autouse=True) +def _initialize_pyrit(): + """Initialize PyRIT with in-memory database once per test session.""" + asyncio.run(initialize_pyrit_async(memory_db_type=IN_MEMORY)) @pytest.fixture From 183071912953c1ff0695881ebb17e511a6bf3bc2 Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Tue, 24 Mar 2026 12:59:30 -0400 Subject: [PATCH 3/4] fix: ruff-format split f-string in test_converter_contract.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure_ai_evaluation/test_converter_contract.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py b/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py index 8685245f1e..a673d324f4 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_converter_contract.py @@ -81,8 +81,7 @@ def test_converter_importable(self, converter_name): converter_class = getattr(pc, converter_name, None) assert converter_class is not None, ( - f"{converter_name} not found in pyrit.prompt_converter — " - f"azure-ai-evaluation depends on this converter" + f"{converter_name} not found in pyrit.prompt_converter — azure-ai-evaluation depends on this converter" ) def test_ascii_smuggler_converter_importable(self): From 895406cb1c0eb8d159250bc6ec33f633000f800b Mon Sep 17 00:00:00 2001 From: Sydney Lister Date: Wed, 25 Mar 2026 09:34:16 -0400 Subject: [PATCH 4/4] Address PR review comments: PromptChatTarget -> PromptTarget, fix imports, add seed model structural tests - Update PromptChatTarget to PromptTarget per PR #1532 deprecation - Move ScenarioStrategy import to top-level in test_foundry_contract.py - Add rationale for explicit inheritance checks in test_import_smoke.py - Expand seed model tests with structural validation (value, data_type, harm_categories, role, metadata, SeedGroup composition) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../test_foundry_contract.py | 4 +- .../azure_ai_evaluation/test_import_smoke.py | 12 +++--- .../test_model_contract.py | 42 +++++++++++++++++++ .../test_prompt_target_contract.py | 39 ++++++++--------- 4 files changed, 68 insertions(+), 29 deletions(-) diff --git a/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py b/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py index cc5c70d8a3..5dc2231a3b 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_foundry_contract.py @@ -14,7 +14,7 @@ from pyrit.executor.attack import AttackScoringConfig from pyrit.models import AttackOutcome, AttackResult from pyrit.models.scenario_result import ScenarioResult -from pyrit.scenario import DatasetConfiguration +from pyrit.scenario import DatasetConfiguration, ScenarioStrategy from pyrit.scenario.foundry import FoundryScenario, FoundryStrategy @@ -27,8 +27,6 @@ def test_foundry_strategy_class_exists(self): def test_foundry_strategy_is_scenario_strategy(self): """FoundryStrategy should extend ScenarioStrategy.""" - from pyrit.scenario import ScenarioStrategy - assert issubclass(FoundryStrategy, ScenarioStrategy) diff --git a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py index 569d22c332..4093d1e8a2 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py +++ b/tests/partner_integration/azure_ai_evaluation/test_import_smoke.py @@ -11,7 +11,7 @@ import pytest -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget from pyrit.score.true_false.true_false_scorer import TrueFalseScorer @@ -54,18 +54,20 @@ def test_redteam_public_api_imports(self): @requires_azure_ai_evaluation class TestCallbackChatTargetInheritance: - """Verify _CallbackChatTarget correctly extends PromptChatTarget. + """Verify _CallbackChatTarget correctly extends PromptTarget. NOTE: These tests intentionally import private (_-prefixed) modules from azure-ai-evaluation. This is correct for contract testing — we need to verify the actual subclass relationships that PyRIT API changes could break. + Explicit inheritance checks are needed because azure-ai-evaluation subclasses + are detected via issubclass() checks in PyRIT orchestrators and scenarios. """ - def test_callback_chat_target_extends_prompt_chat_target(self): - """_CallbackChatTarget must be a subclass of pyrit.prompt_target.PromptChatTarget.""" + def test_callback_chat_target_extends_prompt_target(self): + """_CallbackChatTarget must be a subclass of pyrit.prompt_target.PromptTarget.""" from azure.ai.evaluation.red_team._callback_chat_target import _CallbackChatTarget - assert issubclass(_CallbackChatTarget, PromptChatTarget) + assert issubclass(_CallbackChatTarget, PromptTarget) @requires_azure_ai_evaluation diff --git a/tests/partner_integration/azure_ai_evaluation/test_model_contract.py b/tests/partner_integration/azure_ai_evaluation/test_model_contract.py index eb7b80d3cc..2dddb3b796 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_model_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_model_contract.py @@ -115,14 +115,56 @@ def test_seed_prompt_class_exists(self): """DatasetConfigurationBuilder creates SeedPrompt instances.""" assert SeedPrompt is not None + def test_seed_prompt_accepts_value(self): + """SeedPrompt requires a value field (the actual prompt text).""" + prompt = SeedPrompt(value="test prompt") + assert prompt.value == "test prompt" + + def test_seed_prompt_has_data_type(self): + """SeedPrompt.data_type defaults to 'text' for string values.""" + prompt = SeedPrompt(value="test") + assert prompt.data_type == "text" + + def test_seed_prompt_has_harm_categories(self): + """DatasetConfigurationBuilder sets harm_categories on SeedPrompt.""" + prompt = SeedPrompt(value="test", harm_categories=["violence"]) + assert "violence" in prompt.harm_categories + + def test_seed_prompt_has_role(self): + """SeedPrompt supports role field for conversation context.""" + prompt = SeedPrompt(value="test", role="user") + assert prompt.role == "user" + + def test_seed_prompt_has_metadata(self): + """DatasetConfigurationBuilder attaches metadata to SeedPrompt.""" + prompt = SeedPrompt(value="test", metadata={"key": "val"}) + assert prompt.metadata["key"] == "val" + def test_seed_objective_class_exists(self): """DatasetConfigurationBuilder creates SeedObjective instances.""" assert SeedObjective is not None + def test_seed_objective_accepts_value(self): + """SeedObjective requires a value field (the objective text).""" + obj = SeedObjective(value="test objective") + assert obj.value == "test objective" + + def test_seed_objective_has_harm_categories(self): + """DatasetConfigurationBuilder sets harm_categories on SeedObjective.""" + obj = SeedObjective(value="test", harm_categories=["hate"]) + assert "hate" in obj.harm_categories + def test_seed_group_class_exists(self): """DatasetConfigurationBuilder creates SeedGroup instances.""" assert SeedGroup is not None + def test_seed_group_accepts_seeds(self): + """SeedGroup groups multiple seeds together.""" + prompt = SeedPrompt(value="prompt text", role="user") + obj = SeedObjective(value="objective text") + group = SeedGroup(seeds=[prompt, obj]) + assert len(group.seeds) == 2 + class TestMiscModels: """Validate miscellaneous models used by azure-ai-evaluation.""" diff --git a/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py b/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py index dc835a343d..2688bfbe8a 100644 --- a/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py +++ b/tests/partner_integration/azure_ai_evaluation/test_prompt_target_contract.py @@ -1,9 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Contract tests for PromptChatTarget interface used by azure-ai-evaluation. +"""Contract tests for PromptTarget interface used by azure-ai-evaluation. -The azure-ai-evaluation red team module extends PromptChatTarget in four places: +The azure-ai-evaluation red team module extends PromptTarget in four places: - _CallbackChatTarget (wraps user callbacks) - AzureRAIServiceTarget (sends prompts to RAI service) - RAIServiceEvalChatTarget (evaluation-specific RAI target) @@ -17,41 +17,38 @@ import pytest from pyrit.models import Message, MessagePiece, construct_response_from_request -from pyrit.prompt_target import PromptChatTarget +from pyrit.prompt_target import PromptTarget -class _MinimalTarget(PromptChatTarget): - """Minimal concrete PromptChatTarget for contract testing.""" +class _MinimalTarget(PromptTarget): + """Minimal concrete PromptTarget for contract testing.""" async def send_prompt_async(self, *, message=None, **kwargs): return [] - def is_json_response_supported(self) -> bool: - return False - def _validate_request(self, *, message) -> None: pass -class TestPromptChatTargetContract: - """Validate PromptChatTarget base class interface stability.""" +class TestPromptTargetContract: + """Validate PromptTarget base class interface stability.""" - def test_prompt_chat_target_is_abstract(self): - """PromptChatTarget should not be directly instantiable (has abstract methods).""" + def test_prompt_target_is_abstract(self): + """PromptTarget should not be directly instantiable (has abstract methods).""" with pytest.raises(TypeError): - PromptChatTarget() + PromptTarget() - def test_prompt_chat_target_has_send_prompt_async(self): + def test_prompt_target_has_send_prompt_async(self): """azure-ai-evaluation overrides send_prompt_async in all subclasses.""" - assert hasattr(PromptChatTarget, "send_prompt_async") + assert hasattr(PromptTarget, "send_prompt_async") - def test_prompt_chat_target_subclassable_with_send_prompt_async(self): + def test_prompt_target_subclassable_with_send_prompt_async(self): """azure-ai-evaluation creates subclasses that implement send_prompt_async.""" target = _MinimalTarget() - assert isinstance(target, PromptChatTarget) + assert isinstance(target, PromptTarget) - def test_prompt_chat_target_init_accepts_keyword_args(self): - """PromptChatTarget.__init__ should accept max_requests_per_minute.""" + def test_prompt_target_init_accepts_keyword_args(self): + """PromptTarget.__init__ should accept max_requests_per_minute.""" target = _MinimalTarget(max_requests_per_minute=60) assert target is not None @@ -75,8 +72,8 @@ def test_construct_response_from_request_returns_message(self): assert response.message_pieces[0].converted_value == "test response" assert response.message_pieces[0].api_role == "assistant" - def test_prompt_chat_target_has_memory_attribute(self): - """azure-ai-evaluation accesses self._memory on PromptChatTarget subclasses.""" + def test_prompt_target_has_memory_attribute(self): + """azure-ai-evaluation accesses self._memory on PromptTarget subclasses.""" target = _MinimalTarget() # _memory is set during initialization or via property assert hasattr(target, "_memory")