From 6358a0cffb724dc61b2d0a5dd904064dbe064d55 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 21 Mar 2026 10:58:18 -0700 Subject: [PATCH 1/3] fixing flaky target capabilities tests causing intermittent failures --- tests/unit/target/test_realtime_target.py | 7 +++++++ tests/unit/target/test_supports_multi_turn.py | 9 +++++++++ tests/unit/target/test_target_capabilities.py | 12 ++++++++++++ 3 files changed, 28 insertions(+) diff --git a/tests/unit/target/test_realtime_target.py b/tests/unit/target/test_realtime_target.py index 25f68b7c9..0bf0d1164 100644 --- a/tests/unit/target/test_realtime_target.py +++ b/tests/unit/target/test_realtime_target.py @@ -11,7 +11,14 @@ from pyrit.prompt_target.openai.openai_realtime_target import RealtimeTargetResult +# Env vars that may leak from .env files loaded by other tests in parallel workers. +_CLEAN_UNDERLYING_MODEL_ENV = { + "OPENAI_REALTIME_UNDERLYING_MODEL": "", +} + + @pytest.fixture +@patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def target(sqlite_instance): return RealtimeTarget(api_key="test_key", endpoint="wss://test_url", model_name="test") diff --git a/tests/unit/target/test_supports_multi_turn.py b/tests/unit/target/test_supports_multi_turn.py index 99823a092..8ec81a4b2 100644 --- a/tests/unit/target/test_supports_multi_turn.py +++ b/tests/unit/target/test_supports_multi_turn.py @@ -1,11 +1,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from unittest.mock import patch + import pytest from tests.unit.mocks import MockPromptTarget +# Env vars that may leak from .env files loaded by other tests in parallel workers. +_CLEAN_UNDERLYING_MODEL_ENV = { + "OPENAI_VIDEO_UNDERLYING_MODEL": "", +} + + @pytest.mark.usefixtures("patch_central_database") class TestSupportsMultiTurn: """Test supports_multi_turn capability flag across the target hierarchy.""" @@ -41,6 +49,7 @@ def test_openai_image_target_returns_false(self): ) assert target.capabilities.supports_multi_turn is False + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_video_target_returns_false(self): from pyrit.prompt_target import OpenAIVideoTarget diff --git a/tests/unit/target/test_target_capabilities.py b/tests/unit/target/test_target_capabilities.py index 64f4efa5e..df29529f8 100644 --- a/tests/unit/target/test_target_capabilities.py +++ b/tests/unit/target/test_target_capabilities.py @@ -8,6 +8,17 @@ from pyrit.prompt_target.common.target_capabilities import TargetCapabilities +# Env vars that may leak from .env files loaded by other tests in parallel workers. +# Clear them so that targets use _DEFAULT_CAPABILITIES instead of _KNOWN_CAPABILITIES. +_CLEAN_UNDERLYING_MODEL_ENV = { + "OPENAI_VIDEO_UNDERLYING_MODEL": "", + "OPENAI_REALTIME_UNDERLYING_MODEL": "", + "OPENAI_CHAT_UNDERLYING_MODEL": "", + "OPENAI_IMAGE_UNDERLYING_MODEL": "", + "OPENAI_TTS_UNDERLYING_MODEL": "", +} + + class TestDefaultCapabilitiesDefined: """Verify that every concrete PromptTarget subclass defines _DEFAULT_CAPABILITIES.""" @@ -115,6 +126,7 @@ def test_openai_tts_target_modalities(self): assert target.capabilities.input_modalities == frozenset({frozenset(["text"])}) assert target.capabilities.output_modalities == frozenset({frozenset(["audio_path"])}) + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_video_target_modalities(self): from pyrit.prompt_target import OpenAIVideoTarget From 5a772ebbffc511c3ac1e8a8dc64efbd03477bece Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 21 Mar 2026 11:32:03 -0700 Subject: [PATCH 2/3] Fix ruff E303: remove extra blank lines before module-level constants Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/target/test_realtime_target.py | 1 - tests/unit/target/test_supports_multi_turn.py | 1 - tests/unit/target/test_target_capabilities.py | 1 - 3 files changed, 3 deletions(-) diff --git a/tests/unit/target/test_realtime_target.py b/tests/unit/target/test_realtime_target.py index 0bf0d1164..88f047722 100644 --- a/tests/unit/target/test_realtime_target.py +++ b/tests/unit/target/test_realtime_target.py @@ -10,7 +10,6 @@ from pyrit.prompt_target import RealtimeTarget from pyrit.prompt_target.openai.openai_realtime_target import RealtimeTargetResult - # Env vars that may leak from .env files loaded by other tests in parallel workers. _CLEAN_UNDERLYING_MODEL_ENV = { "OPENAI_REALTIME_UNDERLYING_MODEL": "", diff --git a/tests/unit/target/test_supports_multi_turn.py b/tests/unit/target/test_supports_multi_turn.py index 8ec81a4b2..325c226e3 100644 --- a/tests/unit/target/test_supports_multi_turn.py +++ b/tests/unit/target/test_supports_multi_turn.py @@ -7,7 +7,6 @@ from tests.unit.mocks import MockPromptTarget - # Env vars that may leak from .env files loaded by other tests in parallel workers. _CLEAN_UNDERLYING_MODEL_ENV = { "OPENAI_VIDEO_UNDERLYING_MODEL": "", diff --git a/tests/unit/target/test_target_capabilities.py b/tests/unit/target/test_target_capabilities.py index df29529f8..7dead23a6 100644 --- a/tests/unit/target/test_target_capabilities.py +++ b/tests/unit/target/test_target_capabilities.py @@ -7,7 +7,6 @@ from pyrit.prompt_target.common.target_capabilities import TargetCapabilities - # Env vars that may leak from .env files loaded by other tests in parallel workers. # Clear them so that targets use _DEFAULT_CAPABILITIES instead of _KNOWN_CAPABILITIES. _CLEAN_UNDERLYING_MODEL_ENV = { From 6967954a290952635f53e3b3841a55212739d665 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 21 Mar 2026 11:38:51 -0700 Subject: [PATCH 3/3] Add missing env var patches for all OpenAI target tests - Add OPENAI_COMPLETION_UNDERLYING_MODEL and OPENAI_RESPONSES_UNDERLYING_MODEL to _CLEAN_UNDERLYING_MODEL_ENV in test_target_capabilities.py - Apply @patch.dict to all OpenAI target test methods in TestTargetCapabilitiesModalities (chat, image, tts, realtime, response, completion) - not just video - Add @patch.dict to TTS and completion tests in test_supports_multi_turn.py - Add TTS and completion env vars to test_supports_multi_turn.py dict Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/unit/target/test_supports_multi_turn.py | 4 ++++ tests/unit/target/test_target_capabilities.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/unit/target/test_supports_multi_turn.py b/tests/unit/target/test_supports_multi_turn.py index 325c226e3..f27b71795 100644 --- a/tests/unit/target/test_supports_multi_turn.py +++ b/tests/unit/target/test_supports_multi_turn.py @@ -10,6 +10,8 @@ # Env vars that may leak from .env files loaded by other tests in parallel workers. _CLEAN_UNDERLYING_MODEL_ENV = { "OPENAI_VIDEO_UNDERLYING_MODEL": "", + "OPENAI_TTS_UNDERLYING_MODEL": "", + "OPENAI_COMPLETION_UNDERLYING_MODEL": "", } @@ -59,6 +61,7 @@ def test_openai_video_target_returns_false(self): ) assert target.capabilities.supports_multi_turn is False + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_tts_target_returns_false(self): from pyrit.prompt_target import OpenAITTSTarget @@ -69,6 +72,7 @@ def test_openai_tts_target_returns_false(self): ) assert target.capabilities.supports_multi_turn is False + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_completion_target_returns_false(self): from pyrit.prompt_target import OpenAICompletionTarget diff --git a/tests/unit/target/test_target_capabilities.py b/tests/unit/target/test_target_capabilities.py index 7dead23a6..0ffb2688c 100644 --- a/tests/unit/target/test_target_capabilities.py +++ b/tests/unit/target/test_target_capabilities.py @@ -15,6 +15,8 @@ "OPENAI_CHAT_UNDERLYING_MODEL": "", "OPENAI_IMAGE_UNDERLYING_MODEL": "", "OPENAI_TTS_UNDERLYING_MODEL": "", + "OPENAI_COMPLETION_UNDERLYING_MODEL": "", + "OPENAI_RESPONSES_UNDERLYING_MODEL": "", } @@ -89,6 +91,7 @@ def test_default_capabilities_are_text_only(self): assert caps.input_modalities == frozenset({frozenset(["text"])}) assert caps.output_modalities == frozenset({frozenset(["text"])}) + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_chat_target_modalities(self): from pyrit.prompt_target import OpenAIChatTarget @@ -102,6 +105,7 @@ def test_openai_chat_target_modalities(self): assert target.capabilities.supports_json_output is True assert target.capabilities.supports_multi_message_pieces is True + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_image_target_modalities(self): from pyrit.prompt_target import OpenAIImageTarget @@ -114,6 +118,7 @@ def test_openai_image_target_modalities(self): assert target.capabilities.output_modalities == frozenset({frozenset(["image_path"])}) assert target.capabilities.supports_multi_message_pieces is True + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_tts_target_modalities(self): from pyrit.prompt_target import OpenAITTSTarget @@ -139,6 +144,7 @@ def test_openai_video_target_modalities(self): assert target.capabilities.output_modalities == frozenset({frozenset(["video_path"])}) assert target.capabilities.supports_multi_message_pieces is True + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_realtime_target_modalities(self): from pyrit.prompt_target import RealtimeTarget @@ -152,6 +158,7 @@ def test_openai_realtime_target_modalities(self): assert any("text" in combo for combo in target.capabilities.output_modalities) assert any("audio_path" in combo for combo in target.capabilities.output_modalities) + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_response_target_modalities(self): from pyrit.prompt_target import OpenAIResponseTarget @@ -166,6 +173,7 @@ def test_openai_response_target_modalities(self): assert target.capabilities.supports_json_output is True assert target.capabilities.supports_multi_message_pieces is True + @patch.dict("os.environ", _CLEAN_UNDERLYING_MODEL_ENV) def test_openai_completion_target_modalities(self): from pyrit.prompt_target import OpenAICompletionTarget