Update integ tests to isolate provider-based tests (#396)

Move tests to `tests_integ` as `tests-integ` is not a proper module name.  Also extract all provider ignoring to a new providers file which centralizes the environment variables needed.

Author: zastrowm

.github/workflows/integration-test.yml

@@ -69,4 +69,4 @@ jobs:
           AWS_REGION_NAME: us-east-1 # Needed for LiteLLM
         id: tests
         run: |
-          hatch test tests-integ
+          hatch test tests_integ

pyproject.toml

@@ -195,7 +195,7 @@ test = [
     "hatch test --cover --cov-report html --cov-report xml {args}"
 ]
 test-integ = [
-    "hatch test tests-integ {args}"
+    "hatch test tests_integ {args}"
 ]
 prepare = [
     "hatch fmt --linter",
@@ -230,7 +230,7 @@ ignore_missing_imports = true
 
 [tool.ruff]
 line-length = 120
-include = ["examples/**/*.py", "src/**/*.py", "tests/**/*.py", "tests-integ/**/*.py"]
+include = ["examples/**/*.py", "src/**/*.py", "tests/**/*.py", "tests_integ/**/*.py"]
 
 [tool.ruff.lint]
 select = [

tests-integ/__init__.py renamed to tests_integ/__init__.py

@@ -0,0 +1,46 @@
+import os
+from dataclasses import dataclass
+
+import requests
+from pytest import mark
+
+
+@dataclass
+class ApiKeyProviderInfo:
+    """Provider-based info for providers that require an APIKey via environment variables."""
+
+    def __init__(self, id: str, environment_variable: str) -> None:
+        self.id = id
+        self.environment_variable = environment_variable
+        self.mark = mark.skipif(
+            self.environment_variable not in os.environ,
+            reason=f"{self.environment_variable} environment variable missing",
+        )
+
+
+class OllamaProviderInfo:
+    """Special case ollama as it's dependent on the server being available."""
+
+    def __init__(self):
+        self.id = "ollama"
+
+        is_server_available = False
+        try:
+            is_server_available = requests.get("http://localhost:11434").ok
+        except requests.exceptions.ConnectionError:
+            pass
+
+        self.mark = mark.skipif(
+            not is_server_available,
+            reason="Local Ollama endpoint not available at localhost:11434",
+        )
+
+
+anthropic = ApiKeyProviderInfo(id="anthropic", environment_variable="ANTHROPIC_API_KEY")
+cohere = ApiKeyProviderInfo(id="cohere", environment_variable="CO_API_KEY")
+llama = ApiKeyProviderInfo(id="cohere", environment_variable="LLAMA_API_KEY")
+mistral = ApiKeyProviderInfo(id="mistral", environment_variable="MISTRAL_API_KEY")
+openai = ApiKeyProviderInfo(id="openai", environment_variable="OPENAI_API_KEY")
+writer = ApiKeyProviderInfo(id="writer", environment_variable="WRITER_API_KEY")
+
+ollama = OllamaProviderInfo()

tests-integ/conftest.py renamed to tests_integ/conftest.py

@@ -6,6 +6,10 @@
 import strands
 from strands import Agent
 from strands.models.anthropic import AnthropicModel
+from tests_integ.models import providers
+
+# these tests only run if we have the anthropic api key
+pytestmark = providers.anthropic.mark
 
 
 @pytest.fixture(scope="module")
@@ -53,15 +57,13 @@ class Weather(BaseModel):
     return Weather(time="12:00", weather="sunny")
 
 
-@pytest.mark.skipif("ANTHROPIC_API_KEY" not in os.environ, reason="ANTHROPIC_API_KEY environment variable missing")
 def test_agent_invoke(agent):
     result = agent("What is the time and weather in New York?")
     text = result.message["content"][0]["text"].lower()
 
     assert all(string in text for string in ["12:00", "sunny"])
 
 
-@pytest.mark.skipif("ANTHROPIC_API_KEY" not in os.environ, reason="ANTHROPIC_API_KEY environment variable missing")
 @pytest.mark.asyncio
 async def test_agent_invoke_async(agent):
     result = await agent.invoke_async("What is the time and weather in New York?")
@@ -70,7 +72,6 @@ async def test_agent_invoke_async(agent):
     assert all(string in text for string in ["12:00", "sunny"])
 
 
-@pytest.mark.skipif("ANTHROPIC_API_KEY" not in os.environ, reason="ANTHROPIC_API_KEY environment variable missing")
 @pytest.mark.asyncio
 async def test_agent_stream_async(agent):
     stream = agent.stream_async("What is the time and weather in New York?")
@@ -83,14 +84,12 @@ async def test_agent_stream_async(agent):
     assert all(string in text for string in ["12:00", "sunny"])
 
 
-@pytest.mark.skipif("ANTHROPIC_API_KEY" not in os.environ, reason="ANTHROPIC_API_KEY environment variable missing")
 def test_structured_output(agent, weather):
     tru_weather = agent.structured_output(type(weather), "The time is 12:00 and the weather is sunny")
     exp_weather = weather
     assert tru_weather == exp_weather
 
 
-@pytest.mark.skipif("ANTHROPIC_API_KEY" not in os.environ, reason="ANTHROPIC_API_KEY environment variable missing")
 @pytest.mark.asyncio
 async def test_agent_structured_output_async(agent, weather):
     tru_weather = await agent.structured_output_async(type(weather), "The time is 12:00 and the weather is sunny")

tests-integ/echo_server.py renamed to tests_integ/echo_server.py

@@ -5,6 +5,10 @@
 import strands
 from strands import Agent
 from strands.models.openai import OpenAIModel
+from tests_integ.models import providers
+
+# these tests only run if we have the cohere api key
+pytestmark = providers.cohere.mark
 
 
 @pytest.fixture
@@ -37,10 +41,6 @@ def agent(model, tools):
     return Agent(model=model, tools=tools)
 
 
-@pytest.mark.skipif(
-    "CO_API_KEY" not in os.environ,
-    reason="CO_API_KEY environment variable missing",
-)
 def test_agent(agent):
     result = agent("What is the time and weather in New York?")
     text = result.message["content"][0]["text"].lower()