From af48ad2b5caabb26ec2be3e20f4fd95e99d97327 Mon Sep 17 00:00:00 2001 From: Philip Kiely Date: Tue, 8 Jul 2025 21:33:33 -0700 Subject: [PATCH 1/2] Document Baseten integration --- .../concepts/model-providers/baseten.md | 132 ++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 133 insertions(+) create mode 100644 docs/user-guide/concepts/model-providers/baseten.md diff --git a/docs/user-guide/concepts/model-providers/baseten.md b/docs/user-guide/concepts/model-providers/baseten.md new file mode 100644 index 00000000..d98d047c --- /dev/null +++ b/docs/user-guide/concepts/model-providers/baseten.md @@ -0,0 +1,132 @@ +# Baseten + +[Baseten](https://baseten.co) is an AI inference provider that supports open-source, fine-tuned, and custom models of all modalities. The Strands Agents SDK can be used to run against powered by any OpenAI-compatible LLM hosted on Baseten. + +Baseten offers two options for LLM inference, both of which are compatible with the Strands Agents SDK: + +- **Model APIs**: Access to pre-deployed models like DeepSeek and Llama +- **Dedicated Deployments**: Custom model deployments with dedicated infrastructure + +## Installation + +Baseten is configured as an optional dependency in Strands Agents. To install, run: + +```bash +pip install 'strands-agents[baseten]' +``` + +## Configuration + +### API Key + +You'll need a Baseten API key to use the service. Set it as an environment variable: + +```bash +export BASETEN_API_KEY="your-api-key-here" +``` + +### Model APIs + +Model APIs provide access to popular models through a common API endpoint: + +```python +from strands.models.baseten import BasetenModel + +# DeepSeek R1 model +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-R1-0528", + client_args={ + "api_key": "your-api-key", + }, +) + +# DeepSeek V3 model +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={ + "api_key": "your-api-key", + }, +) + +# Llama 4 Maverick model +model = BasetenModel( + model_id="meta-llama/Llama-4-Maverick-17B-128E-Instruct", + client_args={ + "api_key": "your-api-key", + }, +) + +# Llama 4 Scout model +model = BasetenModel( + model_id="meta-llama/Llama-4-Scout-17B-16E-Instruct", + client_args={ + "api_key": "your-api-key", + }, +) +``` + +**Available Model APIs:** + +* `deepseek-ai/DeepSeek-R1-0528`: DeepSeek R1 0528 model +* `deepseek-ai/DeepSeek-V3-0324`: DeepSeek V3 0324 model +* `meta-llama/Llama-4-Maverick-17B-128E-Instruct`: Llama 4 Maverick 17B model +* `meta-llama/Llama-4-Scout-17B-16E-Instruct`: Llama 4 Scout 17B model + +### Dedicated Deployments + +Dedicated deployments provide custom model hosting with dedicated infrastructure: + +```python +from strands.models.baseten import BasetenModel + +deployment_id = "dq4kr413" # Your deployment ID +environment = "production" # Environment (default: "production") +base_url = f"https://model-{deployment_id}.api.baseten.co/environments/{environment}/sync/v1" + +model = BasetenModel( + model_id=deployment_id, + base_url=base_url, + environment=environment, + client_args={ + "api_key": "your-api-key", + }, +) +``` + +**Environment Options:** + +* `production`: Production environment (default) +* `staging`: Staging environment +* `development`: Development environment + +## Usage + +After installing `strands-agents[baseten]`, you can import and initialize the Strands Agents' Baseten provider as follows: + +```python +from strands import Agent +from strands.models.baseten import BasetenModel + +# Initialize model +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={"api_key": "your-api-key"}, +) + +# Create agent +agent = Agent(model=model) + +# Chat with the model +response = agent("Hello! How are you?") +print(response) +``` + +Baseten models + + +## References + +* [Baseten Documentation](https://docs.baseten.co/) +* [Strands SDK Documentation](https://docs.strands.ai/) + + diff --git a/mkdocs.yml b/mkdocs.yml index 9c3d0c96..6edadcaf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,6 +85,7 @@ nav: - Model Providers: - Amazon Bedrock: user-guide/concepts/model-providers/amazon-bedrock.md - Anthropic: user-guide/concepts/model-providers/anthropic.md + - Baseten: user-guide/concepts/model-providers/baseten.md - LiteLLM: user-guide/concepts/model-providers/litellm.md - LlamaAPI: user-guide/concepts/model-providers/llamaapi.md - MistralAI: user-guide/concepts/model-providers/mistral.md From a7fe222de6eaa70d877b302edba46c90c5b4e949 Mon Sep 17 00:00:00 2001 From: Philip Kiely Date: Mon, 14 Jul 2025 17:50:49 -0700 Subject: [PATCH 2/2] updated user guide --- .../concepts/model-providers/baseten.md | 151 +++++++++++++++--- 1 file changed, 125 insertions(+), 26 deletions(-) diff --git a/docs/user-guide/concepts/model-providers/baseten.md b/docs/user-guide/concepts/model-providers/baseten.md index d98d047c..e45dd5d1 100644 --- a/docs/user-guide/concepts/model-providers/baseten.md +++ b/docs/user-guide/concepts/model-providers/baseten.md @@ -27,7 +27,7 @@ export BASETEN_API_KEY="your-api-key-here" ### Model APIs -Model APIs provide access to popular models through a common API endpoint: +Model APIs provide access to popular models through a common API endpoint. The SDK automatically uses the correct base URL for Model APIs: ```python from strands.models.baseten import BasetenModel @@ -36,7 +36,7 @@ from strands.models.baseten import BasetenModel model = BasetenModel( model_id="deepseek-ai/DeepSeek-R1-0528", client_args={ - "api_key": "your-api-key", + "api_key": os.getenv("BASETEN_API_KEY"), }, ) @@ -44,7 +44,7 @@ model = BasetenModel( model = BasetenModel( model_id="deepseek-ai/DeepSeek-V3-0324", client_args={ - "api_key": "your-api-key", + "api_key": os.getenv("BASETEN_API_KEY"), }, ) @@ -52,7 +52,7 @@ model = BasetenModel( model = BasetenModel( model_id="meta-llama/Llama-4-Maverick-17B-128E-Instruct", client_args={ - "api_key": "your-api-key", + "api_key": os.getenv("BASETEN_API_KEY"), }, ) @@ -60,49 +60,53 @@ model = BasetenModel( model = BasetenModel( model_id="meta-llama/Llama-4-Scout-17B-16E-Instruct", client_args={ - "api_key": "your-api-key", + "api_key": os.getenv("BASETEN_API_KEY"), }, ) ``` -**Available Model APIs:** - -* `deepseek-ai/DeepSeek-R1-0528`: DeepSeek R1 0528 model -* `deepseek-ai/DeepSeek-V3-0324`: DeepSeek V3 0324 model -* `meta-llama/Llama-4-Maverick-17B-128E-Instruct`: Llama 4 Maverick 17B model -* `meta-llama/Llama-4-Scout-17B-16E-Instruct`: Llama 4 Scout 17B model +Check the Baseten documentation for a [complete list of Model APIs](https://docs.baseten.co/development/model-apis/overview). ### Dedicated Deployments -Dedicated deployments provide custom model hosting with dedicated infrastructure: +Dedicated deployments provide custom model hosting with dedicated infrastructure. You'll need to specify the custom base URL: ```python from strands.models.baseten import BasetenModel -deployment_id = "dq4kr413" # Your deployment ID -environment = "production" # Environment (default: "production") -base_url = f"https://model-{deployment_id}.api.baseten.co/environments/{environment}/sync/v1" +# Replace with your model URL +base_url = "https://model-abcd1234.api.baseten.co/environments/production/sync/v1" model = BasetenModel( - model_id=deployment_id, + model_id="", base_url=base_url, - environment=environment, client_args={ - "api_key": "your-api-key", + "api_key": os.getenv("BASETEN_API_KEY"), }, ) ``` -**Environment Options:** +### Model Parameters -* `production`: Production environment (default) -* `staging`: Staging environment -* `development`: Development environment +You can configure model parameters using the `params` argument: + +```python +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={"api_key": os.getenv("BASETEN_API_KEY")}, + params={ + "max_tokens": 1000, + "temperature": 0.7 + }, +) +``` ## Usage After installing `strands-agents[baseten]`, you can import and initialize the Strands Agents' Baseten provider as follows: +### Basic Usage + ```python from strands import Agent from strands.models.baseten import BasetenModel @@ -110,7 +114,7 @@ from strands.models.baseten import BasetenModel # Initialize model model = BasetenModel( model_id="deepseek-ai/DeepSeek-V3-0324", - client_args={"api_key": "your-api-key"}, + client_args={"api_key": os.getenv("BASETEN_API_KEY")}, ) # Create agent @@ -121,12 +125,107 @@ response = agent("Hello! How are you?") print(response) ``` -Baseten models +### Async Usage + +The BasetenModel supports async operations: + +```python +import asyncio +from strands.models.baseten import BasetenModel + +async def chat_with_model(): + model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={"api_key": os.getenv("BASETEN_API_KEY")}, + ) + + messages = [{"role": "user", "content": [{"text": "Hello!"}]}] + + async for event in model.stream(messages): + print(event) + +# Run the async function +asyncio.run(chat_with_model()) +``` + +### Structured Output + +You can get structured output using Pydantic models: + +```python +from pydantic import BaseModel +from strands.models.baseten import BasetenModel + +class MathResult(BaseModel): + answer: int + explanation: str + +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={"api_key": os.getenv("BASETEN_API_KEY")}, +) + +messages = [{"role": "user", "content": [{"text": "What is 5 + 3?"}]}] + +async for result in model.structured_output(MathResult, messages): + print(f"Answer: {result['output'].answer}") + print(f"Explanation: {result['output'].explanation}") +``` + +### With Tools +Baseten models support tool use for function calling: + +```python +from strands.models.baseten import BasetenModel + +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={"api_key": os.getenv("BASETEN_API_KEY")}, +) + +tool_specs = [ + { + "name": "calculator", + "description": "A simple calculator", + "inputSchema": { + "json": { + "type": "object", + "properties": { + "expression": {"type": "string"} + }, + "required": ["expression"] + } + } + } +] + +messages = [{"role": "user", "content": [{"text": "Calculate 2 + 2"}]}] + +async for event in model.stream(messages, tool_specs=tool_specs): + print(event) +``` + +### Configuration Management + +You can update model configuration at runtime: + +```python +model = BasetenModel( + model_id="deepseek-ai/DeepSeek-V3-0324", + client_args={"api_key": os.getenv("BASETEN_API_KEY")}, + params={"max_tokens": 100} +) + +# Update configuration +model.update_config(params={"max_tokens": 200, "temperature": 0.8}) + +# Get current configuration +config = model.get_config() +print(config) +``` ## References * [Baseten Documentation](https://docs.baseten.co/) * [Strands SDK Documentation](https://docs.strands.ai/) - -