From 4cb033b6d3a5fe424d11877b270107d87a723597 Mon Sep 17 00:00:00 2001 From: Christopher Miles Date: Thu, 17 Jul 2025 08:38:44 -0400 Subject: [PATCH] Add more timeout configuration options for AWS Bedrock When submitting large requests or receiving large response, AWS Bedrock request may experience latency that exceeds our default settings. Here we add additional configuration options to allow increasing these timeout settings. Signed-off-by: Christopher Miles --- .../BedrockAwsConnectionProperties.java | 54 ++++++++++++++++++- ...ockConverseProxyChatAutoConfiguration.java | 4 ++ models/spring-ai-bedrock-converse/pom.xml | 6 +++ .../converse/BedrockProxyChatModel.java | 48 +++++++++++++++-- .../ROOT/pages/api/chat/bedrock-converse.adoc | 13 +++-- 5 files changed, 115 insertions(+), 10 deletions(-) diff --git a/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/autoconfigure/BedrockAwsConnectionProperties.java b/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/autoconfigure/BedrockAwsConnectionProperties.java index cc47b675b63..54de46eee3d 100644 --- a/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/autoconfigure/BedrockAwsConnectionProperties.java +++ b/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/autoconfigure/BedrockAwsConnectionProperties.java @@ -53,10 +53,30 @@ public class BedrockAwsConnectionProperties { private String sessionToken; /** - * Set model timeout, Defaults 5 min. + * Maximum duration of the entire API call operation. */ private Duration timeout = Duration.ofMinutes(5L); + /** + * Maximum time to wait while establishing connection with AWS service. + */ + private Duration connectionTimeout = Duration.ofSeconds(5L); + + /** + * Maximum duration spent reading response data. + */ + private Duration asyncReadTimeout = Duration.ofSeconds(30L); + + /** + * Maximum time to wait for a new connection from the pool. + */ + private Duration connectionAcquisitionTimeout = Duration.ofSeconds(30L); + + /** + * Maximum time to wait for response data. + */ + private Duration socketTimeout = Duration.ofSeconds(90L); + public String getRegion() { return this.region; } @@ -89,6 +109,38 @@ public void setTimeout(Duration timeout) { this.timeout = timeout; } + public Duration getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(Duration connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + public Duration getAsyncReadTimeout() { + return asyncReadTimeout; + } + + public void setAsyncReadTimeout(Duration asyncReadTimeout) { + this.asyncReadTimeout = asyncReadTimeout; + } + + public Duration getConnectionAcquisitionTimeout() { + return connectionAcquisitionTimeout; + } + + public void setConnectionAcquisitionTimeout(Duration connectionAcquisitionTimeout) { + this.connectionAcquisitionTimeout = connectionAcquisitionTimeout; + } + + public Duration getSocketTimeout() { + return socketTimeout; + } + + public void setSocketTimeout(Duration socketTimeout) { + this.socketTimeout = socketTimeout; + } + public String getSessionToken() { return this.sessionToken; } diff --git a/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/converse/autoconfigure/BedrockConverseProxyChatAutoConfiguration.java b/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/converse/autoconfigure/BedrockConverseProxyChatAutoConfiguration.java index 311c1bfb807..96be2491325 100644 --- a/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/converse/autoconfigure/BedrockConverseProxyChatAutoConfiguration.java +++ b/auto-configurations/models/spring-ai-autoconfigure-model-bedrock-ai/src/main/java/org/springframework/ai/model/bedrock/converse/autoconfigure/BedrockConverseProxyChatAutoConfiguration.java @@ -76,6 +76,10 @@ public BedrockProxyChatModel bedrockProxyChatModel(AwsCredentialsProvider creden .credentialsProvider(credentialsProvider) .region(regionProvider.getRegion()) .timeout(connectionProperties.getTimeout()) + .connectionTimeout(connectionProperties.getConnectionTimeout()) + .asyncReadTimeout(connectionProperties.getAsyncReadTimeout()) + .connectionAcquisitionTimeout(connectionProperties.getConnectionAcquisitionTimeout()) + .socketTimeout(connectionProperties.getSocketTimeout()) .defaultOptions(chatProperties.getOptions()) .observationRegistry(observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP)) .toolCallingManager(toolCallingManager) diff --git a/models/spring-ai-bedrock-converse/pom.xml b/models/spring-ai-bedrock-converse/pom.xml index cc536e4d556..d084db287a7 100644 --- a/models/spring-ai-bedrock-converse/pom.xml +++ b/models/spring-ai-bedrock-converse/pom.xml @@ -76,6 +76,12 @@ ${bedrockruntime.version} + + software.amazon.awssdk + apache-client + ${bedrockruntime.version} + + org.springframework.ai diff --git a/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java b/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java index 071e77a78cb..c2329825808 100644 --- a/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java +++ b/models/spring-ai-bedrock-converse/src/main/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModel.java @@ -46,6 +46,7 @@ import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.core.document.Document; import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; @@ -782,7 +783,15 @@ public static final class Builder { private Region region = Region.US_EAST_1; - private Duration timeout = Duration.ofMinutes(10); + private Duration timeout = Duration.ofMinutes(5L); + + private Duration connectionTimeout = Duration.ofSeconds(5L); + + private Duration asyncReadTimeout = Duration.ofSeconds(30L); + + private Duration connectionAcquisitionTimeout = Duration.ofSeconds(30L); + + private Duration socketTimeout = Duration.ofSeconds(30L); private ToolCallingManager toolCallingManager; @@ -836,6 +845,30 @@ public Builder timeout(Duration timeout) { return this; } + public Builder connectionTimeout(Duration connectionTimeout) { + Assert.notNull(connectionTimeout, "'connectionTimeout' must not be null."); + this.connectionTimeout = connectionTimeout; + return this; + } + + public Builder asyncReadTimeout(Duration asyncReadTimeout) { + Assert.notNull(asyncReadTimeout, "'asyncReadTimeout' must not be null."); + this.asyncReadTimeout = asyncReadTimeout; + return this; + } + + public Builder connectionAcquisitionTimeout(Duration connectionAcquisitionTimeout) { + Assert.notNull(connectionAcquisitionTimeout, "'connectionAcquisitionTimeout' must not be null."); + this.connectionAcquisitionTimeout = connectionAcquisitionTimeout; + return this; + } + + public Builder socketTimeout(Duration socketTimeout) { + Assert.notNull(socketTimeout, "'socketTimeout' must not be null."); + this.socketTimeout = socketTimeout; + return this; + } + public Builder defaultOptions(BedrockChatOptions defaultOptions) { Assert.notNull(defaultOptions, "'defaultOptions' must not be null."); this.defaultOptions = defaultOptions; @@ -867,9 +900,15 @@ public Builder bedrockRuntimeAsyncClient(BedrockRuntimeAsyncClient bedrockRuntim public BedrockProxyChatModel build() { if (this.bedrockRuntimeClient == null) { + + var httpClientBuilder = ApacheHttpClient.builder() + .connectionAcquisitionTimeout(connectionAcquisitionTimeout) + .connectionTimeout(this.connectionTimeout) + .socketTimeout(this.socketTimeout); + this.bedrockRuntimeClient = BedrockRuntimeClient.builder() .region(this.region) - .httpClientBuilder(null) + .httpClientBuilder(httpClientBuilder) .credentialsProvider(this.credentialsProvider) .overrideConfiguration(c -> c.apiCallTimeout(this.timeout)) .build(); @@ -877,10 +916,11 @@ public BedrockProxyChatModel build() { if (this.bedrockRuntimeAsyncClient == null) { - // TODO: Is it ok to configure the NettyNioAsyncHttpClient explicitly??? var httpClientBuilder = NettyNioAsyncHttpClient.builder() .tcpKeepAlive(true) - .connectionAcquisitionTimeout(Duration.ofSeconds(30)) + .readTimeout(this.asyncReadTimeout) + .connectionTimeout(this.connectionTimeout) + .connectionAcquisitionTimeout(this.connectionAcquisitionTimeout) .maxConcurrency(200); var builder = BedrockRuntimeAsyncClient.builder() diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/bedrock-converse.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/bedrock-converse.adoc index 44966528e06..25258aa19a5 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/bedrock-converse.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/bedrock-converse.adoc @@ -73,11 +73,14 @@ The prefix `spring.ai.bedrock.aws` is the property prefix to configure the conne |==== | Property | Description | Default -| spring.ai.bedrock.aws.region | AWS region to use. | us-east-1 -| spring.ai.bedrock.aws.timeout | AWS timeout to use. | 5m -| spring.ai.bedrock.aws.access-key | AWS access key. | - -| spring.ai.bedrock.aws.secret-key | AWS secret key. | - -| spring.ai.bedrock.aws.session-token | AWS session token for temporary credentials. | - +| spring.ai.bedrock.aws.region | AWS region to use | us-east-1 +| spring.ai.bedrock.aws.timeout | AWS max duration for entire API call | 5m +| spring.ai.bedrock.aws.connectionTimeout | Max duration to wait while establishing connection | 5s +| spring.ai.bedrock.aws.connectionAcquisitionTimeout | Max duration to wait for new connection from the pool | 30s +| spring.ai.bedrock.aws.asyncReadTimeout | Max duration spent reading asynchronous responses | 30s +| spring.ai.bedrock.aws.access-key | AWS access key | - +| spring.ai.bedrock.aws.secret-key | AWS secret key | - +| spring.ai.bedrock.aws.session-token | AWS session token for temporary credentials | - |==== [NOTE]