From 6b4b75795050d02e5186181402d3c9d2942292e0 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 22 Sep 2025 10:37:21 -0400 Subject: [PATCH 1/9] Add new FileIo CRT bindings --- pom.xml | 2 +- .../services/s3/S3CrtAsyncClientBuilder.java | 19 ++ .../s3/crt/S3CrtFileIoConfiguration.java | 203 ++++++++++++++++++ .../internal/crt/DefaultS3CrtAsyncClient.java | 12 +- .../s3/internal/crt/S3CrtAsyncHttpClient.java | 3 +- .../crt/S3NativeClientConfiguration.java | 23 ++ .../crt/S3CrtAsyncHttpClientTest.java | 24 ++- 7 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java diff --git a/pom.xml b/pom.xml index 67a93bbc64ce..ade2761fc045 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,7 @@ 3.1.5 1.17.1 1.37 - 0.38.9 + 0.39.0 5.10.0 diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index b4cedc3ec1c0..18e05eb983c5 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -28,6 +28,7 @@ import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; import software.amazon.awssdk.identity.spi.IdentityProvider; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -361,6 +362,24 @@ default S3CrtAsyncClientBuilder retryConfiguration(Consumer fileIoOptionsBuilder) { + Validate.paramNotNull(fileIoOptionsBuilder, "fileIoOptionsBuilder"); + return fileIoOptions(S3CrtFileIoConfiguration.builder() + .applyMutation(fileIoOptionsBuilder) + .build()); + } @Override S3AsyncClient build(); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java new file mode 100644 index 000000000000..47b07cf1efdb --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java @@ -0,0 +1,203 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crt; + +import java.util.Objects; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Configuration options which controls how client performs file I/O operations. Only applies to file-based workloads. + */ +@SdkPublicApi +@Immutable +@ThreadSafe +public final class S3CrtFileIoConfiguration implements ToCopyableBuilder { + private final Boolean shouldStream; + private final Double diskThroughputGbps; + private final Boolean directIo; + + private S3CrtFileIoConfiguration(DefaultBuilder builder) { + this.shouldStream = builder.shouldStream; + this.diskThroughputGbps = builder.diskThroughputGbps; + this.directIo = builder.directIo; + } + + /** + * Creates a default builder for {@link S3CrtFileIoConfiguration}. + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + /** + * Skip buffering the part in memory before sending the request. + * If set, set the {@code diskThroughputGbps} to reasonably align with the available disk throughput. + * Otherwise, the transfer may fail with connection starvation. + * Defaults to false. + * + * @return if client should skip buffering in memory before sending the request. + */ + public Boolean shouldStream() { + return shouldStream; + } + + /** + * The estimated disk throughput in gigabits per second (Gbps). + * Only applied when {@code shouldStream} is true. + * + * When doing upload with streaming, it's important to set the disk throughput to prevent connection starvation. + * Note: There are possibilities that cannot reach all available disk throughput: + * 1. Disk is busy with other applications + * 2. OS Cache may cap the throughput, use {@code directIo} to get around this. + * + * @return disk throughput value in Gpbs. + */ + public Double diskThroughputGbps() { + return diskThroughputGbps; + } + + /** + * Enable direct I/O to bypass the OS cache. Helpful when the disk I/O outperforms the kernel cache. + * Notes: + * - Only supported on Linux for now. + * - Only supports upload for now. + * - Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT + * for additional info open + * + * @return directIO value + */ + public Boolean directIo() { + return directIo; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + S3CrtFileIoConfiguration that = (S3CrtFileIoConfiguration) o; + + if (!Objects.equals(shouldStream, that.shouldStream)) { + return false; + } + if (!Objects.equals(diskThroughputGbps, that.diskThroughputGbps)) { + return false; + } + return Objects.equals(directIo, that.directIo); + } + + @Override + public int hashCode() { + int result = shouldStream != null ? shouldStream.hashCode() : 0; + result = 31 * result + (diskThroughputGbps != null ? diskThroughputGbps.hashCode() : 0); + result = 31 * result + (directIo != null ? directIo.hashCode() : 0); + return result; + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + public interface Builder extends CopyableBuilder { + /** + * Skip buffering the part in memory before sending the request. + * If set, set the {@code diskThroughputGbps} to reasonably align with the available disk throughput. + * Otherwise, the transfer may fail with connection starvation. + * Defaults to false. + * + * @param shouldStream whether to stream the file + * @return The builder for method chaining. + */ + Builder shouldStream(Boolean shouldStream); + + /** + * The estimated disk throughput in gigabits per second (Gbps). + * Only applied when {@code shouldStream} is true. + * + * When doing upload with streaming, it's important to set the disk throughput to prevent connection starvation. + * Note: There are possibilities that cannot reach all available disk throughput: + * 1. Disk is busy with other applications + * 2. OS Cache may cap the throughput, use {@code directIo} to get around this. + * + * @param diskThroughputGbps the disk throughput in Gbps + * @return The builder for method chaining. + */ + Builder diskThroughputGbps(Double diskThroughputGbps); + + /** + * Enable direct I/O to bypass the OS cache. Helpful when the disk I/O outperforms the kernel cache. + * Notes: + * - Only supported on Linux for now. + * - Only supports upload for now. + * - Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT + * for additional info https://man7.org/linux/man-pages/man2/openat.2.html + * + * @param directIo whether to enable direct I/O + * @return The builder for method chaining. + */ + Builder directIo(Boolean directIo); + + @Override + S3CrtFileIoConfiguration build(); + } + + private static final class DefaultBuilder implements Builder { + private Boolean shouldStream; + private Double diskThroughputGbps; + private Boolean directIo; + + private DefaultBuilder() { + } + + private DefaultBuilder(S3CrtFileIoConfiguration fileIoOptions) { + this.shouldStream = fileIoOptions.shouldStream; + this.diskThroughputGbps = fileIoOptions.diskThroughputGbps; + this.directIo = fileIoOptions.directIo; + } + + @Override + public Builder shouldStream(Boolean shouldStream) { + this.shouldStream = shouldStream; + return this; + } + + @Override + public Builder diskThroughputGbps(Double diskThroughputGbps) { + this.diskThroughputGbps = diskThroughputGbps; + return this; + } + + @Override + public Builder directIo(Boolean directIo) { + this.directIo = directIo; + return this; + } + + @Override + public S3CrtFileIoConfiguration build() { + return new S3CrtFileIoConfiguration(this); + } + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 676e060e3218..40e88563c269 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -67,6 +67,7 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration; import software.amazon.awssdk.services.s3.internal.checksums.ChecksumsEnabledValidator; @@ -222,7 +223,8 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau .readBufferSizeInBytes(builder.readBufferSizeInBytes) .httpConfiguration(builder.httpConfiguration) .thresholdInBytes(builder.thresholdInBytes) - .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes); + .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes) + .fileIoOptions(builder.fileIoOptions); if (builder.retryConfiguration != null) { nativeClientBuilder.standardRetryOptions( @@ -256,7 +258,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Long thresholdInBytes; private Executor futureCompletionExecutor; private Boolean disableS3ExpressSessionAuth; - + private S3CrtFileIoConfiguration fileIoOptions; @Override public DefaultS3CrtClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) { @@ -388,6 +390,12 @@ public DefaultS3CrtClientBuilder disableS3ExpressSessionAuth(Boolean disableS3Ex return this; } + @Override + public S3CrtAsyncClientBuilder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions) { + this.fileIoOptions = fileIoOptions; + return this; + } + @Override public S3CrtAsyncClient build() { return new DefaultS3CrtAsyncClient(this); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index 1fed55813d33..99edc159156e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -112,7 +112,8 @@ private S3ClientOptions createS3ClientOption() { .withMemoryLimitInBytes(s3NativeClientConfiguration.maxNativeMemoryLimitInBytes()) .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps()) .withInitialReadWindowSize(initialWindowSize) - .withReadBackpressureEnabled(true); + .withReadBackpressureEnabled(true) + .withFileIoOptions(s3NativeClientConfiguration.fileIoOptions()); if (s3NativeClientConfiguration.standardRetryOptions() != null) { options.withStandardRetryOptions(s3NativeClientConfiguration.standardRetryOptions()); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index f43cbf84d46e..aeb3d7efb6e8 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -29,9 +29,11 @@ import software.amazon.awssdk.crt.io.TlsCipherPreference; import software.amazon.awssdk.crt.io.TlsContext; import software.amazon.awssdk.crt.io.TlsContextOptions; +import software.amazon.awssdk.crt.s3.FileIoOptions; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; import software.amazon.awssdk.identity.spi.IdentityProvider; import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.SdkAutoCloseable; @@ -43,6 +45,7 @@ @SdkInternalApi public class S3NativeClientConfiguration implements SdkAutoCloseable { static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; + static final double DEFAULT_FILE_IO_THROUGHPUT_IN_GPBS = 10.0; private static final Logger log = Logger.loggerFor(S3NativeClientConfiguration.class); private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 10; @@ -64,6 +67,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable { private final HttpMonitoringOptions httpMonitoringOptions; private final Boolean useEnvironmentVariableProxyOptionsValues; private final long maxNativeMemoryLimitInBytes; + private final FileIoOptions fileIoOptions; public S3NativeClientConfiguration(Builder builder) { this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() : @@ -113,6 +117,7 @@ public S3NativeClientConfiguration(Builder builder) { } this.standardRetryOptions = builder.standardRetryOptions; this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder); + this.fileIoOptions = builder.fileIoOptions == null ? null : resolveFileIoOptions(builder.fileIoOptions); } private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { @@ -122,6 +127,14 @@ private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { return true; } + private static FileIoOptions resolveFileIoOptions(S3CrtFileIoConfiguration s3CrtfileIoConfiguration) { + boolean shouldStream = Validate.getOrDefault(s3CrtfileIoConfiguration.shouldStream(), () -> false); + double diskThroughputInGbps = Validate.getOrDefault(s3CrtfileIoConfiguration.diskThroughputGbps(), + () -> DEFAULT_FILE_IO_THROUGHPUT_IN_GPBS); + boolean directIo = Validate.getOrDefault(s3CrtfileIoConfiguration.directIo(), () -> false); + return new FileIoOptions(shouldStream, diskThroughputInGbps, directIo); + } + public Boolean isUseEnvironmentVariableValues() { return useEnvironmentVariableProxyOptionsValues; } @@ -191,6 +204,10 @@ public Long readBufferSizeInBytes() { return readBufferSizeInBytes; } + public FileIoOptions fileIoOptions() { + return fileIoOptions; + } + @Override public void close() { clientBootstrap.close(); @@ -212,6 +229,7 @@ public static final class Builder { private StandardRetryOptions standardRetryOptions; private Long thresholdInBytes; private Long maxNativeMemoryLimitInBytes; + private S3CrtFileIoConfiguration fileIoOptions; private Builder() { } @@ -274,5 +292,10 @@ public Builder thresholdInBytes(Long thresholdInBytes) { this.thresholdInBytes = thresholdInBytes; return this; } + + public Builder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions) { + this.fileIoOptions = fileIoOptions; + return this; + } } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 776b69c4a10a..9b72627baf77 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -59,13 +59,13 @@ import software.amazon.awssdk.crt.s3.S3ClientOptions; import software.amazon.awssdk.crt.s3.S3MetaRequest; import software.amazon.awssdk.crt.s3.S3MetaRequestOptions; -import software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.async.AsyncExecuteRequest; import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.testutils.RandomTempFile; @@ -443,6 +443,7 @@ void build_shouldPassThroughParameters() { .minimumThroughputTimeout(Duration.ofSeconds(2))) .proxyConfiguration(p -> p.host("127.0.0.1").port(8080)) .build()) + .fileIoOptions(S3CrtFileIoConfiguration.builder().diskThroughputGbps(8.0).shouldStream(true).directIo(true).build()) .build(); try (S3CrtAsyncHttpClient client = (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build()) { @@ -466,6 +467,12 @@ void build_shouldPassThroughParameters() { assertThat(clientOptions.getMaxConnections()).isEqualTo(100); assertThat(clientOptions.getThroughputTargetGbps()).isEqualTo(3.5); assertThat(clientOptions.getMemoryLimitInBytes()).isEqualTo(5L * 1024 * 1024 * 1024); + + assertThat(clientOptions.getFileIoOptions()).isNotNull(); + assertThat(clientOptions.getFileIoOptions().getShouldStream()).isTrue(); + assertThat(clientOptions.getFileIoOptions().getDirectIo()).isTrue(); + assertThat(clientOptions.getFileIoOptions().getDiskThroughputGbps()).isEqualTo(8.0); + } } @@ -504,6 +511,21 @@ void build_nullHttpConfiguration() { } } + @Test + void build_nullFileOptions() { + StaticCredentialsProvider credentialsProvider = + StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")); + S3NativeClientConfiguration configuration = + S3NativeClientConfiguration.builder() + .credentialsProvider(credentialsProvider) + .build(); + try (S3CrtAsyncHttpClient client = + (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build()) { + S3ClientOptions clientOptions = client.s3ClientOptions(); + assertThat(clientOptions.getFileIoOptions()).isNull(); + } + } + private static Stream s3CrtHttpConfigurations() { return Stream.of( Arguments.of(S3CrtHttpConfiguration.builder() From ff6626ae7f5b373efe6c5f487fbfe7cd882c7a3e Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 22 Sep 2025 11:09:53 -0400 Subject: [PATCH 2/9] rename builder method to fileIoConfiguration --- .../awssdk/services/s3/S3CrtAsyncClientBuilder.java | 10 +++++----- .../s3/internal/crt/DefaultS3CrtAsyncClient.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index 18e05eb983c5..e5afb5706bfa 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -367,18 +367,18 @@ default S3CrtAsyncClientBuilder retryConfiguration(Consumer fileIoOptionsBuilder) { + default S3CrtAsyncClientBuilder fileIoConfiguration(Consumer fileIoOptionsBuilder) { Validate.paramNotNull(fileIoOptionsBuilder, "fileIoOptionsBuilder"); - return fileIoOptions(S3CrtFileIoConfiguration.builder() - .applyMutation(fileIoOptionsBuilder) - .build()); + return fileIoConfiguration(S3CrtFileIoConfiguration.builder() + .applyMutation(fileIoOptionsBuilder) + .build()); } @Override diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 40e88563c269..1fcc8534544f 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -391,7 +391,7 @@ public DefaultS3CrtClientBuilder disableS3ExpressSessionAuth(Boolean disableS3Ex } @Override - public S3CrtAsyncClientBuilder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions) { + public S3CrtAsyncClientBuilder fileIoConfiguration(S3CrtFileIoConfiguration fileIoOptions) { this.fileIoOptions = fileIoOptions; return this; } From 1e26d94615895ca9b5ddb5964b8485ea4bd9de6e Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 22 Sep 2025 11:12:39 -0400 Subject: [PATCH 3/9] use fileIoConfiguration everywhere in the SDK --- .../services/s3/internal/crt/DefaultS3CrtAsyncClient.java | 8 ++++---- .../s3/internal/crt/S3NativeClientConfiguration.java | 8 ++++---- .../s3/internal/crt/S3CrtAsyncHttpClientTest.java | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 1fcc8534544f..e1c725696c12 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -224,7 +224,7 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau .httpConfiguration(builder.httpConfiguration) .thresholdInBytes(builder.thresholdInBytes) .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes) - .fileIoOptions(builder.fileIoOptions); + .fileIoConfiguration(builder.fileIoConfiguration); if (builder.retryConfiguration != null) { nativeClientBuilder.standardRetryOptions( @@ -258,7 +258,7 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Long thresholdInBytes; private Executor futureCompletionExecutor; private Boolean disableS3ExpressSessionAuth; - private S3CrtFileIoConfiguration fileIoOptions; + private S3CrtFileIoConfiguration fileIoConfiguration; @Override public DefaultS3CrtClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) { @@ -391,8 +391,8 @@ public DefaultS3CrtClientBuilder disableS3ExpressSessionAuth(Boolean disableS3Ex } @Override - public S3CrtAsyncClientBuilder fileIoConfiguration(S3CrtFileIoConfiguration fileIoOptions) { - this.fileIoOptions = fileIoOptions; + public S3CrtAsyncClientBuilder fileIoConfiguration(S3CrtFileIoConfiguration fileIoConfiguration) { + this.fileIoConfiguration = fileIoConfiguration; return this; } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index aeb3d7efb6e8..eac7c69f9736 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -117,7 +117,7 @@ public S3NativeClientConfiguration(Builder builder) { } this.standardRetryOptions = builder.standardRetryOptions; this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder); - this.fileIoOptions = builder.fileIoOptions == null ? null : resolveFileIoOptions(builder.fileIoOptions); + this.fileIoOptions = builder.fileIoConfiguration == null ? null : resolveFileIoOptions(builder.fileIoConfiguration); } private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { @@ -229,7 +229,7 @@ public static final class Builder { private StandardRetryOptions standardRetryOptions; private Long thresholdInBytes; private Long maxNativeMemoryLimitInBytes; - private S3CrtFileIoConfiguration fileIoOptions; + private S3CrtFileIoConfiguration fileIoConfiguration; private Builder() { } @@ -293,8 +293,8 @@ public Builder thresholdInBytes(Long thresholdInBytes) { return this; } - public Builder fileIoOptions(S3CrtFileIoConfiguration fileIoOptions) { - this.fileIoOptions = fileIoOptions; + public Builder fileIoConfiguration(S3CrtFileIoConfiguration fileIoConfiguration) { + this.fileIoConfiguration = fileIoConfiguration; return this; } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 9b72627baf77..216bb9ee255c 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -443,7 +443,7 @@ void build_shouldPassThroughParameters() { .minimumThroughputTimeout(Duration.ofSeconds(2))) .proxyConfiguration(p -> p.host("127.0.0.1").port(8080)) .build()) - .fileIoOptions(S3CrtFileIoConfiguration.builder().diskThroughputGbps(8.0).shouldStream(true).directIo(true).build()) + .fileIoConfiguration(S3CrtFileIoConfiguration.builder().diskThroughputGbps(8.0).shouldStream(true).directIo(true).build()) .build(); try (S3CrtAsyncHttpClient client = (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build()) { From f685d8ce2b778e66937e103b2e8b5d43faf885c0 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 22 Sep 2025 12:19:13 -0400 Subject: [PATCH 4/9] checkstyle --- .../awssdk/services/s3/crt/S3CrtFileIoConfiguration.java | 3 ++- .../s3/internal/crt/S3CrtAsyncHttpClientTest.java | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java index 47b07cf1efdb..a91e8ac980f9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java @@ -28,7 +28,8 @@ @SdkPublicApi @Immutable @ThreadSafe -public final class S3CrtFileIoConfiguration implements ToCopyableBuilder { +public final class S3CrtFileIoConfiguration + implements ToCopyableBuilder { private final Boolean shouldStream; private final Double diskThroughputGbps; private final Boolean directIo; diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 216bb9ee255c..71c41c67857c 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -65,6 +65,7 @@ import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.testutils.RandomTempFile; @@ -623,4 +624,11 @@ private AsyncExecuteRequest.Builder getExecuteRequestBuilder(Integer port) { .putHeader("custom-header", "foobar") .build()); } + + { + S3AsyncClient client = + S3AsyncClient.crtBuilder() + .fileIoConfiguration(config -> config.directIo(true).shouldStream(true).diskThroughputGbps(20.0)) + .build(); + } } From 0dee658d2e118879c59d944900d20a70c5b8b8b4 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 22 Sep 2025 12:20:50 -0400 Subject: [PATCH 5/9] clean up --- .../services/s3/internal/crt/S3CrtAsyncHttpClientTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 71c41c67857c..7923cc451715 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -624,11 +624,4 @@ private AsyncExecuteRequest.Builder getExecuteRequestBuilder(Integer port) { .putHeader("custom-header", "foobar") .build()); } - - { - S3AsyncClient client = - S3AsyncClient.crtBuilder() - .fileIoConfiguration(config -> config.directIo(true).shouldStream(true).diskThroughputGbps(20.0)) - .build(); - } } From 6d2cd3a258a50f13d9a4f8e8a21af70bd23acb87 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Mon, 22 Sep 2025 12:51:38 -0400 Subject: [PATCH 6/9] fix typo and method param name --- .../amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java | 4 ++-- .../s3/internal/crt/S3NativeClientConfiguration.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index e5afb5706bfa..3f71e2d62631 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -364,10 +364,10 @@ default S3CrtAsyncClientBuilder retryConfiguration(Consumer false); double diskThroughputInGbps = Validate.getOrDefault(s3CrtfileIoConfiguration.diskThroughputGbps(), - () -> DEFAULT_FILE_IO_THROUGHPUT_IN_GPBS); + () -> DEFAULT_FILE_IO_THROUGHPUT_IN_GBPS); + Validate.isPositive(diskThroughputInGbps, "diskThroughputGbps"); boolean directIo = Validate.getOrDefault(s3CrtfileIoConfiguration.directIo(), () -> false); return new FileIoOptions(shouldStream, diskThroughputInGbps, directIo); } From 5001f5f61deee932cc954c8b5fbbb3f83cc912b9 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 26 Sep 2025 12:37:47 -0400 Subject: [PATCH 7/9] do not wrap responseTransformer for multipart for range-get. Rename `shouldStream` to `uploadBufferDisabled` --- .../config/SdkAdvancedAsyncClientOption.java | 16 ++++ .../services/s3/S3CrtAsyncClientBuilder.java | 20 +++++ .../s3/crt/S3CrtFileIoConfiguration.java | 78 ++++++------------- .../internal/crt/DefaultS3CrtAsyncClient.java | 20 ++++- .../crt/S3NativeClientConfiguration.java | 23 ++++-- .../crt/S3CrtAsyncHttpClientTest.java | 10 ++- 6 files changed, 104 insertions(+), 63 deletions(-) diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java index fb2a1135eeba..245a241c891b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java @@ -55,6 +55,22 @@ public final class SdkAdvancedAsyncClientOption extends ClientOption { public static final SdkAdvancedAsyncClientOption FUTURE_COMPLETION_EXECUTOR = new SdkAdvancedAsyncClientOption<>(Executor.class); + /** + * Configure Direct I/O for CRT file-based upload with the s3 async client. Only used with the CRT s3 async client. + *

+ * Enabling direct I/O bypasses the OS cache. Helpful when the disk I/O outperforms the kernel cache. + *

+ * Notes: + *

    + *
  • Only supported on Linux for now.
    • + *
  • Only supports upload for now.
    • + *
  • Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT
    • + *
+ * for additional info https://man7.org/linux/man-pages/man2/openat.2.html + */ + public static final SdkAdvancedAsyncClientOption CRT_UPLOAD_FILE_DIRECT_IO = + new SdkAdvancedAsyncClientOption<>(Boolean.class); + private SdkAdvancedAsyncClientOption(Class valueClass) { super(valueClass); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index 3f71e2d62631..22e2027410a3 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -17,6 +17,7 @@ import java.net.URI; import java.nio.file.Path; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; @@ -25,6 +26,8 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; import software.amazon.awssdk.identity.spi.IdentityProvider; import software.amazon.awssdk.regions.Region; @@ -33,6 +36,7 @@ import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.SdkBuilder; @@ -381,6 +385,22 @@ default S3CrtAsyncClientBuilder fileIoConfiguration(Consumer + */ + S3CrtAsyncClientBuilder putAdvancedOption(SdkAdvancedAsyncClientOption option, T value); + + /** + * + * @param advancedOptions + * @return + */ + S3CrtAsyncClientBuilder advancedOptions(Map, ?> advancedOptions); + @Override S3AsyncClient build(); } \ No newline at end of file diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java index a91e8ac980f9..ff5ce28ef1d4 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.annotations.Immutable; import software.amazon.awssdk.annotations.SdkPublicApi; import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -30,14 +31,12 @@ @ThreadSafe public final class S3CrtFileIoConfiguration implements ToCopyableBuilder { - private final Boolean shouldStream; + private final Boolean uploadBufferDisabled; private final Double diskThroughputGbps; - private final Boolean directIo; private S3CrtFileIoConfiguration(DefaultBuilder builder) { - this.shouldStream = builder.shouldStream; + this.uploadBufferDisabled = builder.uploadBufferDisabled; this.diskThroughputGbps = builder.diskThroughputGbps; - this.directIo = builder.directIo; } /** @@ -48,15 +47,21 @@ public static Builder builder() { } /** - * Skip buffering the part in memory before sending the request. - * If set, set the {@code diskThroughputGbps} to reasonably align with the available disk throughput. + * Skip buffering the part in memory before sending the request. When set to true, the file content will not be buffered + * in memory and instead streamed to the http request. + *

+ * Note: If upload buffering is still enabled, the CRT client will buffer full parts in + * memory, which could lead to out-of-memory for large files when the application does not have enough memory to fully + * buffer those parts in memory. + *

+ * If set to true, also set the {@code diskThroughputGbps} to reasonably align with the available disk throughput. * Otherwise, the transfer may fail with connection starvation. * Defaults to false. - * + * @see SdkAdvancedAsyncClientOption#CRT_UPLOAD_FILE_DIRECT_IO * @return if client should skip buffering in memory before sending the request. */ - public Boolean shouldStream() { - return shouldStream; + public Boolean uploadBufferDisabled() { + return uploadBufferDisabled; } /** @@ -74,20 +79,6 @@ public Double diskThroughputGbps() { return diskThroughputGbps; } - /** - * Enable direct I/O to bypass the OS cache. Helpful when the disk I/O outperforms the kernel cache. - * Notes: - * - Only supported on Linux for now. - * - Only supports upload for now. - * - Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT - * for additional info open - * - * @return directIO value - */ - public Boolean directIo() { - return directIo; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -99,20 +90,16 @@ public boolean equals(Object o) { S3CrtFileIoConfiguration that = (S3CrtFileIoConfiguration) o; - if (!Objects.equals(shouldStream, that.shouldStream)) { - return false; - } - if (!Objects.equals(diskThroughputGbps, that.diskThroughputGbps)) { + if (!Objects.equals(uploadBufferDisabled, that.uploadBufferDisabled)) { return false; } - return Objects.equals(directIo, that.directIo); + return Objects.equals(diskThroughputGbps, that.diskThroughputGbps); } @Override public int hashCode() { - int result = shouldStream != null ? shouldStream.hashCode() : 0; + int result = uploadBufferDisabled != null ? uploadBufferDisabled.hashCode() : 0; result = 31 * result + (diskThroughputGbps != null ? diskThroughputGbps.hashCode() : 0); - result = 31 * result + (directIo != null ? directIo.hashCode() : 0); return result; } @@ -131,7 +118,7 @@ public interface Builder extends CopyableBuilder S3CrtAsyncClientBuilder putAdvancedOption(SdkAdvancedAsyncClientOption option, T value) { + advancedOptions.put(option, value); + return this; + } + + @Override + public S3CrtAsyncClientBuilder advancedOptions(Map, ?> advancedOptions) { + this.advancedOptions.putAll(advancedOptions); + return this; + } + @Override public S3CrtAsyncClient build() { return new DefaultS3CrtAsyncClient(this); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index f835979fc445..469a13232918 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -21,6 +21,7 @@ import java.net.URI; import java.time.Duration; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider; import software.amazon.awssdk.crt.http.HttpMonitoringOptions; import software.amazon.awssdk.crt.http.HttpProxyOptions; @@ -35,6 +36,7 @@ import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; +import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.SdkAutoCloseable; import software.amazon.awssdk.utils.Validate; @@ -117,7 +119,7 @@ public S3NativeClientConfiguration(Builder builder) { } this.standardRetryOptions = builder.standardRetryOptions; this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder); - this.fileIoOptions = builder.fileIoConfiguration == null ? null : resolveFileIoOptions(builder.fileIoConfiguration); + this.fileIoOptions = builder.fileIoConfiguration == null ? null : resolveFileIoOptions(builder); } private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { @@ -127,12 +129,17 @@ private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { return true; } - private static FileIoOptions resolveFileIoOptions(S3CrtFileIoConfiguration s3CrtfileIoConfiguration) { - boolean shouldStream = Validate.getOrDefault(s3CrtfileIoConfiguration.shouldStream(), () -> false); - double diskThroughputInGbps = Validate.getOrDefault(s3CrtfileIoConfiguration.diskThroughputGbps(), + private static FileIoOptions resolveFileIoOptions(Builder builder) { + S3CrtFileIoConfiguration s3CrtFileIoConfiguration = builder.fileIoConfiguration; + boolean shouldStream = Validate.getOrDefault(s3CrtFileIoConfiguration.uploadBufferDisabled(), () -> false); + double diskThroughputInGbps = Validate.getOrDefault(s3CrtFileIoConfiguration.diskThroughputGbps(), () -> DEFAULT_FILE_IO_THROUGHPUT_IN_GBPS); Validate.isPositive(diskThroughputInGbps, "diskThroughputGbps"); - boolean directIo = Validate.getOrDefault(s3CrtfileIoConfiguration.directIo(), () -> false); + + AttributeMap advancedOptions = builder.advancedOptions; + boolean directIo = Validate.getOrDefault( + advancedOptions.get(SdkAdvancedAsyncClientOption.CRT_UPLOAD_FILE_DIRECT_IO), + () -> false); return new FileIoOptions(shouldStream, diskThroughputInGbps, directIo); } @@ -231,6 +238,7 @@ public static final class Builder { private Long thresholdInBytes; private Long maxNativeMemoryLimitInBytes; private S3CrtFileIoConfiguration fileIoConfiguration; + private AttributeMap advancedOptions; private Builder() { } @@ -298,5 +306,10 @@ public Builder fileIoConfiguration(S3CrtFileIoConfiguration fileIoConfiguration) this.fileIoConfiguration = fileIoConfiguration; return this; } + + public Builder advancedOptions(AttributeMap advancedOptions) { + this.advancedOptions = advancedOptions; + return this; + } } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 7923cc451715..d92ebc867401 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -48,6 +48,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.interceptor.trait.HttpChecksum; import software.amazon.awssdk.crt.auth.signing.AwsSigningConfig; import software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting; @@ -69,6 +70,7 @@ import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.utils.AttributeMap; public class S3CrtAsyncHttpClientTest { private static final URI DEFAULT_ENDPOINT = URI.create("https://127.0.0.1:443"); @@ -444,7 +446,13 @@ void build_shouldPassThroughParameters() { .minimumThroughputTimeout(Duration.ofSeconds(2))) .proxyConfiguration(p -> p.host("127.0.0.1").port(8080)) .build()) - .fileIoConfiguration(S3CrtFileIoConfiguration.builder().diskThroughputGbps(8.0).shouldStream(true).directIo(true).build()) + .fileIoConfiguration(S3CrtFileIoConfiguration.builder() + .diskThroughputGbps(8.0) + .uploadBufferDisabled(true) + .build()) + .advancedOptions(AttributeMap.builder() + .put(SdkAdvancedAsyncClientOption.CRT_UPLOAD_FILE_DIRECT_IO, true) + .build()) .build(); try (S3CrtAsyncHttpClient client = (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build()) { From f3b62d1d2b650d12ddcc08d917799bce48916e48 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 26 Sep 2025 12:43:18 -0400 Subject: [PATCH 8/9] advanced option javadoc --- .../services/s3/S3CrtAsyncClientBuilder.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index 22e2027410a3..df2e2daa52b7 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -386,18 +386,21 @@ default S3CrtAsyncClientBuilder fileIoConfiguration(Consumer + * @param option The option to configure. + * @param value The value of the option. + * @param The type of the option. + * @return an instance of this builder */ S3CrtAsyncClientBuilder putAdvancedOption(SdkAdvancedAsyncClientOption option, T value); /** - * - * @param advancedOptions - * @return + * Configure the map of advanced override options. This will override all values currently configured. The values in the + * map must match the key type of the map, or a runtime exception will be raised. + * @param advancedOptions the options to configure + * @return an instance of this builder */ S3CrtAsyncClientBuilder advancedOptions(Map, ?> advancedOptions); From 4f85f71cb5d5676e279bcdc5db09727b2c4d9bc4 Mon Sep 17 00:00:00 2001 From: Olivier Lepage-Applin Date: Fri, 26 Sep 2025 13:31:27 -0400 Subject: [PATCH 9/9] remove unused imports --- .../amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index df2e2daa52b7..04f175639323 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -27,7 +27,6 @@ import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; -import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; import software.amazon.awssdk.identity.spi.IdentityProvider; import software.amazon.awssdk.regions.Region; @@ -36,7 +35,6 @@ import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.SdkBuilder;