diff --git a/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java b/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java index fb2a1135eeba..245a241c891b 100644 --- a/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java +++ b/core/sdk-core/src/main/java/software/amazon/awssdk/core/client/config/SdkAdvancedAsyncClientOption.java @@ -55,6 +55,22 @@ public final class SdkAdvancedAsyncClientOption extends ClientOption { public static final SdkAdvancedAsyncClientOption FUTURE_COMPLETION_EXECUTOR = new SdkAdvancedAsyncClientOption<>(Executor.class); + /** + * Configure Direct I/O for CRT file-based upload with the s3 async client. Only used with the CRT s3 async client. + *

+ * Enabling direct I/O bypasses the OS cache. Helpful when the disk I/O outperforms the kernel cache. + *

+ * Notes: + *

    + *
  • Only supported on Linux for now.
    • + *
  • Only supports upload for now.
    • + *
  • Uses it as a potentially powerful tool that should be used with caution. Read NOTES for O_DIRECT
    • + *
+ * for additional info https://man7.org/linux/man-pages/man2/openat.2.html + */ + public static final SdkAdvancedAsyncClientOption CRT_UPLOAD_FILE_DIRECT_IO = + new SdkAdvancedAsyncClientOption<>(Boolean.class); + private SdkAdvancedAsyncClientOption(Class valueClass) { super(valueClass); } diff --git a/pom.xml b/pom.xml index 67a93bbc64ce..ade2761fc045 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,7 @@ 3.1.5 1.17.1 1.37 - 0.38.9 + 0.39.0 5.10.0 diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index b4cedc3ec1c0..04f175639323 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -17,6 +17,7 @@ import java.net.URI; import java.nio.file.Path; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; @@ -25,9 +26,11 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; import software.amazon.awssdk.identity.spi.IdentityProvider; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -361,6 +364,43 @@ default S3CrtAsyncClientBuilder retryConfiguration(Consumer fileIoOptionsBuilder) { + Validate.paramNotNull(fileIoOptionsBuilder, "fileIoOptionsBuilder"); + return fileIoConfiguration(S3CrtFileIoConfiguration.builder() + .applyMutation(fileIoOptionsBuilder) + .build()); + } + + /** + * Configure an advanced override option. These values are used very rarely, and the majority of SDK customers can ignore + * them. + * + * @param option The option to configure. + * @param value The value of the option. + * @param The type of the option. + * @return an instance of this builder + */ + S3CrtAsyncClientBuilder putAdvancedOption(SdkAdvancedAsyncClientOption option, T value); + + /** + * Configure the map of advanced override options. This will override all values currently configured. The values in the + * map must match the key type of the map, or a runtime exception will be raised. + * @param advancedOptions the options to configure + * @return an instance of this builder + */ + S3CrtAsyncClientBuilder advancedOptions(Map, ?> advancedOptions); @Override S3AsyncClient build(); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java new file mode 100644 index 000000000000..ff5ce28ef1d4 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/crt/S3CrtFileIoConfiguration.java @@ -0,0 +1,170 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crt; + +import java.util.Objects; +import software.amazon.awssdk.annotations.Immutable; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.annotations.ThreadSafe; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Configuration options which controls how client performs file I/O operations. Only applies to file-based workloads. + */ +@SdkPublicApi +@Immutable +@ThreadSafe +public final class S3CrtFileIoConfiguration + implements ToCopyableBuilder { + private final Boolean uploadBufferDisabled; + private final Double diskThroughputGbps; + + private S3CrtFileIoConfiguration(DefaultBuilder builder) { + this.uploadBufferDisabled = builder.uploadBufferDisabled; + this.diskThroughputGbps = builder.diskThroughputGbps; + } + + /** + * Creates a default builder for {@link S3CrtFileIoConfiguration}. + */ + public static Builder builder() { + return new DefaultBuilder(); + } + + /** + * Skip buffering the part in memory before sending the request. When set to true, the file content will not be buffered + * in memory and instead streamed to the http request. + *

+ * Note: If upload buffering is still enabled, the CRT client will buffer full parts in + * memory, which could lead to out-of-memory for large files when the application does not have enough memory to fully + * buffer those parts in memory. + *

+ * If set to true, also set the {@code diskThroughputGbps} to reasonably align with the available disk throughput. + * Otherwise, the transfer may fail with connection starvation. + * Defaults to false. + * @see SdkAdvancedAsyncClientOption#CRT_UPLOAD_FILE_DIRECT_IO + * @return if client should skip buffering in memory before sending the request. + */ + public Boolean uploadBufferDisabled() { + return uploadBufferDisabled; + } + + /** + * The estimated disk throughput in gigabits per second (Gbps). + * Only applied when {@code shouldStream} is true. + * + * When doing upload with streaming, it's important to set the disk throughput to prevent connection starvation. + * Note: There are possibilities that cannot reach all available disk throughput: + * 1. Disk is busy with other applications + * 2. OS Cache may cap the throughput, use {@code directIo} to get around this. + * + * @return disk throughput value in Gpbs. + */ + public Double diskThroughputGbps() { + return diskThroughputGbps; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + S3CrtFileIoConfiguration that = (S3CrtFileIoConfiguration) o; + + if (!Objects.equals(uploadBufferDisabled, that.uploadBufferDisabled)) { + return false; + } + return Objects.equals(diskThroughputGbps, that.diskThroughputGbps); + } + + @Override + public int hashCode() { + int result = uploadBufferDisabled != null ? uploadBufferDisabled.hashCode() : 0; + result = 31 * result + (diskThroughputGbps != null ? diskThroughputGbps.hashCode() : 0); + return result; + } + + @Override + public Builder toBuilder() { + return new DefaultBuilder(this); + } + + public interface Builder extends CopyableBuilder { + /** + * Skip buffering the part in memory before sending the request. + * If set, set the {@code diskThroughputGbps} to reasonably align with the available disk throughput. + * Otherwise, the transfer may fail with connection starvation. + * Defaults to false. + * + * @param shouldStream whether to stream the file + * @return The builder for method chaining. + */ + Builder uploadBufferDisabled(Boolean shouldStream); + + /** + * The estimated disk throughput in gigabits per second (Gbps). + * Only applied when {@code shouldStream} is true. + * + * When doing upload with streaming, it's important to set the disk throughput to prevent connection starvation. + * Note: There are possibilities that cannot reach all available disk throughput: + * 1. Disk is busy with other applications + * 2. OS Cache may cap the throughput, use {@code directIo} to get around this. + * + * @param diskThroughputGbps the disk throughput in Gbps + * @return The builder for method chaining. + */ + Builder diskThroughputGbps(Double diskThroughputGbps); + + @Override + S3CrtFileIoConfiguration build(); + } + + private static final class DefaultBuilder implements Builder { + private Boolean uploadBufferDisabled; + private Double diskThroughputGbps; + + private DefaultBuilder() { + } + + private DefaultBuilder(S3CrtFileIoConfiguration fileIoOptions) { + this.uploadBufferDisabled = fileIoOptions.uploadBufferDisabled; + this.diskThroughputGbps = fileIoOptions.diskThroughputGbps; + } + + @Override + public Builder uploadBufferDisabled(Boolean shouldStream) { + this.uploadBufferDisabled = shouldStream; + return this; + } + + @Override + public Builder diskThroughputGbps(Double diskThroughputGbps) { + this.diskThroughputGbps = diskThroughputGbps; + return this; + } + + @Override + public S3CrtFileIoConfiguration build() { + return new S3CrtFileIoConfiguration(this); + } + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java index 676e060e3218..bc68e72d9aa9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/DefaultS3CrtAsyncClient.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import software.amazon.awssdk.annotations.SdkInternalApi; @@ -47,6 +48,7 @@ import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; import software.amazon.awssdk.core.interceptor.Context; import software.amazon.awssdk.core.interceptor.ExecutionAttribute; @@ -67,6 +69,7 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.S3CrtAsyncClientBuilder; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtRetryConfiguration; import software.amazon.awssdk.services.s3.internal.checksums.ChecksumsEnabledValidator; @@ -78,6 +81,7 @@ import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.CollectionUtils; import software.amazon.awssdk.utils.Validate; @@ -222,7 +226,9 @@ private static S3CrtAsyncHttpClient.Builder initializeS3CrtAsyncHttpClient(Defau .readBufferSizeInBytes(builder.readBufferSizeInBytes) .httpConfiguration(builder.httpConfiguration) .thresholdInBytes(builder.thresholdInBytes) - .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes); + .maxNativeMemoryLimitInBytes(builder.maxNativeMemoryLimitInBytes) + .fileIoConfiguration(builder.fileIoConfiguration) + .advancedOptions(builder.advancedOptions.build()); if (builder.retryConfiguration != null) { nativeClientBuilder.standardRetryOptions( @@ -256,7 +262,9 @@ public static final class DefaultS3CrtClientBuilder implements S3CrtAsyncClientB private Long thresholdInBytes; private Executor futureCompletionExecutor; private Boolean disableS3ExpressSessionAuth; + private S3CrtFileIoConfiguration fileIoConfiguration; + private AttributeMap.Builder advancedOptions = AttributeMap.builder(); @Override public DefaultS3CrtClientBuilder credentialsProvider(AwsCredentialsProvider credentialsProvider) { @@ -388,6 +396,24 @@ public DefaultS3CrtClientBuilder disableS3ExpressSessionAuth(Boolean disableS3Ex return this; } + @Override + public S3CrtAsyncClientBuilder fileIoConfiguration(S3CrtFileIoConfiguration fileIoConfiguration) { + this.fileIoConfiguration = fileIoConfiguration; + return this; + } + + @Override + public S3CrtAsyncClientBuilder putAdvancedOption(SdkAdvancedAsyncClientOption option, T value) { + advancedOptions.put(option, value); + return this; + } + + @Override + public S3CrtAsyncClientBuilder advancedOptions(Map, ?> advancedOptions) { + this.advancedOptions.putAll(advancedOptions); + return this; + } + @Override public S3CrtAsyncClient build() { return new DefaultS3CrtAsyncClient(this); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index 1fed55813d33..99edc159156e 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -112,7 +112,8 @@ private S3ClientOptions createS3ClientOption() { .withMemoryLimitInBytes(s3NativeClientConfiguration.maxNativeMemoryLimitInBytes()) .withThroughputTargetGbps(s3NativeClientConfiguration.targetThroughputInGbps()) .withInitialReadWindowSize(initialWindowSize) - .withReadBackpressureEnabled(true); + .withReadBackpressureEnabled(true) + .withFileIoOptions(s3NativeClientConfiguration.fileIoOptions()); if (s3NativeClientConfiguration.standardRetryOptions() != null) { options.withStandardRetryOptions(s3NativeClientConfiguration.standardRetryOptions()); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java index f43cbf84d46e..469a13232918 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3NativeClientConfiguration.java @@ -21,6 +21,7 @@ import java.net.URI; import java.time.Duration; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.crt.auth.credentials.CredentialsProvider; import software.amazon.awssdk.crt.http.HttpMonitoringOptions; import software.amazon.awssdk.crt.http.HttpProxyOptions; @@ -29,10 +30,13 @@ import software.amazon.awssdk.crt.io.TlsCipherPreference; import software.amazon.awssdk.crt.io.TlsContext; import software.amazon.awssdk.crt.io.TlsContextOptions; +import software.amazon.awssdk.crt.s3.FileIoOptions; import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity; import software.amazon.awssdk.identity.spi.IdentityProvider; import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; +import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.SdkAutoCloseable; import software.amazon.awssdk.utils.Validate; @@ -43,6 +47,7 @@ @SdkInternalApi public class S3NativeClientConfiguration implements SdkAutoCloseable { static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; + static final double DEFAULT_FILE_IO_THROUGHPUT_IN_GBPS = 10.0; private static final Logger log = Logger.loggerFor(S3NativeClientConfiguration.class); private static final long DEFAULT_TARGET_THROUGHPUT_IN_GBPS = 10; @@ -64,6 +69,7 @@ public class S3NativeClientConfiguration implements SdkAutoCloseable { private final HttpMonitoringOptions httpMonitoringOptions; private final Boolean useEnvironmentVariableProxyOptionsValues; private final long maxNativeMemoryLimitInBytes; + private final FileIoOptions fileIoOptions; public S3NativeClientConfiguration(Builder builder) { this.signingRegion = builder.signingRegion == null ? DefaultAwsRegionProviderChain.builder().build().getRegion().id() : @@ -113,6 +119,7 @@ public S3NativeClientConfiguration(Builder builder) { } this.standardRetryOptions = builder.standardRetryOptions; this.useEnvironmentVariableProxyOptionsValues = resolveUseEnvironmentVariableValues(builder); + this.fileIoOptions = builder.fileIoConfiguration == null ? null : resolveFileIoOptions(builder); } private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { @@ -122,6 +129,20 @@ private static Boolean resolveUseEnvironmentVariableValues(Builder builder) { return true; } + private static FileIoOptions resolveFileIoOptions(Builder builder) { + S3CrtFileIoConfiguration s3CrtFileIoConfiguration = builder.fileIoConfiguration; + boolean shouldStream = Validate.getOrDefault(s3CrtFileIoConfiguration.uploadBufferDisabled(), () -> false); + double diskThroughputInGbps = Validate.getOrDefault(s3CrtFileIoConfiguration.diskThroughputGbps(), + () -> DEFAULT_FILE_IO_THROUGHPUT_IN_GBPS); + Validate.isPositive(diskThroughputInGbps, "diskThroughputGbps"); + + AttributeMap advancedOptions = builder.advancedOptions; + boolean directIo = Validate.getOrDefault( + advancedOptions.get(SdkAdvancedAsyncClientOption.CRT_UPLOAD_FILE_DIRECT_IO), + () -> false); + return new FileIoOptions(shouldStream, diskThroughputInGbps, directIo); + } + public Boolean isUseEnvironmentVariableValues() { return useEnvironmentVariableProxyOptionsValues; } @@ -191,6 +212,10 @@ public Long readBufferSizeInBytes() { return readBufferSizeInBytes; } + public FileIoOptions fileIoOptions() { + return fileIoOptions; + } + @Override public void close() { clientBootstrap.close(); @@ -212,6 +237,8 @@ public static final class Builder { private StandardRetryOptions standardRetryOptions; private Long thresholdInBytes; private Long maxNativeMemoryLimitInBytes; + private S3CrtFileIoConfiguration fileIoConfiguration; + private AttributeMap advancedOptions; private Builder() { } @@ -274,5 +301,15 @@ public Builder thresholdInBytes(Long thresholdInBytes) { this.thresholdInBytes = thresholdInBytes; return this; } + + public Builder fileIoConfiguration(S3CrtFileIoConfiguration fileIoConfiguration) { + this.fileIoConfiguration = fileIoConfiguration; + return this; + } + + public Builder advancedOptions(AttributeMap advancedOptions) { + this.advancedOptions = advancedOptions; + return this; + } } } diff --git a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java index 776b69c4a10a..d92ebc867401 100644 --- a/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java +++ b/services/s3/src/test/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClientTest.java @@ -48,6 +48,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; import software.amazon.awssdk.core.checksums.ResponseChecksumValidation; +import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption; import software.amazon.awssdk.core.interceptor.trait.HttpChecksum; import software.amazon.awssdk.crt.auth.signing.AwsSigningConfig; import software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting; @@ -59,15 +60,17 @@ import software.amazon.awssdk.crt.s3.S3ClientOptions; import software.amazon.awssdk.crt.s3.S3MetaRequest; import software.amazon.awssdk.crt.s3.S3MetaRequestOptions; -import software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler; import software.amazon.awssdk.http.SdkHttpMethod; import software.amazon.awssdk.http.SdkHttpRequest; import software.amazon.awssdk.http.async.AsyncExecuteRequest; import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler; import software.amazon.awssdk.http.async.SdkHttpContentPublisher; import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.crt.S3CrtFileIoConfiguration; import software.amazon.awssdk.services.s3.crt.S3CrtHttpConfiguration; import software.amazon.awssdk.testutils.RandomTempFile; +import software.amazon.awssdk.utils.AttributeMap; public class S3CrtAsyncHttpClientTest { private static final URI DEFAULT_ENDPOINT = URI.create("https://127.0.0.1:443"); @@ -443,6 +446,13 @@ void build_shouldPassThroughParameters() { .minimumThroughputTimeout(Duration.ofSeconds(2))) .proxyConfiguration(p -> p.host("127.0.0.1").port(8080)) .build()) + .fileIoConfiguration(S3CrtFileIoConfiguration.builder() + .diskThroughputGbps(8.0) + .uploadBufferDisabled(true) + .build()) + .advancedOptions(AttributeMap.builder() + .put(SdkAdvancedAsyncClientOption.CRT_UPLOAD_FILE_DIRECT_IO, true) + .build()) .build(); try (S3CrtAsyncHttpClient client = (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build()) { @@ -466,6 +476,12 @@ void build_shouldPassThroughParameters() { assertThat(clientOptions.getMaxConnections()).isEqualTo(100); assertThat(clientOptions.getThroughputTargetGbps()).isEqualTo(3.5); assertThat(clientOptions.getMemoryLimitInBytes()).isEqualTo(5L * 1024 * 1024 * 1024); + + assertThat(clientOptions.getFileIoOptions()).isNotNull(); + assertThat(clientOptions.getFileIoOptions().getShouldStream()).isTrue(); + assertThat(clientOptions.getFileIoOptions().getDirectIo()).isTrue(); + assertThat(clientOptions.getFileIoOptions().getDiskThroughputGbps()).isEqualTo(8.0); + } } @@ -504,6 +520,21 @@ void build_nullHttpConfiguration() { } } + @Test + void build_nullFileOptions() { + StaticCredentialsProvider credentialsProvider = + StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")); + S3NativeClientConfiguration configuration = + S3NativeClientConfiguration.builder() + .credentialsProvider(credentialsProvider) + .build(); + try (S3CrtAsyncHttpClient client = + (S3CrtAsyncHttpClient) S3CrtAsyncHttpClient.builder().s3ClientConfiguration(configuration).build()) { + S3ClientOptions clientOptions = client.s3ClientOptions(); + assertThat(clientOptions.getFileIoOptions()).isNull(); + } + } + private static Stream s3CrtHttpConfigurations() { return Stream.of( Arguments.of(S3CrtHttpConfiguration.builder()