Update software.amazon.awssdk:* to v2.42.37 (branch_9x) - abandoned

changelog/unreleased/PR#3593-update-software-amazon-awssdk-to-v2-41-0-branch.yml

@@ -0,0 +1,6 @@
+title: Update software.amazon.awssdk:* to v2.42.37 (branch_9x)
+authors:
+- name: solrbot
+links:
+- name: PR#3593
+  url: https://github.com/apache/solr/pull/3593

solr/licenses/annotations-2.42.37.jar.sha1

@@ -0,0 +1 @@
+b1a13a1f1d3b5445e2f60dcc01c0e1c66662ce32

solr/licenses/apache-client-2.42.37.jar.sha1

@@ -0,0 +1 @@
+34a1dcf7f2d504954b00b5d2df485b445ff0fe4a

solr/licenses/arns-2.42.37.jar.sha1

@@ -0,0 +1 @@
+dfa5f78ee3ef1ae7751d77cbfea79ab9462a4eda

solr/licenses/auth-2.42.37.jar.sha1

@@ -0,0 +1 @@
+1367486a9ad11dc1d86db7b91f1ae6763c80caa9

solr/licenses/aws-core-2.42.37.jar.sha1

@@ -0,0 +1 @@
+81da5401148dc666e61137c2ec4cb6d16a1d93e9

solr/licenses/aws-query-protocol-2.42.37.jar.sha1

@@ -0,0 +1 @@
+e9e2849ad9acc8ac3589d9fb77c6ba9e5e7e327c

solr/licenses/aws-xml-protocol-2.42.37.jar.sha1

@@ -0,0 +1 @@
+495e1f5428bea389bd9bf4d431bc16445d9d6ce9

solr/licenses/checksums-2.42.37.jar.sha1

@@ -0,0 +1 @@
+b74d0ae9d5f24686f687687cf9f0fbf075eeaa63

solr/licenses/checksums-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+9efc58b403257390065e1dac7137a79e261df2a9

solr/licenses/crt-core-2.42.37.jar.sha1

@@ -0,0 +1 @@
+4e3d8cfb9b52b94f60a91849bbfa09c0bc0f1ab9

solr/licenses/endpoints-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+5cf2c9e67fdf32b57417f1975cdff243268e383a

solr/licenses/http-auth-2.42.37.jar.sha1

@@ -0,0 +1 @@
+ad2b97b222ba564bc4e1f5d89a4edfe50e9bcdf8

solr/licenses/http-auth-aws-2.42.37.jar.sha1

@@ -0,0 +1 @@
+98fa9005878817a77ca4ac8bd36173e5fd7762e8

solr/licenses/http-auth-aws-eventstream-2.42.37.jar.sha1

@@ -0,0 +1 @@
+443b35c30db91baad70a365c68243f0e08caf70c

solr/licenses/http-auth-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+e72436b6c5e7601f2d6172b09fe44ab5c97c33e7

solr/licenses/http-client-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+a6a7d757cc5c0606c725228a25c5800ef2380d5d

solr/licenses/identity-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+c0bcbea7717e6220421af3c320f59272e1964a25

solr/licenses/json-utils-2.42.37.jar.sha1

@@ -0,0 +1 @@
+bed7e7e88026ff273838cbd40de3b9eded21e306

solr/licenses/metrics-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+87e9e8c04f82394aa7cb916616e1cbcfc7a954f8

solr/licenses/profiles-2.42.37.jar.sha1

@@ -0,0 +1 @@
+b2e57056153651f93a826a6e0cfc2d935d876ee3

solr/licenses/protocol-core-2.42.37.jar.sha1

@@ -0,0 +1 @@
+49bcc7cdf41b39b84f83f22201f888ac6d60b3e8

solr/licenses/regions-2.42.37.jar.sha1

@@ -0,0 +1 @@
+9f2f7e54e093b5d77638b97f5fa087bbc38f7638

solr/licenses/retries-2.42.37.jar.sha1

@@ -0,0 +1 @@
+360553c72c0e1b1949b8109bf4f202096bf3eac7

solr/licenses/retries-spi-2.42.37.jar.sha1

@@ -0,0 +1 @@
+de34d04f9fc382bea4b91c6e398739512db0b96e

solr/licenses/s3-2.42.37.jar.sha1

@@ -0,0 +1 @@
+1f84320b40a915b39b002bcd08247c6c4186be47

solr/licenses/sdk-core-2.42.37.jar.sha1

@@ -0,0 +1 @@
+07de694e0e72862f093c5c837f27fb384a6b9610

solr/licenses/sts-2.42.37.jar.sha1

@@ -0,0 +1 @@
+e1ce18a7bb2209d6203073357e77d470c4573f06

solr/licenses/third-party-jackson-core-2.42.37.jar.sha1

@@ -0,0 +1 @@
+09ad8d94e91bfa5c6e2ceb0aa9b9f3201ebf9ab9

solr/licenses/url-connection-client-2.42.37.jar.sha1

@@ -0,0 +1 @@
+0d6dc3d6fdf3404e29321698c0c7ea4b0de2c90d

solr/licenses/utils-2.42.37.jar.sha1

@@ -0,0 +1 @@
+9b862317e697a236013adbd9299174595ed279bb

solr/licenses/utils-lite-2.42.37.jar.sha1

@@ -0,0 +1 @@
+7fbe573b0b7764ae29d2b36e63543a372c4cbbfb

solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java

@@ -44,6 +44,7 @@
 import software.amazon.awssdk.core.retry.RetryMode;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.core.sync.ResponseTransformer;
+import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.http.apache.ApacheHttpClient;
 import software.amazon.awssdk.http.apache.ProxyConfiguration;
 import software.amazon.awssdk.regions.Region;
@@ -92,6 +93,9 @@ public class S3StorageClient {
   /** The S3 bucket where we read/write all data. */
   private final String bucketName;
 
+  /** The HTTP client used by the S3 client. Needs to be closed separately. */
+  private final SdkHttpClient httpClient;
+
   S3StorageClient(
       String bucketName,
       String profile,
@@ -100,30 +104,19 @@ public class S3StorageClient {
       boolean proxyUseSystemSettings,
       String endpoint,
       boolean disableRetries) {
-    this(
-        createInternalClient(
-            profile, region, proxyUrl, proxyUseSystemSettings, endpoint, disableRetries),
-        bucketName);
+    this.bucketName = bucketName;
+    this.httpClient = createHttpClient(proxyUrl, proxyUseSystemSettings);
+    this.s3Client = createInternalClient(profile, region, endpoint, disableRetries, httpClient);
   }
 
   @VisibleForTesting
   S3StorageClient(S3Client s3Client, String bucketName) {
     this.s3Client = s3Client;
     this.bucketName = bucketName;
+    this.httpClient = null;
   }
 
-  private static S3Client createInternalClient(
-      String profile,
-      String region,
-      String proxyUrl,
-      boolean proxyUseSystemSettings,
-      String endpoint,
-      boolean disableRetries) {
-    S3Configuration.Builder configBuilder = S3Configuration.builder().pathStyleAccessEnabled(true);
-    if (StrUtils.isNotNullOrEmpty(profile)) {
-      configBuilder.profileName(profile);
-    }
-
+  private static SdkHttpClient createHttpClient(String proxyUrl, boolean proxyUseSystemSettings) {
     ApacheHttpClient.Builder sdkHttpClientBuilder = ApacheHttpClient.builder();
     // If configured, add proxy
     ProxyConfiguration.Builder proxyConfigurationBuilder = ProxyConfiguration.builder();
@@ -135,6 +128,20 @@ private static S3Client createInternalClient(
     sdkHttpClientBuilder.proxyConfiguration(proxyConfigurationBuilder.build());
     sdkHttpClientBuilder.useIdleConnectionReaper(false);
 
+    return sdkHttpClientBuilder.build();
+  }
+
+  private static S3Client createInternalClient(
+      String profile,
+      String region,
+      String endpoint,
+      boolean disableRetries,
+      SdkHttpClient httpClient) {
+    S3Configuration.Builder configBuilder = S3Configuration.builder().pathStyleAccessEnabled(true);
+    if (StrUtils.isNotNullOrEmpty(profile)) {
+      configBuilder.profileName(profile);
+    }
+
     /*
      * Retry logic
      */
@@ -167,7 +174,7 @@ private static S3Client createInternalClient(
             .credentialsProvider(credentialsProviderBuilder.build())
             .overrideConfiguration(builder -> builder.retryStrategy(retryStrategy))
             .serviceConfiguration(configBuilder.build())
-            .httpClient(sdkHttpClientBuilder.build());
+            .httpClient(httpClient);
 
     if (StrUtils.isNotNullOrEmpty(endpoint)) {
       clientBuilder.endpointOverride(URI.create(endpoint));
@@ -374,23 +381,27 @@ InputStream pullStream(String path) throws S3Exception {
       GetObjectRequest.Builder getBuilder =
           GetObjectRequest.builder().bucket(bucketName).key(s3Path);
       // This InputStream instance needs to be closed by the caller
-      return s3Client.getObject(
-          getBuilder.build(),
-          ResponseTransformer.unmanaged(
-              (response, inputStream) -> {
-                final long contentLength = response.contentLength();
-                return new ResumableInputStream(
-                    inputStream,
-                    bytesRead -> {
-                      if (contentLength > 0 && bytesRead >= contentLength) {
-                        // No more bytes to read
-                        return null;
-                      } else if (bytesRead > 0) {
-                        getBuilder.range(String.format(Locale.ROOT, "bytes=%d-", bytesRead));
-                      }
-                      return s3Client.getObject(getBuilder.build());
-                    });
-              }));
+      // Use Duration.ZERO to disable timeout and prevent response-input-stream-timeout-scheduler
+      // thread leak (see https://github.com/aws/aws-sdk-java-v2/issues/6567)
+      software.amazon.awssdk.core.ResponseInputStream<
+              software.amazon.awssdk.services.s3.model.GetObjectResponse>
+          responseStream =
+              s3Client.getObject(
+                  getBuilder.build(), ResponseTransformer.toInputStream(java.time.Duration.ZERO));
+      final long contentLength = responseStream.response().contentLength();
+      return new ResumableInputStream(
+          responseStream,
+          bytesRead -> {
+            if (contentLength > 0 && bytesRead >= contentLength) {
+              // No more bytes to read
+              return null;
+            } else if (bytesRead > 0) {
+              getBuilder.range(String.format(Locale.ROOT, "bytes=%d-", bytesRead));
+            }
+            // Use Duration.ZERO to disable timeout on resumed streams as well
+            return s3Client.getObject(
+                getBuilder.build(), ResponseTransformer.toInputStream(java.time.Duration.ZERO));
+          });
     } catch (SdkException sdke) {
       throw handleAmazonException(sdke);
     }
@@ -419,6 +430,9 @@ OutputStream pushStream(String path) throws S3Exception {
   /** Override {@link Closeable} since we throw no exception. */
   void close() {
     s3Client.close();
+    if (httpClient != null) {
+      httpClient.close();
+    }
   }
 
   /** Any file path that specifies a non-existent file will not be treated as an error. */
@@ -544,7 +558,7 @@ private String getParentDirectory(String path) {
   }
 
   /** Ensures path adheres to some rules: -Doesn't start with a leading slash */
-  String sanitizedPath(String path) throws S3Exception {
+  String sanitizedPath(String path) {
     // Trim space from start and end
     String sanitizedPath = path.trim();
 
@@ -580,7 +594,7 @@ String sanitizedFilePath(String path) throws S3Exception {
    * Ensures directory path adheres to some rules: -Overall Path rules from `sanitizedPath` -Add a
    * trailing slash if one does not exist
    */
-  String sanitizedDirPath(String path) throws S3Exception {
+  String sanitizedDirPath(String path) {
     // Trim space from start and end
     String sanitizedPath = sanitizedPath(path);
 

solr/modules/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java

@@ -42,6 +42,7 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.core.sync.ResponseTransformer;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
 
@@ -107,18 +108,18 @@ public void testLocalDirectoryFunctions() throws Exception {
       repo.createDirectory(path);
       assertTrue(repo.exists(path));
       assertEquals(BackupRepository.PathType.DIRECTORY, repo.getPathType(path));
-      assertEquals("No files should exist in dir yet", repo.listAll(path).length, 0);
+      assertEquals("No files should exist in dir yet", 0, repo.listAll(path).length);
 
       URI subDir = new URI("/test/dir/");
       repo.createDirectory(subDir);
       assertTrue(repo.exists(subDir));
       assertEquals(BackupRepository.PathType.DIRECTORY, repo.getPathType(subDir));
-      assertEquals("No files should exist in subdir yet", repo.listAll(subDir).length, 0);
+      assertEquals("No files should exist in subdir yet", 0, repo.listAll(subDir).length);
 
       assertEquals(
           "subDir should now be returned when listing all in parent dir",
-          repo.listAll(path).length,
-          1);
+          1,
+          repo.listAll(path).length);
 
       repo.deleteDirectory(path);
       assertFalse(repo.exists(path));
@@ -338,7 +339,10 @@ private void pushObject(String path, String content) {
   private File pullObject(String path) throws IOException {
     try (S3Client s3 = S3_MOCK_RULE.createS3ClientV2()) {
       File file = temporaryFolder.newFile();
-      InputStream input = s3.getObject(b -> b.bucket(BUCKET_NAME).key(path));
+      InputStream input =
+          s3.getObject(
+              b -> b.bucket(BUCKET_NAME).key(path),
+              ResponseTransformer.toInputStream(java.time.Duration.ZERO));
       FileUtils.copyInputStreamToFile(input, file);
       return file;
     }