Skip to content

Commit 3032a24

Browse files
iroquetaGonzalo Gallotti Vazquez
iroqueta
authored and
Gonzalo Gallotti Vazquez
committed
The method createNewFile converts all the inputstream to memory. This could cause out of memory errors.
Now the process use a buffer to avoid this. Issue: 205339
1 parent 84349ee commit 3032a24

File tree

6 files changed

+99
-28
lines changed

6 files changed

+99
-28
lines changed

‎gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import com.amazonaws.services.s3.AmazonS3;
1919
import com.amazonaws.services.s3.AmazonS3Client;
20-
import com.amazonaws.util.IOUtils;
2120

2221
import java.io.FileNotFoundException;
2322
import java.io.FileOutputStream;
@@ -220,23 +219,31 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) {
220219
}
221220

222221
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
223-
byte[] bytes;
222+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
224223
try {
225-
bytes = IOUtils.toByteArray(input);
224+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
225+
226226
ObjectMetadata metadata = new ObjectMetadata();
227-
metadata.setContentLength(bytes.length);
227+
metadata.setContentLength(streamInfo.contentLength);
228+
228229
if (externalFileName.endsWith(".tmp")) {
229230
metadata.setContentType("image/jpeg");
230231
}
231232
String upload = "";
232-
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) {
233-
client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
234-
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
235-
}
233+
client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
234+
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
236235
return upload;
237236
} catch (IOException ex) {
238237
logger.error("Error while uploading file to the external provider.", ex);
239238
return "";
239+
} finally {
240+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
241+
try {
242+
streamInfo.tempFile.delete();
243+
} catch (Exception e) {
244+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
245+
}
246+
}
240247
}
241248
}
242249

‎gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,11 @@
2121
import com.genexus.util.StorageUtils;
2222
import com.genexus.StructSdtMessages_Message;
2323
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
24-
import software.amazon.awssdk.utils.IoUtils;
2524

2625
import java.io.*;
2726
import java.net.URI;
2827
import java.net.URL;
2928
import java.net.URLEncoder;
30-
import java.nio.ByteBuffer;
3129
import java.nio.charset.StandardCharsets;
3230
import java.nio.file.Files;
3331
import java.nio.file.Path;
@@ -613,8 +611,10 @@ else if (acl == ResourceAccessControlList.PublicReadWrite)
613611
}
614612

615613
private String uploadWithACL(String externalFileName, InputStream input, ResourceAccessControlList acl) {
614+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
616615
try {
617-
ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input));
616+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
617+
618618
PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
619619
.bucket(bucket)
620620
.key(externalFileName)
@@ -623,7 +623,7 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc
623623
putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACLWithACL(acl));
624624
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
625625

626-
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer));
626+
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength));
627627
if (!response.sdkHttpResponse().isSuccessful()) {
628628
logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error"));
629629
}
@@ -632,6 +632,15 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc
632632
} catch (IOException ex) {
633633
logger.error("Error while uploading file to the external provider.", ex);
634634
return "";
635+
} finally {
636+
// Clean up the temporary file if it was created
637+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
638+
try {
639+
streamInfo.tempFile.delete();
640+
} catch (Exception e) {
641+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
642+
}
643+
}
635644
}
636645
}
637646

@@ -737,15 +746,17 @@ private String uploadWithoutACL(String localFile, String externalFileName) {
737746
}
738747

739748
private String uploadWithoutACL(String externalFileName, InputStream input) {
749+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
740750
try {
741-
ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input));
751+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
752+
742753
PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
743754
.bucket(bucket)
744755
.key(externalFileName)
745756
.contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null);
746757
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
747758

748-
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer));
759+
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength));
749760
if (!response.sdkHttpResponse().isSuccessful()) {
750761
logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error"));
751762
}
@@ -754,6 +765,15 @@ private String uploadWithoutACL(String externalFileName, InputStream input) {
754765
} catch (IOException ex) {
755766
logger.error("Error while uploading file to the external provider.", ex);
756767
return "";
768+
} finally {
769+
// Clean up the temporary file if it was created
770+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
771+
try {
772+
streamInfo.tempFile.delete();
773+
} catch (Exception e) {
774+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
775+
}
776+
}
757777
}
758778
}
759779

‎gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ public String upload(String externalFileName, InputStream input, ResourceAccessC
148148
try (BlobOutputStream blobOutputStream = blob.openOutputStream()) {
149149
byte[] buffer = new byte[8192];
150150
int bytesRead;
151-
while ((bytesRead = streamInfo.inputStream.read(buffer)) != -1) {
151+
while ((bytesRead = input.read(buffer)) != -1) {
152152
blobOutputStream.write(buffer, 0, bytesRead);
153153
}
154154
}

‎gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import com.genexus.util.Encryption;
44
import com.genexus.util.GXService;
55

6+
import java.io.*;
7+
68
public class ExternalProviderHelper {
79

810
public static String getServicePropertyValue(GXService s, String propName, boolean isSecure){
@@ -23,4 +25,30 @@ public static String getEnvironmentVariable(String name, boolean required) throw
2325
}
2426
return value;
2527
}
28+
29+
public static InputStreamWithLength getInputStreamContentLength(InputStream input) throws IOException {
30+
File tempFile = File.createTempFile("upload-", ".tmp");
31+
try (OutputStream out = new FileOutputStream(tempFile)) {
32+
byte[] buffer = new byte[8192];
33+
int bytesRead;
34+
while ((bytesRead = input.read(buffer)) != -1) {
35+
out.write(buffer, 0, bytesRead);
36+
}
37+
}
38+
long size = tempFile.length();
39+
InputStream newInput = new FileInputStream(tempFile);
40+
return new InputStreamWithLength(newInput, size, tempFile);
41+
}
42+
43+
public static class InputStreamWithLength {
44+
public final InputStream inputStream;
45+
public final long contentLength;
46+
public final File tempFile; // nullable
47+
48+
public InputStreamWithLength(InputStream inputStream, long contentLength, File tempFile) {
49+
this.inputStream = inputStream;
50+
this.contentLength = contentLength;
51+
this.tempFile = tempFile;
52+
}
53+
}
2654
}

‎gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import com.google.api.services.storage.model.StorageObject;
1717
import com.google.auth.oauth2.ServiceAccountCredentials;
1818
import com.google.cloud.storage.*;
19-
import org.apache.commons.io.IOUtils;
2019
import org.apache.logging.log4j.LogManager;
2120
import org.apache.logging.log4j.Logger;
2221

@@ -170,16 +169,28 @@ private void setBlobAcl(BlobId blobId, ResourceAccessControlList acl) {
170169
}
171170

172171
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
172+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
173173
try {
174+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
175+
174176
BlobId blobId = BlobId.of(bucket, externalFileName);
175177
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
176-
byte[] targetArray = IOUtils.toByteArray(input);
177-
storageClient.create(blobInfo, targetArray);
178+
179+
storageClient.createFrom(blobInfo, streamInfo.tempFile.toPath());
180+
178181
setBlobAcl(blobId, acl);
179182
return getResourceUrl(blobInfo, acl);
180183
} catch (IOException ex) {
181184
handleIOException(ex);
182185
return "";
186+
} finally {
187+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
188+
try {
189+
streamInfo.tempFile.delete();
190+
} catch (Exception e) {
191+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
192+
}
193+
}
183194
}
184195
}
185196

‎gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import com.ibm.cloud.objectstorage.services.s3.AmazonS3Client;
1515
import com.ibm.cloud.objectstorage.services.s3.AmazonS3ClientBuilder;
1616
import com.ibm.cloud.objectstorage.services.s3.model.*;
17-
import com.ibm.cloud.objectstorage.util.IOUtils;
1817
import org.apache.logging.log4j.LogManager;
1918
import org.apache.logging.log4j.Logger;
2019

@@ -150,24 +149,30 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) {
150149
}
151150

152151
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
153-
byte[] bytes;
154-
try {
155-
bytes = IOUtils.toByteArray(input);
152+
ExternalProviderHelper.InputStreamWithLength streamInfo = null;
153+
try {
154+
streamInfo = ExternalProviderHelper.getInputStreamContentLength(input);
156155
ObjectMetadata metadata = new ObjectMetadata();
157-
metadata.setContentLength(bytes.length);
156+
metadata.setContentLength(streamInfo.contentLength);
158157
if (externalFileName.endsWith(".tmp")) {
159158
metadata.setContentType("image/jpeg");
160159
}
161160
String upload = "";
162-
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) {
163-
client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
164-
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
165-
}
161+
client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
162+
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
166163
return upload;
167164
} catch (IOException ex) {
168165
logger.error("Error while uploading file to the external provider.", ex);
169166
return "";
170-
}
167+
} finally {
168+
if (streamInfo != null && streamInfo.tempFile != null && streamInfo.tempFile.exists()) {
169+
try {
170+
streamInfo.tempFile.delete();
171+
} catch (Exception e) {
172+
logger.warn("Could not delete temporary file: " + streamInfo.tempFile.getAbsolutePath(), e);
173+
}
174+
}
175+
}
171176
}
172177

173178
public String get(String externalFileName, ResourceAccessControlList acl, int expirationMinutes) {

0 commit comments

Comments
 (0)