Skip to content

Commit 8dda4be

Browse files
iroquetairoqueta
committed
The method createNewFile converts all the inputstream to memory. This… (#980)
* The method createNewFile converts all the inputstream to memory. This could cause out of memory errors. Now the process use a buffer to avoid this. Issue: 205339 * The method createNewFile converts all the inputstream to memory. This could cause out of memory errors. Now the process use a buffer to avoid this. Issue: 205339 * Tries to determine the content-type of files uploaded to external storage. Previously, if the file extension was .tmp, it was always set to image/jpeg, which caused issues—such as when uploading a PDF: upon downloading it, the file couldn't be opened properly. * Update Tika version to be java 8 compliant. * Add log4j impl for slf4j to avoid "No SLF4J providers were found" in Tika logs * Implemented better way to delete temporary files (cherry picked from commit 1fa5ad3)
1 parent ae61c12 commit 8dda4be

File tree

11 files changed

+98
-58
lines changed

11 files changed

+98
-58
lines changed

‎common/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
<dependency>
3939
<groupId>commons-io</groupId>
4040
<artifactId>commons-io</artifactId>
41-
<version>2.11.0</version>
41+
<version>2.12.0</version>
4242
</dependency>
4343
<dependency>
4444
<groupId>org.bouncycastle</groupId>

‎common/src/main/java/com/genexus/util/GXFileInfo.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public boolean createNewFile() throws IOException{
6868
return fileSource.createNewFile();
6969
}
7070
public boolean createNewFile(InputStream input) throws IOException{
71-
fromBytes(SpecificImplementation.GXutil.toByteArray(input));
71+
fromInputStream(input);
7272
return true;
7373
}
7474
public boolean delete(){
@@ -180,6 +180,16 @@ public void fromBytes(byte[] data) throws IOException
180180
destination.write(data, 0, data.length);
181181
}
182182
}
183+
private void fromInputStream(InputStream input) throws IOException
184+
{
185+
try (OutputStream output = new BufferedOutputStream(new FileOutputStream(fileSource))) {
186+
byte[] buffer = new byte[8192];
187+
int bytesRead;
188+
while ((bytesRead = input.read(buffer)) != -1) {
189+
output.write(buffer, 0, bytesRead);
190+
}
191+
}
192+
}
183193
public String readAllText(String encoding)throws IOException{
184194
return SpecificImplementation.FileUtils.readFileToString(fileSource, CommonUtil.normalizeEncodingName(encoding));
185195
}

‎gxcloudstorage-awss3-v1/src/main/java/com/genexus/db/driver/ExternalProviderS3V1.java‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import com.amazonaws.services.s3.AmazonS3;
1919
import com.amazonaws.services.s3.AmazonS3Client;
20-
import com.amazonaws.util.IOUtils;
2120

2221
import java.io.FileNotFoundException;
2322
import java.io.FileOutputStream;
@@ -220,19 +219,14 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) {
220219
}
221220

222221
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
223-
byte[] bytes;
224-
try {
225-
bytes = IOUtils.toByteArray(input);
222+
try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) {
226223
ObjectMetadata metadata = new ObjectMetadata();
227-
metadata.setContentLength(bytes.length);
228-
if (externalFileName.endsWith(".tmp")) {
229-
metadata.setContentType("image/jpeg");
230-
}
224+
metadata.setContentLength(streamInfo.contentLength);
225+
metadata.setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType);
226+
231227
String upload = "";
232-
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) {
233-
client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
234-
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
235-
}
228+
client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
229+
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
236230
return upload;
237231
} catch (IOException ex) {
238232
logger.error("Error while uploading file to the external provider.", ex);

‎gxcloudstorage-awss3-v2/src/main/java/com/genexus/db/driver/ExternalProviderS3V2.java‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,11 @@
2121
import com.genexus.util.StorageUtils;
2222
import com.genexus.StructSdtMessages_Message;
2323
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;
24-
import software.amazon.awssdk.utils.IoUtils;
2524

2625
import java.io.*;
2726
import java.net.URI;
2827
import java.net.URL;
2928
import java.net.URLEncoder;
30-
import java.nio.ByteBuffer;
3129
import java.nio.charset.StandardCharsets;
3230
import java.nio.file.Files;
3331
import java.nio.file.Path;
@@ -613,8 +611,7 @@ else if (acl == ResourceAccessControlList.PublicReadWrite)
613611
}
614612

615613
private String uploadWithACL(String externalFileName, InputStream input, ResourceAccessControlList acl) {
616-
try {
617-
ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input));
614+
try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) {
618615
PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
619616
.bucket(bucket)
620617
.key(externalFileName)
@@ -623,7 +620,7 @@ private String uploadWithACL(String externalFileName, InputStream input, Resourc
623620
putObjectRequestBuilder = putObjectRequestBuilder.acl(internalToAWSACLWithACL(acl));
624621
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
625622

626-
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer));
623+
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength));
627624
if (!response.sdkHttpResponse().isSuccessful()) {
628625
logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error"));
629626
}
@@ -737,15 +734,14 @@ private String uploadWithoutACL(String localFile, String externalFileName) {
737734
}
738735

739736
private String uploadWithoutACL(String externalFileName, InputStream input) {
740-
try {
741-
ByteBuffer byteBuffer = ByteBuffer.wrap(IoUtils.toByteArray(input));
737+
try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) {
742738
PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
743739
.bucket(bucket)
744740
.key(externalFileName)
745-
.contentType(externalFileName.endsWith(".tmp") ? "image/jpeg" : null);
741+
.contentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType);
746742
PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
747743

748-
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromByteBuffer(byteBuffer));
744+
PutObjectResponse response = client.putObject(putObjectRequest, RequestBody.fromInputStream(streamInfo.inputStream, streamInfo.contentLength));
749745
if (!response.sdkHttpResponse().isSuccessful()) {
750746
logger.error("Error while uploading file: " + response.sdkHttpResponse().statusText().orElse("Unknown error"));
751747
}

‎gxcloudstorage-azureblob/src/main/java/com/genexus/db/driver/ExternalProviderAzureStorage.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -139,17 +139,14 @@ public String upload(String localFile, String externalFileName, ResourceAccessCo
139139
}
140140

141141
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
142-
143-
try {
142+
try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) {
144143
CloudBlockBlob blob = getCloudBlockBlob(externalFileName, acl);
145-
if (externalFileName.endsWith(".tmp")) {
146-
blob.getProperties().setContentType("image/jpeg");
147-
}
144+
blob.getProperties().setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType);
148145
try (BlobOutputStream blobOutputStream = blob.openOutputStream()) {
149-
int next = input.read();
150-
while (next != -1) {
151-
blobOutputStream.write(next);
152-
next = input.read();
146+
byte[] buffer = new byte[8192];
147+
int bytesRead;
148+
while ((bytesRead = streamInfo.inputStream.read(buffer)) != -1) {
149+
blobOutputStream.write(buffer, 0, bytesRead);
153150
}
154151
}
155152
return getResourceUrl(externalFileName, acl, DEFAULT_EXPIRATION_MINUTES);

‎gxcloudstorage-common/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@
3434
<artifactId>log4j-api</artifactId>
3535
<version>${log4j.version}</version>
3636
</dependency>
37+
<dependency>
38+
<groupId>org.apache.logging.log4j</groupId>
39+
<artifactId>log4j-slf4j2-impl</artifactId>
40+
<version>${log4j.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>org.apache.tika</groupId>
44+
<artifactId>tika-core</artifactId>
45+
<version>2.9.4</version>
46+
</dependency>
3747
</dependencies>
3848

3949
</project>

‎gxcloudstorage-common/src/main/java/com/genexus/db/driver/ExternalProviderHelper.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,13 @@
33
import com.genexus.util.Encryption;
44
import com.genexus.util.GXService;
55

6+
import java.io.*;
7+
import org.apache.tika.Tika;
8+
69
public class ExternalProviderHelper {
710

11+
public static final int BUFFER_MARK_LIMIT = 128 * 1024;
12+
813
public static String getServicePropertyValue(GXService s, String propName, boolean isSecure){
914
String value = s.getProperties().get(propName);
1015
if (value != null){
@@ -23,4 +28,49 @@ public static String getEnvironmentVariable(String name, boolean required) throw
2328
}
2429
return value;
2530
}
31+
32+
public static InputStreamWithLength getInputStreamContentLength(InputStream input) throws IOException {
33+
File tempFile = File.createTempFile("upload-", ".tmp");
34+
try (OutputStream out = new FileOutputStream(tempFile)) {
35+
byte[] buffer = new byte[8192];
36+
int bytesRead;
37+
while ((bytesRead = input.read(buffer)) != -1) {
38+
out.write(buffer, 0, bytesRead);
39+
}
40+
}
41+
long size = tempFile.length();
42+
InputStream bufferedInput = new BufferedInputStream(new FileInputStream(tempFile));
43+
bufferedInput.mark(BUFFER_MARK_LIMIT);
44+
Tika tika = new Tika();
45+
String detectedContentType = tika.detect(bufferedInput);
46+
bufferedInput.reset();
47+
return new InputStreamWithLength(bufferedInput, size, tempFile, detectedContentType);
48+
}
49+
50+
public static class InputStreamWithLength implements AutoCloseable {
51+
public final InputStream inputStream;
52+
public final long contentLength;
53+
public final File tempFile; // nullable
54+
public final String detectedContentType;
55+
56+
public InputStreamWithLength(InputStream inputStream, long contentLength, File tempFile, String detectedContentType) {
57+
this.inputStream = inputStream;
58+
this.contentLength = contentLength;
59+
this.tempFile = tempFile;
60+
this.detectedContentType = detectedContentType;
61+
}
62+
63+
@Override
64+
public void close() throws IOException {
65+
try {
66+
if (inputStream != null) {
67+
inputStream.close();
68+
}
69+
} finally {
70+
if (tempFile != null && tempFile.exists()) {
71+
tempFile.delete();
72+
}
73+
}
74+
}
75+
}
2676
}

‎gxcloudstorage-googlecloudstorage/src/main/java/com/genexus/db/driver/ExternalProviderGoogle.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import com.google.api.services.storage.model.StorageObject;
1717
import com.google.auth.oauth2.ServiceAccountCredentials;
1818
import com.google.cloud.storage.*;
19-
import org.apache.commons.io.IOUtils;
2019
import org.apache.logging.log4j.LogManager;
2120
import org.apache.logging.log4j.Logger;
2221

@@ -170,11 +169,12 @@ private void setBlobAcl(BlobId blobId, ResourceAccessControlList acl) {
170169
}
171170

172171
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
173-
try {
172+
try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) {
174173
BlobId blobId = BlobId.of(bucket, externalFileName);
175174
BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
176-
byte[] targetArray = IOUtils.toByteArray(input);
177-
storageClient.create(blobInfo, targetArray);
175+
176+
storageClient.createFrom(blobInfo, streamInfo.tempFile.toPath());
177+
178178
setBlobAcl(blobId, acl);
179179
return getResourceUrl(blobInfo, acl);
180180
} catch (IOException ex) {

‎gxcloudstorage-ibmcos/src/main/java/com/genexus/db/driver/ExternalProviderIBM.java‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import com.ibm.cloud.objectstorage.services.s3.AmazonS3Client;
1515
import com.ibm.cloud.objectstorage.services.s3.AmazonS3ClientBuilder;
1616
import com.ibm.cloud.objectstorage.services.s3.model.*;
17-
import com.ibm.cloud.objectstorage.util.IOUtils;
1817
import org.apache.logging.log4j.LogManager;
1918
import org.apache.logging.log4j.Logger;
2019

@@ -150,19 +149,13 @@ else if (acl == ResourceAccessControlList.PublicReadWrite) {
150149
}
151150

152151
public String upload(String externalFileName, InputStream input, ResourceAccessControlList acl) {
153-
byte[] bytes;
154-
try {
155-
bytes = IOUtils.toByteArray(input);
152+
try (ExternalProviderHelper.InputStreamWithLength streamInfo = ExternalProviderHelper.getInputStreamContentLength(input)) {
156153
ObjectMetadata metadata = new ObjectMetadata();
157-
metadata.setContentLength(bytes.length);
158-
if (externalFileName.endsWith(".tmp")) {
159-
metadata.setContentType("image/jpeg");
160-
}
154+
metadata.setContentLength(streamInfo.contentLength);
155+
metadata.setContentType((externalFileName.endsWith(".tmp") && "application/octet-stream".equals(streamInfo.detectedContentType)) ? "image/jpeg" : streamInfo.detectedContentType);
161156
String upload = "";
162-
try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes)) {
163-
client.putObject(new PutObjectRequest(bucket, externalFileName, byteArrayInputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
164-
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
165-
}
157+
client.putObject(new PutObjectRequest(bucket, externalFileName, streamInfo.inputStream, metadata).withCannedAcl(internalToAWSACL(acl)));
158+
upload = getResourceUrl(externalFileName, acl, defaultExpirationMinutes);
166159
return upload;
167160
} catch (IOException ex) {
168161
logger.error("Error while uploading file to the external provider.", ex);

‎java/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,6 @@
5959
<artifactId>javax.jms</artifactId>
6060
<version>3.1.2.2</version>
6161
</dependency>
62-
<dependency>
63-
<groupId>commons-io</groupId>
64-
<artifactId>commons-io</artifactId>
65-
<version>2.11.0</version>
66-
</dependency>
6762
<dependency>
6863
<groupId>commons-net</groupId>
6964
<artifactId>commons-net</artifactId>

0 commit comments

Comments
 (0)