From 6136e4198fdfca1311ae9900f88dfb74f06e2df4 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 28 Mar 2026 03:10:58 +0800 Subject: [PATCH 1/4] Implement conditional read support in S3 endpoint. Add tests for If-Match, If-None-Match, If-Modified-Since, and If-Unmodified-Since headers in both GET and HEAD requests. Enhance response handling for conditional requests in ObjectEndpoint. --- .../s3/awssdk/v2/AbstractS3SDKV2Tests.java | 112 +++++++++++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 149 +++++++++++++++--- .../apache/hadoop/ozone/s3/util/S3Consts.java | 3 + .../ozone/s3/endpoint/TestObjectGet.java | 88 +++++++++++ .../ozone/s3/endpoint/TestObjectHead.java | 67 ++++++++ 5 files changed, 398 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 35a53dd328bb..402fb7862a2e 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -338,6 +338,118 @@ public void testPutObjectIfMatchMissingKeyFail() { b -> b.bucket(bucketName).key(keyName))); } + @Test + public void testGetObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject( + b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + ResponseBytes response = s3Client.getObjectAsBytes( + b -> b.bucket(bucketName).key(keyName).ifMatch(initialResponse.eTag())); + + assertEquals(content, response.asUtf8String()); + } + + @Test + public void testGetObjectIfMatchFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.getObjectAsBytes( + b -> b.bucket(bucketName).key(keyName).ifMatch("wrong-etag"))); + + assertEquals(412, exception.statusCode()); + assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode()); + } + + @Test + public void testGetObjectIfNoneMatchReturnsNotModified() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject( + b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.getObjectAsBytes( + b -> b.bucket(bucketName).key(keyName) + .ifNoneMatch(initialResponse.eTag()))); + + assertEquals(304, exception.statusCode()); + } + + @Test + public void testGetObjectIfModifiedSinceReturnsNotModified() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + HeadObjectResponse headObjectResponse = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName)); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.getObjectAsBytes( + b -> b.bucket(bucketName).key(keyName) + .ifModifiedSince(headObjectResponse.lastModified() + .plusSeconds(60)))); + + assertEquals(304, exception.statusCode()); + } + + @Test + public void testHeadObjectIfMatch() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + + PutObjectResponse initialResponse = s3Client.putObject( + b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + HeadObjectResponse response = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName).ifMatch(initialResponse.eTag())); + + assertEquals(Long.valueOf(content.length()), response.contentLength()); + } + + @Test + public void testHeadObjectIfUnmodifiedSinceFail() { + final String bucketName = getBucketName(); + final String keyName = getKeyName(); + final String content = "bar"; + s3Client.createBucket(b -> b.bucket(bucketName)); + s3Client.putObject(b -> b.bucket(bucketName).key(keyName), + RequestBody.fromString(content)); + + HeadObjectResponse headObjectResponse = s3Client.headObject( + b -> b.bucket(bucketName).key(keyName)); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.headObject( + b -> b.bucket(bucketName).key(keyName) + .ifUnmodifiedSince(headObjectResponse.lastModified() + .minusSeconds(60)))); + + assertEquals(412, exception.statusCode()); + } + @Test public void testPutObjectEmpty() throws Exception { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index ad67ebd25908..2eda0cd1efb4 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -55,6 +55,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -108,6 +109,7 @@ import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; import org.apache.hadoop.ozone.s3.util.S3StorageType; import org.apache.hadoop.ozone.s3.util.S3Utils; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.util.Time; import org.apache.http.HttpStatus; import org.apache.ratis.util.function.CheckedRunnable; @@ -449,6 +451,17 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) isFile(keyPath, keyDetails); + Response conditionalResponse = createConditionalReadResponse( + keyPath, keyDetails); + if (conditionalResponse != null) { + long metadataLatencyNs = getMetrics().updateGetKeyMetadataStats( + startNanos); + perf.appendMetaLatencyNanos(metadataLatencyNs); + long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos); + perf.appendOpLatencyNanos(opLatencyNs); + return conditionalResponse; + } + long length = keyDetails.getDataSize(); LOG.debug("Data length of the key {} is {}", keyPath, length); @@ -509,15 +522,7 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath) } responseBuilder.header(ACCEPT_RANGE_HEADER, RANGE_HEADER_SUPPORTED_UNIT); - - String eTag = keyDetails.getMetadata().get(OzoneConsts.ETAG); - if (eTag != null) { - responseBuilder.header(HttpHeaders.ETAG, wrapInQuotes(eTag)); - String partsCount = extractPartsCount(eTag); - if (partsCount != null) { - responseBuilder.header(MP_PARTS_COUNT, partsCount); - } - } + addEntityTagHeader(responseBuilder, keyDetails); MultivaluedMap queryParams = getContext().getUriInfo().getQueryParameters(); @@ -569,6 +574,20 @@ static void addTagCountIfAny( } } + static void addEntityTagHeader(ResponseBuilder responseBuilder, OzoneKey key) { + String eTag = key.getMetadata().get(OzoneConsts.ETAG); + if (eTag != null) { + // Should not return ETag header if the ETag is not set + // doing so will result in "null" string being returned instead + // which breaks some AWS SDK implementation + responseBuilder.header(HttpHeaders.ETAG, wrapInQuotes(eTag)); + String partsCount = extractPartsCount(eTag); + if (partsCount != null) { + responseBuilder.header(MP_PARTS_COUNT, partsCount); + } + } + } + /** * Rest endpoint to check existence of an object in a bucket. *

@@ -591,6 +610,13 @@ public Response head( key = getClientProtocol().headS3Object(bucketName, keyPath); isFile(keyPath, key); + Response conditionalResponse = createConditionalReadResponse( + keyPath, key); + if (conditionalResponse != null) { + getMetrics().updateHeadKeySuccessStats(startNanos); + auditReadSuccess(s3GAction); + return conditionalResponse; + } // TODO: return the specified range bytes of this object. } catch (OMException ex) { auditReadFailure(s3GAction, ex); @@ -618,18 +644,7 @@ public Response head( .header(HttpHeaders.CONTENT_LENGTH, key.getDataSize()) .header(HttpHeaders.CONTENT_TYPE, "binary/octet-stream") .header(STORAGE_CLASS_HEADER, s3StorageType.toString()); - - String eTag = key.getMetadata().get(OzoneConsts.ETAG); - if (eTag != null) { - // Should not return ETag header if the ETag is not set - // doing so will result in "null" string being returned instead - // which breaks some AWS SDK implementation - response.header(HttpHeaders.ETAG, wrapInQuotes(eTag)); - String partsCount = extractPartsCount(eTag); - if (partsCount != null) { - response.header(MP_PARTS_COUNT, partsCount); - } - } + addEntityTagHeader(response, key); addLastModifiedDate(response, key); addCustomMetadataHeaders(response, key); @@ -1208,6 +1223,98 @@ static String parseETag(String headerValue) { return stripQuotes(headerValue.trim()); } + private Response createConditionalReadResponse(String keyPath, OzoneKey key) + throws OS3Exception { + String currentETag = key.getMetadata().get(OzoneConsts.ETAG); + String ifMatch = getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER); + if (ifMatch != null && !eTagMatches(ifMatch, currentETag)) { + throw newError(PRECOND_FAILED, keyPath); + } + + String ifUnmodifiedSince = getHeaders().getHeaderString( + S3Consts.IF_UNMODIFIED_SINCE_HEADER); + if (ifMatch == null && ifUnmodifiedSince != null + && !matchesIfUnmodifiedSince(key, ifUnmodifiedSince)) { + throw newError(PRECOND_FAILED, keyPath); + } + + String ifNoneMatch = getHeaders().getHeaderString( + S3Consts.IF_NONE_MATCH_HEADER); + if (ifNoneMatch != null) { + if (eTagMatches(ifNoneMatch, currentETag)) { + return buildNotModifiedResponse(key); + } + return null; + } + + String ifModifiedSince = getHeaders().getHeaderString( + S3Consts.IF_MODIFIED_SINCE_HEADER); + if (ifModifiedSince != null + && !matchesIfModifiedSince(key, ifModifiedSince)) { + return buildNotModifiedResponse(key); + } + + return null; + } + + private static Response buildNotModifiedResponse(OzoneKey key) { + ResponseBuilder responseBuilder = Response.status(Status.NOT_MODIFIED); + addEntityTagHeader(responseBuilder, key); + addLastModifiedDate(responseBuilder, key); + return responseBuilder.build(); + } + + static boolean eTagMatches(String headerValue, String currentETag) { + if (headerValue == null) { + return false; + } + for (String candidate : headerValue.split(",")) { + String trimmedCandidate = candidate.trim(); + if ("*".equals(trimmedCandidate)) { + return true; + } + if (currentETag != null + && currentETag.equals(parseETag(trimmedCandidate))) { + return true; + } + } + return false; + } + + private static boolean matchesIfModifiedSince(OzoneKey key, + String headerValue) { + Instant since = parseConditionalInstant(headerValue); + if (since == null) { + return true; + } + Instant lastModified = key.getModificationTime().truncatedTo( + ChronoUnit.SECONDS); + return lastModified.isAfter(since); + } + + private static boolean matchesIfUnmodifiedSince(OzoneKey key, + String headerValue) { + Instant since = parseConditionalInstant(headerValue); + if (since == null) { + return true; + } + Instant lastModified = key.getModificationTime().truncatedTo( + ChronoUnit.SECONDS); + return !lastModified.isAfter(since); + } + + private static Instant parseConditionalInstant(String headerValue) { + if (headerValue == null) { + return null; + } + try { + return Instant.ofEpochMilli(OzoneUtils.formatDate(headerValue)) + .truncatedTo(ChronoUnit.SECONDS); + } catch (IllegalArgumentException | java.text.ParseException ex) { + return null; + } + } + /** Request context shared among {@code ObjectOperationHandler}s. */ final class ObjectRequestContext extends S3RequestContext { private final String bucketName; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java index e90612cc56ed..67f6b4c7d41d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java @@ -101,6 +101,9 @@ public final class S3Consts { // Conditional request headers public static final String IF_NONE_MATCH_HEADER = "If-None-Match"; public static final String IF_MATCH_HEADER = "If-Match"; + public static final String IF_MODIFIED_SINCE_HEADER = "If-Modified-Since"; + public static final String IF_UNMODIFIED_SINCE_HEADER = + "If-Unmodified-Since"; //Never Constructed private S3Consts() { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index 4e0e631fdcae..ad2b3a37be17 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -22,28 +22,38 @@ import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MATCH_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MODIFIED_SINCE_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_NONE_MATCH_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_UNMODIFIED_SINCE_HEADER; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.OzoneClientTestUtils; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.util.RFC1123Util; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -112,6 +122,79 @@ public void testGet() throws IOException, OS3Exception { assertNull(response.getHeaderString(TAG_COUNT_HEADER)); } + @Test + public void testGetIfMatch() throws IOException, OS3Exception { + Response response = get(rest, BUCKET_NAME, KEY_NAME); + String eTag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(eTag); + + when(headers.getHeaderString(IF_MATCH_HEADER)).thenReturn(eTag); + + response = get(rest, BUCKET_NAME, KEY_NAME); + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + } + + @Test + public void testGetIfMatchFailure() throws IOException { + when(headers.getHeaderString(IF_MATCH_HEADER)).thenReturn("\"wrong-etag\""); + + assertErrorResponse(PRECOND_FAILED, () -> get(rest, BUCKET_NAME, KEY_NAME)); + } + + @Test + public void testGetIfNoneMatchReturnsNotModified() throws IOException, OS3Exception { + Response response = get(rest, BUCKET_NAME, KEY_NAME); + String eTag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(eTag); + + when(headers.getHeaderString(IF_NONE_MATCH_HEADER)).thenReturn(eTag); + + response = get(rest, BUCKET_NAME, KEY_NAME); + assertEquals(Response.Status.NOT_MODIFIED.getStatusCode(), + response.getStatus()); + } + + @Test + public void testGetIfModifiedSinceReturnsNotModified() + throws IOException, OS3Exception { + when(headers.getHeaderString(IF_MODIFIED_SINCE_HEADER)) + .thenReturn(formatHttpDate(bucket.getKey(KEY_NAME) + .getModificationTime().plusSeconds(60))); + + Response response = get(rest, BUCKET_NAME, KEY_NAME); + assertEquals(Response.Status.NOT_MODIFIED.getStatusCode(), + response.getStatus()); + } + + @Test + public void testGetIgnoresIfUnmodifiedSinceAfterMatchingIfMatch() + throws IOException, OS3Exception { + Response response = get(rest, BUCKET_NAME, KEY_NAME); + String eTag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(eTag); + + when(headers.getHeaderString(IF_MATCH_HEADER)).thenReturn(eTag); + when(headers.getHeaderString(IF_UNMODIFIED_SINCE_HEADER)) + .thenReturn(formatHttpDate(bucket.getKey(KEY_NAME) + .getModificationTime().minusSeconds(60))); + + response = get(rest, BUCKET_NAME, KEY_NAME); + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + } + + @Test + public void testGetIgnoresIfModifiedSinceWhenIfNoneMatchPresent() + throws IOException, OS3Exception { + when(headers.getHeaderString(IF_NONE_MATCH_HEADER)) + .thenReturn("\"different-etag\""); + when(headers.getHeaderString(IF_MODIFIED_SINCE_HEADER)) + .thenReturn(formatHttpDate(bucket.getKey(KEY_NAME) + .getModificationTime().plusSeconds(60))); + + Response response = get(rest, BUCKET_NAME, KEY_NAME); + assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + } + @Test public void getKeyWithTag() throws IOException, OS3Exception { //WHEN @@ -241,4 +324,9 @@ public void testGetWhenKeyIsDirectoryAndDoesNotEndWithASlash() assertErrorResponse(NO_SUCH_KEY, () -> get(rest, BUCKET_NAME, keyPath)); } + + private static String formatHttpDate(Instant instant) { + return RFC1123Util.FORMAT.format( + instant.atZone(ZoneId.of(OzoneConsts.OZONE_TIME_ZONE))); + } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index 972b28c9d095..67042c088e27 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -21,18 +21,32 @@ import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertStatus; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MATCH_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_NONE_MATCH_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_UNMODIFIED_SINCE_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.X_AMZ_CONTENT_SHA256; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.io.OutputStream; +import java.time.Instant; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.util.RFC1123Util; import org.apache.http.HttpStatus; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -44,15 +58,20 @@ public class TestObjectHead { private String bucketName = "b1"; private ObjectEndpoint keyEndpoint; private OzoneBucket bucket; + private HttpHeaders headers; @BeforeEach public void setup() throws IOException { OzoneClient clientStub = new OzoneClientStub(); clientStub.getObjectStore().createS3Bucket(bucketName); bucket = clientStub.getObjectStore().getS3Bucket(bucketName); + headers = mock(HttpHeaders.class); + when(headers.getHeaderString(X_AMZ_CONTENT_SHA256)) + .thenReturn("UNSIGNED-PAYLOAD"); keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() .setClient(clientStub) + .setHeaders(headers) .build(); } @@ -78,6 +97,49 @@ public void testHeadFailByBadName() throws Exception { assertStatus(HttpStatus.SC_NOT_FOUND, () -> keyEndpoint.head(bucketName, "badKeyName")); } + @Test + public void testHeadIfMatch() throws Exception { + assertSucceeds(() -> put(keyEndpoint, bucketName, "etag-key", "head-content")); + + Response response = keyEndpoint.head(bucketName, "etag-key"); + String eTag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(eTag); + + when(headers.getHeaderString(IF_MATCH_HEADER)).thenReturn(eTag); + + response = keyEndpoint.head(bucketName, "etag-key"); + assertEquals(HttpStatus.SC_OK, response.getStatus()); + } + + @Test + public void testHeadIfNoneMatchReturnsNotModified() throws Exception { + assertSucceeds(() -> put(keyEndpoint, bucketName, "etag-key", "head-content")); + + Response response = keyEndpoint.head(bucketName, "etag-key"); + String eTag = response.getHeaderString(HttpHeaders.ETAG); + assertNotNull(eTag); + + when(headers.getHeaderString(IF_NONE_MATCH_HEADER)).thenReturn(eTag); + + response = keyEndpoint.head(bucketName, "etag-key"); + assertEquals(Response.Status.NOT_MODIFIED.getStatusCode(), + response.getStatus()); + } + + @Test + public void testHeadIfUnmodifiedSinceFailure() throws Exception { + assertSucceeds(() -> put(keyEndpoint, bucketName, "etag-key", "head-content")); + + when(headers.getHeaderString(IF_UNMODIFIED_SINCE_HEADER)) + .thenReturn(formatHttpDate(bucket.getKey("etag-key") + .getModificationTime().minusSeconds(60))); + + OS3Exception ex = org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils + .assertErrorResponse(PRECOND_FAILED, + () -> keyEndpoint.head(bucketName, "etag-key")); + assertNotNull(ex); + } + @Test public void testHeadWhenKeyIsAFileAndKeyPathDoesNotEndWithASlash() throws IOException, OS3Exception { @@ -137,4 +199,9 @@ private byte[] createKey(String keyPath) throws IOException { } return bytes; } + + private static String formatHttpDate(Instant instant) { + return RFC1123Util.FORMAT.format( + instant.atZone(ZoneId.of(OzoneConsts.OZONE_TIME_ZONE))); + } } From 7dc1d18790615c0e0cf71e3414c7c5b113b2c252 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 28 Mar 2026 03:33:11 +0800 Subject: [PATCH 2/4] fix checkstyle --- .../java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index ad2b3a37be17..8f310b74062d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -22,12 +22,12 @@ import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MATCH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_MODIFIED_SINCE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_NONE_MATCH_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.IF_UNMODIFIED_SINCE_HEADER; -import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_COUNT_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.TAG_HEADER; From 149f590409ad8e5f036ffac6d429445cdbe7901a Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 3 Apr 2026 02:28:32 +0800 Subject: [PATCH 3/4] Refactor S3 conditional request handling. --- .../ozone/s3/endpoint/EndpointBase.java | 49 ---- .../ozone/s3/endpoint/ObjectEndpoint.java | 150 ++--------- .../s3/endpoint/ObjectEndpointStreaming.java | 20 +- .../s3/endpoint/S3ConditionalRequest.java | 246 ++++++++++++++++++ 4 files changed, 271 insertions(+), 194 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 37e91ef74cd5..deec54d9a1d6 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -52,7 +52,6 @@ import java.io.UnsupportedEncodingException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -61,7 +60,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.OptionalLong; import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -107,7 +105,6 @@ import org.apache.hadoop.ozone.s3.signature.SignatureInfo; import org.apache.hadoop.ozone.s3.util.AuditUtils; import org.apache.hadoop.ozone.s3.util.S3Utils; -import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.util.Time; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; @@ -727,52 +724,6 @@ protected static int parsePartNumberMarker(String partNumberMarker) { return partMarker; } - // Parses date string and return long representation. Returns an - // empty if DateStr is null or invalid. Dates in the future are - // considered invalid. - private static OptionalLong parseAndValidateDate(String ozoneDateStr) { - long ozoneDateInMs; - if (ozoneDateStr == null) { - return OptionalLong.empty(); - } - try { - ozoneDateInMs = OzoneUtils.formatDate(ozoneDateStr); - } catch (ParseException e) { - // if time not parseable, then return empty() - return OptionalLong.empty(); - } - - long currentDate = System.currentTimeMillis(); - if (ozoneDateInMs <= currentDate) { - return OptionalLong.of(ozoneDateInMs); - } else { - // dates in the future are invalid, so return empty() - return OptionalLong.empty(); - } - } - - public static boolean checkCopySourceModificationTime( - Long lastModificationTime, - String copySourceIfModifiedSinceStr, - String copySourceIfUnmodifiedSinceStr) { - long copySourceIfModifiedSince = Long.MIN_VALUE; - long copySourceIfUnmodifiedSince = Long.MAX_VALUE; - - OptionalLong modifiedDate = - parseAndValidateDate(copySourceIfModifiedSinceStr); - if (modifiedDate.isPresent()) { - copySourceIfModifiedSince = modifiedDate.getAsLong(); - } - - OptionalLong unmodifiedDate = - parseAndValidateDate(copySourceIfUnmodifiedSinceStr); - if (unmodifiedDate.isPresent()) { - copySourceIfUnmodifiedSince = unmodifiedDate.getAsLong(); - } - return (copySourceIfModifiedSince <= lastModificationTime) && - (lastModificationTime <= copySourceIfUnmodifiedSince); - } - /** * Create a {@link S3ChunkInputStreamInfo} that contains the necessary information to handle * the S3 chunk upload. diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 2eda0cd1efb4..19f469d5a3be 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -55,7 +55,6 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -109,7 +108,6 @@ import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; import org.apache.hadoop.ozone.s3.util.S3StorageType; import org.apache.hadoop.ozone.s3.util.S3Utils; -import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.util.Time; import org.apache.http.HttpStatus; import org.apache.ratis.util.function.CheckedRunnable; @@ -282,37 +280,8 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr return Response.ok().status(HttpStatus.SC_OK).build(); } - String ifNoneMatch = getHeaders().getHeaderString( - S3Consts.IF_NONE_MATCH_HEADER); - String ifMatch = getHeaders().getHeaderString( - S3Consts.IF_MATCH_HEADER); - - if (ifNoneMatch != null && StringUtils.isBlank(ifNoneMatch)) { - OS3Exception ex = newError(INVALID_REQUEST, keyPath); - ex.setErrorMessage("If-None-Match header cannot be empty."); - throw ex; - } - if (ifMatch != null && StringUtils.isBlank(ifMatch)) { - OS3Exception ex = newError(INVALID_REQUEST, keyPath); - ex.setErrorMessage("If-Match header cannot be empty."); - throw ex; - } - - String ifNoneMatchTrimmed = ifNoneMatch == null ? null : ifNoneMatch.trim(); - String ifMatchTrimmed = ifMatch == null ? null : ifMatch.trim(); - - if (ifNoneMatchTrimmed != null && ifMatchTrimmed != null) { - OS3Exception ex = newError(INVALID_REQUEST, keyPath); - ex.setErrorMessage("If-Match and If-None-Match cannot be specified together."); - throw ex; - } - - if (ifNoneMatchTrimmed != null - && !"*".equals(stripQuotes(ifNoneMatchTrimmed))) { - OS3Exception ex = newError(INVALID_REQUEST, keyPath); - ex.setErrorMessage("Only If-None-Match: * is supported for conditional put."); - throw ex; - } + S3ConditionalRequest.WriteConditions writeConditions = + S3ConditionalRequest.parseWriteConditions(getHeaders(), keyPath); // Normal put object S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body, @@ -331,8 +300,7 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr Pair keyWriteResult = ObjectEndpointStreaming .put(bucket, keyPath, length, replicationConfig, getChunkSize(), customMetadata, tags, multiDigestInputStream, getHeaders(), - signatureInfo.isSignPayload(), perf, ifNoneMatchTrimmed, - ifMatchTrimmed); + signatureInfo.isSignPayload(), perf, writeConditions); md5Hash = keyWriteResult.getKey(); putLength = keyWriteResult.getValue(); } else { @@ -340,8 +308,7 @@ customMetadata, tags, multiDigestInputStream, getHeaders(), validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload()); try (OzoneOutputStream output = openKeyForPut( volume.getName(), bucketName, keyPath, length, - replicationConfig, customMetadata, tags, ifNoneMatchTrimmed, - ifMatchTrimmed)) { + replicationConfig, customMetadata, tags, writeConditions)) { long metadataLatencyNs = getMetrics().updatePutKeyMetadataStats(startNanos); perf.appendMetaLatencyNanos(metadataLatencyNs); @@ -931,7 +898,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket, getHeaders().getHeaderString(COPY_SOURCE_IF_MODIFIED_SINCE); String copySourceIfUnmodifiedSince = getHeaders().getHeaderString(COPY_SOURCE_IF_UNMODIFIED_SINCE); - if (!checkCopySourceModificationTime(sourceKeyModificationTime, + if (!S3ConditionalRequest.checkCopySourceModificationTime( + sourceKeyModificationTime, copySourceIfModifiedSince, copySourceIfUnmodifiedSince)) { throw newError(PRECOND_FAILED, sourceBucket + "/" + sourceKey); } @@ -1194,16 +1162,17 @@ private CopyObjectResponse copyObject(OzoneVolume volume, @SuppressWarnings("checkstyle:ParameterNumber") private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, String keyPath, long length, ReplicationConfig replicationConfig, Map customMetadata, - Map tags, String ifNoneMatch, String ifMatch) + Map tags, + S3ConditionalRequest.WriteConditions writeConditions) throws IOException { - if (ifNoneMatch != null && "*".equals(stripQuotes(ifNoneMatch.trim()))) { + if (writeConditions.hasIfNoneMatch()) { return getClientProtocol().createKeyIfNotExists( volumeName, bucketName, keyPath, length, replicationConfig, customMetadata, tags); - } else if (ifMatch != null) { - String expectedETag = parseETag(ifMatch); + } else if (writeConditions.hasIfMatch()) { return getClientProtocol().rewriteKeyIfMatch( - volumeName, bucketName, keyPath, length, expectedETag, + volumeName, bucketName, keyPath, length, + writeConditions.getExpectedETag(), replicationConfig, customMetadata, tags); } else { return getClientProtocol().createKey( @@ -1217,102 +1186,13 @@ private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, St * quotes if present. */ static String parseETag(String headerValue) { - if (headerValue == null) { - return null; - } - return stripQuotes(headerValue.trim()); + return S3ConditionalRequest.parseETag(headerValue); } private Response createConditionalReadResponse(String keyPath, OzoneKey key) throws OS3Exception { - String currentETag = key.getMetadata().get(OzoneConsts.ETAG); - String ifMatch = getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER); - if (ifMatch != null && !eTagMatches(ifMatch, currentETag)) { - throw newError(PRECOND_FAILED, keyPath); - } - - String ifUnmodifiedSince = getHeaders().getHeaderString( - S3Consts.IF_UNMODIFIED_SINCE_HEADER); - if (ifMatch == null && ifUnmodifiedSince != null - && !matchesIfUnmodifiedSince(key, ifUnmodifiedSince)) { - throw newError(PRECOND_FAILED, keyPath); - } - - String ifNoneMatch = getHeaders().getHeaderString( - S3Consts.IF_NONE_MATCH_HEADER); - if (ifNoneMatch != null) { - if (eTagMatches(ifNoneMatch, currentETag)) { - return buildNotModifiedResponse(key); - } - return null; - } - - String ifModifiedSince = getHeaders().getHeaderString( - S3Consts.IF_MODIFIED_SINCE_HEADER); - if (ifModifiedSince != null - && !matchesIfModifiedSince(key, ifModifiedSince)) { - return buildNotModifiedResponse(key); - } - - return null; - } - - private static Response buildNotModifiedResponse(OzoneKey key) { - ResponseBuilder responseBuilder = Response.status(Status.NOT_MODIFIED); - addEntityTagHeader(responseBuilder, key); - addLastModifiedDate(responseBuilder, key); - return responseBuilder.build(); - } - - static boolean eTagMatches(String headerValue, String currentETag) { - if (headerValue == null) { - return false; - } - for (String candidate : headerValue.split(",")) { - String trimmedCandidate = candidate.trim(); - if ("*".equals(trimmedCandidate)) { - return true; - } - if (currentETag != null - && currentETag.equals(parseETag(trimmedCandidate))) { - return true; - } - } - return false; - } - - private static boolean matchesIfModifiedSince(OzoneKey key, - String headerValue) { - Instant since = parseConditionalInstant(headerValue); - if (since == null) { - return true; - } - Instant lastModified = key.getModificationTime().truncatedTo( - ChronoUnit.SECONDS); - return lastModified.isAfter(since); - } - - private static boolean matchesIfUnmodifiedSince(OzoneKey key, - String headerValue) { - Instant since = parseConditionalInstant(headerValue); - if (since == null) { - return true; - } - Instant lastModified = key.getModificationTime().truncatedTo( - ChronoUnit.SECONDS); - return !lastModified.isAfter(since); - } - - private static Instant parseConditionalInstant(String headerValue) { - if (headerValue == null) { - return null; - } - try { - return Instant.ofEpochMilli(OzoneUtils.formatDate(headerValue)) - .truncatedTo(ChronoUnit.SECONDS); - } catch (IllegalArgumentException | java.text.ParseException ex) { - return null; - } + return S3ConditionalRequest.evaluateReadPreconditions(getHeaders(), + keyPath, key); } /** Request context shared among {@code ObjectOperationHandler}s. */ diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java index 2eb2dd21f30c..63f8281177f4 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java @@ -73,13 +73,14 @@ public static Pair put( int chunkSize, Map keyMetadata, Map tags, MultiDigestInputStream body, HttpHeaders headers, boolean isSignedPayload, - PerformanceStringBuilder perf, String ifNoneMatch, String ifMatch) + PerformanceStringBuilder perf, + S3ConditionalRequest.WriteConditions writeConditions) throws IOException, OS3Exception { try { return putKeyWithStream(bucket, keyPath, length, chunkSize, replicationConfig, keyMetadata, tags, body, - headers, isSignedPayload, perf, ifNoneMatch, ifMatch); + headers, isSignedPayload, perf, writeConditions); } catch (IOException ex) { LOG.error("Exception occurred in PutObject", ex); if (ex instanceof OMException) { @@ -115,16 +116,15 @@ public static Pair putKeyWithStream( HttpHeaders headers, boolean isSignedPayload, PerformanceStringBuilder perf, - String ifNoneMatch, - String ifMatch) + S3ConditionalRequest.WriteConditions writeConditions) throws IOException, OS3Exception { long startNanos = Time.monotonicNowNanos(); final String amzContentSha256Header = validateSignatureHeader(headers, keyPath, isSignedPayload); long writeLen; String md5Hash; try (OzoneDataStreamOutput streamOutput = openStreamKeyForPut(bucket, - keyPath, length, replicationConfig, keyMetadata, tags, ifNoneMatch, - ifMatch)) { + keyPath, length, replicationConfig, keyMetadata, tags, + writeConditions)) { long metadataLatencyNs = METRICS.updatePutKeyMetadataStats(startNanos); writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length); md5Hash = DatatypeConverter.printHexBinary(body.getMessageDigest(OzoneConsts.MD5_HASH).digest()) @@ -163,14 +163,14 @@ public static Pair putKeyWithStream( private static OzoneDataStreamOutput openStreamKeyForPut(OzoneBucket bucket, String keyPath, long length, ReplicationConfig replicationConfig, Map keyMetadata, Map tags, - String ifNoneMatch, String ifMatch) throws IOException { - if (ifNoneMatch != null && "*".equals(ObjectEndpoint.parseETag(ifNoneMatch))) { + S3ConditionalRequest.WriteConditions writeConditions) throws IOException { + if (writeConditions.hasIfNoneMatch()) { return bucket.createStreamKeyIfNotExists(keyPath, length, replicationConfig, keyMetadata, tags); } - if (ifMatch != null) { + if (writeConditions.hasIfMatch()) { return bucket.rewriteStreamKeyIfMatch(keyPath, length, - ObjectEndpoint.parseETag(ifMatch), replicationConfig, keyMetadata, + writeConditions.getExpectedETag(), replicationConfig, keyMetadata, tags); } return bucket.createStreamKey(keyPath, length, replicationConfig, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java new file mode 100644 index 000000000000..976ac9f12f1f --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3ConditionalRequest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; +import static org.apache.hadoop.ozone.s3.util.S3Utils.stripQuotes; + +import java.text.ParseException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.OptionalLong; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.Response.Status; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.util.S3Consts; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; + +/** + * Shared parsing and evaluation for S3 conditional request headers. + */ +final class S3ConditionalRequest { + + private S3ConditionalRequest() { + } + + static Response evaluateReadPreconditions(HttpHeaders headers, + String keyPath, OzoneKey key) throws OS3Exception { + String currentETag = key.getMetadata().get(OzoneConsts.ETAG); + String ifMatch = headers.getHeaderString(S3Consts.IF_MATCH_HEADER); + if (ifMatch != null && !eTagMatches(ifMatch, currentETag)) { + throw newError(PRECOND_FAILED, keyPath); + } + + String ifUnmodifiedSince = headers.getHeaderString( + S3Consts.IF_UNMODIFIED_SINCE_HEADER); + if (ifMatch == null && ifUnmodifiedSince != null + && !matchesIfUnmodifiedSince(key, ifUnmodifiedSince)) { + throw newError(PRECOND_FAILED, keyPath); + } + + String ifNoneMatch = headers.getHeaderString( + S3Consts.IF_NONE_MATCH_HEADER); + if (ifNoneMatch != null) { + if (eTagMatches(ifNoneMatch, currentETag)) { + return buildNotModifiedResponse(key); + } + return null; + } + + String ifModifiedSince = headers.getHeaderString( + S3Consts.IF_MODIFIED_SINCE_HEADER); + if (ifModifiedSince != null && !matchesIfModifiedSince(key, + ifModifiedSince)) { + return buildNotModifiedResponse(key); + } + + return null; + } + + static boolean checkCopySourceModificationTime(Long lastModificationTime, + String copySourceIfModifiedSinceStr, + String copySourceIfUnmodifiedSinceStr) { + long copySourceIfModifiedSince = Long.MIN_VALUE; + long copySourceIfUnmodifiedSince = Long.MAX_VALUE; + + OptionalLong modifiedDate = + parseAndValidatePastOrPresentDate(copySourceIfModifiedSinceStr); + if (modifiedDate.isPresent()) { + copySourceIfModifiedSince = modifiedDate.getAsLong(); + } + + OptionalLong unmodifiedDate = + parseAndValidatePastOrPresentDate(copySourceIfUnmodifiedSinceStr); + if (unmodifiedDate.isPresent()) { + copySourceIfUnmodifiedSince = unmodifiedDate.getAsLong(); + } + return (copySourceIfModifiedSince <= lastModificationTime) + && (lastModificationTime <= copySourceIfUnmodifiedSince); + } + + static WriteConditions parseWriteConditions(HttpHeaders headers, + String keyPath) throws OS3Exception { + String ifNoneMatch = headers.getHeaderString(S3Consts.IF_NONE_MATCH_HEADER); + if (ifNoneMatch != null && StringUtils.isBlank(ifNoneMatch)) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-None-Match header cannot be empty."); + throw ex; + } + + String ifMatch = headers.getHeaderString(S3Consts.IF_MATCH_HEADER); + if (ifMatch != null && StringUtils.isBlank(ifMatch)) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage("If-Match header cannot be empty."); + throw ex; + } + + String trimmedIfNoneMatch = ifNoneMatch == null ? null : ifNoneMatch.trim(); + String trimmedIfMatch = ifMatch == null ? null : ifMatch.trim(); + + if (trimmedIfNoneMatch != null && trimmedIfMatch != null) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage( + "If-Match and If-None-Match cannot be specified together."); + throw ex; + } + + if (trimmedIfNoneMatch != null + && !"*".equals(stripQuotes(trimmedIfNoneMatch))) { + OS3Exception ex = newError(INVALID_REQUEST, keyPath); + ex.setErrorMessage( + "Only If-None-Match: * is supported for conditional put."); + throw ex; + } + + return new WriteConditions(trimmedIfNoneMatch, trimmedIfMatch); + } + + static String parseETag(String headerValue) { + if (headerValue == null) { + return null; + } + return stripQuotes(headerValue.trim()); + } + + private static Response buildNotModifiedResponse(OzoneKey key) { + ResponseBuilder responseBuilder = Response.status(Status.NOT_MODIFIED); + ObjectEndpoint.addEntityTagHeader(responseBuilder, key); + ObjectEndpoint.addLastModifiedDate(responseBuilder, key); + return responseBuilder.build(); + } + + private static boolean eTagMatches(String headerValue, String currentETag) { + if (headerValue == null) { + return false; + } + for (String candidate : headerValue.split(",")) { + String trimmedCandidate = candidate.trim(); + if ("*".equals(trimmedCandidate)) { + return true; + } + if (currentETag != null + && currentETag.equals(parseETag(trimmedCandidate))) { + return true; + } + } + return false; + } + + private static boolean matchesIfModifiedSince(OzoneKey key, + String headerValue) { + Instant since = parseConditionalInstant(headerValue); + if (since == null) { + return true; + } + Instant lastModified = key.getModificationTime() + .truncatedTo(ChronoUnit.SECONDS); + return lastModified.isAfter(since); + } + + private static boolean matchesIfUnmodifiedSince(OzoneKey key, + String headerValue) { + Instant since = parseConditionalInstant(headerValue); + if (since == null) { + return true; + } + Instant lastModified = key.getModificationTime() + .truncatedTo(ChronoUnit.SECONDS); + return !lastModified.isAfter(since); + } + + private static Instant parseConditionalInstant(String headerValue) { + if (headerValue == null) { + return null; + } + try { + return Instant.ofEpochMilli(OzoneUtils.formatDate(headerValue)) + .truncatedTo(ChronoUnit.SECONDS); + } catch (IllegalArgumentException | ParseException ex) { + return null; + } + } + + private static OptionalLong parseAndValidatePastOrPresentDate( + String ozoneDateStr) { + if (ozoneDateStr == null) { + return OptionalLong.empty(); + } + + long ozoneDateInMs; + try { + ozoneDateInMs = OzoneUtils.formatDate(ozoneDateStr); + } catch (ParseException e) { + return OptionalLong.empty(); + } + + long currentDate = System.currentTimeMillis(); + if (ozoneDateInMs <= currentDate) { + return OptionalLong.of(ozoneDateInMs); + } + return OptionalLong.empty(); + } + + static final class WriteConditions { + private final String ifNoneMatch; + private final String ifMatch; + + private WriteConditions(String ifNoneMatch, String ifMatch) { + this.ifNoneMatch = ifNoneMatch; + this.ifMatch = ifMatch; + } + + boolean hasIfNoneMatch() { + return ifNoneMatch != null; + } + + boolean hasIfMatch() { + return ifMatch != null; + } + + String getExpectedETag() { + return parseETag(ifMatch); + } + } +} From f2dc8f33355934572cd854b980aa00e623f1629f Mon Sep 17 00:00:00 2001 From: peterxcli Date: Fri, 3 Apr 2026 03:03:43 +0800 Subject: [PATCH 4/4] chore --- .../org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index 67042c088e27..6f4aaf110d94 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; +import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertStatus; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; @@ -134,9 +135,8 @@ public void testHeadIfUnmodifiedSinceFailure() throws Exception { .thenReturn(formatHttpDate(bucket.getKey("etag-key") .getModificationTime().minusSeconds(60))); - OS3Exception ex = org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils - .assertErrorResponse(PRECOND_FAILED, - () -> keyEndpoint.head(bucketName, "etag-key")); + OS3Exception ex = assertErrorResponse(PRECOND_FAILED, + () -> keyEndpoint.head(bucketName, "etag-key")); assertNotNull(ex); }