Skip to content

HDDS-13920. Conditional Reads (GetObject, HeadObject) #10031

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
peterxcli wants to merge 5 commits into
base: master
Choose a base branch
from
Open

HDDS-13920. Conditional Reads (GetObject, HeadObject) #10031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6136e41
Implement conditional read support in S3 endpoint.
peterxcli Mar 27, 2026
7dc1d18
fix checkstyle
peterxcli Mar 27, 2026
149f590
Refactor S3 conditional request handling.
peterxcli Apr 2, 2026
f2dc8f3
chore
peterxcli Apr 2, 2026
2f44d1b
Merge branch 'master' into HDDS-13920-conditional-reads
peterxcli Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 112 additions & 0 deletions ...tion-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,118 @@ public void testPutObjectIfMatchMissingKeyFail() {
b -> b.bucket(bucketName).key(keyName)));
}

@Test
public void testGetObjectIfMatch() {
final String bucketName = getBucketName();
final String keyName = getKeyName();
final String content = "bar";
s3Client.createBucket(b -> b.bucket(bucketName));

PutObjectResponse initialResponse = s3Client.putObject(
b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString(content));

ResponseBytes<GetObjectResponse> response = s3Client.getObjectAsBytes(
b -> b.bucket(bucketName).key(keyName).ifMatch(initialResponse.eTag()));

assertEquals(content, response.asUtf8String());
}

@Test
public void testGetObjectIfMatchFail() {
final String bucketName = getBucketName();
final String keyName = getKeyName();
final String content = "bar";
s3Client.createBucket(b -> b.bucket(bucketName));
s3Client.putObject(b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString(content));

S3Exception exception = assertThrows(S3Exception.class,
() -> s3Client.getObjectAsBytes(
b -> b.bucket(bucketName).key(keyName).ifMatch("wrong-etag")));

assertEquals(412, exception.statusCode());
assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
}

@Test
public void testGetObjectIfNoneMatchReturnsNotModified() {
final String bucketName = getBucketName();
final String keyName = getKeyName();
final String content = "bar";
s3Client.createBucket(b -> b.bucket(bucketName));

PutObjectResponse initialResponse = s3Client.putObject(
b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString(content));

S3Exception exception = assertThrows(S3Exception.class,
() -> s3Client.getObjectAsBytes(
b -> b.bucket(bucketName).key(keyName)
.ifNoneMatch(initialResponse.eTag())));

assertEquals(304, exception.statusCode());
}

@Test
public void testGetObjectIfModifiedSinceReturnsNotModified() {
final String bucketName = getBucketName();
final String keyName = getKeyName();
final String content = "bar";
s3Client.createBucket(b -> b.bucket(bucketName));
s3Client.putObject(b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString(content));

HeadObjectResponse headObjectResponse = s3Client.headObject(
b -> b.bucket(bucketName).key(keyName));

S3Exception exception = assertThrows(S3Exception.class,
() -> s3Client.getObjectAsBytes(
b -> b.bucket(bucketName).key(keyName)
.ifModifiedSince(headObjectResponse.lastModified()
.plusSeconds(60))));

assertEquals(304, exception.statusCode());
}

@Test
public void testHeadObjectIfMatch() {
final String bucketName = getBucketName();
final String keyName = getKeyName();
final String content = "bar";
s3Client.createBucket(b -> b.bucket(bucketName));

PutObjectResponse initialResponse = s3Client.putObject(
b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString(content));

HeadObjectResponse response = s3Client.headObject(
b -> b.bucket(bucketName).key(keyName).ifMatch(initialResponse.eTag()));

assertEquals(Long.valueOf(content.length()), response.contentLength());
}

@Test
public void testHeadObjectIfUnmodifiedSinceFail() {
final String bucketName = getBucketName();
final String keyName = getKeyName();
final String content = "bar";
s3Client.createBucket(b -> b.bucket(bucketName));
s3Client.putObject(b -> b.bucket(bucketName).key(keyName),
RequestBody.fromString(content));

HeadObjectResponse headObjectResponse = s3Client.headObject(
b -> b.bucket(bucketName).key(keyName));

S3Exception exception = assertThrows(S3Exception.class,
() -> s3Client.headObject(
b -> b.bucket(bucketName).key(keyName)
.ifUnmodifiedSince(headObjectResponse.lastModified()
.minusSeconds(60))));

assertEquals(412, exception.statusCode());
}

@Test
public void testPutObjectEmpty() throws Exception {
final String bucketName = getBucketName();
Expand Down
49 changes: 0 additions & 49 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
Expand All @@ -61,7 +60,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
Expand Down Expand Up @@ -107,7 +105,6 @@
import org.apache.hadoop.ozone.s3.signature.SignatureInfo;
import org.apache.hadoop.ozone.s3.util.AuditUtils;
import org.apache.hadoop.ozone.s3.util.S3Utils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.util.Time;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
Expand Down Expand Up @@ -727,52 +724,6 @@ protected static int parsePartNumberMarker(String partNumberMarker) {
return partMarker;
}

// Parses date string and return long representation. Returns an
// empty if DateStr is null or invalid. Dates in the future are
// considered invalid.
private static OptionalLong parseAndValidateDate(String ozoneDateStr) {
long ozoneDateInMs;
if (ozoneDateStr == null) {
return OptionalLong.empty();
}
try {
ozoneDateInMs = OzoneUtils.formatDate(ozoneDateStr);
} catch (ParseException e) {
// if time not parseable, then return empty()
return OptionalLong.empty();
}

long currentDate = System.currentTimeMillis();
if (ozoneDateInMs <= currentDate) {
return OptionalLong.of(ozoneDateInMs);
} else {
// dates in the future are invalid, so return empty()
return OptionalLong.empty();
}
}

public static boolean checkCopySourceModificationTime(
Long lastModificationTime,
String copySourceIfModifiedSinceStr,
String copySourceIfUnmodifiedSinceStr) {
long copySourceIfModifiedSince = Long.MIN_VALUE;
long copySourceIfUnmodifiedSince = Long.MAX_VALUE;

OptionalLong modifiedDate =
parseAndValidateDate(copySourceIfModifiedSinceStr);
if (modifiedDate.isPresent()) {
copySourceIfModifiedSince = modifiedDate.getAsLong();
}

OptionalLong unmodifiedDate =
parseAndValidateDate(copySourceIfUnmodifiedSinceStr);
if (unmodifiedDate.isPresent()) {
copySourceIfUnmodifiedSince = unmodifiedDate.getAsLong();
}
return (copySourceIfModifiedSince <= lastModificationTime) &&
(lastModificationTime <= copySourceIfUnmodifiedSince);
}

/**
* Create a {@link S3ChunkInputStreamInfo} that contains the necessary information to handle
* the S3 chunk upload.
Expand Down
119 changes: 53 additions & 66 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,37 +280,8 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr
return Response.ok().status(HttpStatus.SC_OK).build();
}

String ifNoneMatch = getHeaders().getHeaderString(
S3Consts.IF_NONE_MATCH_HEADER);
String ifMatch = getHeaders().getHeaderString(
S3Consts.IF_MATCH_HEADER);

if (ifNoneMatch != null && StringUtils.isBlank(ifNoneMatch)) {
OS3Exception ex = newError(INVALID_REQUEST, keyPath);
ex.setErrorMessage("If-None-Match header cannot be empty.");
throw ex;
}
if (ifMatch != null && StringUtils.isBlank(ifMatch)) {
OS3Exception ex = newError(INVALID_REQUEST, keyPath);
ex.setErrorMessage("If-Match header cannot be empty.");
throw ex;
}

String ifNoneMatchTrimmed = ifNoneMatch == null ? null : ifNoneMatch.trim();
String ifMatchTrimmed = ifMatch == null ? null : ifMatch.trim();

if (ifNoneMatchTrimmed != null && ifMatchTrimmed != null) {
OS3Exception ex = newError(INVALID_REQUEST, keyPath);
ex.setErrorMessage("If-Match and If-None-Match cannot be specified together.");
throw ex;
}

if (ifNoneMatchTrimmed != null
&& !"*".equals(stripQuotes(ifNoneMatchTrimmed))) {
OS3Exception ex = newError(INVALID_REQUEST, keyPath);
ex.setErrorMessage("Only If-None-Match: * is supported for conditional put.");
throw ex;
}
S3ConditionalRequest.WriteConditions writeConditions =
S3ConditionalRequest.parseWriteConditions(getHeaders(), keyPath);

// Normal put object
S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body,
Expand All @@ -329,17 +300,15 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr
Pair<String, Long> keyWriteResult = ObjectEndpointStreaming
.put(bucket, keyPath, length, replicationConfig, getChunkSize(),
customMetadata, tags, multiDigestInputStream, getHeaders(),
signatureInfo.isSignPayload(), perf, ifNoneMatchTrimmed,
ifMatchTrimmed);
signatureInfo.isSignPayload(), perf, writeConditions);
md5Hash = keyWriteResult.getKey();
putLength = keyWriteResult.getValue();
} else {
final String amzContentSha256Header =
validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload());
try (OzoneOutputStream output = openKeyForPut(
volume.getName(), bucketName, keyPath, length,
replicationConfig, customMetadata, tags, ifNoneMatchTrimmed,
ifMatchTrimmed)) {
replicationConfig, customMetadata, tags, writeConditions)) {
long metadataLatencyNs =
getMetrics().updatePutKeyMetadataStats(startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
Expand Down Expand Up @@ -449,6 +418,17 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath)

isFile(keyPath, keyDetails);

Response conditionalResponse = createConditionalReadResponse(
keyPath, keyDetails);
if (conditionalResponse != null) {
long metadataLatencyNs = getMetrics().updateGetKeyMetadataStats(
startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
long opLatencyNs = getMetrics().updateGetKeySuccessStats(startNanos);
perf.appendOpLatencyNanos(opLatencyNs);
return conditionalResponse;
}

long length = keyDetails.getDataSize();

LOG.debug("Data length of the key {} is {}", keyPath, length);
Expand Down Expand Up @@ -509,15 +489,7 @@ Response handleGetRequest(ObjectRequestContext context, String keyPath)
}

responseBuilder.header(ACCEPT_RANGE_HEADER, RANGE_HEADER_SUPPORTED_UNIT);

String eTag = keyDetails.getMetadata().get(OzoneConsts.ETAG);
if (eTag != null) {
responseBuilder.header(HttpHeaders.ETAG, wrapInQuotes(eTag));
String partsCount = extractPartsCount(eTag);
if (partsCount != null) {
responseBuilder.header(MP_PARTS_COUNT, partsCount);
}
}
addEntityTagHeader(responseBuilder, keyDetails);

MultivaluedMap<String, String> queryParams =
getContext().getUriInfo().getQueryParameters();
Expand Down Expand Up @@ -569,6 +541,20 @@ static void addTagCountIfAny(
}
}

static void addEntityTagHeader(ResponseBuilder responseBuilder, OzoneKey key) {
String eTag = key.getMetadata().get(OzoneConsts.ETAG);
if (eTag != null) {
// Should not return ETag header if the ETag is not set
// doing so will result in "null" string being returned instead
// which breaks some AWS SDK implementation
responseBuilder.header(HttpHeaders.ETAG, wrapInQuotes(eTag));
String partsCount = extractPartsCount(eTag);
if (partsCount != null) {
responseBuilder.header(MP_PARTS_COUNT, partsCount);
}
}
}

/**
* Rest endpoint to check existence of an object in a bucket.
* <p>
Expand All @@ -591,6 +577,13 @@ public Response head(
key = getClientProtocol().headS3Object(bucketName, keyPath);

isFile(keyPath, key);
Response conditionalResponse = createConditionalReadResponse(
keyPath, key);
if (conditionalResponse != null) {
getMetrics().updateHeadKeySuccessStats(startNanos);
auditReadSuccess(s3GAction);
return conditionalResponse;
}
// TODO: return the specified range bytes of this object.
} catch (OMException ex) {
auditReadFailure(s3GAction, ex);
Expand Down Expand Up @@ -618,18 +611,7 @@ public Response head(
.header(HttpHeaders.CONTENT_LENGTH, key.getDataSize())
.header(HttpHeaders.CONTENT_TYPE, "binary/octet-stream")
.header(STORAGE_CLASS_HEADER, s3StorageType.toString());

String eTag = key.getMetadata().get(OzoneConsts.ETAG);
if (eTag != null) {
// Should not return ETag header if the ETag is not set
// doing so will result in "null" string being returned instead
// which breaks some AWS SDK implementation
response.header(HttpHeaders.ETAG, wrapInQuotes(eTag));
String partsCount = extractPartsCount(eTag);
if (partsCount != null) {
response.header(MP_PARTS_COUNT, partsCount);
}
}
addEntityTagHeader(response, key);

addLastModifiedDate(response, key);
addCustomMetadataHeaders(response, key);
Expand Down Expand Up @@ -916,7 +898,8 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket,
getHeaders().getHeaderString(COPY_SOURCE_IF_MODIFIED_SINCE);
String copySourceIfUnmodifiedSince =
getHeaders().getHeaderString(COPY_SOURCE_IF_UNMODIFIED_SINCE);
if (!checkCopySourceModificationTime(sourceKeyModificationTime,
if (!S3ConditionalRequest.checkCopySourceModificationTime(
sourceKeyModificationTime,
copySourceIfModifiedSince, copySourceIfUnmodifiedSince)) {
throw newError(PRECOND_FAILED, sourceBucket + "/" + sourceKey);
}
Expand Down Expand Up @@ -1179,16 +1162,17 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
@SuppressWarnings("checkstyle:ParameterNumber")
private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, String keyPath, long length,
ReplicationConfig replicationConfig, Map<String, String> customMetadata,
Map<String, String> tags, String ifNoneMatch, String ifMatch)
Map<String, String> tags,
S3ConditionalRequest.WriteConditions writeConditions)
throws IOException {
if (ifNoneMatch != null && "*".equals(stripQuotes(ifNoneMatch.trim()))) {
if (writeConditions.hasIfNoneMatch()) {
return getClientProtocol().createKeyIfNotExists(
volumeName, bucketName, keyPath, length, replicationConfig,
customMetadata, tags);
} else if (ifMatch != null) {
String expectedETag = parseETag(ifMatch);
} else if (writeConditions.hasIfMatch()) {
return getClientProtocol().rewriteKeyIfMatch(
volumeName, bucketName, keyPath, length, expectedETag,
volumeName, bucketName, keyPath, length,
writeConditions.getExpectedETag(),
replicationConfig, customMetadata, tags);
} else {
return getClientProtocol().createKey(
Expand All @@ -1202,10 +1186,13 @@ private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, St
* quotes if present.
*/
static String parseETag(String headerValue) {
if (headerValue == null) {
return null;
}
return stripQuotes(headerValue.trim());
return S3ConditionalRequest.parseETag(headerValue);
}

private Response createConditionalReadResponse(String keyPath, OzoneKey key)
throws OS3Exception {
return S3ConditionalRequest.evaluateReadPreconditions(getHeaders(),
keyPath, key);
}

/** Request context shared among {@code ObjectOperationHandler}s. */
Expand Down
Loading