From c075c909347e82e418e381ba98a2fe45d806af7e Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Thu, 2 Apr 2026 10:57:52 -0400 Subject: [PATCH 01/12] =?UTF-8?q?HDDS-14949=20=E2=80=94=20Migrate=20gRPC?= =?UTF-8?q?=20transport=20to=20the=20Ratis-shaded=20gRPC/Netty/Protobuf=20?= =?UTF-8?q?stack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hadoop-hdds/client/pom.xml | 19 + hadoop-hdds/common/pom.xml | 25 + .../org/apache/hadoop/hdds/HddsUtils.java | 6 +- hadoop-hdds/container-service/pom.xml | 18 + hadoop-hdds/datanode-grpc-client/pom.xml | 164 ++ .../main/proto/DatanodeClientProtocol.proto | 0 .../src/main/resources/proto.lock | 1831 +++++++++++++++++ hadoop-hdds/framework/pom.xml | 30 +- .../GrpcMetricsServerRequestInterceptor.java | 12 +- .../GrpcMetricsServerResponseInterceptor.java | 12 +- .../GrpcMetricsServerTransportFilter.java | 4 +- hadoop-hdds/interface-client/pom.xml | 76 +- .../src/main/resources/proto.lock | 1829 +--------------- hadoop-hdds/interface-server/pom.xml | 80 +- .../src/main/resources/proto.lock | 470 +---- hadoop-hdds/pom.xml | 2 + hadoop-hdds/scm-grpc-client/pom.xml | 164 ++ .../src/main/proto/InterSCMProtocol.proto | 0 .../src/main/proto/SCMRatisProtocol.proto | 0 .../src/main/proto/SCMUpdateProtocol.proto | 0 .../src/main/resources/proto.lock | 472 +++++ hadoop-hdds/server-scm/pom.xml | 34 +- hadoop-ozone/cli-repair/pom.xml | 2 +- hadoop-ozone/cli-shell/pom.xml | 19 + hadoop-ozone/common/pom.xml | 34 - .../om/ha/GrpcOMFailoverProxyProvider.java | 4 +- .../ozone/om/protocolPB/GrpcOmTransport.java | 173 +- .../grpc/ClientAddressClientInterceptor.java | 14 +- .../grpc/ClientAddressServerInterceptor.java | 12 +- .../protocolPB/grpc/GrpcClientConstants.java | 4 +- ...TestGrpcOmTransportConcurrentFailover.java | 78 +- .../om/protocolPB/TestS3GrpcOmTransport.java | 181 +- .../TestClientAddressClientInterceptor.java | 14 +- .../TestClientAddressServerInterceptor.java | 12 +- hadoop-ozone/csi/pom.xml | 131 +- .../hadoop/ozone/csi/ControllerService.java | 2 +- .../apache/hadoop/ozone/csi/CsiServer.java | 10 +- .../hadoop/ozone/csi/IdentityService.java | 4 +- .../apache/hadoop/ozone/csi/NodeService.java | 2 +- hadoop-ozone/integration-test-recon/pom.xml | 17 +- hadoop-ozone/interface-client/pom.xml | 28 + hadoop-ozone/ozone-manager/pom.xml | 35 - .../ozone/om/GrpcOzoneManagerServer.java | 18 +- .../ozone/om/OzoneManagerServiceGrpc.java | 65 +- .../hadoop/ozone/om/TestOMMetadataReader.java | 2 +- .../TestOMClientRequestWithUserInfo.java | 2 +- hadoop-ozone/ozonefs-common/pom.xml | 19 + hadoop-ozone/ozonefs-hadoop2/pom.xml | 17 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 23 +- hadoop-ozone/recon/pom.xml | 48 +- hadoop-ozone/s3gateway/pom.xml | 21 +- .../datanode/proto/ContainerProtos.class | Bin 0 -> 56561 bytes pom.xml | 294 +++ 53 files changed, 3744 insertions(+), 2789 deletions(-) create mode 100644 hadoop-hdds/datanode-grpc-client/pom.xml rename hadoop-hdds/{interface-client => datanode-grpc-client}/src/main/proto/DatanodeClientProtocol.proto (100%) create mode 100644 hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock create mode 100644 hadoop-hdds/scm-grpc-client/pom.xml rename hadoop-hdds/{interface-server => scm-grpc-client}/src/main/proto/InterSCMProtocol.proto (100%) rename hadoop-hdds/{interface-server => scm-grpc-client}/src/main/proto/SCMRatisProtocol.proto (100%) rename hadoop-hdds/{interface-server => scm-grpc-client}/src/main/proto/SCMUpdateProtocol.proto (100%) create mode 100644 hadoop-hdds/scm-grpc-client/src/main/resources/proto.lock create mode 100644 org/apache/hadoop/hdds/protocol/datanode/proto/ContainerProtos.class diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index 99b7664cdd55..b697f7088116 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -52,6 +52,10 @@ org.apache.ozone hdds-config + + org.apache.ozone + hdds-datanode-grpc-client + org.apache.ozone hdds-erasurecode @@ -101,6 +105,21 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + + com.google.protobuf:protobuf-java + + + + + com.github.spotbugs spotbugs-maven-plugin diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index daf7008fa83b..ba1d2259b61c 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -112,6 +112,10 @@ org.apache.ozone hdds-config + + org.apache.ozone + hdds-datanode-grpc-client + org.apache.ozone hdds-interface-client @@ -248,6 +252,7 @@ org.apache.maven.plugins maven-compiler-plugin + false org.apache.ozone @@ -263,6 +268,26 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + + org.apache.ozone:hdds-config + org.apache.ratis:ratis-thirdparty-misc + + + + + org.apache.maven.plugins maven-enforcer-plugin diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index fce0f295e33d..61eddfb4ec1b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -34,8 +34,9 @@ import com.google.common.base.Preconditions; import com.google.common.net.HostAndPort; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.ServiceException; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.ratis.thirdparty.com.google.protobuf.ServiceException; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import java.io.File; @@ -86,7 +87,6 @@ import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat; import org.apache.ratis.util.SizeInBytes; import org.slf4j.Logger; diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index a0034eb78f4f..da0f5a45f981 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -336,6 +336,24 @@ true + + analyze + + + + org.apache.hadoop:hadoop-auth + + org.apache.ratis:ratis-netty + + + diff --git a/hadoop-hdds/datanode-grpc-client/pom.xml b/hadoop-hdds/datanode-grpc-client/pom.xml new file mode 100644 index 000000000000..01315c208a7b --- /dev/null +++ b/hadoop-hdds/datanode-grpc-client/pom.xml @@ -0,0 +1,164 @@ + + + + 4.0.0 + + org.apache.ozone + hdds + 2.2.0-SNAPSHOT + + hdds-datanode-grpc-client + 2.2.0-SNAPSHOT + jar + Apache Ozone HDDS DataNode gRPC Client + DataNode XCeiver and IntraDatanode gRPC stubs generated from DatanodeClientProtocol.proto. + Package names in generated sources are rewritten by maven-antrun-plugin (Ant replace task, pure Java) + from vanilla com.google.protobuf / io.grpc / com.google.common to their ratis-thirdparty counterparts + before compilation, so target/classes/ already contains the shaded namespace and downstream modules + in the same Maven reactor can compile against the correct types. + + + + yyyyMMddHHmmss + + true + true + true + + + + + + org.apache.ratis + ratis-thirdparty-misc + + + + javax.annotation + javax.annotation-api + provided + + + + + + + + com.gradle + develocity-maven-extension + + + + + maven-compiler-plugin + + compiles generated + rewritten sources + + + + maven-antrun-plugin + + rewrites generated source files + + + + protobuf-maven-plugin + + generates sources + + + + + + + + + + + com.salesforce.servicelibs + proto-backwards-compatibility + + + org.apache.maven.plugins + maven-compiler-plugin + + none + false + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + compile-proto-datanode + + compile + compile-custom + + generate-sources + + + com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier} + ${project.build.directory}/generated-sources/proto-java-ratis + ${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/${maven.build.timestamp} + false + grpc-java + io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier} + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + + rewrite-generated-sources-to-ratis-ns + + run + + process-sources + + + + + + + + + + + + + \ No newline at end of file diff --git a/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/datanode-grpc-client/src/main/proto/DatanodeClientProtocol.proto similarity index 100% rename from hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto rename to hadoop-hdds/datanode-grpc-client/src/main/proto/DatanodeClientProtocol.proto diff --git a/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock b/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock new file mode 100644 index 000000000000..9e99c1a330c2 --- /dev/null +++ b/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock @@ -0,0 +1,1831 @@ +{ + "definitions": [ + { + "protopath": "DatanodeClientProtocol.proto", + "def": { + "enums": [ + { + "name": "Type", + "enum_fields": [ + { + "name": "CreateContainer", + "integer": 1 + }, + { + "name": "ReadContainer", + "integer": 2 + }, + { + "name": "UpdateContainer", + "integer": 3 + }, + { + "name": "DeleteContainer", + "integer": 4 + }, + { + "name": "ListContainer", + "integer": 5 + }, + { + "name": "PutBlock", + "integer": 6 + }, + { + "name": "GetBlock", + "integer": 7 + }, + { + "name": "DeleteBlock", + "integer": 8 + }, + { + "name": "ListBlock", + "integer": 9 + }, + { + "name": "ReadChunk", + "integer": 10 + }, + { + "name": "DeleteChunk", + "integer": 11 + }, + { + "name": "WriteChunk", + "integer": 12 + }, + { + "name": "ListChunk", + "integer": 13 + }, + { + "name": "CompactChunk", + "integer": 14 + }, + { + "name": "PutSmallFile", + "integer": 15 + }, + { + "name": "GetSmallFile", + "integer": 16 + }, + { + "name": "CloseContainer", + "integer": 17 + }, + { + "name": "GetCommittedBlockLength", + "integer": 18 + }, + { + "name": "StreamInit", + "integer": 19 + }, + { + "name": "StreamWrite", + "integer": 20 + }, + { + "name": "FinalizeBlock", + "integer": 21 + }, + { + "name": "Echo", + "integer": 22 + }, + { + "name": "GetContainerChecksumInfo", + "integer": 23 + } + ] + }, + { + "name": "Result", + "enum_fields": [ + { + "name": "SUCCESS", + "integer": 1 + }, + { + "name": "UNSUPPORTED_REQUEST", + "integer": 2 + }, + { + "name": "MALFORMED_REQUEST", + "integer": 3 + }, + { + "name": "CONTAINER_INTERNAL_ERROR", + "integer": 4 + }, + { + "name": "INVALID_CONFIG", + "integer": 5 + }, + { + "name": "INVALID_FILE_HASH_FOUND", + "integer": 6 + }, + { + "name": "CONTAINER_EXISTS", + "integer": 7 + }, + { + "name": "NO_SUCH_ALGORITHM", + "integer": 8 + }, + { + "name": "CONTAINER_NOT_FOUND", + "integer": 9 + }, + { + "name": "IO_EXCEPTION", + "integer": 10 + }, + { + "name": "UNABLE_TO_READ_METADATA_DB", + "integer": 11 + }, + { + "name": "NO_SUCH_BLOCK", + "integer": 12 + }, + { + "name": "OVERWRITE_FLAG_REQUIRED", + "integer": 13 + }, + { + "name": "UNABLE_TO_FIND_DATA_DIR", + "integer": 14 + }, + { + "name": "INVALID_WRITE_SIZE", + "integer": 15 + }, + { + "name": "CHECKSUM_MISMATCH", + "integer": 16 + }, + { + "name": "UNABLE_TO_FIND_CHUNK", + "integer": 17 + }, + { + "name": "PROTOC_DECODING_ERROR", + "integer": 18 + }, + { + "name": "INVALID_ARGUMENT", + "integer": 19 + }, + { + "name": "PUT_SMALL_FILE_ERROR", + "integer": 20 + }, + { + "name": "GET_SMALL_FILE_ERROR", + "integer": 21 + }, + { + "name": "CLOSED_CONTAINER_IO", + "integer": 22 + }, + { + "name": "ERROR_IN_COMPACT_DB", + "integer": 24 + }, + { + "name": "UNCLOSED_CONTAINER_IO", + "integer": 25 + }, + { + "name": "DELETE_ON_OPEN_CONTAINER", + "integer": 26 + }, + { + "name": "CLOSED_CONTAINER_RETRY", + "integer": 27 + }, + { + "name": "INVALID_CONTAINER_STATE", + "integer": 28 + }, + { + "name": "DISK_OUT_OF_SPACE", + "integer": 29 + }, + { + "name": "CONTAINER_ALREADY_EXISTS", + "integer": 30 + }, + { + "name": "CONTAINER_METADATA_ERROR", + "integer": 31 + }, + { + "name": "CONTAINER_FILES_CREATE_ERROR", + "integer": 32 + }, + { + "name": "CONTAINER_CHECKSUM_ERROR", + "integer": 33 + }, + { + "name": "UNKNOWN_CONTAINER_TYPE", + "integer": 34 + }, + { + "name": "BLOCK_NOT_COMMITTED", + "integer": 35 + }, + { + "name": "CONTAINER_UNHEALTHY", + "integer": 36 + }, + { + "name": "UNKNOWN_BCSID", + "integer": 37 + }, + { + "name": "BCSID_MISMATCH", + "integer": 38 + }, + { + "name": "CONTAINER_NOT_OPEN", + "integer": 39 + }, + { + "name": "CONTAINER_MISSING", + "integer": 40 + }, + { + "name": "BLOCK_TOKEN_VERIFICATION_FAILED", + "integer": 41 + }, + { + "name": "ERROR_IN_DB_SYNC", + "integer": 42 + }, + { + "name": "CHUNK_FILE_INCONSISTENCY", + "integer": 43 + }, + { + "name": "DELETE_ON_NON_EMPTY_CONTAINER", + "integer": 44 + }, + { + "name": "EXPORT_CONTAINER_METADATA_FAILED", + "integer": 45 + }, + { + "name": "IMPORT_CONTAINER_METADATA_FAILED", + "integer": 46 + }, + { + "name": "BLOCK_ALREADY_FINALIZED", + "integer": 47 + }, + { + "name": "CONTAINER_ID_MISMATCH", + "integer": 48 + } + ] + }, + { + "name": "ContainerDataProto.State", + "enum_fields": [ + { + "name": "OPEN", + "integer": 1 + }, + { + "name": "CLOSING", + "integer": 2 + }, + { + "name": "QUASI_CLOSED", + "integer": 3 + }, + { + "name": "CLOSED", + "integer": 4 + }, + { + "name": "UNHEALTHY", + "integer": 5 + }, + { + "name": "INVALID", + "integer": 6 + }, + { + "name": "DELETED", + "integer": 7 + }, + { + "name": "RECOVERING", + "integer": 8 + } + ] + }, + { + "name": "ContainerType", + "enum_fields": [ + { + "name": "KeyValueContainer", + "integer": 1 + } + ] + }, + { + "name": "ChecksumType", + "enum_fields": [ + { + "name": "NONE", + "integer": 1 + }, + { + "name": "CRC32", + "integer": 2 + }, + { + "name": "CRC32C", + "integer": 3 + }, + { + "name": "SHA256", + "integer": 4 + }, + { + "name": "MD5", + "integer": 5 + } + ] + }, + { + "name": "ReadChunkVersion", + "enum_fields": [ + { + "name": "V0" + }, + { + "name": "V1", + "integer": 1 + } + ] + }, + { + "name": "CopyContainerCompressProto", + "enum_fields": [ + { + "name": "NO_COMPRESSION", + "integer": 1 + }, + { + "name": "GZIP", + "integer": 2 + }, + { + "name": "LZ4", + "integer": 3 + }, + { + "name": "SNAPPY", + "integer": 4 + }, + { + "name": "ZSTD", + "integer": 5 + } + ] + } + ], + "messages": [ + { + "name": "DatanodeBlockID", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "localID", + "type": "int64", + "required": true + }, + { + "id": 3, + "name": "blockCommitSequenceId", + "type": "uint64", + "optional": true, + "options": [ + { + "name": "default", + "value": "0" + } + ] + }, + { + "id": 4, + "name": "replicaIndex", + "type": "int32", + "optional": true + } + ] + }, + { + "name": "KeyValue", + "fields": [ + { + "id": 1, + "name": "key", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "value", + "type": "string", + "optional": true + } + ] + }, + { + "name": "ContainerCommandRequestProto", + "fields": [ + { + "id": 1, + "name": "cmdType", + "type": "Type", + "required": true + }, + { + "id": 2, + "name": "traceID", + "type": "string", + "optional": true + }, + { + "id": 3, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 4, + "name": "datanodeUuid", + "type": "string", + "required": true + }, + { + "id": 5, + "name": "pipelineID", + "type": "string", + "optional": true + }, + { + "id": 6, + "name": "createContainer", + "type": "CreateContainerRequestProto", + "optional": true + }, + { + "id": 7, + "name": "readContainer", + "type": "ReadContainerRequestProto", + "optional": true + }, + { + "id": 8, + "name": "updateContainer", + "type": "UpdateContainerRequestProto", + "optional": true + }, + { + "id": 9, + "name": "deleteContainer", + "type": "DeleteContainerRequestProto", + "optional": true + }, + { + "id": 10, + "name": "listContainer", + "type": "ListContainerRequestProto", + "optional": true + }, + { + "id": 11, + "name": "closeContainer", + "type": "CloseContainerRequestProto", + "optional": true + }, + { + "id": 12, + "name": "putBlock", + "type": "PutBlockRequestProto", + "optional": true + }, + { + "id": 13, + "name": "getBlock", + "type": "GetBlockRequestProto", + "optional": true + }, + { + "id": 14, + "name": "deleteBlock", + "type": "DeleteBlockRequestProto", + "optional": true, + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + }, + { + "id": 15, + "name": "listBlock", + "type": "ListBlockRequestProto", + "optional": true + }, + { + "id": 16, + "name": "readChunk", + "type": "ReadChunkRequestProto", + "optional": true + }, + { + "id": 17, + "name": "writeChunk", + "type": "WriteChunkRequestProto", + "optional": true + }, + { + "id": 18, + "name": "deleteChunk", + "type": "DeleteChunkRequestProto", + "optional": true, + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + }, + { + "id": 19, + "name": "listChunk", + "type": "ListChunkRequestProto", + "optional": true + }, + { + "id": 20, + "name": "putSmallFile", + "type": "PutSmallFileRequestProto", + "optional": true + }, + { + "id": 21, + "name": "getSmallFile", + "type": "GetSmallFileRequestProto", + "optional": true + }, + { + "id": 22, + "name": "getCommittedBlockLength", + "type": "GetCommittedBlockLengthRequestProto", + "optional": true + }, + { + "id": 23, + "name": "encodedToken", + "type": "string", + "optional": true + }, + { + "id": 24, + "name": "version", + "type": "uint32", + "optional": true + }, + { + "id": 25, + "name": "finalizeBlock", + "type": "FinalizeBlockRequestProto", + "optional": true + }, + { + "id": 26, + "name": "echo", + "type": "EchoRequestProto", + "optional": true + }, + { + "id": 27, + "name": "getContainerChecksumInfo", + "type": "GetContainerChecksumInfoRequestProto", + "optional": true + } + ] + }, + { + "name": "ContainerCommandResponseProto", + "fields": [ + { + "id": 1, + "name": "cmdType", + "type": "Type", + "required": true + }, + { + "id": 2, + "name": "traceID", + "type": "string", + "optional": true + }, + { + "id": 3, + "name": "result", + "type": "Result", + "required": true + }, + { + "id": 4, + "name": "message", + "type": "string", + "optional": true + }, + { + "id": 5, + "name": "createContainer", + "type": "CreateContainerResponseProto", + "optional": true + }, + { + "id": 6, + "name": "readContainer", + "type": "ReadContainerResponseProto", + "optional": true + }, + { + "id": 7, + "name": "updateContainer", + "type": "UpdateContainerResponseProto", + "optional": true + }, + { + "id": 8, + "name": "deleteContainer", + "type": "DeleteContainerResponseProto", + "optional": true + }, + { + "id": 9, + "name": "listContainer", + "type": "ListContainerResponseProto", + "optional": true + }, + { + "id": 10, + "name": "closeContainer", + "type": "CloseContainerResponseProto", + "optional": true + }, + { + "id": 11, + "name": "putBlock", + "type": "PutBlockResponseProto", + "optional": true + }, + { + "id": 12, + "name": "getBlock", + "type": "GetBlockResponseProto", + "optional": true + }, + { + "id": 13, + "name": "deleteBlock", + "type": "DeleteBlockResponseProto", + "optional": true + }, + { + "id": 14, + "name": "listBlock", + "type": "ListBlockResponseProto", + "optional": true + }, + { + "id": 15, + "name": "writeChunk", + "type": "WriteChunkResponseProto", + "optional": true + }, + { + "id": 16, + "name": "readChunk", + "type": "ReadChunkResponseProto", + "optional": true + }, + { + "id": 17, + "name": "deleteChunk", + "type": "DeleteChunkResponseProto", + "optional": true + }, + { + "id": 18, + "name": "listChunk", + "type": "ListChunkResponseProto", + "optional": true + }, + { + "id": 19, + "name": "putSmallFile", + "type": "PutSmallFileResponseProto", + "optional": true + }, + { + "id": 20, + "name": "getSmallFile", + "type": "GetSmallFileResponseProto", + "optional": true + }, + { + "id": 21, + "name": "getCommittedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "optional": true + }, + { + "id": 22, + "name": "finalizeBlock", + "type": "FinalizeBlockResponseProto", + "optional": true + }, + { + "id": 23, + "name": "echo", + "type": "EchoResponseProto", + "optional": true + }, + { + "id": 24, + "name": "getContainerChecksumInfo", + "type": "GetContainerChecksumInfoResponseProto", + "optional": true + } + ] + }, + { + "name": "ContainerDataProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 4, + "name": "containerPath", + "type": "string", + "optional": true + }, + { + "id": 6, + "name": "bytesUsed", + "type": "int64", + "optional": true + }, + { + "id": 7, + "name": "size", + "type": "int64", + "optional": true + }, + { + "id": 8, + "name": "blockCount", + "type": "int64", + "optional": true + }, + { + "id": 9, + "name": "state", + "type": "State", + "optional": true, + "options": [ + { + "name": "default", + "value": "OPEN" + } + ] + }, + { + "id": 10, + "name": "containerType", + "type": "ContainerType", + "optional": true, + "options": [ + { + "name": "default", + "value": "KeyValueContainer" + } + ] + } + ] + }, + { + "name": "Container2BCSIDMapProto", + "maps": [ + { + "key_type": "int64", + "field": { + "id": 1, + "name": "container2BCSID", + "type": "int64" + } + } + ] + }, + { + "name": "CreateContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 3, + "name": "containerType", + "type": "ContainerType", + "optional": true, + "options": [ + { + "name": "default", + "value": "KeyValueContainer" + } + ] + }, + { + "id": 4, + "name": "replicaIndex", + "type": "int32", + "optional": true + }, + { + "id": 5, + "name": "state", + "type": "ContainerDataProto.State", + "optional": true + } + ] + }, + { + "name": "CreateContainerResponseProto" + }, + { + "name": "ReadContainerRequestProto" + }, + { + "name": "ReadContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "containerData", + "type": "ContainerDataProto", + "optional": true + } + ] + }, + { + "name": "UpdateContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 3, + "name": "forceUpdate", + "type": "bool", + "optional": true, + "options": [ + { + "name": "default", + "value": "false" + } + ] + } + ] + }, + { + "name": "UpdateContainerResponseProto" + }, + { + "name": "DeleteContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "forceDelete", + "type": "bool", + "optional": true, + "options": [ + { + "name": "default", + "value": "false" + } + ] + } + ] + }, + { + "name": "DeleteContainerResponseProto" + }, + { + "name": "ListContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "count", + "type": "uint32", + "optional": true + } + ] + }, + { + "name": "ListContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "containerData", + "type": "ContainerDataProto", + "is_repeated": true + } + ] + }, + { + "name": "CloseContainerRequestProto" + }, + { + "name": "CloseContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "hash", + "type": "string", + "optional": true + }, + { + "id": 2, + "name": "containerID", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "BlockData", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "flags", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 4, + "name": "chunks", + "type": "ChunkInfo", + "is_repeated": true + }, + { + "id": 5, + "name": "size", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "PutBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "required": true + }, + { + "id": 2, + "name": "eof", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "PutBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "committedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "required": true + } + ] + }, + { + "name": "FinalizeBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "FinalizeBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "required": true + } + ] + }, + { + "name": "GetBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "GetBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "required": true + } + ] + }, + { + "name": "DeleteBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "GetCommittedBlockLengthRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "GetCommittedBlockLengthResponseProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "blockLength", + "type": "int64", + "required": true + } + ] + }, + { + "name": "DeleteBlockResponseProto" + }, + { + "name": "ListBlockRequestProto", + "fields": [ + { + "id": 2, + "name": "startLocalID", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "count", + "type": "uint32", + "required": true + } + ] + }, + { + "name": "ListBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "is_repeated": true + } + ] + }, + { + "name": "EchoRequestProto", + "fields": [ + { + "id": 1, + "name": "payload", + "type": "bytes", + "optional": true + }, + { + "id": 2, + "name": "payloadSizeResp", + "type": "int32", + "optional": true + }, + { + "id": 3, + "name": "sleepTimeMs", + "type": "int32", + "optional": true + }, + { + "id": 4, + "name": "readOnly", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "EchoResponseProto", + "fields": [ + { + "id": 1, + "name": "payload", + "type": "bytes", + "optional": true + } + ] + }, + { + "name": "GetContainerChecksumInfoRequestProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "GetContainerChecksumInfoResponseProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "containerChecksumInfo", + "type": "bytes", + "optional": true + } + ] + }, + { + "name": "ChunkInfo", + "fields": [ + { + "id": 1, + "name": "chunkName", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "offset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "len", + "type": "uint64", + "required": true + }, + { + "id": 4, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 5, + "name": "checksumData", + "type": "ChecksumData", + "required": true + }, + { + "id": 6, + "name": "stripeChecksum", + "type": "bytes", + "optional": true + } + ] + }, + { + "name": "ChunkInfoList", + "fields": [ + { + "id": 1, + "name": "chunks", + "type": "ChunkInfo", + "is_repeated": true + } + ] + }, + { + "name": "ChecksumData", + "fields": [ + { + "id": 1, + "name": "type", + "type": "ChecksumType", + "required": true + }, + { + "id": 2, + "name": "bytesPerChecksum", + "type": "uint32", + "required": true + }, + { + "id": 3, + "name": "checksums", + "type": "bytes", + "is_repeated": true + } + ] + }, + { + "name": "WriteChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "optional": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "optional": true + }, + { + "id": 4, + "name": "block", + "type": "PutBlockRequestProto", + "optional": true + } + ] + }, + { + "name": "WriteChunkResponseProto", + "fields": [ + { + "id": 1, + "name": "committedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "optional": true + } + ] + }, + { + "name": "ReadChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "required": true + }, + { + "id": 3, + "name": "readChunkVersion", + "type": "ReadChunkVersion", + "optional": true + } + ] + }, + { + "name": "ReadChunkResponseProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "required": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "oneof_parent": "responseData" + }, + { + "id": 4, + "name": "dataBuffers", + "type": "DataBuffers", + "oneof_parent": "responseData" + } + ] + }, + { + "name": "DataBuffers", + "fields": [ + { + "id": 1, + "name": "buffers", + "type": "bytes", + "is_repeated": true + } + ] + }, + { + "name": "DeleteChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "required": true + } + ] + }, + { + "name": "DeleteChunkResponseProto" + }, + { + "name": "ListChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "prevChunkName", + "type": "string", + "required": true + }, + { + "id": 3, + "name": "count", + "type": "uint32", + "required": true + } + ] + }, + { + "name": "ListChunkResponseProto", + "fields": [ + { + "id": 1, + "name": "chunkData", + "type": "ChunkInfo", + "is_repeated": true + } + ] + }, + { + "name": "PutSmallFileRequestProto", + "fields": [ + { + "id": 1, + "name": "block", + "type": "PutBlockRequestProto", + "required": true + }, + { + "id": 2, + "name": "chunkInfo", + "type": "ChunkInfo", + "required": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "required": true + } + ] + }, + { + "name": "PutSmallFileResponseProto", + "fields": [ + { + "id": 1, + "name": "committedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "required": true + } + ] + }, + { + "name": "GetSmallFileRequestProto", + "fields": [ + { + "id": 1, + "name": "block", + "type": "GetBlockRequestProto", + "required": true + }, + { + "id": 2, + "name": "readChunkVersion", + "type": "ReadChunkVersion", + "optional": true + } + ] + }, + { + "name": "GetSmallFileResponseProto", + "fields": [ + { + "id": 1, + "name": "data", + "type": "ReadChunkResponseProto", + "required": true + } + ] + }, + { + "name": "CopyContainerRequestProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "readOffset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "len", + "type": "uint64", + "optional": true + }, + { + "id": 4, + "name": "version", + "type": "uint32", + "optional": true + }, + { + "id": 5, + "name": "compression", + "type": "CopyContainerCompressProto", + "optional": true + } + ] + }, + { + "name": "CopyContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "readOffset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "len", + "type": "uint64", + "required": true + }, + { + "id": 4, + "name": "eof", + "type": "bool", + "required": true + }, + { + "id": 5, + "name": "data", + "type": "bytes", + "required": true + }, + { + "id": 6, + "name": "checksum", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "SendContainerRequest", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "offset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "required": true + }, + { + "id": 4, + "name": "checksum", + "type": "int64", + "optional": true + }, + { + "id": 5, + "name": "compression", + "type": "CopyContainerCompressProto", + "optional": true + }, + { + "id": 6, + "name": "size", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "SendContainerResponse" + }, + { + "name": "ChunkMerkleTree", + "fields": [ + { + "id": 1, + "name": "offset", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "length", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "dataChecksum", + "type": "int64", + "optional": true + }, + { + "id": 4, + "name": "checksumMatches", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "BlockMerkleTree", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "dataChecksum", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "chunkMerkleTree", + "type": "ChunkMerkleTree", + "is_repeated": true + }, + { + "id": 4, + "name": "deleted", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "ContainerMerkleTree", + "fields": [ + { + "id": 1, + "name": "dataChecksum", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "blockMerkleTree", + "type": "BlockMerkleTree", + "is_repeated": true + } + ] + }, + { + "name": "ContainerChecksumInfo", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "containerMerkleTree", + "type": "ContainerMerkleTree", + "optional": true + } + ] + }, + { + "name": "ContainerCreateInfo", + "fields": [ + { + "id": 1, + "name": "state", + "type": "ContainerDataProto.State", + "required": true + }, + { + "id": 2, + "name": "replicaIndex", + "type": "int32", + "optional": true, + "options": [ + { + "name": "default", + "value": "-1" + } + ] + } + ] + } + ], + "services": [ + { + "name": "XceiverClientProtocolService", + "rpcs": [ + { + "name": "send", + "in_type": "ContainerCommandRequestProto", + "out_type": "ContainerCommandResponseProto", + "in_streamed": true, + "out_streamed": true + } + ] + }, + { + "name": "IntraDatanodeProtocolService", + "rpcs": [ + { + "name": "download", + "in_type": "CopyContainerRequestProto", + "out_type": "CopyContainerResponseProto", + "out_streamed": true + }, + { + "name": "upload", + "in_type": "SendContainerRequest", + "out_type": "SendContainerResponse", + "in_streamed": true + } + ] + } + ], + "package": { + "name": "hadoop.hdds.datanode" + }, + "options": [ + { + "name": "java_package", + "value": "org.apache.hadoop.hdds.protocol.datanode.proto" + }, + { + "name": "java_outer_classname", + "value": "ContainerProtos" + }, + { + "name": "java_generate_equals_and_hash", + "value": "true" + } + ] + } + } + ] +} diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 34dcdde921fc..795dba1e2844 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -86,16 +86,6 @@ io.dropwizard.metrics metrics-core - - io.grpc - grpc-api - - - com.google.code.findbugs - jsr305 - - - io.opentelemetry opentelemetry-api @@ -415,6 +405,26 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + + org.eclipse.jetty:jetty-http + + + + + diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java index 45041fec5045..874136f5b1b1 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.grpc.metrics; -import com.google.protobuf.AbstractMessage; -import io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; +import org.apache.ratis.thirdparty.com.google.protobuf.AbstractMessage; +import org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.ServerCall; +import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor; /** * Interceptor to gather metrics based on grpc server request. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java index f8c8ad6027e8..58dedb4af0de 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.grpc.metrics; -import com.google.protobuf.AbstractMessage; -import io.grpc.ForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; +import org.apache.ratis.thirdparty.com.google.protobuf.AbstractMessage; +import org.apache.ratis.thirdparty.io.grpc.ForwardingServerCall; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.ServerCall; +import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor; /** * Interceptor to gather metrics based on grpc server response. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java index 434dd9017b26..2f4fac78a58b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java @@ -17,8 +17,8 @@ package org.apache.hadoop.ozone.grpc.metrics; -import io.grpc.Attributes; -import io.grpc.ServerTransportFilter; +import org.apache.ratis.thirdparty.io.grpc.Attributes; +import org.apache.ratis.thirdparty.io.grpc.ServerTransportFilter; /** * Transport filter class for tracking active client connections. diff --git a/hadoop-hdds/interface-client/pom.xml b/hadoop-hdds/interface-client/pom.xml index ca9a6aa95bcd..6cb900a06115 100644 --- a/hadoop-hdds/interface-client/pom.xml +++ b/hadoop-hdds/interface-client/pom.xml @@ -26,6 +26,8 @@ Apache Ozone Distributed Data Store Client interface + + yyyyMMddHHmmss true @@ -37,10 +39,6 @@ com.google.protobuf protobuf-java - - org.apache.ratis - ratis-thirdparty-misc - javax.annotation @@ -49,6 +47,33 @@ + + + + com.gradle + develocity-maven-extension + + + + + + maven-compiler-plugin + + compiles generated sources + + + + protobuf-maven-plugin + + generates sources + + + + + + + + com.salesforce.servicelibs @@ -65,25 +90,6 @@ org.xolstice.maven.plugins protobuf-maven-plugin - - compile-proto-for-ratis - - compile - test-compile - compile-custom - test-compile-custom - - - com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier} - - DatanodeClientProtocol.proto - - target/generated-sources/proto-java-for-ratis - false - grpc-java - io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier} - - compile-proto-${protobuf.version} @@ -92,35 +98,13 @@ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - - DatanodeClientProtocol.proto - target/generated-sources/proto-java-for-protobuf-${protobuf.version} + ${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/protobuf-${protobuf.version}-${maven.build.timestamp} false - - org.apache.maven.plugins - maven-antrun-plugin - - - - run - - generate-sources - - - - - - - - - - - diff --git a/hadoop-hdds/interface-client/src/main/resources/proto.lock b/hadoop-hdds/interface-client/src/main/resources/proto.lock index 70ff576a7084..3c6248ad0c13 100644 --- a/hadoop-hdds/interface-client/src/main/resources/proto.lock +++ b/hadoop-hdds/interface-client/src/main/resources/proto.lock @@ -1,1832 +1,5 @@ { "definitions": [ - { - "protopath": "DatanodeClientProtocol.proto", - "def": { - "enums": [ - { - "name": "Type", - "enum_fields": [ - { - "name": "CreateContainer", - "integer": 1 - }, - { - "name": "ReadContainer", - "integer": 2 - }, - { - "name": "UpdateContainer", - "integer": 3 - }, - { - "name": "DeleteContainer", - "integer": 4 - }, - { - "name": "ListContainer", - "integer": 5 - }, - { - "name": "PutBlock", - "integer": 6 - }, - { - "name": "GetBlock", - "integer": 7 - }, - { - "name": "DeleteBlock", - "integer": 8 - }, - { - "name": "ListBlock", - "integer": 9 - }, - { - "name": "ReadChunk", - "integer": 10 - }, - { - "name": "DeleteChunk", - "integer": 11 - }, - { - "name": "WriteChunk", - "integer": 12 - }, - { - "name": "ListChunk", - "integer": 13 - }, - { - "name": "CompactChunk", - "integer": 14 - }, - { - "name": "PutSmallFile", - "integer": 15 - }, - { - "name": "GetSmallFile", - "integer": 16 - }, - { - "name": "CloseContainer", - "integer": 17 - }, - { - "name": "GetCommittedBlockLength", - "integer": 18 - }, - { - "name": "StreamInit", - "integer": 19 - }, - { - "name": "StreamWrite", - "integer": 20 - }, - { - "name": "FinalizeBlock", - "integer": 21 - }, - { - "name": "Echo", - "integer": 22 - }, - { - "name": "GetContainerChecksumInfo", - "integer": 23 - } - ] - }, - { - "name": "Result", - "enum_fields": [ - { - "name": "SUCCESS", - "integer": 1 - }, - { - "name": "UNSUPPORTED_REQUEST", - "integer": 2 - }, - { - "name": "MALFORMED_REQUEST", - "integer": 3 - }, - { - "name": "CONTAINER_INTERNAL_ERROR", - "integer": 4 - }, - { - "name": "INVALID_CONFIG", - "integer": 5 - }, - { - "name": "INVALID_FILE_HASH_FOUND", - "integer": 6 - }, - { - "name": "CONTAINER_EXISTS", - "integer": 7 - }, - { - "name": "NO_SUCH_ALGORITHM", - "integer": 8 - }, - { - "name": "CONTAINER_NOT_FOUND", - "integer": 9 - }, - { - "name": "IO_EXCEPTION", - "integer": 10 - }, - { - "name": "UNABLE_TO_READ_METADATA_DB", - "integer": 11 - }, - { - "name": "NO_SUCH_BLOCK", - "integer": 12 - }, - { - "name": "OVERWRITE_FLAG_REQUIRED", - "integer": 13 - }, - { - "name": "UNABLE_TO_FIND_DATA_DIR", - "integer": 14 - }, - { - "name": "INVALID_WRITE_SIZE", - "integer": 15 - }, - { - "name": "CHECKSUM_MISMATCH", - "integer": 16 - }, - { - "name": "UNABLE_TO_FIND_CHUNK", - "integer": 17 - }, - { - "name": "PROTOC_DECODING_ERROR", - "integer": 18 - }, - { - "name": "INVALID_ARGUMENT", - "integer": 19 - }, - { - "name": "PUT_SMALL_FILE_ERROR", - "integer": 20 - }, - { - "name": "GET_SMALL_FILE_ERROR", - "integer": 21 - }, - { - "name": "CLOSED_CONTAINER_IO", - "integer": 22 - }, - { - "name": "ERROR_IN_COMPACT_DB", - "integer": 24 - }, - { - "name": "UNCLOSED_CONTAINER_IO", - "integer": 25 - }, - { - "name": "DELETE_ON_OPEN_CONTAINER", - "integer": 26 - }, - { - "name": "CLOSED_CONTAINER_RETRY", - "integer": 27 - }, - { - "name": "INVALID_CONTAINER_STATE", - "integer": 28 - }, - { - "name": "DISK_OUT_OF_SPACE", - "integer": 29 - }, - { - "name": "CONTAINER_ALREADY_EXISTS", - "integer": 30 - }, - { - "name": "CONTAINER_METADATA_ERROR", - "integer": 31 - }, - { - "name": "CONTAINER_FILES_CREATE_ERROR", - "integer": 32 - }, - { - "name": "CONTAINER_CHECKSUM_ERROR", - "integer": 33 - }, - { - "name": "UNKNOWN_CONTAINER_TYPE", - "integer": 34 - }, - { - "name": "BLOCK_NOT_COMMITTED", - "integer": 35 - }, - { - "name": "CONTAINER_UNHEALTHY", - "integer": 36 - }, - { - "name": "UNKNOWN_BCSID", - "integer": 37 - }, - { - "name": "BCSID_MISMATCH", - "integer": 38 - }, - { - "name": "CONTAINER_NOT_OPEN", - "integer": 39 - }, - { - "name": "CONTAINER_MISSING", - "integer": 40 - }, - { - "name": "BLOCK_TOKEN_VERIFICATION_FAILED", - "integer": 41 - }, - { - "name": "ERROR_IN_DB_SYNC", - "integer": 42 - }, - { - "name": "CHUNK_FILE_INCONSISTENCY", - "integer": 43 - }, - { - "name": "DELETE_ON_NON_EMPTY_CONTAINER", - "integer": 44 - }, - { - "name": "EXPORT_CONTAINER_METADATA_FAILED", - "integer": 45 - }, - { - "name": "IMPORT_CONTAINER_METADATA_FAILED", - "integer": 46 - }, - { - "name": "BLOCK_ALREADY_FINALIZED", - "integer": 47 - }, - { - "name": "CONTAINER_ID_MISMATCH", - "integer": 48 - } - ] - }, - { - "name": "ContainerDataProto.State", - "enum_fields": [ - { - "name": "OPEN", - "integer": 1 - }, - { - "name": "CLOSING", - "integer": 2 - }, - { - "name": "QUASI_CLOSED", - "integer": 3 - }, - { - "name": "CLOSED", - "integer": 4 - }, - { - "name": "UNHEALTHY", - "integer": 5 - }, - { - "name": "INVALID", - "integer": 6 - }, - { - "name": "DELETED", - "integer": 7 - }, - { - "name": "RECOVERING", - "integer": 8 - } - ] - }, - { - "name": "ContainerType", - "enum_fields": [ - { - "name": "KeyValueContainer", - "integer": 1 - } - ] - }, - { - "name": "ChecksumType", - "enum_fields": [ - { - "name": "NONE", - "integer": 1 - }, - { - "name": "CRC32", - "integer": 2 - }, - { - "name": "CRC32C", - "integer": 3 - }, - { - "name": "SHA256", - "integer": 4 - }, - { - "name": "MD5", - "integer": 5 - } - ] - }, - { - "name": "ReadChunkVersion", - "enum_fields": [ - { - "name": "V0" - }, - { - "name": "V1", - "integer": 1 - } - ] - }, - { - "name": "CopyContainerCompressProto", - "enum_fields": [ - { - "name": "NO_COMPRESSION", - "integer": 1 - }, - { - "name": "GZIP", - "integer": 2 - }, - { - "name": "LZ4", - "integer": 3 - }, - { - "name": "SNAPPY", - "integer": 4 - }, - { - "name": "ZSTD", - "integer": 5 - } - ] - } - ], - "messages": [ - { - "name": "DatanodeBlockID", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "localID", - "type": "int64", - "required": true - }, - { - "id": 3, - "name": "blockCommitSequenceId", - "type": "uint64", - "optional": true, - "options": [ - { - "name": "default", - "value": "0" - } - ] - }, - { - "id": 4, - "name": "replicaIndex", - "type": "int32", - "optional": true - } - ] - }, - { - "name": "KeyValue", - "fields": [ - { - "id": 1, - "name": "key", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "value", - "type": "string", - "optional": true - } - ] - }, - { - "name": "ContainerCommandRequestProto", - "fields": [ - { - "id": 1, - "name": "cmdType", - "type": "Type", - "required": true - }, - { - "id": 2, - "name": "traceID", - "type": "string", - "optional": true - }, - { - "id": 3, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 4, - "name": "datanodeUuid", - "type": "string", - "required": true - }, - { - "id": 5, - "name": "pipelineID", - "type": "string", - "optional": true - }, - { - "id": 6, - "name": "createContainer", - "type": "CreateContainerRequestProto", - "optional": true - }, - { - "id": 7, - "name": "readContainer", - "type": "ReadContainerRequestProto", - "optional": true - }, - { - "id": 8, - "name": "updateContainer", - "type": "UpdateContainerRequestProto", - "optional": true - }, - { - "id": 9, - "name": "deleteContainer", - "type": "DeleteContainerRequestProto", - "optional": true - }, - { - "id": 10, - "name": "listContainer", - "type": "ListContainerRequestProto", - "optional": true - }, - { - "id": 11, - "name": "closeContainer", - "type": "CloseContainerRequestProto", - "optional": true - }, - { - "id": 12, - "name": "putBlock", - "type": "PutBlockRequestProto", - "optional": true - }, - { - "id": 13, - "name": "getBlock", - "type": "GetBlockRequestProto", - "optional": true - }, - { - "id": 14, - "name": "deleteBlock", - "type": "DeleteBlockRequestProto", - "optional": true, - "options": [ - { - "name": "deprecated", - "value": "true" - } - ] - }, - { - "id": 15, - "name": "listBlock", - "type": "ListBlockRequestProto", - "optional": true - }, - { - "id": 16, - "name": "readChunk", - "type": "ReadChunkRequestProto", - "optional": true - }, - { - "id": 17, - "name": "writeChunk", - "type": "WriteChunkRequestProto", - "optional": true - }, - { - "id": 18, - "name": "deleteChunk", - "type": "DeleteChunkRequestProto", - "optional": true, - "options": [ - { - "name": "deprecated", - "value": "true" - } - ] - }, - { - "id": 19, - "name": "listChunk", - "type": "ListChunkRequestProto", - "optional": true - }, - { - "id": 20, - "name": "putSmallFile", - "type": "PutSmallFileRequestProto", - "optional": true - }, - { - "id": 21, - "name": "getSmallFile", - "type": "GetSmallFileRequestProto", - "optional": true - }, - { - "id": 22, - "name": "getCommittedBlockLength", - "type": "GetCommittedBlockLengthRequestProto", - "optional": true - }, - { - "id": 23, - "name": "encodedToken", - "type": "string", - "optional": true - }, - { - "id": 24, - "name": "version", - "type": "uint32", - "optional": true - }, - { - "id": 25, - "name": "finalizeBlock", - "type": "FinalizeBlockRequestProto", - "optional": true - }, - { - "id": 26, - "name": "echo", - "type": "EchoRequestProto", - "optional": true - }, - { - "id": 27, - "name": "getContainerChecksumInfo", - "type": "GetContainerChecksumInfoRequestProto", - "optional": true - } - ] - }, - { - "name": "ContainerCommandResponseProto", - "fields": [ - { - "id": 1, - "name": "cmdType", - "type": "Type", - "required": true - }, - { - "id": 2, - "name": "traceID", - "type": "string", - "optional": true - }, - { - "id": 3, - "name": "result", - "type": "Result", - "required": true - }, - { - "id": 4, - "name": "message", - "type": "string", - "optional": true - }, - { - "id": 5, - "name": "createContainer", - "type": "CreateContainerResponseProto", - "optional": true - }, - { - "id": 6, - "name": "readContainer", - "type": "ReadContainerResponseProto", - "optional": true - }, - { - "id": 7, - "name": "updateContainer", - "type": "UpdateContainerResponseProto", - "optional": true - }, - { - "id": 8, - "name": "deleteContainer", - "type": "DeleteContainerResponseProto", - "optional": true - }, - { - "id": 9, - "name": "listContainer", - "type": "ListContainerResponseProto", - "optional": true - }, - { - "id": 10, - "name": "closeContainer", - "type": "CloseContainerResponseProto", - "optional": true - }, - { - "id": 11, - "name": "putBlock", - "type": "PutBlockResponseProto", - "optional": true - }, - { - "id": 12, - "name": "getBlock", - "type": "GetBlockResponseProto", - "optional": true - }, - { - "id": 13, - "name": "deleteBlock", - "type": "DeleteBlockResponseProto", - "optional": true - }, - { - "id": 14, - "name": "listBlock", - "type": "ListBlockResponseProto", - "optional": true - }, - { - "id": 15, - "name": "writeChunk", - "type": "WriteChunkResponseProto", - "optional": true - }, - { - "id": 16, - "name": "readChunk", - "type": "ReadChunkResponseProto", - "optional": true - }, - { - "id": 17, - "name": "deleteChunk", - "type": "DeleteChunkResponseProto", - "optional": true - }, - { - "id": 18, - "name": "listChunk", - "type": "ListChunkResponseProto", - "optional": true - }, - { - "id": 19, - "name": "putSmallFile", - "type": "PutSmallFileResponseProto", - "optional": true - }, - { - "id": 20, - "name": "getSmallFile", - "type": "GetSmallFileResponseProto", - "optional": true - }, - { - "id": 21, - "name": "getCommittedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "optional": true - }, - { - "id": 22, - "name": "finalizeBlock", - "type": "FinalizeBlockResponseProto", - "optional": true - }, - { - "id": 23, - "name": "echo", - "type": "EchoResponseProto", - "optional": true - }, - { - "id": 24, - "name": "getContainerChecksumInfo", - "type": "GetContainerChecksumInfoResponseProto", - "optional": true - } - ] - }, - { - "name": "ContainerDataProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 4, - "name": "containerPath", - "type": "string", - "optional": true - }, - { - "id": 6, - "name": "bytesUsed", - "type": "int64", - "optional": true - }, - { - "id": 7, - "name": "size", - "type": "int64", - "optional": true - }, - { - "id": 8, - "name": "blockCount", - "type": "int64", - "optional": true - }, - { - "id": 9, - "name": "state", - "type": "State", - "optional": true, - "options": [ - { - "name": "default", - "value": "OPEN" - } - ] - }, - { - "id": 10, - "name": "containerType", - "type": "ContainerType", - "optional": true, - "options": [ - { - "name": "default", - "value": "KeyValueContainer" - } - ] - } - ] - }, - { - "name": "Container2BCSIDMapProto", - "maps": [ - { - "key_type": "int64", - "field": { - "id": 1, - "name": "container2BCSID", - "type": "int64" - } - } - ] - }, - { - "name": "CreateContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 3, - "name": "containerType", - "type": "ContainerType", - "optional": true, - "options": [ - { - "name": "default", - "value": "KeyValueContainer" - } - ] - }, - { - "id": 4, - "name": "replicaIndex", - "type": "int32", - "optional": true - }, - { - "id": 5, - "name": "state", - "type": "ContainerDataProto.State", - "optional": true - } - ] - }, - { - "name": "CreateContainerResponseProto" - }, - { - "name": "ReadContainerRequestProto" - }, - { - "name": "ReadContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "containerData", - "type": "ContainerDataProto", - "optional": true - } - ] - }, - { - "name": "UpdateContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 3, - "name": "forceUpdate", - "type": "bool", - "optional": true, - "options": [ - { - "name": "default", - "value": "false" - } - ] - } - ] - }, - { - "name": "UpdateContainerResponseProto" - }, - { - "name": "DeleteContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "forceDelete", - "type": "bool", - "optional": true, - "options": [ - { - "name": "default", - "value": "false" - } - ] - } - ] - }, - { - "name": "DeleteContainerResponseProto" - }, - { - "name": "ListContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "count", - "type": "uint32", - "optional": true - } - ] - }, - { - "name": "ListContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "containerData", - "type": "ContainerDataProto", - "is_repeated": true - } - ] - }, - { - "name": "CloseContainerRequestProto" - }, - { - "name": "CloseContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "hash", - "type": "string", - "optional": true - }, - { - "id": 2, - "name": "containerID", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "BlockData", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "flags", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 4, - "name": "chunks", - "type": "ChunkInfo", - "is_repeated": true - }, - { - "id": 5, - "name": "size", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "PutBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "required": true - }, - { - "id": 2, - "name": "eof", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "PutBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "committedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "required": true - } - ] - }, - { - "name": "FinalizeBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "FinalizeBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "required": true - } - ] - }, - { - "name": "GetBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "GetBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "required": true - } - ] - }, - { - "name": "DeleteBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "GetCommittedBlockLengthRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "GetCommittedBlockLengthResponseProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "blockLength", - "type": "int64", - "required": true - } - ] - }, - { - "name": "DeleteBlockResponseProto" - }, - { - "name": "ListBlockRequestProto", - "fields": [ - { - "id": 2, - "name": "startLocalID", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "count", - "type": "uint32", - "required": true - } - ] - }, - { - "name": "ListBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "is_repeated": true - } - ] - }, - { - "name": "EchoRequestProto", - "fields": [ - { - "id": 1, - "name": "payload", - "type": "bytes", - "optional": true - }, - { - "id": 2, - "name": "payloadSizeResp", - "type": "int32", - "optional": true - }, - { - "id": 3, - "name": "sleepTimeMs", - "type": "int32", - "optional": true - }, - { - "id": 4, - "name": "readOnly", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "EchoResponseProto", - "fields": [ - { - "id": 1, - "name": "payload", - "type": "bytes", - "optional": true - } - ] - }, - { - "name": "GetContainerChecksumInfoRequestProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "GetContainerChecksumInfoResponseProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "containerChecksumInfo", - "type": "bytes", - "optional": true - } - ] - }, - { - "name": "ChunkInfo", - "fields": [ - { - "id": 1, - "name": "chunkName", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "offset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "len", - "type": "uint64", - "required": true - }, - { - "id": 4, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 5, - "name": "checksumData", - "type": "ChecksumData", - "required": true - }, - { - "id": 6, - "name": "stripeChecksum", - "type": "bytes", - "optional": true - } - ] - }, - { - "name": "ChunkInfoList", - "fields": [ - { - "id": 1, - "name": "chunks", - "type": "ChunkInfo", - "is_repeated": true - } - ] - }, - { - "name": "ChecksumData", - "fields": [ - { - "id": 1, - "name": "type", - "type": "ChecksumType", - "required": true - }, - { - "id": 2, - "name": "bytesPerChecksum", - "type": "uint32", - "required": true - }, - { - "id": 3, - "name": "checksums", - "type": "bytes", - "is_repeated": true - } - ] - }, - { - "name": "WriteChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "optional": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "optional": true - }, - { - "id": 4, - "name": "block", - "type": "PutBlockRequestProto", - "optional": true - } - ] - }, - { - "name": "WriteChunkResponseProto", - "fields": [ - { - "id": 1, - "name": "committedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "optional": true - } - ] - }, - { - "name": "ReadChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "required": true - }, - { - "id": 3, - "name": "readChunkVersion", - "type": "ReadChunkVersion", - "optional": true - } - ] - }, - { - "name": "ReadChunkResponseProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "required": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "oneof_parent": "responseData" - }, - { - "id": 4, - "name": "dataBuffers", - "type": "DataBuffers", - "oneof_parent": "responseData" - } - ] - }, - { - "name": "DataBuffers", - "fields": [ - { - "id": 1, - "name": "buffers", - "type": "bytes", - "is_repeated": true - } - ] - }, - { - "name": "DeleteChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "required": true - } - ] - }, - { - "name": "DeleteChunkResponseProto" - }, - { - "name": "ListChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "prevChunkName", - "type": "string", - "required": true - }, - { - "id": 3, - "name": "count", - "type": "uint32", - "required": true - } - ] - }, - { - "name": "ListChunkResponseProto", - "fields": [ - { - "id": 1, - "name": "chunkData", - "type": "ChunkInfo", - "is_repeated": true - } - ] - }, - { - "name": "PutSmallFileRequestProto", - "fields": [ - { - "id": 1, - "name": "block", - "type": "PutBlockRequestProto", - "required": true - }, - { - "id": 2, - "name": "chunkInfo", - "type": "ChunkInfo", - "required": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "required": true - } - ] - }, - { - "name": "PutSmallFileResponseProto", - "fields": [ - { - "id": 1, - "name": "committedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "required": true - } - ] - }, - { - "name": "GetSmallFileRequestProto", - "fields": [ - { - "id": 1, - "name": "block", - "type": "GetBlockRequestProto", - "required": true - }, - { - "id": 2, - "name": "readChunkVersion", - "type": "ReadChunkVersion", - "optional": true - } - ] - }, - { - "name": "GetSmallFileResponseProto", - "fields": [ - { - "id": 1, - "name": "data", - "type": "ReadChunkResponseProto", - "required": true - } - ] - }, - { - "name": "CopyContainerRequestProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "readOffset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "len", - "type": "uint64", - "optional": true - }, - { - "id": 4, - "name": "version", - "type": "uint32", - "optional": true - }, - { - "id": 5, - "name": "compression", - "type": "CopyContainerCompressProto", - "optional": true - } - ] - }, - { - "name": "CopyContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "readOffset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "len", - "type": "uint64", - "required": true - }, - { - "id": 4, - "name": "eof", - "type": "bool", - "required": true - }, - { - "id": 5, - "name": "data", - "type": "bytes", - "required": true - }, - { - "id": 6, - "name": "checksum", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "SendContainerRequest", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "offset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "required": true - }, - { - "id": 4, - "name": "checksum", - "type": "int64", - "optional": true - }, - { - "id": 5, - "name": "compression", - "type": "CopyContainerCompressProto", - "optional": true - }, - { - "id": 6, - "name": "size", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "SendContainerResponse" - }, - { - "name": "ChunkMerkleTree", - "fields": [ - { - "id": 1, - "name": "offset", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "length", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "dataChecksum", - "type": "int64", - "optional": true - }, - { - "id": 4, - "name": "checksumMatches", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "BlockMerkleTree", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "dataChecksum", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "chunkMerkleTree", - "type": "ChunkMerkleTree", - "is_repeated": true - }, - { - "id": 4, - "name": "deleted", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "ContainerMerkleTree", - "fields": [ - { - "id": 1, - "name": "dataChecksum", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "blockMerkleTree", - "type": "BlockMerkleTree", - "is_repeated": true - } - ] - }, - { - "name": "ContainerChecksumInfo", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "containerMerkleTree", - "type": "ContainerMerkleTree", - "optional": true - } - ] - }, - { - "name": "ContainerCreateInfo", - "fields": [ - { - "id": 1, - "name": "state", - "type": "ContainerDataProto.State", - "required": true - }, - { - "id": 2, - "name": "replicaIndex", - "type": "int32", - "optional": true, - "options": [ - { - "name": "default", - "value": "-1" - } - ] - } - ] - } - ], - "services": [ - { - "name": "XceiverClientProtocolService", - "rpcs": [ - { - "name": "send", - "in_type": "ContainerCommandRequestProto", - "out_type": "ContainerCommandResponseProto", - "in_streamed": true, - "out_streamed": true - } - ] - }, - { - "name": "IntraDatanodeProtocolService", - "rpcs": [ - { - "name": "download", - "in_type": "CopyContainerRequestProto", - "out_type": "CopyContainerResponseProto", - "out_streamed": true - }, - { - "name": "upload", - "in_type": "SendContainerRequest", - "out_type": "SendContainerResponse", - "in_streamed": true - } - ] - } - ], - "package": { - "name": "hadoop.hdds.datanode" - }, - "options": [ - { - "name": "java_package", - "value": "org.apache.hadoop.hdds.protocol.datanode.proto" - }, - { - "name": "java_outer_classname", - "value": "ContainerProtos" - }, - { - "name": "java_generate_equals_and_hash", - "value": "true" - } - ] - } - }, { "protopath": "ReconfigureProtocol.proto", "def": { @@ -3785,4 +1958,4 @@ } } ] -} \ No newline at end of file +} diff --git a/hadoop-hdds/interface-server/pom.xml b/hadoop-hdds/interface-server/pom.xml index b914a3d4ab4c..b795917a79cd 100644 --- a/hadoop-hdds/interface-server/pom.xml +++ b/hadoop-hdds/interface-server/pom.xml @@ -26,6 +26,8 @@ Apache Ozone Distributed Data Store Server interface + + yyyyMMddHHmmss true @@ -42,10 +44,6 @@ org.apache.ozone hdds-interface-client - - org.apache.ratis - ratis-thirdparty-misc - javax.annotation @@ -54,6 +52,33 @@ + + + + com.gradle + develocity-maven-extension + + + + + + maven-compiler-plugin + + compiles generated sources + + + + protobuf-maven-plugin + + generates sources + + + + + + + + com.salesforce.servicelibs @@ -70,27 +95,6 @@ org.xolstice.maven.plugins protobuf-maven-plugin - - compile-proto-for-ratis - - compile - test-compile - compile-custom - test-compile-custom - - - com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier} - - InterSCMProtocol.proto - SCMUpdateProtocol.proto - SCMRatisProtocol.proto - - target/generated-sources/proto-java-for-ratis - false - grpc-java - io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier} - - compile-proto-${protobuf.version} @@ -99,37 +103,13 @@ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - - InterSCMProtocol.proto - SCMUpdateProtocol.proto - SCMRatisProtocol.proto - target/generated-sources/proto-java-for-protobuf-${protobuf.version} + ${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/protobuf-${protobuf.version}-${maven.build.timestamp} false - - org.apache.maven.plugins - maven-antrun-plugin - - - - run - - generate-sources - - - - - - - - - - - diff --git a/hadoop-hdds/interface-server/src/main/resources/proto.lock b/hadoop-hdds/interface-server/src/main/resources/proto.lock index 822a2f88ebe8..67a2b8f24b19 100644 --- a/hadoop-hdds/interface-server/src/main/resources/proto.lock +++ b/hadoop-hdds/interface-server/src/main/resources/proto.lock @@ -1,473 +1,5 @@ { "definitions": [ - { - "protopath": "InterSCMProtocol.proto", - "def": { - "messages": [ - { - "name": "CopyDBCheckpointRequestProto", - "fields": [ - { - "id": 1, - "name": "flush", - "type": "bool", - "required": true - } - ] - }, - { - "name": "CopyDBCheckpointResponseProto", - "fields": [ - { - "id": 1, - "name": "clusterId", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "len", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "eof", - "type": "bool", - "required": true - }, - { - "id": 4, - "name": "data", - "type": "bytes", - "required": true - }, - { - "id": 6, - "name": "readOffset", - "type": "uint64", - "required": true - }, - { - "id": 7, - "name": "checksum", - "type": "int64", - "optional": true - } - ] - } - ], - "services": [ - { - "name": "InterSCMProtocolService", - "rpcs": [ - { - "name": "download", - "in_type": "CopyDBCheckpointRequestProto", - "out_type": "CopyDBCheckpointResponseProto", - "out_streamed": true - } - ] - } - ], - "package": { - "name": "hadoop.hdds.security" - }, - "options": [ - { - "name": "java_package", - "value": "org.apache.hadoop.hdds.protocol.scm.proto" - }, - { - "name": "java_outer_classname", - "value": "InterSCMProtocolProtos" - }, - { - "name": "java_generate_equals_and_hash", - "value": "true" - } - ] - } - }, - { - "protopath": "SCMRatisProtocol.proto", - "def": { - "enums": [ - { - "name": "RequestType", - "enum_fields": [ - { - "name": "PIPELINE", - "integer": 1 - }, - { - "name": "CONTAINER", - "integer": 2 - }, - { - "name": "BLOCK", - "integer": 3 - }, - { - "name": "SEQUENCE_ID", - "integer": 4 - }, - { - "name": "CERT_STORE", - "integer": 5 - }, - { - "name": "MOVE", - "integer": 6 - }, - { - "name": "STATEFUL_SERVICE_CONFIG", - "integer": 7 - }, - { - "name": "FINALIZE", - "integer": 8 - }, - { - "name": "SECRET_KEY", - "integer": 9 - }, - { - "name": "CERT_ROTATE", - "integer": 10 - } - ] - } - ], - "messages": [ - { - "name": "Method", - "fields": [ - { - "id": 1, - "name": "name", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "args", - "type": "MethodArgument", - "is_repeated": true - } - ] - }, - { - "name": "MethodArgument", - "fields": [ - { - "id": 1, - "name": "type", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "value", - "type": "bytes", - "required": true - } - ] - }, - { - "name": "ListArgument", - "fields": [ - { - "id": 1, - "name": "type", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "value", - "type": "bytes", - "is_repeated": true - } - ] - }, - { - "name": "SCMRatisRequestProto", - "fields": [ - { - "id": 1, - "name": "type", - "type": "RequestType", - "required": true - }, - { - "id": 2, - "name": "method", - "type": "Method", - "required": true - } - ] - }, - { - "name": "SCMRatisResponseProto", - "fields": [ - { - "id": 2, - "name": "type", - "type": "string", - "required": true - }, - { - "id": 3, - "name": "value", - "type": "bytes", - "required": true - } - ] - } - ], - "package": { - "name": "hadoop.hdds.security" - }, - "options": [ - { - "name": "java_package", - "value": "org.apache.hadoop.hdds.protocol.proto" - }, - { - "name": "java_outer_classname", - "value": "SCMRatisProtocol" - }, - { - "name": "java_generate_equals_and_hash", - "value": "true" - } - ] - } - }, - { - "protopath": "SCMUpdateProtocol.proto", - "def": { - "enums": [ - { - "name": "Type", - "enum_fields": [ - { - "name": "CRLUpdate", - "integer": 1 - }, - { - "name": "PipelineUpdate", - "integer": 2 - } - ] - } - ], - "messages": [ - { - "name": "CRLInfoProto", - "fields": [ - { - "id": 1, - "name": "x509CRL", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "creationTimestamp", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "crlSequenceID", - "type": "int64", - "required": true - } - ] - }, - { - "name": "ClientId", - "fields": [ - { - "id": 1, - "name": "msb", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "lsb", - "type": "int64", - "required": true - } - ] - }, - { - "name": "SubscribeRequest", - "fields": [ - { - "id": 1, - "name": "hostname", - "type": "string", - "optional": true - } - ] - }, - { - "name": "SubscribeResponse", - "fields": [ - { - "id": 1, - "name": "clientId", - "type": "ClientId", - "required": true - } - ] - }, - { - "name": "UpdateRequest", - "fields": [ - { - "id": 1, - "name": "updateType", - "type": "Type", - "required": true - }, - { - "id": 2, - "name": "traceID", - "type": "string", - "optional": true - }, - { - "id": 3, - "name": "clientId", - "type": "ClientId", - "required": true - }, - { - "id": 4, - "name": "crlUpdateRequest", - "type": "CRLUpdateRequest", - "optional": true - } - ] - }, - { - "name": "UpdateResponse", - "fields": [ - { - "id": 1, - "name": "updateType", - "type": "Type", - "required": true - }, - { - "id": 2, - "name": "traceID", - "type": "string", - "optional": true - }, - { - "id": 3, - "name": "crlUpdateResponse", - "type": "CRLUpdateResponse", - "optional": true - } - ] - }, - { - "name": "CRLUpdateRequest", - "fields": [ - { - "id": 1, - "name": "receivedCrlId", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "pendingCrlIds", - "type": "int64", - "is_repeated": true - } - ] - }, - { - "name": "CRLUpdateResponse", - "fields": [ - { - "id": 1, - "name": "crlInfo", - "type": "CRLInfoProto", - "optional": true - } - ] - }, - { - "name": "UnsubscribeRequest", - "fields": [ - { - "id": 1, - "name": "clientId", - "type": "ClientId", - "required": true - } - ] - }, - { - "name": "UnsubscribeResponse" - } - ], - "services": [ - { - "name": "SCMUpdateService", - "rpcs": [ - { - "name": "subscribe", - "in_type": "SubscribeRequest", - "out_type": "SubscribeResponse" - }, - { - "name": "updateStatus", - "in_type": "UpdateRequest", - "out_type": "UpdateResponse", - "in_streamed": true, - "out_streamed": true - }, - { - "name": "unsubscribe", - "in_type": "UnsubscribeRequest", - "out_type": "UnsubscribeResponse" - } - ] - } - ], - "package": { - "name": "hadoop.hdds.scm" - }, - "options": [ - { - "name": "java_package", - "value": "org.apache.hadoop.hdds.protocol.scm.proto" - }, - { - "name": "java_outer_classname", - "value": "SCMUpdateServiceProtos" - }, - { - "name": "java_generate_equals_and_hash", - "value": "true" - }, - { - "name": "java_generic_services", - "value": "true" - } - ] - } - }, { "protopath": "ScmSecretKeyProtocol.proto", "def": { @@ -3762,4 +3294,4 @@ } } ] -} \ No newline at end of file +} diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml index cfd22b1bfc7a..2bf317f33110 100644 --- a/hadoop-hdds/pom.xml +++ b/hadoop-hdds/pom.xml @@ -35,6 +35,7 @@ container-service crypto-api crypto-default + datanode-grpc-client docs erasurecode framework @@ -45,6 +46,7 @@ managed-rocksdb rocks-native rocksdb-checkpoint-differ + scm-grpc-client server-scm test-utils diff --git a/hadoop-hdds/scm-grpc-client/pom.xml b/hadoop-hdds/scm-grpc-client/pom.xml new file mode 100644 index 000000000000..d80a873447e1 --- /dev/null +++ b/hadoop-hdds/scm-grpc-client/pom.xml @@ -0,0 +1,164 @@ + + + + 4.0.0 + + org.apache.ozone + hdds + 2.2.0-SNAPSHOT + + hdds-scm-grpc-client + 2.2.0-SNAPSHOT + jar + Apache Ozone HDDS SCM gRPC Client + SCM internal gRPC stubs generated from InterSCMProtocol.proto, SCMUpdateProtocol.proto, and + SCMRatisProtocol.proto. Package names in generated sources are rewritten by maven-antrun-plugin (Ant + replace task, pure Java) from vanilla com.google.protobuf / io.grpc / com.google.common to their + ratis-thirdparty counterparts before compilation. Replaces the antrun source-rewriting approach + previously in hdds-interface-server. + + + + yyyyMMddHHmmss + + true + true + true + + + + + + org.apache.ratis + ratis-thirdparty-misc + + + + javax.annotation + javax.annotation-api + provided + + + + + + + + com.gradle + develocity-maven-extension + + + + + maven-compiler-plugin + + compiles generated + rewritten sources + + + + maven-antrun-plugin + + rewrites generated source files + + + + protobuf-maven-plugin + + generates sources + + + + + + + + + + + com.salesforce.servicelibs + proto-backwards-compatibility + + + org.apache.maven.plugins + maven-compiler-plugin + + none + false + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + compile-proto-scm + + compile + compile-custom + + generate-sources + + + com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier} + ${project.build.directory}/generated-sources/proto-java-ratis + ${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/${maven.build.timestamp} + false + grpc-java + io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier} + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + + rewrite-generated-sources-to-ratis-ns + + run + + process-sources + + + + + + + + + + + + + diff --git a/hadoop-hdds/interface-server/src/main/proto/InterSCMProtocol.proto b/hadoop-hdds/scm-grpc-client/src/main/proto/InterSCMProtocol.proto similarity index 100% rename from hadoop-hdds/interface-server/src/main/proto/InterSCMProtocol.proto rename to hadoop-hdds/scm-grpc-client/src/main/proto/InterSCMProtocol.proto diff --git a/hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto b/hadoop-hdds/scm-grpc-client/src/main/proto/SCMRatisProtocol.proto similarity index 100% rename from hadoop-hdds/interface-server/src/main/proto/SCMRatisProtocol.proto rename to hadoop-hdds/scm-grpc-client/src/main/proto/SCMRatisProtocol.proto diff --git a/hadoop-hdds/interface-server/src/main/proto/SCMUpdateProtocol.proto b/hadoop-hdds/scm-grpc-client/src/main/proto/SCMUpdateProtocol.proto similarity index 100% rename from hadoop-hdds/interface-server/src/main/proto/SCMUpdateProtocol.proto rename to hadoop-hdds/scm-grpc-client/src/main/proto/SCMUpdateProtocol.proto diff --git a/hadoop-hdds/scm-grpc-client/src/main/resources/proto.lock b/hadoop-hdds/scm-grpc-client/src/main/resources/proto.lock new file mode 100644 index 000000000000..72c30be919da --- /dev/null +++ b/hadoop-hdds/scm-grpc-client/src/main/resources/proto.lock @@ -0,0 +1,472 @@ +{ + "definitions": [ + { + "protopath": "InterSCMProtocol.proto", + "def": { + "messages": [ + { + "name": "CopyDBCheckpointRequestProto", + "fields": [ + { + "id": 1, + "name": "flush", + "type": "bool", + "required": true + } + ] + }, + { + "name": "CopyDBCheckpointResponseProto", + "fields": [ + { + "id": 1, + "name": "clusterId", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "len", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "eof", + "type": "bool", + "required": true + }, + { + "id": 4, + "name": "data", + "type": "bytes", + "required": true + }, + { + "id": 6, + "name": "readOffset", + "type": "uint64", + "required": true + }, + { + "id": 7, + "name": "checksum", + "type": "int64", + "optional": true + } + ] + } + ], + "services": [ + { + "name": "InterSCMProtocolService", + "rpcs": [ + { + "name": "download", + "in_type": "CopyDBCheckpointRequestProto", + "out_type": "CopyDBCheckpointResponseProto", + "out_streamed": true + } + ] + } + ], + "package": { + "name": "hadoop.hdds.security" + }, + "options": [ + { + "name": "java_package", + "value": "org.apache.hadoop.hdds.protocol.scm.proto" + }, + { + "name": "java_outer_classname", + "value": "InterSCMProtocolProtos" + }, + { + "name": "java_generate_equals_and_hash", + "value": "true" + } + ] + } + }, + { + "protopath": "SCMRatisProtocol.proto", + "def": { + "enums": [ + { + "name": "RequestType", + "enum_fields": [ + { + "name": "PIPELINE", + "integer": 1 + }, + { + "name": "CONTAINER", + "integer": 2 + }, + { + "name": "BLOCK", + "integer": 3 + }, + { + "name": "SEQUENCE_ID", + "integer": 4 + }, + { + "name": "CERT_STORE", + "integer": 5 + }, + { + "name": "MOVE", + "integer": 6 + }, + { + "name": "STATEFUL_SERVICE_CONFIG", + "integer": 7 + }, + { + "name": "FINALIZE", + "integer": 8 + }, + { + "name": "SECRET_KEY", + "integer": 9 + }, + { + "name": "CERT_ROTATE", + "integer": 10 + } + ] + } + ], + "messages": [ + { + "name": "Method", + "fields": [ + { + "id": 1, + "name": "name", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "args", + "type": "MethodArgument", + "is_repeated": true + } + ] + }, + { + "name": "MethodArgument", + "fields": [ + { + "id": 1, + "name": "type", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "value", + "type": "bytes", + "required": true + } + ] + }, + { + "name": "ListArgument", + "fields": [ + { + "id": 1, + "name": "type", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "value", + "type": "bytes", + "is_repeated": true + } + ] + }, + { + "name": "SCMRatisRequestProto", + "fields": [ + { + "id": 1, + "name": "type", + "type": "RequestType", + "required": true + }, + { + "id": 2, + "name": "method", + "type": "Method", + "required": true + } + ] + }, + { + "name": "SCMRatisResponseProto", + "fields": [ + { + "id": 2, + "name": "type", + "type": "string", + "required": true + }, + { + "id": 3, + "name": "value", + "type": "bytes", + "required": true + } + ] + } + ], + "package": { + "name": "hadoop.hdds.security" + }, + "options": [ + { + "name": "java_package", + "value": "org.apache.hadoop.hdds.protocol.proto" + }, + { + "name": "java_outer_classname", + "value": "SCMRatisProtocol" + }, + { + "name": "java_generate_equals_and_hash", + "value": "true" + } + ] + } + }, + { + "protopath": "SCMUpdateProtocol.proto", + "def": { + "enums": [ + { + "name": "Type", + "enum_fields": [ + { + "name": "CRLUpdate", + "integer": 1 + }, + { + "name": "PipelineUpdate", + "integer": 2 + } + ] + } + ], + "messages": [ + { + "name": "CRLInfoProto", + "fields": [ + { + "id": 1, + "name": "x509CRL", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "creationTimestamp", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "crlSequenceID", + "type": "int64", + "required": true + } + ] + }, + { + "name": "ClientId", + "fields": [ + { + "id": 1, + "name": "msb", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "lsb", + "type": "int64", + "required": true + } + ] + }, + { + "name": "SubscribeRequest", + "fields": [ + { + "id": 1, + "name": "hostname", + "type": "string", + "optional": true + } + ] + }, + { + "name": "SubscribeResponse", + "fields": [ + { + "id": 1, + "name": "clientId", + "type": "ClientId", + "required": true + } + ] + }, + { + "name": "UpdateRequest", + "fields": [ + { + "id": 1, + "name": "updateType", + "type": "Type", + "required": true + }, + { + "id": 2, + "name": "traceID", + "type": "string", + "optional": true + }, + { + "id": 3, + "name": "clientId", + "type": "ClientId", + "required": true + }, + { + "id": 4, + "name": "crlUpdateRequest", + "type": "CRLUpdateRequest", + "optional": true + } + ] + }, + { + "name": "UpdateResponse", + "fields": [ + { + "id": 1, + "name": "updateType", + "type": "Type", + "required": true + }, + { + "id": 2, + "name": "traceID", + "type": "string", + "optional": true + }, + { + "id": 3, + "name": "crlUpdateResponse", + "type": "CRLUpdateResponse", + "optional": true + } + ] + }, + { + "name": "CRLUpdateRequest", + "fields": [ + { + "id": 1, + "name": "receivedCrlId", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "pendingCrlIds", + "type": "int64", + "is_repeated": true + } + ] + }, + { + "name": "CRLUpdateResponse", + "fields": [ + { + "id": 1, + "name": "crlInfo", + "type": "CRLInfoProto", + "optional": true + } + ] + }, + { + "name": "UnsubscribeRequest", + "fields": [ + { + "id": 1, + "name": "clientId", + "type": "ClientId", + "required": true + } + ] + }, + { + "name": "UnsubscribeResponse" + } + ], + "services": [ + { + "name": "SCMUpdateService", + "rpcs": [ + { + "name": "subscribe", + "in_type": "SubscribeRequest", + "out_type": "SubscribeResponse" + }, + { + "name": "updateStatus", + "in_type": "UpdateRequest", + "out_type": "UpdateResponse", + "in_streamed": true, + "out_streamed": true + }, + { + "name": "unsubscribe", + "in_type": "UnsubscribeRequest", + "out_type": "UnsubscribeResponse" + } + ] + } + ], + "package": { + "name": "hadoop.hdds.scm" + }, + "options": [ + { + "name": "java_package", + "value": "org.apache.hadoop.hdds.protocol.scm.proto" + }, + { + "name": "java_outer_classname", + "value": "SCMUpdateServiceProtos" + }, + { + "name": "java_generate_equals_and_hash", + "value": "true" + }, + { + "name": "java_generic_services", + "value": "true" + } + ] + } + } + ] +} diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index a169bea52048..038ce24f18bb 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -38,11 +38,6 @@ com.fasterxml.jackson.core jackson-core - - com.fasterxml.jackson.core - jackson-databind - - com.google.guava guava @@ -71,11 +66,6 @@ org.apache.commons commons-collections4 - - org.apache.commons - commons-compress - - org.apache.commons commons-lang3 @@ -92,11 +82,6 @@ org.apache.ozone hdds-cli-common - - org.apache.ozone - hdds-client - - org.apache.ozone hdds-common @@ -125,6 +110,10 @@ org.apache.ozone hdds-managed-rocksdb + + org.apache.ozone + hdds-scm-grpc-client + org.apache.ozone hdds-server-framework @@ -179,6 +168,16 @@ slf4j-reload4j runtime + + com.fasterxml.jackson.core + jackson-databind + test + + + org.apache.commons + commons-compress + test + @@ -203,6 +202,11 @@ hadoop-hdfs-client test + + org.apache.ozone + hdds-client + test + org.apache.ozone hdds-common diff --git a/hadoop-ozone/cli-repair/pom.xml b/hadoop-ozone/cli-repair/pom.xml index cb56bf6d7ce1..4f13cec2f8bd 100644 --- a/hadoop-ozone/cli-repair/pom.xml +++ b/hadoop-ozone/cli-repair/pom.xml @@ -72,7 +72,7 @@ org.apache.ozone - hdds-interface-client + hdds-datanode-grpc-client org.apache.ozone diff --git a/hadoop-ozone/cli-shell/pom.xml b/hadoop-ozone/cli-shell/pom.xml index d824ac082bad..2a7dca1b8601 100644 --- a/hadoop-ozone/cli-shell/pom.xml +++ b/hadoop-ozone/cli-shell/pom.xml @@ -205,6 +205,25 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + + org.apache.ratis:ratis-common + + + + + diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index 13a53cdc7a88..7de31464d4e6 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -47,24 +47,6 @@ com.google.protobuf protobuf-java - - io.grpc - grpc-api - - - com.google.code.findbugs - jsr305 - - - - - io.grpc - grpc-netty - - - io.netty - netty-handler - jakarta.annotation jakarta.annotation-api @@ -125,12 +107,6 @@ org.slf4j slf4j-api - - - io.grpc - grpc-stub - provided - io.netty netty-codec-http2 @@ -147,16 +123,6 @@ runtime - - io.grpc - grpc-inprocess - test - - - io.grpc - grpc-testing - test - org.apache.ozone hdds-test-utils diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java index 41cd45956547..c29d145d6d95 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/GrpcOMFailoverProxyProvider.java @@ -20,8 +20,6 @@ import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -38,6 +36,8 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransport; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java index e794107cd54d..c6b17e04e6df 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java @@ -23,14 +23,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.net.HostAndPort; -import io.grpc.Context; -import io.grpc.ManagedChannel; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.NettyChannelBuilder; -import io.netty.handler.ssl.SslContextBuilder; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.lang.reflect.Constructor; import java.net.InetAddress; import java.security.cert.X509Certificate; @@ -38,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -55,12 +53,30 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; import org.apache.hadoop.ozone.om.ha.GrpcOMFailoverProxyProvider; -import org.apache.hadoop.ozone.om.protocolPB.grpc.ClientAddressClientInterceptor; -import org.apache.hadoop.ozone.om.protocolPB.grpc.GrpcClientConstants; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerServiceGrpc; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ratis.thirdparty.io.grpc.CallOptions; +import org.apache.ratis.thirdparty.io.grpc.ClientCall; +import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor; +import org.apache.ratis.thirdparty.io.grpc.ClientInterceptors; +import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall; +import org.apache.ratis.thirdparty.io.grpc.ManagedChannel; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; +import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder; +import org.apache.ratis.thirdparty.io.grpc.stub.ClientCalls; +import org.apache.ratis.thirdparty.io.netty.channel.Channel; +import org.apache.ratis.thirdparty.io.netty.channel.EventLoopGroup; +import org.apache.ratis.thirdparty.io.netty.channel.epoll.Epoll; +import org.apache.ratis.thirdparty.io.netty.channel.epoll.EpollEventLoopGroup; +import org.apache.ratis.thirdparty.io.netty.channel.epoll.EpollSocketChannel; +import org.apache.ratis.thirdparty.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.ratis.thirdparty.io.netty.channel.socket.nio.NioSocketChannel; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,6 +88,7 @@ public class GrpcOmTransport implements OmTransport { LoggerFactory.getLogger(GrpcOmTransport.class); private static final String CLIENT_NAME = "GrpcOmTransport"; + private static final String SERVICE_NAME = "hadoop.ozone.OzoneManagerService"; private static final int SHUTDOWN_WAIT_INTERVAL = 100; private static final int SHUTDOWN_MAX_WAIT_SECONDS = 5; private final AtomicBoolean isRunning = new AtomicBoolean(false); @@ -79,8 +96,20 @@ public class GrpcOmTransport implements OmTransport { // gRPC specific private static List caCerts = null; - private final Map clients; + private static final Metadata.Key CLIENT_HOSTNAME_METADATA_KEY = + Metadata.Key.of("CLIENT_HOSTNAME", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key CLIENT_IP_ADDRESS_METADATA_KEY = + Metadata.Key.of("CLIENT_IP_ADDRESS", Metadata.ASCII_STRING_MARSHALLER); + + private static final MethodDescriptor + SUBMIT_REQUEST_METHOD = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY) + .setFullMethodName(MethodDescriptor.generateFullMethodName( + SERVICE_NAME, "submitRequest")) + .setRequestMarshaller(new Proto2Marshaller<>(OMRequest::parseFrom)) + .setResponseMarshaller(new Proto2Marshaller<>(OMResponse::parseFrom)) + .build(); + private final Map channels; private final ConfigurationSource conf; @@ -88,6 +117,8 @@ public class GrpcOmTransport implements OmTransport { private AtomicInteger globalFailoverCount; private final int maxSize; private final SecurityConfig secConfig; + private EventLoopGroup eventLoopGroup; + private Class channelType; private RetryPolicy retryPolicy; private final GrpcOMFailoverProxyProvider @@ -102,7 +133,6 @@ public GrpcOmTransport(ConfigurationSource conf, throws IOException { this.channels = new HashMap<>(); - this.clients = new HashMap<>(); this.conf = conf; this.host = new AtomicReference<>(); this.globalFailoverCount = new AtomicInteger(); @@ -130,6 +160,20 @@ public void start() throws IOException { return; } + ThreadFactory factory = new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat(CLIENT_NAME + "-ELG-%d") + .build(); + + if (Epoll.isAvailable()) { + eventLoopGroup = new EpollEventLoopGroup(0, factory); + channelType = EpollSocketChannel.class; + } else { + eventLoopGroup = new NioEventLoopGroup(0, factory); + channelType = NioSocketChannel.class; + } + LOG.info("{} channel type {}", CLIENT_NAME, channelType.getSimpleName()); + for (String nodeId : omFailoverProxyProvider.getOMProxyMap().getNodeIds()) { String hostaddr = omFailoverProxyProvider.getGrpcProxyAddress(nodeId); HostAndPort hp = HostAndPort.fromString(hostaddr); @@ -137,6 +181,8 @@ public void start() throws IOException { NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(hp.getHost(), hp.getPort()) .usePlaintext() + .eventLoopGroup(eventLoopGroup) + .channelType(channelType) .proxyDetector(uri -> null) .maxInboundMessageSize(maxSize); @@ -158,11 +204,7 @@ public void start() throws IOException { } channels.put(hostaddr, - channelBuilder.intercept(new ClientAddressClientInterceptor()) - .build()); - clients.put(hostaddr, - OzoneManagerServiceGrpc - .newBlockingStub(channels.get(hostaddr))); + channelBuilder.build()); } int maxFailovers = conf.getInt( OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, @@ -184,13 +226,12 @@ public OMResponse submitRequest(OMRequest payload) throws IOException { expectedFailoverCount = globalFailoverCount.get(); try { InetAddress inetAddress = InetAddress.getLocalHost(); - Context.current() - .withValue(GrpcClientConstants.CLIENT_IP_ADDRESS_CTX_KEY, - inetAddress.getHostAddress()) - .withValue(GrpcClientConstants.CLIENT_HOSTNAME_CTX_KEY, - inetAddress.getHostName()) - .run(() -> resp.set(clients.get(host.get()) - .submitRequest(payload))); + final ManagedChannel channel = channels.get(host.get()); + if (channel == null) { + throw new OMException(ResultCodes.INTERNAL_ERROR); + } + resp.set(submitRequest(channel, payload, + inetAddress.getHostName(), inetAddress.getHostAddress())); } catch (StatusRuntimeException e) { LOG.error("Failed to submit request", e); if (e.getStatus().getCode() == Status.Code.UNAVAILABLE) { @@ -330,6 +371,15 @@ public void shutdown() { LOG.warn("Channels {} did not terminate within timeout.", failedChannels); } + if (eventLoopGroup != null) { + try { + eventLoopGroup.shutdownGracefully().sync(); + } catch (InterruptedException e) { + LOG.error("Interrupted while shutting down event loop group", e); + Thread.currentThread().interrupt(); + } + } + LOG.info("{}: stopped", CLIENT_NAME); } @@ -338,6 +388,75 @@ public void close() throws IOException { shutdown(); } + private OMResponse submitRequest(ManagedChannel channel, OMRequest request, + String clientHostname, String clientIpAddress) { + Metadata headers = new Metadata(); + if (clientHostname != null) { + headers.put(CLIENT_HOSTNAME_METADATA_KEY, clientHostname); + } + if (clientIpAddress != null) { + headers.put(CLIENT_IP_ADDRESS_METADATA_KEY, clientIpAddress); + } + + org.apache.ratis.thirdparty.io.grpc.Channel intercepted = + ClientInterceptors.intercept(channel, new FixedHeadersInterceptor(headers)); + return ClientCalls.blockingUnaryCall(intercepted, SUBMIT_REQUEST_METHOD, + CallOptions.DEFAULT, request); + } + + private static final class FixedHeadersInterceptor implements ClientInterceptor { + private final Metadata headers; + + private FixedHeadersInterceptor(Metadata headers) { + this.headers = headers; + } + + @Override + public ClientCall interceptCall( + MethodDescriptor methodDescriptor, + CallOptions callOptions, + org.apache.ratis.thirdparty.io.grpc.Channel channel) { + return new ForwardingClientCall.SimpleForwardingClientCall( + channel.newCall(methodDescriptor, callOptions)) { + @Override + public void start(Listener responseListener, Metadata metadata) { + metadata.merge(headers); + super.start(responseListener, metadata); + } + }; + } + } + + private static final class Proto2Marshaller implements MethodDescriptor.Marshaller { + private final Proto2Parser parser; + + private Proto2Marshaller(Proto2Parser parser) { + this.parser = parser; + } + + @Override + public InputStream stream(T value) { + if (!(value instanceof com.google.protobuf.MessageLite)) { + throw new IllegalArgumentException("Expected protobuf request/response"); + } + return new ByteArrayInputStream(((com.google.protobuf.MessageLite) value).toByteArray()); + } + + @Override + public T parse(InputStream stream) { + try { + return parser.parse(stream); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + @FunctionalInterface + private interface Proto2Parser { + T parse(InputStream stream) throws IOException; + } + /** * GrpcOmTransport configuration in Java style configuration class. */ @@ -364,9 +483,7 @@ public void startClient(ManagedChannel testChannel) throws IOException { for (String nodeId : omFailoverProxyProvider.getOMProxyMap().getNodeIds()) { String hostaddr = omFailoverProxyProvider.getGrpcProxyAddress(nodeId); - clients.put(hostaddr, - OzoneManagerServiceGrpc - .newBlockingStub(testChannel)); + channels.put(hostaddr, testChannel); } LOG.info("{}: started", CLIENT_NAME); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressClientInterceptor.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressClientInterceptor.java index 85190d0b93f7..1ba921af04dc 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressClientInterceptor.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressClientInterceptor.java @@ -17,13 +17,13 @@ package org.apache.hadoop.ozone.om.protocolPB.grpc; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.ForwardingClientCall; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.CallOptions; +import org.apache.ratis.thirdparty.io.grpc.Channel; +import org.apache.ratis.thirdparty.io.grpc.ClientCall; +import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor; +import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; /** * GRPC client side interceptor to provide client hostname and IP address. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressServerInterceptor.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressServerInterceptor.java index f51eb2b6dc92..753fe9ff8eac 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressServerInterceptor.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/ClientAddressServerInterceptor.java @@ -17,12 +17,12 @@ package org.apache.hadoop.ozone.om.protocolPB.grpc; -import io.grpc.Context; -import io.grpc.Contexts; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; +import org.apache.ratis.thirdparty.io.grpc.Context; +import org.apache.ratis.thirdparty.io.grpc.Contexts; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.ServerCall; +import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor; /** * GRPC server side interceptor to retrieve the client IP and hostname. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/GrpcClientConstants.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/GrpcClientConstants.java index e59ec98ec784..fd5331b2d8eb 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/GrpcClientConstants.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/grpc/GrpcClientConstants.java @@ -17,8 +17,8 @@ package org.apache.hadoop.ozone.om.protocolPB.grpc; -import io.grpc.Context; -import io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.Context; +import org.apache.ratis.thirdparty.io.grpc.Metadata; /** * Constants to store grpc-client specific header values. diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestGrpcOmTransportConcurrentFailover.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestGrpcOmTransportConcurrentFailover.java index c8890eaa49c8..1f5b31b63167 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestGrpcOmTransportConcurrentFailover.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestGrpcOmTransportConcurrentFailover.java @@ -26,12 +26,8 @@ import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.Mockito.mock; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.StreamObserver; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.Map; import java.util.StringJoiner; @@ -48,8 +44,15 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerServiceGrpc; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.Server; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -63,6 +66,17 @@ public class TestGrpcOmTransportConcurrentFailover { private static final Logger LOG = LoggerFactory.getLogger(TestGrpcOmTransportConcurrentFailover.class); + private static final String SERVICE_NAME = "hadoop.ozone.OzoneManagerService"; + + private static final MethodDescriptor + SUBMIT_REQUEST_METHOD = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY) + .setFullMethodName(MethodDescriptor.generateFullMethodName( + SERVICE_NAME, "submitRequest")) + .setRequestMarshaller(new Proto2Marshaller<>(OMRequest::parseFrom)) + .setResponseMarshaller(new Proto2Marshaller<>(OMResponse::parseFrom)) + .build(); + private static final String OM_SERVICE_ID = "om-service-test"; private static final String NODE_ID_BASE = "om"; private static final int NUM_OMS = 3; @@ -237,11 +251,12 @@ private static class MockOMServer { private final AtomicInteger successCount = new AtomicInteger(0); private final AtomicInteger failureCount = new AtomicInteger(0); private final AtomicBoolean isLeader = new AtomicBoolean(false); - private final OzoneManagerServiceGrpc.OzoneManagerServiceImplBase serviceImpl = - mock(OzoneManagerServiceGrpc.OzoneManagerServiceImplBase.class, - delegatesTo(new OzoneManagerServiceGrpc.OzoneManagerServiceImplBase() { + private final ServerCalls.UnaryMethod submitRequestImpl = + mock(ServerCalls.UnaryMethod.class, + delegatesTo(new ServerCalls.UnaryMethod() { @Override - public void submitRequest(OMRequest request, StreamObserver responseObserver) { + public void invoke(OMRequest request, + StreamObserver responseObserver) { requestCount.incrementAndGet(); if (!isLeader.get()) { @@ -272,8 +287,14 @@ public void submitRequest(OMRequest request, StreamObserver response } public void start() throws Exception { - server = ServerBuilder.forPort(port) - .addService(serviceImpl) + ServerServiceDefinition service = ServerServiceDefinition.builder(SERVICE_NAME) + .addMethod(SUBMIT_REQUEST_METHOD, + ServerCalls.asyncUnaryCall(submitRequestImpl)) + .build(); + + server = NettyServerBuilder.forPort(port) + .directExecutor() + .addService(service) .build() .start(); } @@ -301,5 +322,38 @@ public int getFailureCount() { return failureCount.get(); } } + + private static final class Proto2Marshaller implements MethodDescriptor.Marshaller { + private final Parser parser; + + private Proto2Marshaller(Parser parser) { + this.parser = parser; + } + + @Override + public InputStream stream(T value) { + if (value instanceof com.google.protobuf.Message) { + return ((com.google.protobuf.Message) value).toByteString().newInput(); + } + if (value instanceof com.google.protobuf.MessageLite) { + return ((com.google.protobuf.MessageLite) value).toByteString().newInput(); + } + throw new IllegalArgumentException("Unsupported protobuf type: " + value.getClass()); + } + + @Override + public T parse(InputStream stream) { + try { + return parser.parse(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @FunctionalInterface + private interface Parser { + T parse(InputStream stream) throws IOException; + } + } } diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java index 176d9b6d03bd..ca665ef7acc6 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java @@ -18,29 +18,37 @@ package org.apache.hadoop.ozone.om.protocolPB; import static org.apache.hadoop.ozone.ClientVersion.CURRENT_VERSION; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_PORT_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.AdditionalAnswers.delegatesTo; import static org.mockito.Mockito.mock; import com.google.protobuf.ServiceException; -import io.grpc.ManagedChannel; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.grpc.testing.GrpcCleanupRule; import java.io.IOException; +import java.io.InputStream; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerServiceGrpc; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.Server; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -50,12 +58,23 @@ * Tests for GrpcOmTransport client. */ public class TestS3GrpcOmTransport { - private final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); - private static final Logger LOG = LoggerFactory.getLogger(TestS3GrpcOmTransport.class); + private static final String SERVICE_NAME = "hadoop.ozone.OzoneManagerService"; + + private static final MethodDescriptor + SUBMIT_REQUEST_METHOD = MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY) + .setFullMethodName(MethodDescriptor.generateFullMethodName( + SERVICE_NAME, "submitRequest")) + .setRequestMarshaller(new Proto2Marshaller<>(OMRequest::parseFrom)) + .setResponseMarshaller(new Proto2Marshaller<>(OMResponse::parseFrom)) + .build(); + private static final String LEADER_OM_NODE_ID = "TestOM"; + private static final String OM_SERVICE_ID = "om-service-test"; + private static final String OM_NODE_ID = "om0"; private final OMResponse omResponse = OMResponse.newBuilder() .setSuccess(true) @@ -73,44 +92,33 @@ public class TestS3GrpcOmTransport { private String omServiceId; private UserGroupInformation ugi; - private ManagedChannel channel; - - private String serverName; - - private final OzoneManagerServiceGrpc.OzoneManagerServiceImplBase - serviceImpl = - mock(OzoneManagerServiceGrpc.OzoneManagerServiceImplBase.class, - delegatesTo( - new OzoneManagerServiceGrpc.OzoneManagerServiceImplBase() { - @Override - public void submitRequest(org.apache.hadoop.ozone.protocol.proto - .OzoneManagerProtocolProtos - .OMRequest request, - io.grpc.stub.StreamObserver - responseObserver) { - try { - if (doFailover) { - if (completeFailover) { - doFailover = false; - } - failoverCount++; - throw createNotLeaderException(); - } else { - responseObserver.onNext(omResponse); - responseObserver.onCompleted(); - } - } catch (Throwable e) { - IOException ex = new IOException(e.getCause()); - responseObserver.onError(io.grpc.Status - .INTERNAL - .withDescription(ex.getMessage()) - .asRuntimeException()); + private Server server; + + private final ServerCalls.UnaryMethod submitRequestImpl = + mock(ServerCalls.UnaryMethod.class, + delegatesTo(new ServerCalls.UnaryMethod() { + @Override + public void invoke(OMRequest request, + StreamObserver responseObserver) { + try { + if (doFailover) { + if (completeFailover) { + doFailover = false; } + failoverCount++; + throw createNotLeaderException(); + } else { + responseObserver.onNext(omResponse); + responseObserver.onCompleted(); } - })); + } catch (Throwable e) { + IOException ex = new IOException(e.getCause()); + responseObserver.onError(Status.INTERNAL + .withDescription(ex.getMessage()) + .asRuntimeException()); + } + } + })); private GrpcOmTransport client; @@ -128,28 +136,42 @@ private ServiceException createNotLeaderException() { @BeforeEach public void setUp() throws Exception { failoverCount = 0; - // Generate a unique in-process server name. - serverName = InProcessServerBuilder.generateName(); - // Create a server, add service, start, - // and register for automatic graceful shutdown. - grpcCleanup.register(InProcessServerBuilder - .forName(serverName) + ServerServiceDefinition service = ServerServiceDefinition.builder(SERVICE_NAME) + .addMethod(SUBMIT_REQUEST_METHOD, + ServerCalls.asyncUnaryCall(submitRequestImpl)) + .build(); + + server = NettyServerBuilder.forPort(0) .directExecutor() - .addService(serviceImpl) + .addService(service) .build() - .start()); + .start(); - // Create a client channel and register for automatic graceful shutdown. - channel = grpcCleanup.register( - InProcessChannelBuilder.forName(serverName).directExecutor().build()); - - omServiceId = ""; + omServiceId = OM_SERVICE_ID; conf = new OzoneConfiguration(); + conf.set(OZONE_OM_SERVICE_IDS_KEY, omServiceId); + conf.set(ConfUtils.addKeySuffixes(OZONE_OM_NODES_KEY, omServiceId), OM_NODE_ID); + conf.set(ConfUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, omServiceId, OM_NODE_ID), "localhost"); + conf.setInt(ConfUtils.addKeySuffixes(OZONE_OM_GRPC_PORT_KEY, omServiceId, OM_NODE_ID), + server.getPort()); ugi = UserGroupInformation.getCurrentUser(); doFailover = false; } + @AfterEach + public void tearDown() throws Exception { + if (client != null) { + client.close(); + client = null; + } + if (server != null) { + server.shutdownNow(); + server.awaitTermination(); + server = null; + } + } + @Test public void testSubmitRequestToServer() throws Exception { ServiceListRequest req = ServiceListRequest.newBuilder().build(); @@ -162,7 +184,6 @@ public void testSubmitRequestToServer() throws Exception { .build(); client = new GrpcOmTransport(conf, ugi, omServiceId); - client.startClient(channel); final OMResponse resp = client.submitRequest(omRequest); assertEquals(resp.getStatus(), org.apache.hadoop.ozone.protocol @@ -182,7 +203,6 @@ public void testGrpcFailoverProxy() throws Exception { .build(); client = new GrpcOmTransport(conf, ugi, omServiceId); - client.startClient(channel); doFailover = true; // first invocation generates a NotALeaderException @@ -208,7 +228,6 @@ public void testGrpcFailoverProxyExhaustRetry() throws Exception { conf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 0); client = new GrpcOmTransport(conf, ugi, omServiceId); - client.startClient(channel); doFailover = true; // first invocation generates a NotALeaderException @@ -229,7 +248,6 @@ public void testGrpcFailoverProxyCalculatesFailoverCountPerRequest() throws Exce conf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, maxFailoverAttempts); conf.setLong(OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY, 50); client = new GrpcOmTransport(conf, ugi, omServiceId); - client.startClient(channel); assertThrows(Exception.class, () -> client.submitRequest(arbitraryOmRequest())); assertEquals(maxFailoverAttempts, failoverCount); @@ -255,14 +273,6 @@ public void testGrpcFailoverExceedMaxMesgLen() throws Exception { conf.setInt(OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH, 1); client = new GrpcOmTransport(conf, ugi, omServiceId); - int maxSize = conf.getInt(OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH, - OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT); - channel = grpcCleanup.register( - InProcessChannelBuilder - .forName(serverName) - .maxInboundMetadataSize(maxSize) - .directExecutor().build()); - client.startClient(channel); doFailover = true; // GrpcOMFailoverProvider returns Fail retry due to mesg response @@ -281,4 +291,37 @@ private static OMRequest arbitraryOmRequest() { .setServiceListRequest(req) .build(); } + + private static final class Proto2Marshaller implements MethodDescriptor.Marshaller { + private final Parser parser; + + private Proto2Marshaller(Parser parser) { + this.parser = parser; + } + + @Override + public InputStream stream(T value) { + if (value instanceof com.google.protobuf.Message) { + return ((com.google.protobuf.Message) value).toByteString().newInput(); + } + if (value instanceof com.google.protobuf.MessageLite) { + return ((com.google.protobuf.MessageLite) value).toByteString().newInput(); + } + throw new IllegalArgumentException("Unsupported protobuf type: " + value.getClass()); + } + + @Override + public T parse(InputStream stream) { + try { + return parser.parse(stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @FunctionalInterface + private interface Parser { + T parse(InputStream stream) throws IOException; + } + } } diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressClientInterceptor.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressClientInterceptor.java index 0a862c2eff00..8b2a9d67d064 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressClientInterceptor.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressClientInterceptor.java @@ -23,13 +23,13 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.Context; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.CallOptions; +import org.apache.ratis.thirdparty.io.grpc.Channel; +import org.apache.ratis.thirdparty.io.grpc.ClientCall; +import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor; +import org.apache.ratis.thirdparty.io.grpc.Context; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressServerInterceptor.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressServerInterceptor.java index 240c1db65206..49a7e912ed5a 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressServerInterceptor.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/grpc/TestClientAddressServerInterceptor.java @@ -23,12 +23,12 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; -import io.grpc.Context; -import io.grpc.Contexts; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; +import org.apache.ratis.thirdparty.io.grpc.Context; +import org.apache.ratis.thirdparty.io.grpc.Contexts; +import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.ServerCall; +import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml index 045b2cae1ebf..99f3d775980a 100644 --- a/hadoop-ozone/csi/pom.xml +++ b/hadoop-ozone/csi/pom.xml @@ -33,15 +33,6 @@ - - com.google.guava - guava - - - com.google.protobuf - protobuf-java - ${protobuf.version} - commons-io commons-io @@ -50,50 +41,6 @@ info.picocli picocli - - io.grpc - grpc-api - - - com.google.code.findbugs - jsr305 - - - - - io.grpc - grpc-netty - - - io.grpc - grpc-protobuf - - - com.google.code.findbugs - jsr305 - - - com.google.protobuf - protobuf-java - - - - - io.grpc - grpc-stub - - - io.netty - netty-transport - - - io.netty - netty-transport-classes-epoll - - - io.netty - netty-transport-native-unix-common - org.apache.ozone hdds-cli-common @@ -146,6 +93,10 @@ org.apache.ozone ozone-common + + org.apache.ratis + ratis-thirdparty-misc + org.slf4j slf4j-api @@ -156,48 +107,12 @@ 3.0.2 provided - - com.google.protobuf - protobuf-java-util - ${protobuf.version} - provided - - - com.google.code.findbugs - jsr305 - - - com.google.j2objc - j2objc-annotations - - - com.google.protobuf - protobuf-java - - - javax.annotation javax.annotation-api provided - - io.netty - netty-codec-http2 - runtime - - - io.netty - netty-handler-proxy - runtime - - - io.netty - netty-transport-native-epoll - linux-x86_64 - runtime - org.slf4j slf4j-reload4j @@ -227,6 +142,21 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + org.apache.ozone:hdds-server-framework + org.apache.ratis:ratis-thirdparty-misc + + + + + org.xolstice.maven.plugins protobuf-maven-plugin @@ -272,6 +202,29 @@ + + org.apache.maven.plugins + maven-antrun-plugin + + + shade-proto-for-ratis + + run + + process-sources + + + + + + + + + + + + + com.github.spotbugs spotbugs-maven-plugin diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java index f0fa375c3fc6..7f321b999627 100644 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java +++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/ControllerService.java @@ -29,7 +29,7 @@ import csi.v1.Csi.DeleteVolumeRequest; import csi.v1.Csi.DeleteVolumeResponse; import csi.v1.Csi.Volume; -import io.grpc.stub.StreamObserver; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import java.io.IOException; import org.apache.hadoop.ozone.client.OzoneClient; diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java index cf1133a53401..b1987a7768fb 100644 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java +++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/CsiServer.java @@ -17,11 +17,11 @@ package org.apache.hadoop.ozone.csi; -import io.grpc.Server; -import io.grpc.netty.NettyServerBuilder; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.epoll.EpollServerDomainSocketChannel; -import io.netty.channel.unix.DomainSocketAddress; +import org.apache.ratis.thirdparty.io.grpc.Server; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.netty.channel.epoll.EpollEventLoopGroup; +import org.apache.ratis.thirdparty.io.netty.channel.epoll.EpollServerDomainSocketChannel; +import org.apache.ratis.thirdparty.io.netty.channel.unix.DomainSocketAddress; import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java index b65de8cb5f43..1b41d1ae8a57 100644 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java +++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/IdentityService.java @@ -19,14 +19,14 @@ import static csi.v1.Csi.PluginCapability.Service.Type.CONTROLLER_SERVICE; -import com.google.protobuf.BoolValue; +import org.apache.ratis.thirdparty.com.google.protobuf.BoolValue; import csi.v1.Csi.GetPluginCapabilitiesResponse; import csi.v1.Csi.GetPluginInfoResponse; import csi.v1.Csi.PluginCapability; import csi.v1.Csi.PluginCapability.Service; import csi.v1.Csi.ProbeResponse; import csi.v1.IdentityGrpc.IdentityImplBase; -import io.grpc.stub.StreamObserver; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.apache.hadoop.ozone.util.OzoneVersionInfo; /** diff --git a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java index 7220c31137b3..7787121267fa 100644 --- a/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java +++ b/hadoop-ozone/csi/src/main/java/org/apache/hadoop/ozone/csi/NodeService.java @@ -26,7 +26,7 @@ import csi.v1.Csi.NodeUnpublishVolumeRequest; import csi.v1.Csi.NodeUnpublishVolumeResponse; import csi.v1.NodeGrpc.NodeImplBase; -import io.grpc.stub.StreamObserver; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; diff --git a/hadoop-ozone/integration-test-recon/pom.xml b/hadoop-ozone/integration-test-recon/pom.xml index 812ee6f8667d..dff96a102161 100644 --- a/hadoop-ozone/integration-test-recon/pom.xml +++ b/hadoop-ozone/integration-test-recon/pom.xml @@ -217,12 +217,17 @@ org.apache.maven.plugins maven-dependency-plugin - - - org.mockito:mockito-inline:jar - org.mockito:mockito-junit-jupiter:jar - - + + + analyze + + + org.mockito:mockito-inline:jar + org.mockito:mockito-junit-jupiter:jar + + + + diff --git a/hadoop-ozone/interface-client/pom.xml b/hadoop-ozone/interface-client/pom.xml index 60ddefa7cee4..4cd8352a722c 100644 --- a/hadoop-ozone/interface-client/pom.xml +++ b/hadoop-ozone/interface-client/pom.xml @@ -87,6 +87,34 @@ + + + + com.gradle + develocity-maven-extension + + + + + + maven-compiler-plugin + + compiles generated sources + + + + protobuf-maven-plugin + + generates sources + + + + + + + + com.salesforce.servicelibs diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index fae36afa538f..9a7f2861d22f 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -58,36 +58,6 @@ info.picocli picocli - - io.grpc - grpc-api - - - com.google.code.findbugs - jsr305 - - - - - io.grpc - grpc-netty - - - io.grpc - grpc-stub - - - io.netty - netty-common - - - io.netty - netty-handler - - - io.netty - netty-transport - jakarta.annotation jakarta.annotation-api @@ -238,11 +208,6 @@ hdds-docs provided - - io.netty - netty-tcnative-boringssl-static - runtime - org.slf4j slf4j-reload4j diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java index faf6d623f3ff..28894aac46d6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java @@ -29,15 +29,15 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_WORKERGROUP_SIZE_KEY; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.grpc.Server; -import io.grpc.ServerInterceptors; -import io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.NettyServerBuilder; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.SslProvider; +import org.apache.ratis.thirdparty.io.grpc.Server; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors; +import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.netty.channel.EventLoopGroup; +import org.apache.ratis.thirdparty.io.netty.channel.nio.NioEventLoopGroup; +import org.apache.ratis.thirdparty.io.netty.channel.socket.nio.NioServerSocketChannel; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider; import java.io.IOException; import java.util.OptionalInt; import java.util.concurrent.LinkedBlockingQueue; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java index 05abb25c376b..877c22b197ad 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java @@ -17,15 +17,23 @@ package org.apache.hadoop.ozone.om; +import com.google.protobuf.MessageLite; import com.google.protobuf.RpcController; -import io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.BindableService; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; +import org.apache.ratis.thirdparty.io.grpc.Status; +import org.apache.ratis.thirdparty.io.grpc.stub.ServerCalls; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.ipc_.RPC; import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerServiceGrpc.OzoneManagerServiceImplBase; import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB; import org.apache.hadoop.ozone.util.UUIDUtil; import org.slf4j.Logger; @@ -33,10 +41,26 @@ /** * Grpc Service for handling S3 gateway OzoneManagerProtocol client requests. + * Implements BindableService directly using shaded gRPC with a custom byte-stream + * marshaller so that vanilla protobuf OMRequest/OMResponse can be used without + * shading OzoneManagerProtocolProtos (which imports HddsProtos vanilla types). */ -public class OzoneManagerServiceGrpc extends OzoneManagerServiceImplBase { +public class OzoneManagerServiceGrpc implements BindableService { private static final Logger LOG = LoggerFactory.getLogger(OzoneManagerServiceGrpc.class); + + static final String SERVICE_NAME = "hadoop.ozone.OzoneManagerService"; + + private static final MethodDescriptor + SUBMIT_REQUEST_DESCRIPTOR = + MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNARY) + .setFullMethodName( + MethodDescriptor.generateFullMethodName(SERVICE_NAME, "submitRequest")) + .setRequestMarshaller(proto2Marshaller(OMRequest::parseFrom)) + .setResponseMarshaller(proto2Marshaller(OMResponse::parseFrom)) + .build(); + /** * RpcController is not used and hence is set to null. */ @@ -49,9 +73,15 @@ public class OzoneManagerServiceGrpc extends OzoneManagerServiceImplBase { } @Override - public void submitRequest(OMRequest request, - io.grpc.stub.StreamObserver - responseObserver) { + public ServerServiceDefinition bindService() { + return ServerServiceDefinition.builder(SERVICE_NAME) + .addMethod(SUBMIT_REQUEST_DESCRIPTOR, + ServerCalls.asyncUnaryCall(this::submitRequest)) + .build(); + } + + private void submitRequest(OMRequest request, + StreamObserver responseObserver) { LOG.debug("OzoneManagerServiceGrpc: OzoneManagerServiceImplBase " + "processing s3g client submit request - for command {}", request.getCmdType().name()); @@ -89,4 +119,27 @@ private static byte[] getClientId() { return UUIDUtil.randomUUIDBytes(); } + private static MethodDescriptor.Marshaller proto2Marshaller( + Proto2Parser parser) { + return new MethodDescriptor.Marshaller() { + @Override + public InputStream stream(T value) { + return new ByteArrayInputStream(value.toByteArray()); + } + + @Override + public T parse(InputStream stream) { + try { + return parser.parse(stream); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }; + } + + @FunctionalInterface + private interface Proto2Parser { + T parse(InputStream stream) throws IOException; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java index 903b0720943d..b2eec10d69c6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMMetadataReader.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; -import io.grpc.Context; +import org.apache.ratis.thirdparty.io.grpc.Context; import org.apache.hadoop.ipc_.Server; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java index 145ffe9854fe..cb90cd6d6e17 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestOMClientRequestWithUserInfo.java @@ -27,7 +27,7 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; -import io.grpc.Context; +import org.apache.ratis.thirdparty.io.grpc.Context; import java.io.IOException; import java.net.InetAddress; import java.nio.file.Path; diff --git a/hadoop-ozone/ozonefs-common/pom.xml b/hadoop-ozone/ozonefs-common/pom.xml index aecaa66cd4c0..72ec8f089e04 100644 --- a/hadoop-ozone/ozonefs-common/pom.xml +++ b/hadoop-ozone/ozonefs-common/pom.xml @@ -104,6 +104,25 @@ none + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + + org.apache.httpcomponents:httpclient + + + + + diff --git a/hadoop-ozone/ozonefs-hadoop2/pom.xml b/hadoop-ozone/ozonefs-hadoop2/pom.xml index fc8a5c6cd8e6..c2389c56a23f 100644 --- a/hadoop-ozone/ozonefs-hadoop2/pom.xml +++ b/hadoop-ozone/ozonefs-hadoop2/pom.xml @@ -120,14 +120,17 @@ org.apache.maven.plugins maven-dependency-plugin - - - org.assertj:* - org.junit.jupiter:* - org.mockito:* - - + + analyze + + + org.assertj:* + org.junit.jupiter:* + org.mockito:* + + + include-dependencies diff --git a/hadoop-ozone/ozonefs-hadoop3/pom.xml b/hadoop-ozone/ozonefs-hadoop3/pom.xml index 5a751541481d..0dc6217837ec 100644 --- a/hadoop-ozone/ozonefs-hadoop3/pom.xml +++ b/hadoop-ozone/ozonefs-hadoop3/pom.xml @@ -60,17 +60,20 @@ org.apache.maven.plugins maven-dependency-plugin - - - org.apache.ratis:ratis-thirdparty-misc - - - org.assertj:* - org.junit.jupiter:* - org.mockito:* - - + + analyze + + + org.apache.ratis:ratis-thirdparty-misc + + + org.assertj:* + org.junit.jupiter:* + org.mockito:* + + + include-dependencies diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index ecaf38d8de64..bdc1a876ecbf 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -244,7 +244,8 @@ org.springframework spring-core - + org.springframework @@ -331,11 +332,46 @@ org.apache.maven.plugins maven-dependency-plugin - - - javax.annotation:javax.annotation-api:jar - - + + + + analyze + + + aopalliance:aopalliance:jar + com.google.inject.extensions:guice-assistedinject:jar + com.jolbox:bonecp:jar + info.picocli:picocli:jar + javax.annotation:javax.annotation-api:jar + org.apache.derby:derby:jar + org.apache.ozone:ozone-reconcodegen:jar + org.apache.ratis:ratis-proto:jar + org.glassfish.hk2.external:jakarta.inject:jar + org.glassfish.jersey.inject:jersey-hk2:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.mockito:mockito-inline:jar + org.rocksdb:rocksdbjni:jar + org.slf4j:slf4j-reload4j:jar + org.springframework:spring-core:jar + org.springframework:spring-jdbc:jar + org.springframework:spring-tx:jar + org.xerial:sqlite-jdbc:jar + + + com.fasterxml.jackson.core:jackson-databind:jar + org.apache.commons:commons-collections4:jar + org.apache.ozone:hdds-container-service:jar + org.apache.ozone:hdds-managed-rocksdb:jar + org.springframework:spring-core:jar + + + + org.apache.maven.plugins diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index bebe6d1d07ba..9716389055dc 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml @@ -125,7 +125,8 @@ org.apache.ozone hdds-client - + org.apache.ozone @@ -280,6 +281,24 @@ true + + analyze + + + + org.apache.ozone:hdds-client + org.eclipse.jetty:jetty-servlet + org.eclipse.jetty:jetty-webapp + org.apache.ozone:ozone-interface-client + org.glassfish.jersey.core:jersey-common + + + diff --git a/org/apache/hadoop/hdds/protocol/datanode/proto/ContainerProtos.class b/org/apache/hadoop/hdds/protocol/datanode/proto/ContainerProtos.class new file mode 100644 index 0000000000000000000000000000000000000000..2bfbf1b3625c3874692c254e6b9651a3aa9684f6 GIT binary patch literal 56561 zcmc(I349$@_5a+Nm&{}`Nnc-@Hc9u_g>+5(l60leG)o(rjl86#Ef8Ll*ETd8NeUDf zKms4RW?C|B7z$(;D#W$krzb~6j4$9pS#S=z3<(5nRMds|KaoD_PqJt z?>*<-bN4%w@4WrhuPcgD6&lV{R7KO3kfr1(xkx!_aQI+FVkpslC|PkR(K9$WRB@=M zXQX0icyM&Ed$6yfCo!5B80<;XUn*(`2SyXU1Ighw3>ZPm&fbC1ke+1|@&OKXRaAdlHlZ%ARK6NgPb>s$ATVLA)7TZCFa4Vj`vX zxHaxcj&u+A4vh{DBW1^=HLY`Uk1TfoO;ocir2yAP^CVR}lglf#Gmk{!dzq@?08Z?UD!JdWz&ctt%+ETt6dx!srL+xiB( z4|80Z!HIR8Z7FlGjwc@%r>NgNOPTLqzuH4b1}4^Tp`}Fq>zB?c>bKZZmiUj!cyge} zY2@w6k)gqX5wDs)!BWco$E2KD)OER~tiZZ9`Kp&R#&eD&M@Bt@a+Rg5#;VpGKe4Fk z8cSJ=HEs7rSUWg$G{dW;=k?IH(o(9hzNZAG7qwnzDeJM;rydV2@ozSzH(JUjtaM{Q zc2VaOEoF;;owp}P`RU&efzfCQhNNyf+xAC@c~OYh&68YRbc|@g#PWdl(VtI{^m?x+@ijH zmeP;)^*8OVgXPAyBL@y7heteyxIs%9!a8pDXFta$>NRXBBUmqgZK^$)=TYY&pgH+V;E0%H}*3kdxN_JrU1oH!y@*vjJ|L7{A7PbAVr96za z^*_2Y$_btNHB0$AR@Psih)35MEiE#)!)p2eN3#C-Yo*5j7)P5++tpcb|LmZd!D z-?JR$gr5DjrF_S~XXVVIuHUni?_*v4t-k5(*SBwfU@1Su>iTQ4p5&s&Kem)-u*Q?< zb^H5!M`5>|=-reYI5>L9Pn-LRrTi3YKZ#yh( zloZz1U(Y9J7IpoDrTh`=>Tj3VWt=c5{$weC#;W?8QcHRX(|80O8c8vd9#Z;|Jzdj<3E03 zyUn+g|7#(|fBeds1znM9ACDDzwlvtdqZ+P>ENY{J9 zx}RmC60Ekrz90*|sCAizX8YF~&#orr?70@2=U?kA^rF@aEVK}7?XL$*qn*&zi!8Jl zEA6ib^JW)yUTUEeu+ILbR&3AJzJ=i@u$NhAIab@>u$i?0QS+4+T7@US!g%r z^|uChd>z|X8*i*@P7Kjo25G<74~z~U?Gg*xW1+p+LH?GOX@pKe(xoLj-9l$zBm6Bb zy|_hvyDW4T*4JOFbU5)uGRw^iwBJJASk-O5R`3#9QOl%-4qz?)ZH+s0wS)criGdzJ zi(t;AKo-J?j!g~Pkv(UXv(l`q?6Dk{3f zLhnVFBBgjZc@QS&$>Ey5zWVb=lLI5Ya5E1YW%;D;Fi*I7N@@5@fDOesK9qt`CXrK?EHO1x}+jq5ny~%-5(tW_? z<<&#@l5%7Qy^_2dUrAow!)h)%Xil?pv0982nfcxJyrZ$M1T9d@wa7G6hr6MPKKMZ; zKP)hd_v725xC`hVjVGZM4Rj|PdrGtgs64a@t%=Mqt>NTQUvGD!aiAx8en|)(4ki2G!Z&2kA=ytcr*#kGOWcl% zCAsh*Kekq`M3#{>(qI+3B2LpwjBGk(nYm)F9(v8k4uz7u zY+PB^9oW7;era4{X0y+8#TeOP7Iq684kh`7XpP)a-mCn`iKa2cHo;0P!m(V=A=@Rw zxeu}};F5xDxn+B#;EjChS<>E03JJj~dGXY%1lW#S*a&UZ+td;iImyh!UZpJ42+ImN z3-)LR2hLBT4qvG*31`bsA`ZiN9U@yz^E~I$O38FWuu`so{6>fC333UdBr>rl*xe(! zCn!r%HcLosWmr;YS`ZD+#n+Mg?nZ&AeNNC4+Xfn9^m)l zOG-(>9^H}W7Imb`3?<=SKuHLEq!HrEkL4Dfb zM8`Le&R86r(r6$moH2)uaUL`f84gm6&3)d(6CLlebT=C ziK^Mig9o_$*a`BkkB&TiwllH}J$Pu$k8PGyfxny{gdea(-?4zCEt8I^9-+XEJ)C~g zWA|ug#3=|b+h{*6U9hb&jsE0l0xxG^#iN1p^W@4qtFdCUpxf@;+7hFO*pM>a%-er- zG&#~alI$UC94%s*p%FN#C+idqFOT-qmEe(qQL^IDBDH1?ei1fF`qRRgN2|%SWpy0q z+7xPSt8a-ka|)4V9_~^LV~gZ=IAj>QX}WEe^K)XtiCV;BCZzAmhRVY%$zAy}T~b++uNuW9Pou_v5^b2hf@s%dJh3+G~?uD+?h z12XB6Lep%ouWj8`-;QaG=&?JHIm^*P-0+O-gYnw-Iei3cSzKA$CNNUuS)ImDHcrdd znKNX+yUT6_3wvSNt!+{_py=cGB6D`e%)p&=M7eBF#o?(`yJ>*($GB!2EknI+pIwLDj zz1%EKn`Cjs5USmSxR1d*I0e|{X;_X)=5pMw{JVUPS?0Zi5ZwwbB+DEQddNeGkwc`O zGxnOW`dakyOOTmI1_NBpk#(j{dQrCHmLFRp4-O3G_FQrMJ=N@edzv|;n`}C!t&j0WWqvHc1SA?aOVef*oI%G~LD4p|ERB>m<~ZCGZ82xK zbN4CGaHm=9@yRCAWFESLiQ&M+5|8J2ARI0f`0=3-BJ-W$gm0C=aFv;=z37@`~Y z)1zf(iQp!#py-KaF}Ez@EQAdlm{kprHaR=-Yy_v>BHl45h&Gusx!fFYBrVpmGdN+Z-26RgqW8D(@bL+_Lf=)`i>H_jOb!> zdgfvzy_D!?7N!$zab>Iqa4)^jTtZ@8nwfW`EO%tCS?tVG`SlGk(~2JZ28?9x_y7wA zGN801`sv9`9(PyyuD0gfUIaN z7|shE190n!xT7MSk54ozM*B_6<|J{o#te<%6d0a`TT_bxaVVOPRugzaRLsGQlQ6UZ&vSOd+~IE4<^3{_I|hWT+Y0gdF^vQK z1ePWF90Kos!Hs#yLOY-H`-cIS`M)2+M7oPTC`ooy@z0Zb;TO;Pqo+%x~AQ+bk6o z<2|y8IgeIVmhH&Q$cY}i4Vk5B-)7>B&X`HyYb3CxMkYV}{Lh&Tz`YUvG?L8uBK7z- zNPDU+@JWtjuAv$k|ABNi#~PhC}H@Nf!t` z8D497Z0MvJ867o?ox*Hyk$snp28BJ)U=}%3bD=~$HCiSgXB~PsLv(;Mk?w+$GMz0| z+$1Kwr3yGVLvbSb{!*+jfPG`RQf>na$XHt*|{0xU^kK z*rA6X)Akd?A)Hy*7=-P5%QWoGB-pP9$?j}n!+sjfD@b*dng|RTFsZ4YW&%j<0_MyVX_#gx?Ji{|1bIcM#EmQAe!Z|i7mZGjq}XO?xg)NF&|I$FV~n!2v$`i`2qnvR;T zx@`~u;90wbZB4DUJ0S)@PkaKj8%nG1YG|t2PMp!$4kttf_@T?!(AZMfMY$T=Avk85 z5!*tVJKngr9zq1nsNGRtyEER|+|}F|Z?5U6-2rh@V$Mk8s@>7qvJ;}E6!uTsTRU27 zyXxRLstyjLm}eu{QFa+M?b|z>>svY?Vt}fxv!e^Bo0__acWF(`z)!Z>TIE>YGR&J91iptt*f=A3!ebGfN)u)%$%8# zt-ZdZeNT8c9~Q12@s65~`tTg=xw^*q&aPH4uC<{n4!YEb=Ze8s(}e4F4^%sent9?= zr@E;_=0_I5QITUWc2&Hq76_oODPn<`->EqIbRo6{DygM)H)mp3$DX$Oa1?uvL{S$G zN^o6sV+TH1TqH)UdltDE$DhqcUZ)5z!EJ~9(+!oSII>)&F*$Z|_yiJJjxi8g%z-$BstYwx2-F_r=>Q$oEVRzkVa8s3ltj% zzt*?Z?g_7mEHvl3)!YJX_04S^d$_t@8Ch&Ds6P$&BJQtln`4Eq!l=gP2~ewXY*0Jw zI))Yp=eB!6{|X$9u20|wyVJ7PEME#+LiX@O1}cc7j6>xxS`PRA>|vU~pi>2->Y+Q# zGjnR&Yb#^W0pgQn{Lfl?svO@@6RTbaeVlG;&2`n`oaIq7>}Cu!zS0 zSPMn%Nt?t9Q(Ro<&<)|n)(+heo*X{6w>ud*0*C(adE1D`H7mzgczh!7gt%*gvdRf2B2FG| zZ4!O-qH?qb#UB3(Xx0dC0wwSg6(4<(89iKVw>29yjh?~t2Jnt-#N$Rw+T5R0&CgOo z*iTu5BHd>0ks(r?$Kp0^)=Y{k^SFkUhNT`TN89Sm)$kSY)%2_6g8zotSIM1Q{Pc$Z zwrNfs5D$$Uy9FT~eGp!AnC$1o-8T+K;5*{{8we)48hzM8*PxFeWuE<}$+jcCeLc=6 z$Z>{-;lX~S+)=)-$$d)$rX%pK!sdNb`98$5sU*X1j9h1-kD}|5QgARi>Usgb;8Z@f zMS&YXW}%Ox8<8>%iejH_#~lc~mJ)%&h~JKk_V!iax#;G6bQ8K+N1wFNE$CB7DHI>W z8!tct@41vWvYgy|G4RsJXW%1Kwn0R)y} zcpnA^1APT4Iphs31Kkgb*V30$4D=uf9rWcB1AP@#wL3EWj){RDfft6_;7EeLUt*xI zLrA2phz#^7gru+{GSD%=rLBkz^iB9NZAE0DZ$V~ZMP#6-Acn*RhJn6=6laxWpznbM zYXahN13e8@!>y7G^h2b$t0V*c7({0PGtjeOle;J~&`+K0WLadOpR?>td=9+AuZaxw zJp2O1I~!^S`V~^V);|V%3E=hI<#q%82E5KjQUm=CJRq!o4D>Qm$m+*HDWrI=ehl;n zW*KwDEAW~bUHcg5&rk%5XaoHP{MF2E+!^R~D9{OIe4EZde*>az_Dwni{T<53=Y3_{@ULksSvN2xqrNZ{!ZV`&3up6P@yE(bR6o2YC!cv2V`G2GE{gS zkp4hp%7Bl;C^@zGa*v@Ff@5~Tb)_x%lZ9heS^8gPYADa3=Vd&+63n?NHHKkA8K0|`U! zoY7EEB3$f@(NMSA)YOV?;CtdSL#-n&5Rab>wE>L4P@n0c0YOZ4H^&V1ETqVG0KuoVFdu~+@M>_fxaehk51ae?7|D?e#Djz2fE87!wSv<@GIS9N(dUwCSK zu<()V;3|QQW`g`xoW*O_K#Mpf;QY%S(mCd}n0e1}F#DR%KtcJ;1SkkfawJl&BW)0# zZRDluNPX6xFB_Evgd1)uKh!qD9prRkf(bI^tDF$#tR$P!6x;daqL9ki%N=JL6sPURib07HVSU9+9=d-^%|k2RId@Lta@#lC{R_4J#e+? z*lN*r)ne3DSBd3Ti3N#KS1m?ewb-1i#ZI+a>^7^#AjI7>=l1FiL=EG_Zdd}7?{^gT zIG@14lRB7vN!H#lLx>L^fkg%0a3ZsPc6vp3zgFWd5*QL22jEZR)%GPuMv^1C`WN_Q zPVn6e*rI93`15+Y`d9dS9Frw?Cnhj6G+hbi@Z!#!uD%IVM~?nDW{!*3>gqq?17rbN z-1W$?{dWxc39@^BtgEnq-8c26wGNXl$K)_lxNn3w$~cd8^}q04peZY#ttRjSMb{Ag z$%6j{v!gt0u)2=sigp0H7MjAkbk{q&221!Ut2LL`QJk+e$zxs1gALb!zD--h=~_Pg zUCpWGI29AtlRIJY9qRw*CLJvKIEkaF78YU7?gBE|dqq^`jpV&Z>QvST1p zqG@h$)pJ0)wj#I}_`Q64V~|YLzzd$wbPaY#gL=V}-7yfnFfmBi)&}o&#bR7v(-R2i zu!Fw>tZT5(dse_m@G6&c!NLhKy0$)e9Pp<+>2cusOxIxlI%phtvU5&|P7KnuEy3eJ zEJla}uSdEDoAN>9z>A!7!NLhKx>hTm0Uyswgq%mxPjLxj19^Fo%s_I5S`? zkeA8XK7Rs)t~Caq0dZyU6)%4;MAzV`B`+=#Dwi{V;e;4n>qN?~fYB$E!q>dlOG~;2XINeTn^HOR7kDVUF!?pTHV_#!Z-wXP&}UK+FcP#~zF;DV zuEDNY&}ioiaVr<1UH(AVjs%Z(IW=ec{0R`cc0T->jDY>6ePsZ5B)WD%@aDr4a>f)( zf7i8l!55Fv_GN;TLD#NEO85Uv zN4`U%pKg)7wj?g0Ka7;V{{wv-4mWQnexqw}y*udASY9S)Gnr~;YULnw?Q_A$N=^&6HsNqU`n#^d_3)reV=1#^6db{2 z2k6?Dg4Z`5rZYVYlP&oJUAr@Qeaopi!^u`irdAF@*X{{k-<%dgeM^7WwflnCx0Km2 z3hFyMK-V6es;!Vr-@+y^U5aNu(zS=DYAYm@oHL(og=E3#+SjLOY|p%#oz~y^`YfBS zJvK#S`vm%MUZP{WU%ak;Gx(7fnN7$_yHrBjc#TOYX2&S# zRkH(h?We&v1l=NBzru!~>}Os3Merkb88K(_+^o-a?fKxdaN&ifw7xUTH%E2t#o+ys zciEf|7WqKw8oa0#^oTvJ96kWNzv!xKzYl(>=vgx7hUml~T}uUT_i%70ZTHgOb?uMA z+r5;TvwU85fUf;Hc)NFt;HzEwQDR+tEqJ?^mB*PpH|uj<`&;mxFjC2MfAa6acfuSt z&WK3nkGcjINB0JtH;8YXrgd4_KB%p0aFI0Vj#8G=IolWef$17tEDgGgl3oxWzgfOL zu7}j%{kM17oGTXjK8Q9~Itz%|;K!f{!PjC39|wP7KmR)xrCVw9l!e_ZQh7>!A(7`-^M^aPC;* z3#W(R&GNuA@nPYUoBU}hQnLN=Cp`q8iU>Rt_b#1t$KnikxZ&%-^iWOk{X(a&5Wjf> zmwh~*=ppzHM$oHy9^8&;NLeruL=SBbz5;X#!D~u(Ctvb+=JXJJ!Xt3p*J(u`$ zneb2#9SVN`giKlRjXL&mF+Ftl)ZIG7pJ;-2*6q82Sky!CCB?v30_`SF8R4dCeZVFH zb13-!vmQDJKKB!l3hSh-g^w@3OHq^yl``}W#e{#Aq4Skok{)65Fp~$F+{ffzCLdsO zH_ zFK6-zOkT|7g-o8ufNL$K=m2`4%SM#N>}N`FbW_%j6F;`D!L#$>htKd?}OP!{iH@{7xnx zW%9X99%1qjllz%`Hj@uBxrfOKCZEaVeM~-$$(>AYXL2i(o0#0l&9n4HJt93q?Oa&!f>!}mZB za1pv#L23e|3I)M`ijuSZ0rbwh;UB8HAO4?9el*l>__sv?YLA)(DG$9DUCJ`A02yaS z=^696j7UA89;6x7Lo8z@WTT}%h~DF7hA+bcaGDZQrgH>2K+sDF;FGb8U=GQyD4~1M zWp`l>T&-WeGHj@4tA{g@N7Ims4#+;WKNESb3^^d<9FZXhWt>AYSe6Y?2}0DdTLEA>SqAY>*+}E#s_}Aup71Zj~V~l5uX4AupD3ZjvG2Bja2zLtY}| ztdb$$E8|=%LtZN5TrERhCgWToL%vVOSuR6fF5_GxLtY`{jLMMjmvPRQA+MBi&XFOn zl5v*GkROn7&XggqmT?xzkROzBhGobP$v6vS$PddnO&RhU8K*8oeniF@k|D2^aUvP= zIti!pwhZ}E38(Uw40*kb^B*$g4KmI*WXO-nIA50`KQ7~ZRffD##`%g2`3V{4xD0ud zjPqp~@@5(5Z)M0&$~a$=A#ag!z92(>O2+w|4Ebpp=g(xw&&W8Rl_77HasEh#{H%=g zX&Lf!GS2VHke`=vJ|#nbLB{!n40)T3^Oy{IyNvT2GUOL!oR7$mUy^Y?Btw2##(BRC zd54ViUK#RE8RuOxk>B^l>j z8S>XM&N3PDH!{vyGURV%oHJy|-^nA#=xgZxd*I`qSH*WO*S?0Xeh_`=UUb=m=-T_y4cbHK6RPqk`t-f% z=)>sNJ@=raI)i^Mz9&?Kz7XG&Q-p4h@5wDfUy46~?zk77OS6XDtRpn*NDj+-H_3W0 zg%449jtlRn@c!@|hQE*CA0S0Ru~5`Q@jZGGdN{tvC_-P0?+L?C-+%;udkhlz@o`A< ziqI2~m__JGNb-x&w;{2L(03szC_>+dq_7D60Fr4%=tq!*i_kNWOfNz|fh1Cdeg;WV z5&8urGm6kJAt^3GFF-Q02)zi&tRnPlNJ@&(Zy_lyLcfQktO$)kGP?+kQwz_g7UsGZ z9;6l?43|nQ{3EgOPsBp74UEC2frZ#Kun?ODKVs9sLTnmXh)n|vv1wo-HVrJqrh$dn zCf7o28d!);0}HWfU?DaQEX1aPh1eLd5E}y)Vq?HUW=sz?Mt6-#P-7BdLx_x5Np-zO zjQJ}u<_%)Zn?MJ~{1Xx|<}K=?GbyX#vhJg-`|Rla4+Wlf588J>`Y#ukofOz93occq zykXAw?G)c`$1&&$ar_q1_De+Df6}5_SzgZ6CYrYin#ub~tT!?Yr+5R!G&q=AhRJh{ z+e$H8nQ>|kEpZD&nl5q^MQ*aKBigDu#jj`hd>3Cu@l{UgYZ=CJF{>$NwOx81Wn00J z1un9jBFi1NB@9#OVxkliwb}A1+kA$c<|5}%bsEO2(@DL9S96QhA{M2jW_89;PbM zJp+A&3gjPbuY`c z(9Lxh&2^Vu;xdwJ1u1bADY1g){W8lNb@Sd%^WJXTwTvP^&yb5;QN#qW0-WEaC!xR}!^=5*2O zoy51SS?*!E*0{NL(OkQ1olmAZ#~E_1i)^FFHk-eO)Oa(iJED0r%N28TokDY+V)L_F z+s=@cF0zgy>ujG;epX*K3}5BqPonsfY-ZNHH#20li`+nw8-&S|+DbI1t>f3EW>2NI zS2KonE<=nm#OwyfS|mrI&LR{L#L&Rvl&FFct#^r5QlgbxP|O0mwLvM+jX)O)9cd?NZM!M|T$Xo}o41VSEwl5|2!&jb zcMr+CkLI1l@@{tX&Y*c`*fj~cpxiSkemcXS=;8}0zEJG$x+p%M;kUSWo#J&H5A$4@ zUecuzcOykDR4!4Jmp`QBk9|uCk3Ff(9Q&?PKK4E3)Uh8a-D5viE*krpa?RM!(f+X) z(CFB&)QYj+sx@Q3Q!gDGSFa!YgL=oIeSy5S*p=sS9!+OI?`zZ0e%i*HV}2q10vi?9`R|>eN;G?$n3%vs2gTA4pxV-;%mP z|90vo{rS|*`d?F@F>+J48go;(8EaCv8_lUZjk8jB86QpEXWW*$-*`0ju<=al5##Tv z$MW)0$MRySC-dr4PvxDS`hMPE>gl{|QqSalHuY@Yv#DR?jisK;dpq@_IX(4~xjOZG zvnKVj*`E5NnM}Q6UY>f*yfO8=3A+^^CRP0{*v)f{`JpCpS-h8`79kssWd#=H_V{ z$Y+6^1F{aX0rHt3!v!bJ0=WR>y&z8mc{<1uNN0dt4CyS8OCT)+c{ZeTL7oTc0+1I% zx(MXOkS+!J1W1>Gyd2V%Ag_Y70^~K2Vtpzhtp<4=q#Hoq22{EJKza(uJ0WcbxdqZTkWYm)4sr*iyFlIz=^l{xf;|l&pAPbwAa_BU0C_*8 zJs>9`JqYq4NY4iOFr@t;4?sEu@;Q)>fIJH6xgehheAvFDAio3TcS8DZkS~PvVvyku zo7w{MdqKVoz8q!-q{w$=Q2l)$--VX8?A^kGQcR+d<$ah0}FUVhk^nQ>Zfb=1dzY6IiAb$Qh1i1-Ph>uV!q^E-14rvF-osjMZ`7}tO z{e|{HdIrd6LJI9Qlz_AwxJ9o(It0%*Z7uzO Dn${hdds.version} test-jar + + org.apache.ozone + hdds-datanode-grpc-client + ${hdds.version} + org.apache.ozone hdds-docs @@ -1210,6 +1215,11 @@ ${hdds.rocks.native.version} test-jar + + org.apache.ozone + hdds-scm-grpc-client + ${hdds.version} + org.apache.ozone hdds-server-framework @@ -1725,6 +1735,37 @@ sortpom-maven-plugin ${sortpom-maven-plugin.version} + + com.gradle + develocity-maven-extension + + + + + + maven-antrun-plugin + + mutates generated sources + + + + maven-compiler-plugin + + compiles generated sources + + + + protobuf-maven-plugin + + generates sources + + + + + + com.google.code.maven-replacer-plugin replacer @@ -2246,11 +2287,32 @@ com.fasterxml.jackson.core:jackson-databind:jar org.apache.commons:commons-compress:jar org.apache.ozone:hdds-client:jar + org.apache.ozone:hdds-datanode-grpc-client:jar + org.apache.ozone:hdds-scm-grpc-client:jar org.apache.ozone:ozone-interface-client:jar org.glassfish.jersey.core:jersey-common:jar org.rocksdb:rocksdbjni:jar org.springframework:spring-core:jar + + + org.rocksdb:rocksdbjni:jar + + org.apache.ozone:hdds-config:jar + + + org.apache.ozone:hdds-datanode-grpc-client:jar + org.apache.ozone:hdds-scm-grpc-client:jar + @@ -2285,6 +2347,15 @@ org.apache.maven.plugins maven-clean-plugin + + false ${project.basedir} @@ -2305,6 +2376,39 @@ maven-antrun-plugin ${maven-antrun-plugin.version} + + + pre-clean-force-delete-target + + run + + pre-clean + + + + + + + + + + create-testdirs @@ -2318,6 +2422,196 @@ + + + pre-verify-delete-test-classes + + run + + pre-integration-test + + + + + + + + + + + + pre-verify-refresh-classes-from-jar + + run + + pre-integration-test + + + + + + + + + + + + + + + + + + pre-compile-delete-classes + + run + + process-resources + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + pre-test-compile-refresh-classes + + run + + process-test-resources + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 43464b77715761ef0237f3965e3f4d3f294fea1d Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Thu, 2 Apr 2026 12:18:58 -0400 Subject: [PATCH 02/12] =?UTF-8?q?HDDS-14949=20=E2=80=94=20fixed=20CI=20bui?= =?UTF-8?q?ld?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 41 ++++++----------------------------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 8c6281b85d12..301ad67f5650 100644 --- a/pom.xml +++ b/pom.xml @@ -2515,39 +2515,25 @@ hdds-scm-grpc-client) from their installed .m2/ JARs before every module's compile phase. The LSP can corrupt these modules' target/classes/ directories in the window between when Maven compiles them and when a later downstream - module reads them. The delete + unzip restores them to the last-installed - (known-good) state; for the foundation modules themselves, Maven's subsequent - compile goal overwrites the extracted files with fresh compilation anyway. - Prerequisite: run "mvn install -pl -am" after modifying a foundation - module so that the .m2/ JAR reflects the latest changes. + module reads them. unzip -o overwrites any LSP-corrupted files with the + known-good installed versions; if the JAR does not yet exist (e.g. a fresh CI + build) unzip exits silently (failonerror=false) and the freshly Maven-compiled + class files are left undisturbed. + Prerequisite for local dev: run "mvn install -pl -am" after modifying + a foundation module so that the .m2/ JAR reflects the latest changes. --> - - - - - - - - - - - - - - - @@ -2576,33 +2562,18 @@ - - - - - - - - - - - - - - - From d0dfa6d463aba8169450b022453d200b7be5c6fa Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Thu, 2 Apr 2026 12:58:43 -0400 Subject: [PATCH 03/12] =?UTF-8?q?HDDS-14949=20=E2=80=94=20guard=20pre-veri?= =?UTF-8?q?fy-refresh=20against=20absent=20JAR;=20disable=20for=20grpc-cli?= =?UTF-8?q?ent=20modules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-verify-refresh-classes-from-jar antrun execution (bound to pre-integration-test) did an unconditional rm -rf target/classes/ followed by unzip from the module's local JAR. On CI with mvn clean verify (not install), the JAR is present in target/ but on pom-packaging modules or in any edge-case where the JAR is absent the directory was left empty, causing downstream reactor modules to fail with "class file for ContainerProtos$ContainerCommandRequestProto not found" when they tried to compile against the now-empty target/classes/. Two-part fix: 1. Root pom.xml: replace the unconditional rm+unzip pair with a single /bin/sh one-liner that only runs if the JAR file actually exists, so target/classes/ is left intact when no JAR was produced. 2. hdds-datanode-grpc-client and hdds-scm-grpc-client: override pre-verify-refresh-classes-from-jar with phase=none, because both modules set mdep.analyze.skip=true (no dependency analysis runs), so wiping target/classes/ at pre-integration-test serves no purpose and only risks leaving the directory empty for downstream compilation. Made-with: Cursor --- hadoop-hdds/datanode-grpc-client/pom.xml | 11 +++++++++++ hadoop-hdds/scm-grpc-client/pom.xml | 11 +++++++++++ pom.xml | 21 ++++++++++++--------- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/datanode-grpc-client/pom.xml b/hadoop-hdds/datanode-grpc-client/pom.xml index 01315c208a7b..07a009af1632 100644 --- a/hadoop-hdds/datanode-grpc-client/pom.xml +++ b/hadoop-hdds/datanode-grpc-client/pom.xml @@ -135,6 +135,17 @@ org.apache.maven.plugins maven-antrun-plugin + + + pre-verify-refresh-classes-from-jar + none + + pre-verify-refresh-classes-from-jar + none + + + + From 051104e150bde5e32dc453e8887e3072c3c10901 Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Thu, 2 Apr 2026 13:31:47 -0400 Subject: [PATCH 04/12] HDDS-14949: restore foundation classes from local target/ JAR instead of .m2/ The pre-compile-delete-classes and pre-test-compile-refresh-classes antrun executions were restoring hdds-datanode-grpc-client/target/classes/ by unzipping from the .m2/ repository JAR. On a CI clean-build (mvn clean verify) no Ozone artifacts are pre-installed to .m2/, so those unzip operations silently failed, leaving no safety net if anything clears the directory after compilation. Switch the source paths from ${settings.localRepository}/org/apache/ozone/... to ${maven.multiModuleProjectDirectory}/hadoop-hdds/.../target/X.jar. These local JARs are created during each foundation module's own package phase (which runs before any downstream module's process-resources), so they are always present on CI. The unzip still silently no-ops when the JAR is not yet built (e.g. the foundation module itself hasn't been packaged yet in the current reactor pass). This makes the class-file restoration reliable on both CI and local machines without requiring a prior mvn install. Made-with: Cursor --- pom.xml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 6d2ffcee6e32..c1fb74caa38e 100644 --- a/pom.xml +++ b/pom.xml @@ -2515,31 +2515,35 @@ - + - + - + @@ -2567,19 +2571,19 @@ - + - + - + From fe268cef0d8455dc0a95c9588e6bb96fb11576cb Mon Sep 17 00:00:00 2001 From: Andrey Yarovoy Date: Thu, 2 Apr 2026 14:11:03 -0400 Subject: [PATCH 05/12] =?UTF-8?q?HDDS-14949=20=E2=80=94=20fixed=20CI=20bui?= =?UTF-8?q?ld?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hadoop-hdds/common/pom.xml | 4 - hadoop-hdds/container-service/pom.xml | 4 + hadoop-hdds/datanode-grpc-client/pom.xml | 30 +- .../src/main/resources/proto.lock | 1832 +---------------- hadoop-hdds/framework/pom.xml | 5 + hadoop-hdds/interface-client/pom.xml | 75 +- .../main/proto/DatanodeClientProtocol.proto | 0 .../src/main/resources/proto.lock | 1829 +++++++++++++++- pom.xml | 10 +- 9 files changed, 1943 insertions(+), 1846 deletions(-) rename hadoop-hdds/{datanode-grpc-client => interface-client}/src/main/proto/DatanodeClientProtocol.proto (100%) diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index ba1d2259b61c..5d91b1cb43eb 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -112,10 +112,6 @@ org.apache.ozone hdds-config - - org.apache.ozone - hdds-datanode-grpc-client - org.apache.ozone hdds-interface-client diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index da0f5a45f981..9d225466cafa 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -138,6 +138,10 @@ org.apache.ozone hdds-config + + org.apache.ozone + hdds-datanode-grpc-client + org.apache.ozone hdds-interface-client diff --git a/hadoop-hdds/datanode-grpc-client/pom.xml b/hadoop-hdds/datanode-grpc-client/pom.xml index 07a009af1632..543be0eae63b 100644 --- a/hadoop-hdds/datanode-grpc-client/pom.xml +++ b/hadoop-hdds/datanode-grpc-client/pom.xml @@ -24,10 +24,11 @@ jar Apache Ozone HDDS DataNode gRPC Client DataNode XCeiver and IntraDatanode gRPC stubs generated from DatanodeClientProtocol.proto. - Package names in generated sources are rewritten by maven-antrun-plugin (Ant replace task, pure Java) - from vanilla com.google.protobuf / io.grpc / com.google.common to their ratis-thirdparty counterparts - before compilation, so target/classes/ already contains the shaded namespace and downstream modules - in the same Maven reactor can compile against the correct types. + The proto file lives in hdds-interface-client (which also generates the shaded ContainerProtos + message classes); this module generates only the gRPC service stubs. Package names in generated + sources are rewritten by maven-antrun-plugin (Ant replace task, pure Java) from vanilla + com.google.protobuf / io.grpc / com.google.common to their ratis-thirdparty counterparts + before compilation. @@ -39,6 +40,14 @@ + + + org.apache.ozone + hdds-interface-client + compile-custom generate-sources @@ -120,6 +134,8 @@ Output directly to proto-java-ratis so that both Maven and the IDE/LSP compile the same ratis-shaded source tree, eliminating any race between Maven's javac and the LSP writing vanilla class files to target/classes/. + DatanodeClientProtocol.proto lives in hdds-interface-client; point protoSourceRoot + there so protoc finds it without duplicating the file. --> com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier} ${project.build.directory}/generated-sources/proto-java-ratis @@ -127,6 +143,10 @@ false grpc-java io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier} + ${project.basedir}/../interface-client/src/main/proto + + DatanodeClientProtocol.proto + diff --git a/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock b/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock index 9e99c1a330c2..338eb7a3db19 100644 --- a/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock +++ b/hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock @@ -1,1831 +1,3 @@ { - "definitions": [ - { - "protopath": "DatanodeClientProtocol.proto", - "def": { - "enums": [ - { - "name": "Type", - "enum_fields": [ - { - "name": "CreateContainer", - "integer": 1 - }, - { - "name": "ReadContainer", - "integer": 2 - }, - { - "name": "UpdateContainer", - "integer": 3 - }, - { - "name": "DeleteContainer", - "integer": 4 - }, - { - "name": "ListContainer", - "integer": 5 - }, - { - "name": "PutBlock", - "integer": 6 - }, - { - "name": "GetBlock", - "integer": 7 - }, - { - "name": "DeleteBlock", - "integer": 8 - }, - { - "name": "ListBlock", - "integer": 9 - }, - { - "name": "ReadChunk", - "integer": 10 - }, - { - "name": "DeleteChunk", - "integer": 11 - }, - { - "name": "WriteChunk", - "integer": 12 - }, - { - "name": "ListChunk", - "integer": 13 - }, - { - "name": "CompactChunk", - "integer": 14 - }, - { - "name": "PutSmallFile", - "integer": 15 - }, - { - "name": "GetSmallFile", - "integer": 16 - }, - { - "name": "CloseContainer", - "integer": 17 - }, - { - "name": "GetCommittedBlockLength", - "integer": 18 - }, - { - "name": "StreamInit", - "integer": 19 - }, - { - "name": "StreamWrite", - "integer": 20 - }, - { - "name": "FinalizeBlock", - "integer": 21 - }, - { - "name": "Echo", - "integer": 22 - }, - { - "name": "GetContainerChecksumInfo", - "integer": 23 - } - ] - }, - { - "name": "Result", - "enum_fields": [ - { - "name": "SUCCESS", - "integer": 1 - }, - { - "name": "UNSUPPORTED_REQUEST", - "integer": 2 - }, - { - "name": "MALFORMED_REQUEST", - "integer": 3 - }, - { - "name": "CONTAINER_INTERNAL_ERROR", - "integer": 4 - }, - { - "name": "INVALID_CONFIG", - "integer": 5 - }, - { - "name": "INVALID_FILE_HASH_FOUND", - "integer": 6 - }, - { - "name": "CONTAINER_EXISTS", - "integer": 7 - }, - { - "name": "NO_SUCH_ALGORITHM", - "integer": 8 - }, - { - "name": "CONTAINER_NOT_FOUND", - "integer": 9 - }, - { - "name": "IO_EXCEPTION", - "integer": 10 - }, - { - "name": "UNABLE_TO_READ_METADATA_DB", - "integer": 11 - }, - { - "name": "NO_SUCH_BLOCK", - "integer": 12 - }, - { - "name": "OVERWRITE_FLAG_REQUIRED", - "integer": 13 - }, - { - "name": "UNABLE_TO_FIND_DATA_DIR", - "integer": 14 - }, - { - "name": "INVALID_WRITE_SIZE", - "integer": 15 - }, - { - "name": "CHECKSUM_MISMATCH", - "integer": 16 - }, - { - "name": "UNABLE_TO_FIND_CHUNK", - "integer": 17 - }, - { - "name": "PROTOC_DECODING_ERROR", - "integer": 18 - }, - { - "name": "INVALID_ARGUMENT", - "integer": 19 - }, - { - "name": "PUT_SMALL_FILE_ERROR", - "integer": 20 - }, - { - "name": "GET_SMALL_FILE_ERROR", - "integer": 21 - }, - { - "name": "CLOSED_CONTAINER_IO", - "integer": 22 - }, - { - "name": "ERROR_IN_COMPACT_DB", - "integer": 24 - }, - { - "name": "UNCLOSED_CONTAINER_IO", - "integer": 25 - }, - { - "name": "DELETE_ON_OPEN_CONTAINER", - "integer": 26 - }, - { - "name": "CLOSED_CONTAINER_RETRY", - "integer": 27 - }, - { - "name": "INVALID_CONTAINER_STATE", - "integer": 28 - }, - { - "name": "DISK_OUT_OF_SPACE", - "integer": 29 - }, - { - "name": "CONTAINER_ALREADY_EXISTS", - "integer": 30 - }, - { - "name": "CONTAINER_METADATA_ERROR", - "integer": 31 - }, - { - "name": "CONTAINER_FILES_CREATE_ERROR", - "integer": 32 - }, - { - "name": "CONTAINER_CHECKSUM_ERROR", - "integer": 33 - }, - { - "name": "UNKNOWN_CONTAINER_TYPE", - "integer": 34 - }, - { - "name": "BLOCK_NOT_COMMITTED", - "integer": 35 - }, - { - "name": "CONTAINER_UNHEALTHY", - "integer": 36 - }, - { - "name": "UNKNOWN_BCSID", - "integer": 37 - }, - { - "name": "BCSID_MISMATCH", - "integer": 38 - }, - { - "name": "CONTAINER_NOT_OPEN", - "integer": 39 - }, - { - "name": "CONTAINER_MISSING", - "integer": 40 - }, - { - "name": "BLOCK_TOKEN_VERIFICATION_FAILED", - "integer": 41 - }, - { - "name": "ERROR_IN_DB_SYNC", - "integer": 42 - }, - { - "name": "CHUNK_FILE_INCONSISTENCY", - "integer": 43 - }, - { - "name": "DELETE_ON_NON_EMPTY_CONTAINER", - "integer": 44 - }, - { - "name": "EXPORT_CONTAINER_METADATA_FAILED", - "integer": 45 - }, - { - "name": "IMPORT_CONTAINER_METADATA_FAILED", - "integer": 46 - }, - { - "name": "BLOCK_ALREADY_FINALIZED", - "integer": 47 - }, - { - "name": "CONTAINER_ID_MISMATCH", - "integer": 48 - } - ] - }, - { - "name": "ContainerDataProto.State", - "enum_fields": [ - { - "name": "OPEN", - "integer": 1 - }, - { - "name": "CLOSING", - "integer": 2 - }, - { - "name": "QUASI_CLOSED", - "integer": 3 - }, - { - "name": "CLOSED", - "integer": 4 - }, - { - "name": "UNHEALTHY", - "integer": 5 - }, - { - "name": "INVALID", - "integer": 6 - }, - { - "name": "DELETED", - "integer": 7 - }, - { - "name": "RECOVERING", - "integer": 8 - } - ] - }, - { - "name": "ContainerType", - "enum_fields": [ - { - "name": "KeyValueContainer", - "integer": 1 - } - ] - }, - { - "name": "ChecksumType", - "enum_fields": [ - { - "name": "NONE", - "integer": 1 - }, - { - "name": "CRC32", - "integer": 2 - }, - { - "name": "CRC32C", - "integer": 3 - }, - { - "name": "SHA256", - "integer": 4 - }, - { - "name": "MD5", - "integer": 5 - } - ] - }, - { - "name": "ReadChunkVersion", - "enum_fields": [ - { - "name": "V0" - }, - { - "name": "V1", - "integer": 1 - } - ] - }, - { - "name": "CopyContainerCompressProto", - "enum_fields": [ - { - "name": "NO_COMPRESSION", - "integer": 1 - }, - { - "name": "GZIP", - "integer": 2 - }, - { - "name": "LZ4", - "integer": 3 - }, - { - "name": "SNAPPY", - "integer": 4 - }, - { - "name": "ZSTD", - "integer": 5 - } - ] - } - ], - "messages": [ - { - "name": "DatanodeBlockID", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "localID", - "type": "int64", - "required": true - }, - { - "id": 3, - "name": "blockCommitSequenceId", - "type": "uint64", - "optional": true, - "options": [ - { - "name": "default", - "value": "0" - } - ] - }, - { - "id": 4, - "name": "replicaIndex", - "type": "int32", - "optional": true - } - ] - }, - { - "name": "KeyValue", - "fields": [ - { - "id": 1, - "name": "key", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "value", - "type": "string", - "optional": true - } - ] - }, - { - "name": "ContainerCommandRequestProto", - "fields": [ - { - "id": 1, - "name": "cmdType", - "type": "Type", - "required": true - }, - { - "id": 2, - "name": "traceID", - "type": "string", - "optional": true - }, - { - "id": 3, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 4, - "name": "datanodeUuid", - "type": "string", - "required": true - }, - { - "id": 5, - "name": "pipelineID", - "type": "string", - "optional": true - }, - { - "id": 6, - "name": "createContainer", - "type": "CreateContainerRequestProto", - "optional": true - }, - { - "id": 7, - "name": "readContainer", - "type": "ReadContainerRequestProto", - "optional": true - }, - { - "id": 8, - "name": "updateContainer", - "type": "UpdateContainerRequestProto", - "optional": true - }, - { - "id": 9, - "name": "deleteContainer", - "type": "DeleteContainerRequestProto", - "optional": true - }, - { - "id": 10, - "name": "listContainer", - "type": "ListContainerRequestProto", - "optional": true - }, - { - "id": 11, - "name": "closeContainer", - "type": "CloseContainerRequestProto", - "optional": true - }, - { - "id": 12, - "name": "putBlock", - "type": "PutBlockRequestProto", - "optional": true - }, - { - "id": 13, - "name": "getBlock", - "type": "GetBlockRequestProto", - "optional": true - }, - { - "id": 14, - "name": "deleteBlock", - "type": "DeleteBlockRequestProto", - "optional": true, - "options": [ - { - "name": "deprecated", - "value": "true" - } - ] - }, - { - "id": 15, - "name": "listBlock", - "type": "ListBlockRequestProto", - "optional": true - }, - { - "id": 16, - "name": "readChunk", - "type": "ReadChunkRequestProto", - "optional": true - }, - { - "id": 17, - "name": "writeChunk", - "type": "WriteChunkRequestProto", - "optional": true - }, - { - "id": 18, - "name": "deleteChunk", - "type": "DeleteChunkRequestProto", - "optional": true, - "options": [ - { - "name": "deprecated", - "value": "true" - } - ] - }, - { - "id": 19, - "name": "listChunk", - "type": "ListChunkRequestProto", - "optional": true - }, - { - "id": 20, - "name": "putSmallFile", - "type": "PutSmallFileRequestProto", - "optional": true - }, - { - "id": 21, - "name": "getSmallFile", - "type": "GetSmallFileRequestProto", - "optional": true - }, - { - "id": 22, - "name": "getCommittedBlockLength", - "type": "GetCommittedBlockLengthRequestProto", - "optional": true - }, - { - "id": 23, - "name": "encodedToken", - "type": "string", - "optional": true - }, - { - "id": 24, - "name": "version", - "type": "uint32", - "optional": true - }, - { - "id": 25, - "name": "finalizeBlock", - "type": "FinalizeBlockRequestProto", - "optional": true - }, - { - "id": 26, - "name": "echo", - "type": "EchoRequestProto", - "optional": true - }, - { - "id": 27, - "name": "getContainerChecksumInfo", - "type": "GetContainerChecksumInfoRequestProto", - "optional": true - } - ] - }, - { - "name": "ContainerCommandResponseProto", - "fields": [ - { - "id": 1, - "name": "cmdType", - "type": "Type", - "required": true - }, - { - "id": 2, - "name": "traceID", - "type": "string", - "optional": true - }, - { - "id": 3, - "name": "result", - "type": "Result", - "required": true - }, - { - "id": 4, - "name": "message", - "type": "string", - "optional": true - }, - { - "id": 5, - "name": "createContainer", - "type": "CreateContainerResponseProto", - "optional": true - }, - { - "id": 6, - "name": "readContainer", - "type": "ReadContainerResponseProto", - "optional": true - }, - { - "id": 7, - "name": "updateContainer", - "type": "UpdateContainerResponseProto", - "optional": true - }, - { - "id": 8, - "name": "deleteContainer", - "type": "DeleteContainerResponseProto", - "optional": true - }, - { - "id": 9, - "name": "listContainer", - "type": "ListContainerResponseProto", - "optional": true - }, - { - "id": 10, - "name": "closeContainer", - "type": "CloseContainerResponseProto", - "optional": true - }, - { - "id": 11, - "name": "putBlock", - "type": "PutBlockResponseProto", - "optional": true - }, - { - "id": 12, - "name": "getBlock", - "type": "GetBlockResponseProto", - "optional": true - }, - { - "id": 13, - "name": "deleteBlock", - "type": "DeleteBlockResponseProto", - "optional": true - }, - { - "id": 14, - "name": "listBlock", - "type": "ListBlockResponseProto", - "optional": true - }, - { - "id": 15, - "name": "writeChunk", - "type": "WriteChunkResponseProto", - "optional": true - }, - { - "id": 16, - "name": "readChunk", - "type": "ReadChunkResponseProto", - "optional": true - }, - { - "id": 17, - "name": "deleteChunk", - "type": "DeleteChunkResponseProto", - "optional": true - }, - { - "id": 18, - "name": "listChunk", - "type": "ListChunkResponseProto", - "optional": true - }, - { - "id": 19, - "name": "putSmallFile", - "type": "PutSmallFileResponseProto", - "optional": true - }, - { - "id": 20, - "name": "getSmallFile", - "type": "GetSmallFileResponseProto", - "optional": true - }, - { - "id": 21, - "name": "getCommittedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "optional": true - }, - { - "id": 22, - "name": "finalizeBlock", - "type": "FinalizeBlockResponseProto", - "optional": true - }, - { - "id": 23, - "name": "echo", - "type": "EchoResponseProto", - "optional": true - }, - { - "id": 24, - "name": "getContainerChecksumInfo", - "type": "GetContainerChecksumInfoResponseProto", - "optional": true - } - ] - }, - { - "name": "ContainerDataProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 4, - "name": "containerPath", - "type": "string", - "optional": true - }, - { - "id": 6, - "name": "bytesUsed", - "type": "int64", - "optional": true - }, - { - "id": 7, - "name": "size", - "type": "int64", - "optional": true - }, - { - "id": 8, - "name": "blockCount", - "type": "int64", - "optional": true - }, - { - "id": 9, - "name": "state", - "type": "State", - "optional": true, - "options": [ - { - "name": "default", - "value": "OPEN" - } - ] - }, - { - "id": 10, - "name": "containerType", - "type": "ContainerType", - "optional": true, - "options": [ - { - "name": "default", - "value": "KeyValueContainer" - } - ] - } - ] - }, - { - "name": "Container2BCSIDMapProto", - "maps": [ - { - "key_type": "int64", - "field": { - "id": 1, - "name": "container2BCSID", - "type": "int64" - } - } - ] - }, - { - "name": "CreateContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 3, - "name": "containerType", - "type": "ContainerType", - "optional": true, - "options": [ - { - "name": "default", - "value": "KeyValueContainer" - } - ] - }, - { - "id": 4, - "name": "replicaIndex", - "type": "int32", - "optional": true - }, - { - "id": 5, - "name": "state", - "type": "ContainerDataProto.State", - "optional": true - } - ] - }, - { - "name": "CreateContainerResponseProto" - }, - { - "name": "ReadContainerRequestProto" - }, - { - "name": "ReadContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "containerData", - "type": "ContainerDataProto", - "optional": true - } - ] - }, - { - "name": "UpdateContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 3, - "name": "forceUpdate", - "type": "bool", - "optional": true, - "options": [ - { - "name": "default", - "value": "false" - } - ] - } - ] - }, - { - "name": "UpdateContainerResponseProto" - }, - { - "name": "DeleteContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "forceDelete", - "type": "bool", - "optional": true, - "options": [ - { - "name": "default", - "value": "false" - } - ] - } - ] - }, - { - "name": "DeleteContainerResponseProto" - }, - { - "name": "ListContainerRequestProto", - "fields": [ - { - "id": 2, - "name": "count", - "type": "uint32", - "optional": true - } - ] - }, - { - "name": "ListContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "containerData", - "type": "ContainerDataProto", - "is_repeated": true - } - ] - }, - { - "name": "CloseContainerRequestProto" - }, - { - "name": "CloseContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "hash", - "type": "string", - "optional": true - }, - { - "id": 2, - "name": "containerID", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "BlockData", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "flags", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 4, - "name": "chunks", - "type": "ChunkInfo", - "is_repeated": true - }, - { - "id": 5, - "name": "size", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "PutBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "required": true - }, - { - "id": 2, - "name": "eof", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "PutBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "committedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "required": true - } - ] - }, - { - "name": "FinalizeBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "FinalizeBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "required": true - } - ] - }, - { - "name": "GetBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "GetBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "required": true - } - ] - }, - { - "name": "DeleteBlockRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "GetCommittedBlockLengthRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - } - ] - }, - { - "name": "GetCommittedBlockLengthResponseProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "blockLength", - "type": "int64", - "required": true - } - ] - }, - { - "name": "DeleteBlockResponseProto" - }, - { - "name": "ListBlockRequestProto", - "fields": [ - { - "id": 2, - "name": "startLocalID", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "count", - "type": "uint32", - "required": true - } - ] - }, - { - "name": "ListBlockResponseProto", - "fields": [ - { - "id": 1, - "name": "blockData", - "type": "BlockData", - "is_repeated": true - } - ] - }, - { - "name": "EchoRequestProto", - "fields": [ - { - "id": 1, - "name": "payload", - "type": "bytes", - "optional": true - }, - { - "id": 2, - "name": "payloadSizeResp", - "type": "int32", - "optional": true - }, - { - "id": 3, - "name": "sleepTimeMs", - "type": "int32", - "optional": true - }, - { - "id": 4, - "name": "readOnly", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "EchoResponseProto", - "fields": [ - { - "id": 1, - "name": "payload", - "type": "bytes", - "optional": true - } - ] - }, - { - "name": "GetContainerChecksumInfoRequestProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "GetContainerChecksumInfoResponseProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "containerChecksumInfo", - "type": "bytes", - "optional": true - } - ] - }, - { - "name": "ChunkInfo", - "fields": [ - { - "id": 1, - "name": "chunkName", - "type": "string", - "required": true - }, - { - "id": 2, - "name": "offset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "len", - "type": "uint64", - "required": true - }, - { - "id": 4, - "name": "metadata", - "type": "KeyValue", - "is_repeated": true - }, - { - "id": 5, - "name": "checksumData", - "type": "ChecksumData", - "required": true - }, - { - "id": 6, - "name": "stripeChecksum", - "type": "bytes", - "optional": true - } - ] - }, - { - "name": "ChunkInfoList", - "fields": [ - { - "id": 1, - "name": "chunks", - "type": "ChunkInfo", - "is_repeated": true - } - ] - }, - { - "name": "ChecksumData", - "fields": [ - { - "id": 1, - "name": "type", - "type": "ChecksumType", - "required": true - }, - { - "id": 2, - "name": "bytesPerChecksum", - "type": "uint32", - "required": true - }, - { - "id": 3, - "name": "checksums", - "type": "bytes", - "is_repeated": true - } - ] - }, - { - "name": "WriteChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "optional": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "optional": true - }, - { - "id": 4, - "name": "block", - "type": "PutBlockRequestProto", - "optional": true - } - ] - }, - { - "name": "WriteChunkResponseProto", - "fields": [ - { - "id": 1, - "name": "committedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "optional": true - } - ] - }, - { - "name": "ReadChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "required": true - }, - { - "id": 3, - "name": "readChunkVersion", - "type": "ReadChunkVersion", - "optional": true - } - ] - }, - { - "name": "ReadChunkResponseProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "required": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "oneof_parent": "responseData" - }, - { - "id": 4, - "name": "dataBuffers", - "type": "DataBuffers", - "oneof_parent": "responseData" - } - ] - }, - { - "name": "DataBuffers", - "fields": [ - { - "id": 1, - "name": "buffers", - "type": "bytes", - "is_repeated": true - } - ] - }, - { - "name": "DeleteChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "chunkData", - "type": "ChunkInfo", - "required": true - } - ] - }, - { - "name": "DeleteChunkResponseProto" - }, - { - "name": "ListChunkRequestProto", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "DatanodeBlockID", - "required": true - }, - { - "id": 2, - "name": "prevChunkName", - "type": "string", - "required": true - }, - { - "id": 3, - "name": "count", - "type": "uint32", - "required": true - } - ] - }, - { - "name": "ListChunkResponseProto", - "fields": [ - { - "id": 1, - "name": "chunkData", - "type": "ChunkInfo", - "is_repeated": true - } - ] - }, - { - "name": "PutSmallFileRequestProto", - "fields": [ - { - "id": 1, - "name": "block", - "type": "PutBlockRequestProto", - "required": true - }, - { - "id": 2, - "name": "chunkInfo", - "type": "ChunkInfo", - "required": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "required": true - } - ] - }, - { - "name": "PutSmallFileResponseProto", - "fields": [ - { - "id": 1, - "name": "committedBlockLength", - "type": "GetCommittedBlockLengthResponseProto", - "required": true - } - ] - }, - { - "name": "GetSmallFileRequestProto", - "fields": [ - { - "id": 1, - "name": "block", - "type": "GetBlockRequestProto", - "required": true - }, - { - "id": 2, - "name": "readChunkVersion", - "type": "ReadChunkVersion", - "optional": true - } - ] - }, - { - "name": "GetSmallFileResponseProto", - "fields": [ - { - "id": 1, - "name": "data", - "type": "ReadChunkResponseProto", - "required": true - } - ] - }, - { - "name": "CopyContainerRequestProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "readOffset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "len", - "type": "uint64", - "optional": true - }, - { - "id": 4, - "name": "version", - "type": "uint32", - "optional": true - }, - { - "id": 5, - "name": "compression", - "type": "CopyContainerCompressProto", - "optional": true - } - ] - }, - { - "name": "CopyContainerResponseProto", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "readOffset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "len", - "type": "uint64", - "required": true - }, - { - "id": 4, - "name": "eof", - "type": "bool", - "required": true - }, - { - "id": 5, - "name": "data", - "type": "bytes", - "required": true - }, - { - "id": 6, - "name": "checksum", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "SendContainerRequest", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "required": true - }, - { - "id": 2, - "name": "offset", - "type": "uint64", - "required": true - }, - { - "id": 3, - "name": "data", - "type": "bytes", - "required": true - }, - { - "id": 4, - "name": "checksum", - "type": "int64", - "optional": true - }, - { - "id": 5, - "name": "compression", - "type": "CopyContainerCompressProto", - "optional": true - }, - { - "id": 6, - "name": "size", - "type": "int64", - "optional": true - } - ] - }, - { - "name": "SendContainerResponse" - }, - { - "name": "ChunkMerkleTree", - "fields": [ - { - "id": 1, - "name": "offset", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "length", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "dataChecksum", - "type": "int64", - "optional": true - }, - { - "id": 4, - "name": "checksumMatches", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "BlockMerkleTree", - "fields": [ - { - "id": 1, - "name": "blockID", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "dataChecksum", - "type": "int64", - "optional": true - }, - { - "id": 3, - "name": "chunkMerkleTree", - "type": "ChunkMerkleTree", - "is_repeated": true - }, - { - "id": 4, - "name": "deleted", - "type": "bool", - "optional": true - } - ] - }, - { - "name": "ContainerMerkleTree", - "fields": [ - { - "id": 1, - "name": "dataChecksum", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "blockMerkleTree", - "type": "BlockMerkleTree", - "is_repeated": true - } - ] - }, - { - "name": "ContainerChecksumInfo", - "fields": [ - { - "id": 1, - "name": "containerID", - "type": "int64", - "optional": true - }, - { - "id": 2, - "name": "containerMerkleTree", - "type": "ContainerMerkleTree", - "optional": true - } - ] - }, - { - "name": "ContainerCreateInfo", - "fields": [ - { - "id": 1, - "name": "state", - "type": "ContainerDataProto.State", - "required": true - }, - { - "id": 2, - "name": "replicaIndex", - "type": "int32", - "optional": true, - "options": [ - { - "name": "default", - "value": "-1" - } - ] - } - ] - } - ], - "services": [ - { - "name": "XceiverClientProtocolService", - "rpcs": [ - { - "name": "send", - "in_type": "ContainerCommandRequestProto", - "out_type": "ContainerCommandResponseProto", - "in_streamed": true, - "out_streamed": true - } - ] - }, - { - "name": "IntraDatanodeProtocolService", - "rpcs": [ - { - "name": "download", - "in_type": "CopyContainerRequestProto", - "out_type": "CopyContainerResponseProto", - "out_streamed": true - }, - { - "name": "upload", - "in_type": "SendContainerRequest", - "out_type": "SendContainerResponse", - "in_streamed": true - } - ] - } - ], - "package": { - "name": "hadoop.hdds.datanode" - }, - "options": [ - { - "name": "java_package", - "value": "org.apache.hadoop.hdds.protocol.datanode.proto" - }, - { - "name": "java_outer_classname", - "value": "ContainerProtos" - }, - { - "name": "java_generate_equals_and_hash", - "value": "true" - } - ] - } - } - ] -} + "definitions": [] +} \ No newline at end of file diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 795dba1e2844..b81fa9484ecd 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -312,6 +312,11 @@ test-jar test + + org.apache.ozone + hdds-datanode-grpc-client + test + org.apache.ozone hdds-managed-rocksdb diff --git a/hadoop-hdds/interface-client/pom.xml b/hadoop-hdds/interface-client/pom.xml index 6cb900a06115..ead525a2529c 100644 --- a/hadoop-hdds/interface-client/pom.xml +++ b/hadoop-hdds/interface-client/pom.xml @@ -23,13 +23,17 @@ 2.2.0-SNAPSHOT jar Apache Ozone HDDS Client Interface - Apache Ozone Distributed Data Store Client interface + Apache Ozone Distributed Data Store Client interface. + Also contains the ContainerProtos message classes generated from DatanodeClientProtocol.proto + and rewritten to the ratis-thirdparty shaded namespace so that hdds-common and other + foundational modules can depend on the shaded types without pulling in the gRPC stubs. yyyyMMddHHmmss true + true true @@ -39,6 +43,16 @@ com.google.protobuf protobuf-java + + + org.apache.ratis + ratis-thirdparty-misc + javax.annotation @@ -62,6 +76,12 @@ compiles generated sources + + maven-antrun-plugin + + rewrites generated source files + + protobuf-maven-plugin @@ -84,6 +104,7 @@ maven-compiler-plugin none + false @@ -101,6 +122,58 @@ target/generated-sources/proto-java-for-protobuf-${protobuf.version} ${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/protobuf-${protobuf.version}-${maven.build.timestamp} false + + + DatanodeClientProtocol.proto + + + + + + compile-proto-datanode-ratis + + compile + + generate-sources + + com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier} + ${project.build.directory}/generated-sources/proto-java-ratis + ${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/ratis-${maven.build.timestamp} + false + + DatanodeClientProtocol.proto + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + + rewrite-generated-sources-to-ratis-ns + + run + + process-sources + + + + + diff --git a/hadoop-hdds/datanode-grpc-client/src/main/proto/DatanodeClientProtocol.proto b/hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto similarity index 100% rename from hadoop-hdds/datanode-grpc-client/src/main/proto/DatanodeClientProtocol.proto rename to hadoop-hdds/interface-client/src/main/proto/DatanodeClientProtocol.proto diff --git a/hadoop-hdds/interface-client/src/main/resources/proto.lock b/hadoop-hdds/interface-client/src/main/resources/proto.lock index 3c6248ad0c13..70ff576a7084 100644 --- a/hadoop-hdds/interface-client/src/main/resources/proto.lock +++ b/hadoop-hdds/interface-client/src/main/resources/proto.lock @@ -1,5 +1,1832 @@ { "definitions": [ + { + "protopath": "DatanodeClientProtocol.proto", + "def": { + "enums": [ + { + "name": "Type", + "enum_fields": [ + { + "name": "CreateContainer", + "integer": 1 + }, + { + "name": "ReadContainer", + "integer": 2 + }, + { + "name": "UpdateContainer", + "integer": 3 + }, + { + "name": "DeleteContainer", + "integer": 4 + }, + { + "name": "ListContainer", + "integer": 5 + }, + { + "name": "PutBlock", + "integer": 6 + }, + { + "name": "GetBlock", + "integer": 7 + }, + { + "name": "DeleteBlock", + "integer": 8 + }, + { + "name": "ListBlock", + "integer": 9 + }, + { + "name": "ReadChunk", + "integer": 10 + }, + { + "name": "DeleteChunk", + "integer": 11 + }, + { + "name": "WriteChunk", + "integer": 12 + }, + { + "name": "ListChunk", + "integer": 13 + }, + { + "name": "CompactChunk", + "integer": 14 + }, + { + "name": "PutSmallFile", + "integer": 15 + }, + { + "name": "GetSmallFile", + "integer": 16 + }, + { + "name": "CloseContainer", + "integer": 17 + }, + { + "name": "GetCommittedBlockLength", + "integer": 18 + }, + { + "name": "StreamInit", + "integer": 19 + }, + { + "name": "StreamWrite", + "integer": 20 + }, + { + "name": "FinalizeBlock", + "integer": 21 + }, + { + "name": "Echo", + "integer": 22 + }, + { + "name": "GetContainerChecksumInfo", + "integer": 23 + } + ] + }, + { + "name": "Result", + "enum_fields": [ + { + "name": "SUCCESS", + "integer": 1 + }, + { + "name": "UNSUPPORTED_REQUEST", + "integer": 2 + }, + { + "name": "MALFORMED_REQUEST", + "integer": 3 + }, + { + "name": "CONTAINER_INTERNAL_ERROR", + "integer": 4 + }, + { + "name": "INVALID_CONFIG", + "integer": 5 + }, + { + "name": "INVALID_FILE_HASH_FOUND", + "integer": 6 + }, + { + "name": "CONTAINER_EXISTS", + "integer": 7 + }, + { + "name": "NO_SUCH_ALGORITHM", + "integer": 8 + }, + { + "name": "CONTAINER_NOT_FOUND", + "integer": 9 + }, + { + "name": "IO_EXCEPTION", + "integer": 10 + }, + { + "name": "UNABLE_TO_READ_METADATA_DB", + "integer": 11 + }, + { + "name": "NO_SUCH_BLOCK", + "integer": 12 + }, + { + "name": "OVERWRITE_FLAG_REQUIRED", + "integer": 13 + }, + { + "name": "UNABLE_TO_FIND_DATA_DIR", + "integer": 14 + }, + { + "name": "INVALID_WRITE_SIZE", + "integer": 15 + }, + { + "name": "CHECKSUM_MISMATCH", + "integer": 16 + }, + { + "name": "UNABLE_TO_FIND_CHUNK", + "integer": 17 + }, + { + "name": "PROTOC_DECODING_ERROR", + "integer": 18 + }, + { + "name": "INVALID_ARGUMENT", + "integer": 19 + }, + { + "name": "PUT_SMALL_FILE_ERROR", + "integer": 20 + }, + { + "name": "GET_SMALL_FILE_ERROR", + "integer": 21 + }, + { + "name": "CLOSED_CONTAINER_IO", + "integer": 22 + }, + { + "name": "ERROR_IN_COMPACT_DB", + "integer": 24 + }, + { + "name": "UNCLOSED_CONTAINER_IO", + "integer": 25 + }, + { + "name": "DELETE_ON_OPEN_CONTAINER", + "integer": 26 + }, + { + "name": "CLOSED_CONTAINER_RETRY", + "integer": 27 + }, + { + "name": "INVALID_CONTAINER_STATE", + "integer": 28 + }, + { + "name": "DISK_OUT_OF_SPACE", + "integer": 29 + }, + { + "name": "CONTAINER_ALREADY_EXISTS", + "integer": 30 + }, + { + "name": "CONTAINER_METADATA_ERROR", + "integer": 31 + }, + { + "name": "CONTAINER_FILES_CREATE_ERROR", + "integer": 32 + }, + { + "name": "CONTAINER_CHECKSUM_ERROR", + "integer": 33 + }, + { + "name": "UNKNOWN_CONTAINER_TYPE", + "integer": 34 + }, + { + "name": "BLOCK_NOT_COMMITTED", + "integer": 35 + }, + { + "name": "CONTAINER_UNHEALTHY", + "integer": 36 + }, + { + "name": "UNKNOWN_BCSID", + "integer": 37 + }, + { + "name": "BCSID_MISMATCH", + "integer": 38 + }, + { + "name": "CONTAINER_NOT_OPEN", + "integer": 39 + }, + { + "name": "CONTAINER_MISSING", + "integer": 40 + }, + { + "name": "BLOCK_TOKEN_VERIFICATION_FAILED", + "integer": 41 + }, + { + "name": "ERROR_IN_DB_SYNC", + "integer": 42 + }, + { + "name": "CHUNK_FILE_INCONSISTENCY", + "integer": 43 + }, + { + "name": "DELETE_ON_NON_EMPTY_CONTAINER", + "integer": 44 + }, + { + "name": "EXPORT_CONTAINER_METADATA_FAILED", + "integer": 45 + }, + { + "name": "IMPORT_CONTAINER_METADATA_FAILED", + "integer": 46 + }, + { + "name": "BLOCK_ALREADY_FINALIZED", + "integer": 47 + }, + { + "name": "CONTAINER_ID_MISMATCH", + "integer": 48 + } + ] + }, + { + "name": "ContainerDataProto.State", + "enum_fields": [ + { + "name": "OPEN", + "integer": 1 + }, + { + "name": "CLOSING", + "integer": 2 + }, + { + "name": "QUASI_CLOSED", + "integer": 3 + }, + { + "name": "CLOSED", + "integer": 4 + }, + { + "name": "UNHEALTHY", + "integer": 5 + }, + { + "name": "INVALID", + "integer": 6 + }, + { + "name": "DELETED", + "integer": 7 + }, + { + "name": "RECOVERING", + "integer": 8 + } + ] + }, + { + "name": "ContainerType", + "enum_fields": [ + { + "name": "KeyValueContainer", + "integer": 1 + } + ] + }, + { + "name": "ChecksumType", + "enum_fields": [ + { + "name": "NONE", + "integer": 1 + }, + { + "name": "CRC32", + "integer": 2 + }, + { + "name": "CRC32C", + "integer": 3 + }, + { + "name": "SHA256", + "integer": 4 + }, + { + "name": "MD5", + "integer": 5 + } + ] + }, + { + "name": "ReadChunkVersion", + "enum_fields": [ + { + "name": "V0" + }, + { + "name": "V1", + "integer": 1 + } + ] + }, + { + "name": "CopyContainerCompressProto", + "enum_fields": [ + { + "name": "NO_COMPRESSION", + "integer": 1 + }, + { + "name": "GZIP", + "integer": 2 + }, + { + "name": "LZ4", + "integer": 3 + }, + { + "name": "SNAPPY", + "integer": 4 + }, + { + "name": "ZSTD", + "integer": 5 + } + ] + } + ], + "messages": [ + { + "name": "DatanodeBlockID", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "localID", + "type": "int64", + "required": true + }, + { + "id": 3, + "name": "blockCommitSequenceId", + "type": "uint64", + "optional": true, + "options": [ + { + "name": "default", + "value": "0" + } + ] + }, + { + "id": 4, + "name": "replicaIndex", + "type": "int32", + "optional": true + } + ] + }, + { + "name": "KeyValue", + "fields": [ + { + "id": 1, + "name": "key", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "value", + "type": "string", + "optional": true + } + ] + }, + { + "name": "ContainerCommandRequestProto", + "fields": [ + { + "id": 1, + "name": "cmdType", + "type": "Type", + "required": true + }, + { + "id": 2, + "name": "traceID", + "type": "string", + "optional": true + }, + { + "id": 3, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 4, + "name": "datanodeUuid", + "type": "string", + "required": true + }, + { + "id": 5, + "name": "pipelineID", + "type": "string", + "optional": true + }, + { + "id": 6, + "name": "createContainer", + "type": "CreateContainerRequestProto", + "optional": true + }, + { + "id": 7, + "name": "readContainer", + "type": "ReadContainerRequestProto", + "optional": true + }, + { + "id": 8, + "name": "updateContainer", + "type": "UpdateContainerRequestProto", + "optional": true + }, + { + "id": 9, + "name": "deleteContainer", + "type": "DeleteContainerRequestProto", + "optional": true + }, + { + "id": 10, + "name": "listContainer", + "type": "ListContainerRequestProto", + "optional": true + }, + { + "id": 11, + "name": "closeContainer", + "type": "CloseContainerRequestProto", + "optional": true + }, + { + "id": 12, + "name": "putBlock", + "type": "PutBlockRequestProto", + "optional": true + }, + { + "id": 13, + "name": "getBlock", + "type": "GetBlockRequestProto", + "optional": true + }, + { + "id": 14, + "name": "deleteBlock", + "type": "DeleteBlockRequestProto", + "optional": true, + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + }, + { + "id": 15, + "name": "listBlock", + "type": "ListBlockRequestProto", + "optional": true + }, + { + "id": 16, + "name": "readChunk", + "type": "ReadChunkRequestProto", + "optional": true + }, + { + "id": 17, + "name": "writeChunk", + "type": "WriteChunkRequestProto", + "optional": true + }, + { + "id": 18, + "name": "deleteChunk", + "type": "DeleteChunkRequestProto", + "optional": true, + "options": [ + { + "name": "deprecated", + "value": "true" + } + ] + }, + { + "id": 19, + "name": "listChunk", + "type": "ListChunkRequestProto", + "optional": true + }, + { + "id": 20, + "name": "putSmallFile", + "type": "PutSmallFileRequestProto", + "optional": true + }, + { + "id": 21, + "name": "getSmallFile", + "type": "GetSmallFileRequestProto", + "optional": true + }, + { + "id": 22, + "name": "getCommittedBlockLength", + "type": "GetCommittedBlockLengthRequestProto", + "optional": true + }, + { + "id": 23, + "name": "encodedToken", + "type": "string", + "optional": true + }, + { + "id": 24, + "name": "version", + "type": "uint32", + "optional": true + }, + { + "id": 25, + "name": "finalizeBlock", + "type": "FinalizeBlockRequestProto", + "optional": true + }, + { + "id": 26, + "name": "echo", + "type": "EchoRequestProto", + "optional": true + }, + { + "id": 27, + "name": "getContainerChecksumInfo", + "type": "GetContainerChecksumInfoRequestProto", + "optional": true + } + ] + }, + { + "name": "ContainerCommandResponseProto", + "fields": [ + { + "id": 1, + "name": "cmdType", + "type": "Type", + "required": true + }, + { + "id": 2, + "name": "traceID", + "type": "string", + "optional": true + }, + { + "id": 3, + "name": "result", + "type": "Result", + "required": true + }, + { + "id": 4, + "name": "message", + "type": "string", + "optional": true + }, + { + "id": 5, + "name": "createContainer", + "type": "CreateContainerResponseProto", + "optional": true + }, + { + "id": 6, + "name": "readContainer", + "type": "ReadContainerResponseProto", + "optional": true + }, + { + "id": 7, + "name": "updateContainer", + "type": "UpdateContainerResponseProto", + "optional": true + }, + { + "id": 8, + "name": "deleteContainer", + "type": "DeleteContainerResponseProto", + "optional": true + }, + { + "id": 9, + "name": "listContainer", + "type": "ListContainerResponseProto", + "optional": true + }, + { + "id": 10, + "name": "closeContainer", + "type": "CloseContainerResponseProto", + "optional": true + }, + { + "id": 11, + "name": "putBlock", + "type": "PutBlockResponseProto", + "optional": true + }, + { + "id": 12, + "name": "getBlock", + "type": "GetBlockResponseProto", + "optional": true + }, + { + "id": 13, + "name": "deleteBlock", + "type": "DeleteBlockResponseProto", + "optional": true + }, + { + "id": 14, + "name": "listBlock", + "type": "ListBlockResponseProto", + "optional": true + }, + { + "id": 15, + "name": "writeChunk", + "type": "WriteChunkResponseProto", + "optional": true + }, + { + "id": 16, + "name": "readChunk", + "type": "ReadChunkResponseProto", + "optional": true + }, + { + "id": 17, + "name": "deleteChunk", + "type": "DeleteChunkResponseProto", + "optional": true + }, + { + "id": 18, + "name": "listChunk", + "type": "ListChunkResponseProto", + "optional": true + }, + { + "id": 19, + "name": "putSmallFile", + "type": "PutSmallFileResponseProto", + "optional": true + }, + { + "id": 20, + "name": "getSmallFile", + "type": "GetSmallFileResponseProto", + "optional": true + }, + { + "id": 21, + "name": "getCommittedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "optional": true + }, + { + "id": 22, + "name": "finalizeBlock", + "type": "FinalizeBlockResponseProto", + "optional": true + }, + { + "id": 23, + "name": "echo", + "type": "EchoResponseProto", + "optional": true + }, + { + "id": 24, + "name": "getContainerChecksumInfo", + "type": "GetContainerChecksumInfoResponseProto", + "optional": true + } + ] + }, + { + "name": "ContainerDataProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 4, + "name": "containerPath", + "type": "string", + "optional": true + }, + { + "id": 6, + "name": "bytesUsed", + "type": "int64", + "optional": true + }, + { + "id": 7, + "name": "size", + "type": "int64", + "optional": true + }, + { + "id": 8, + "name": "blockCount", + "type": "int64", + "optional": true + }, + { + "id": 9, + "name": "state", + "type": "State", + "optional": true, + "options": [ + { + "name": "default", + "value": "OPEN" + } + ] + }, + { + "id": 10, + "name": "containerType", + "type": "ContainerType", + "optional": true, + "options": [ + { + "name": "default", + "value": "KeyValueContainer" + } + ] + } + ] + }, + { + "name": "Container2BCSIDMapProto", + "maps": [ + { + "key_type": "int64", + "field": { + "id": 1, + "name": "container2BCSID", + "type": "int64" + } + } + ] + }, + { + "name": "CreateContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 3, + "name": "containerType", + "type": "ContainerType", + "optional": true, + "options": [ + { + "name": "default", + "value": "KeyValueContainer" + } + ] + }, + { + "id": 4, + "name": "replicaIndex", + "type": "int32", + "optional": true + }, + { + "id": 5, + "name": "state", + "type": "ContainerDataProto.State", + "optional": true + } + ] + }, + { + "name": "CreateContainerResponseProto" + }, + { + "name": "ReadContainerRequestProto" + }, + { + "name": "ReadContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "containerData", + "type": "ContainerDataProto", + "optional": true + } + ] + }, + { + "name": "UpdateContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 3, + "name": "forceUpdate", + "type": "bool", + "optional": true, + "options": [ + { + "name": "default", + "value": "false" + } + ] + } + ] + }, + { + "name": "UpdateContainerResponseProto" + }, + { + "name": "DeleteContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "forceDelete", + "type": "bool", + "optional": true, + "options": [ + { + "name": "default", + "value": "false" + } + ] + } + ] + }, + { + "name": "DeleteContainerResponseProto" + }, + { + "name": "ListContainerRequestProto", + "fields": [ + { + "id": 2, + "name": "count", + "type": "uint32", + "optional": true + } + ] + }, + { + "name": "ListContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "containerData", + "type": "ContainerDataProto", + "is_repeated": true + } + ] + }, + { + "name": "CloseContainerRequestProto" + }, + { + "name": "CloseContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "hash", + "type": "string", + "optional": true + }, + { + "id": 2, + "name": "containerID", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "BlockData", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "flags", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 4, + "name": "chunks", + "type": "ChunkInfo", + "is_repeated": true + }, + { + "id": 5, + "name": "size", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "PutBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "required": true + }, + { + "id": 2, + "name": "eof", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "PutBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "committedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "required": true + } + ] + }, + { + "name": "FinalizeBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "FinalizeBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "required": true + } + ] + }, + { + "name": "GetBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "GetBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "required": true + } + ] + }, + { + "name": "DeleteBlockRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "GetCommittedBlockLengthRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + } + ] + }, + { + "name": "GetCommittedBlockLengthResponseProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "blockLength", + "type": "int64", + "required": true + } + ] + }, + { + "name": "DeleteBlockResponseProto" + }, + { + "name": "ListBlockRequestProto", + "fields": [ + { + "id": 2, + "name": "startLocalID", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "count", + "type": "uint32", + "required": true + } + ] + }, + { + "name": "ListBlockResponseProto", + "fields": [ + { + "id": 1, + "name": "blockData", + "type": "BlockData", + "is_repeated": true + } + ] + }, + { + "name": "EchoRequestProto", + "fields": [ + { + "id": 1, + "name": "payload", + "type": "bytes", + "optional": true + }, + { + "id": 2, + "name": "payloadSizeResp", + "type": "int32", + "optional": true + }, + { + "id": 3, + "name": "sleepTimeMs", + "type": "int32", + "optional": true + }, + { + "id": 4, + "name": "readOnly", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "EchoResponseProto", + "fields": [ + { + "id": 1, + "name": "payload", + "type": "bytes", + "optional": true + } + ] + }, + { + "name": "GetContainerChecksumInfoRequestProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "GetContainerChecksumInfoResponseProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "containerChecksumInfo", + "type": "bytes", + "optional": true + } + ] + }, + { + "name": "ChunkInfo", + "fields": [ + { + "id": 1, + "name": "chunkName", + "type": "string", + "required": true + }, + { + "id": 2, + "name": "offset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "len", + "type": "uint64", + "required": true + }, + { + "id": 4, + "name": "metadata", + "type": "KeyValue", + "is_repeated": true + }, + { + "id": 5, + "name": "checksumData", + "type": "ChecksumData", + "required": true + }, + { + "id": 6, + "name": "stripeChecksum", + "type": "bytes", + "optional": true + } + ] + }, + { + "name": "ChunkInfoList", + "fields": [ + { + "id": 1, + "name": "chunks", + "type": "ChunkInfo", + "is_repeated": true + } + ] + }, + { + "name": "ChecksumData", + "fields": [ + { + "id": 1, + "name": "type", + "type": "ChecksumType", + "required": true + }, + { + "id": 2, + "name": "bytesPerChecksum", + "type": "uint32", + "required": true + }, + { + "id": 3, + "name": "checksums", + "type": "bytes", + "is_repeated": true + } + ] + }, + { + "name": "WriteChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "optional": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "optional": true + }, + { + "id": 4, + "name": "block", + "type": "PutBlockRequestProto", + "optional": true + } + ] + }, + { + "name": "WriteChunkResponseProto", + "fields": [ + { + "id": 1, + "name": "committedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "optional": true + } + ] + }, + { + "name": "ReadChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "required": true + }, + { + "id": 3, + "name": "readChunkVersion", + "type": "ReadChunkVersion", + "optional": true + } + ] + }, + { + "name": "ReadChunkResponseProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "required": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "oneof_parent": "responseData" + }, + { + "id": 4, + "name": "dataBuffers", + "type": "DataBuffers", + "oneof_parent": "responseData" + } + ] + }, + { + "name": "DataBuffers", + "fields": [ + { + "id": 1, + "name": "buffers", + "type": "bytes", + "is_repeated": true + } + ] + }, + { + "name": "DeleteChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "chunkData", + "type": "ChunkInfo", + "required": true + } + ] + }, + { + "name": "DeleteChunkResponseProto" + }, + { + "name": "ListChunkRequestProto", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "DatanodeBlockID", + "required": true + }, + { + "id": 2, + "name": "prevChunkName", + "type": "string", + "required": true + }, + { + "id": 3, + "name": "count", + "type": "uint32", + "required": true + } + ] + }, + { + "name": "ListChunkResponseProto", + "fields": [ + { + "id": 1, + "name": "chunkData", + "type": "ChunkInfo", + "is_repeated": true + } + ] + }, + { + "name": "PutSmallFileRequestProto", + "fields": [ + { + "id": 1, + "name": "block", + "type": "PutBlockRequestProto", + "required": true + }, + { + "id": 2, + "name": "chunkInfo", + "type": "ChunkInfo", + "required": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "required": true + } + ] + }, + { + "name": "PutSmallFileResponseProto", + "fields": [ + { + "id": 1, + "name": "committedBlockLength", + "type": "GetCommittedBlockLengthResponseProto", + "required": true + } + ] + }, + { + "name": "GetSmallFileRequestProto", + "fields": [ + { + "id": 1, + "name": "block", + "type": "GetBlockRequestProto", + "required": true + }, + { + "id": 2, + "name": "readChunkVersion", + "type": "ReadChunkVersion", + "optional": true + } + ] + }, + { + "name": "GetSmallFileResponseProto", + "fields": [ + { + "id": 1, + "name": "data", + "type": "ReadChunkResponseProto", + "required": true + } + ] + }, + { + "name": "CopyContainerRequestProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "readOffset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "len", + "type": "uint64", + "optional": true + }, + { + "id": 4, + "name": "version", + "type": "uint32", + "optional": true + }, + { + "id": 5, + "name": "compression", + "type": "CopyContainerCompressProto", + "optional": true + } + ] + }, + { + "name": "CopyContainerResponseProto", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "readOffset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "len", + "type": "uint64", + "required": true + }, + { + "id": 4, + "name": "eof", + "type": "bool", + "required": true + }, + { + "id": 5, + "name": "data", + "type": "bytes", + "required": true + }, + { + "id": 6, + "name": "checksum", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "SendContainerRequest", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "required": true + }, + { + "id": 2, + "name": "offset", + "type": "uint64", + "required": true + }, + { + "id": 3, + "name": "data", + "type": "bytes", + "required": true + }, + { + "id": 4, + "name": "checksum", + "type": "int64", + "optional": true + }, + { + "id": 5, + "name": "compression", + "type": "CopyContainerCompressProto", + "optional": true + }, + { + "id": 6, + "name": "size", + "type": "int64", + "optional": true + } + ] + }, + { + "name": "SendContainerResponse" + }, + { + "name": "ChunkMerkleTree", + "fields": [ + { + "id": 1, + "name": "offset", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "length", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "dataChecksum", + "type": "int64", + "optional": true + }, + { + "id": 4, + "name": "checksumMatches", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "BlockMerkleTree", + "fields": [ + { + "id": 1, + "name": "blockID", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "dataChecksum", + "type": "int64", + "optional": true + }, + { + "id": 3, + "name": "chunkMerkleTree", + "type": "ChunkMerkleTree", + "is_repeated": true + }, + { + "id": 4, + "name": "deleted", + "type": "bool", + "optional": true + } + ] + }, + { + "name": "ContainerMerkleTree", + "fields": [ + { + "id": 1, + "name": "dataChecksum", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "blockMerkleTree", + "type": "BlockMerkleTree", + "is_repeated": true + } + ] + }, + { + "name": "ContainerChecksumInfo", + "fields": [ + { + "id": 1, + "name": "containerID", + "type": "int64", + "optional": true + }, + { + "id": 2, + "name": "containerMerkleTree", + "type": "ContainerMerkleTree", + "optional": true + } + ] + }, + { + "name": "ContainerCreateInfo", + "fields": [ + { + "id": 1, + "name": "state", + "type": "ContainerDataProto.State", + "required": true + }, + { + "id": 2, + "name": "replicaIndex", + "type": "int32", + "optional": true, + "options": [ + { + "name": "default", + "value": "-1" + } + ] + } + ] + } + ], + "services": [ + { + "name": "XceiverClientProtocolService", + "rpcs": [ + { + "name": "send", + "in_type": "ContainerCommandRequestProto", + "out_type": "ContainerCommandResponseProto", + "in_streamed": true, + "out_streamed": true + } + ] + }, + { + "name": "IntraDatanodeProtocolService", + "rpcs": [ + { + "name": "download", + "in_type": "CopyContainerRequestProto", + "out_type": "CopyContainerResponseProto", + "out_streamed": true + }, + { + "name": "upload", + "in_type": "SendContainerRequest", + "out_type": "SendContainerResponse", + "in_streamed": true + } + ] + } + ], + "package": { + "name": "hadoop.hdds.datanode" + }, + "options": [ + { + "name": "java_package", + "value": "org.apache.hadoop.hdds.protocol.datanode.proto" + }, + { + "name": "java_outer_classname", + "value": "ContainerProtos" + }, + { + "name": "java_generate_equals_and_hash", + "value": "true" + } + ] + } + }, { "protopath": "ReconfigureProtocol.proto", "def": { @@ -1958,4 +3785,4 @@ } } ] -} +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index c1fb74caa38e..73783591b872 100644 --- a/pom.xml +++ b/pom.xml @@ -2514,7 +2514,7 @@ + com.google.guava guava @@ -66,6 +72,12 @@ org.apache.commons commons-collections4 + + org.apache.commons + commons-compress + + org.apache.commons commons-lang3 @@ -82,6 +94,12 @@ org.apache.ozone hdds-cli-common + + org.apache.ozone + hdds-client + + org.apache.ozone hdds-common @@ -168,16 +186,6 @@ slf4j-reload4j runtime - - com.fasterxml.jackson.core - jackson-databind - test - - - org.apache.commons - commons-compress - test - @@ -202,11 +210,6 @@ hadoop-hdfs-client test - - org.apache.ozone - hdds-client - test - org.apache.ozone hdds-common diff --git a/pom.xml b/pom.xml index 73783591b872..5d72e92649b0 100644 --- a/pom.xml +++ b/pom.xml @@ -2308,6 +2308,24 @@ flags hdds-config as unused even when it is the active annotation processor. --> org.apache.ozone:hdds-config:jar + + com.fasterxml.jackson.core:jackson-databind:jar + + org.apache.commons:commons-compress:jar + + org.apache.ozone:hdds-client:jar org.apache.ozone:hdds-datanode-grpc-client:jar