HDDS-14949 — Migrate gRPC transport to the Ratis-shaded gRPC/Netty/Protobuf stack

.gitignore

@@ -16,6 +16,7 @@
 .project
 .settings
 *.factorypath
+*.class
 target
 build
 dependency-reduced-pom.xml

hadoop-hdds/client/pom.xml

@@ -52,6 +52,10 @@
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-config</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-datanode-grpc-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-erasurecode</artifactId>
@@ -101,6 +105,21 @@
 
   <build>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUsedUndeclaredDependencies combine.children="append">
+                <!-- protobuf-java is used via hdds-interface-client transitively -->
+                <ignoredUsedUndeclaredDependency>com.google.protobuf:protobuf-java</ignoredUsedUndeclaredDependency>
+              </ignoredUsedUndeclaredDependencies>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>com.github.spotbugs</groupId>
         <artifactId>spotbugs-maven-plugin</artifactId>

hadoop-hdds/common/pom.xml

@@ -248,6 +248,7 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
+          <useIncrementalCompilation>false</useIncrementalCompilation>
           <annotationProcessorPaths>
             <path>
               <groupId>org.apache.ozone</groupId>
@@ -263,6 +264,26 @@
           </compilerArgs>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUnusedDeclaredDependencies combine.children="append">
+                <!--
+                  hdds-config is used only as an annotationProcessorPath (ConfigFileGenerator).
+                  Its annotation types are @Retention(SOURCE) so they don't appear in compiled
+                  bytecode; the dependency analyzer therefore flags it as unused.
+                -->
+                <ignoredUnusedDeclaredDependency>org.apache.ozone:hdds-config</ignoredUnusedDeclaredDependency>
+                <ignoredUnusedDeclaredDependency>org.apache.ratis:ratis-thirdparty-misc</ignoredUnusedDeclaredDependency>
+              </ignoredUnusedDeclaredDependencies>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -34,8 +34,6 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.net.HostAndPort;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.ServiceException;
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
 import java.io.File;
@@ -87,6 +85,8 @@
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.SecretManager;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.ratis.thirdparty.com.google.protobuf.ServiceException;
 import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
 import org.apache.ratis.util.SizeInBytes;
 import org.slf4j.Logger;

hadoop-hdds/container-service/pom.xml

@@ -138,6 +138,10 @@
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-config</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-datanode-grpc-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-interface-client</artifactId>
@@ -336,6 +340,24 @@
               <overWriteSnapshots>true</overWriteSnapshots>
             </configuration>
           </execution>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUnusedDeclaredDependencies combine.children="append">
+                <!--
+                  hadoop-auth is used by Hadoop delegation tokens at runtime; its types
+                  are accessed through hadoop-common's service-loader mechanism and not
+                  directly imported in compiled bytecode.
+                -->
+                <ignoredUnusedDeclaredDependency>org.apache.hadoop:hadoop-auth</ignoredUnusedDeclaredDependency>
+                <!--
+                  ratis-netty is a Ratis transport module loaded at runtime via service
+                  discovery. Its classes are not referenced directly in compiled bytecode.
+                -->
+                <ignoredUnusedDeclaredDependency>org.apache.ratis:ratis-netty</ignoredUnusedDeclaredDependency>
+              </ignoredUnusedDeclaredDependencies>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
     </plugins>

hadoop-hdds/datanode-grpc-client/pom.xml

@@ -0,0 +1,195 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.ozone</groupId>
+    <artifactId>hdds</artifactId>
+    <version>2.2.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>hdds-datanode-grpc-client</artifactId>
+  <version>2.2.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+  <name>Apache Ozone HDDS DataNode gRPC Client</name>
+  <description>DataNode XCeiver and IntraDatanode gRPC stubs generated from DatanodeClientProtocol.proto.
+    The proto file lives in hdds-interface-client (which also generates the shaded ContainerProtos
+    message classes); this module generates only the gRPC service stubs.  Package names in generated
+    sources are rewritten by maven-antrun-plugin (Ant replace task, pure Java) from vanilla
+    com.google.protobuf / io.grpc / com.google.common to their ratis-thirdparty counterparts
+    before compilation.</description>
+
+  <properties>
+    <!-- used in temp dir names; must be filesystem/protoc-safe (no ':') -->
+    <maven.build.timestamp.format>yyyyMMddHHmmss</maven.build.timestamp.format>
+    <!-- no testable code in this module -->
+    <maven.test.skip>true</maven.test.skip>
+    <mdep.analyze.skip>true</mdep.analyze.skip>
+    <spotbugs.skip>true</spotbugs.skip>
+  </properties>
+
+  <dependencies>
+    <!--
+      ContainerProtos message classes (shaded) live in hdds-interface-client.  The generated
+      gRPC stubs reference those types, so hdds-interface-client must be on the compile classpath.
+    -->
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-interface-client</artifactId>
+    </dependency>
+    <!--
+      Shaded gRPC / Netty / protobuf runtime. The generated sources are rewritten to reference
+      org.apache.ratis.thirdparty.* directly, so ratis-thirdparty-misc must be on the compile
+      classpath and is exposed transitively to all consumers of this module.
+    -->
+    <dependency>
+      <groupId>org.apache.ratis</groupId>
+      <artifactId>ratis-thirdparty-misc</artifactId>
+    </dependency>
+    <dependency>
+      <!-- for @Generated annotation in generated sources (Java 11+) -->
+      <groupId>javax.annotation</groupId>
+      <artifactId>javax.annotation-api</artifactId>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>com.gradle</groupId>
+          <artifactId>develocity-maven-extension</artifactId>
+          <configuration>
+            <develocity>
+              <plugins>
+                <plugin>
+                  <artifactId>maven-compiler-plugin</artifactId>
+                  <outputs>
+                    <notCacheableBecause>compiles generated + rewritten sources</notCacheableBecause>
+                  </outputs>
+                </plugin>
+                <plugin>
+                  <artifactId>maven-antrun-plugin</artifactId>
+                  <outputs>
+                    <notCacheableBecause>rewrites generated source files</notCacheableBecause>
+                  </outputs>
+                </plugin>
+                <plugin>
+                  <artifactId>protobuf-maven-plugin</artifactId>
+                  <outputs>
+                    <notCacheableBecause>generates sources</notCacheableBecause>
+                  </outputs>
+                </plugin>
+              </plugins>
+            </develocity>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>com.salesforce.servicelibs</groupId>
+        <artifactId>proto-backwards-compatibility</artifactId>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <proc>none</proc>
+          <useIncrementalCompilation>false</useIncrementalCompilation>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.xolstice.maven.plugins</groupId>
+        <artifactId>protobuf-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>compile-proto-datanode</id>
+            <goals>
+              <!--
+                compile-custom only: generates gRPC service stubs (XceiverClientProtocolServiceGrpc,
+                IntraDatanodeProtocolServiceGrpc).  The proto messages (ContainerProtos) are compiled
+                in hdds-interface-client with the same ratis-compatible protoc; no need to regenerate
+                them here.
+              -->
+              <goal>compile-custom</goal>
+            </goals>
+            <phase>generate-sources</phase>
+            <configuration>
+              <!--
+                Use the same protoc/grpc-java versions as ratis-thirdparty so that the generated
+                API surface is compatible with the shaded runtime in ratis-thirdparty-misc.
+                Output directly to proto-java-ratis so that both Maven and the IDE/LSP compile the
+                same ratis-shaded source tree, eliminating any race between Maven's javac and the
+                LSP writing vanilla class files to target/classes/.
+                DatanodeClientProtocol.proto lives in hdds-interface-client; point protoSourceRoot
+                there so protoc finds it without duplicating the file.
+              -->
+              <protocArtifact>com.google.protobuf:protoc:${ratis-thirdparty.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+              <outputDirectory>${project.build.directory}/generated-sources/proto-java-ratis</outputDirectory>
+              <temporaryProtoFileDirectory>${java.io.tmpdir}/ozone-protoc-dependencies/${project.artifactId}/${maven.build.timestamp}</temporaryProtoFileDirectory>
+              <clearOutputDirectory>false</clearOutputDirectory>
+              <pluginId>grpc-java</pluginId>
+              <pluginArtifact>io.grpc:protoc-gen-grpc-java:${ratis-thirdparty.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+              <protoSourceRoot>${project.basedir}/../interface-client/src/main/proto</protoSourceRoot>
+              <includes>
+                <include>DatanodeClientProtocol.proto</include>
+              </includes>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <!--
+              mdep.analyze.skip=true for this module so pre-verify-refresh-classes-from-jar
+              (inherited from the root POM) is not needed.  Binding it to "none" prevents the
+              inherited execution from wiping target/classes/ at pre-integration-test, which
+              would leave the directory temporarily empty and cause downstream reactor modules
+              (e.g. hdds-common) to fail with "class file for ContainerProtos$... not found".
+            -->
+            <id>pre-verify-refresh-classes-from-jar</id>
+            <phase>none</phase>
+          </execution>
+          <execution>
+            <!--
+              Rewrite vanilla package names produced by protoc/grpc-java in-place inside
+              proto-java-ratis/ (the protobuf-maven-plugin outputDirectory).  Because the
+              IDE/LSP also uses proto-java-ratis/ as its Java source root, it will compile
+              the same shaded sources and write shaded class files to target/classes/, which
+              means both Maven javac and the LSP produce identical bytecode – no more race
+              where the LSP overwrites Maven's correctly-shaded class files.
+            -->
+            <id>rewrite-generated-sources-to-ratis-ns</id>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <phase>process-sources</phase>
+            <configuration>
+              <target>
+                <replace dir="${project.build.directory}/generated-sources/proto-java-ratis" token="com.google.common" value="org.apache.ratis.thirdparty.com.google.common" />
+                <replace dir="${project.build.directory}/generated-sources/proto-java-ratis" token="com.google.protobuf" value="org.apache.ratis.thirdparty.com.google.protobuf" />
+                <replace dir="${project.build.directory}/generated-sources/proto-java-ratis" token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc" />
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

hadoop-hdds/datanode-grpc-client/src/main/resources/proto.lock

@@ -0,0 +1,3 @@
+{
+  "definitions": []
+}

hadoop-hdds/framework/pom.xml

@@ -86,16 +86,6 @@
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>io.grpc</groupId>
-      <artifactId>grpc-api</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>com.google.code.findbugs</groupId>
-          <artifactId>jsr305</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-api</artifactId>
@@ -322,6 +312,11 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>hdds-datanode-grpc-client</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-managed-rocksdb</artifactId>
@@ -415,6 +410,26 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <configuration>
+              <ignoredUnusedDeclaredDependencies combine.children="append">
+                <!--
+                  jetty-http is a runtime dependency of the embedded Jetty server.
+                  The main classes do not directly import jetty-http types (they use
+                  jetty-server/util which pull in jetty-http transitively), so the
+                  analyzer flags it as unused at compile time.
+                -->
+                <ignoredUnusedDeclaredDependency>org.eclipse.jetty:jetty-http</ignoredUnusedDeclaredDependency>
+              </ignoredUnusedDeclaredDependencies>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>