[SPARK-51348][BUILD][SQL] Upgrade Hive to 4.1

assembly/pom.xml

@@ -255,16 +255,49 @@
           <artifactId>spark-hive_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-util</artifactId>
+          <scope>runtime</scope>
+        </dependency>
       </dependencies>
     </profile>
     <profile>
       <id>hive-thriftserver</id>
+      <properties>
+        <hive.llap.scope>provided</hive.llap.scope>
+      </properties>
       <dependencies>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-server</artifactId>
+          <scope>runtime</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-security</artifactId>
+          <scope>runtime</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-servlet</artifactId>
+          <scope>runtime</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.datasketches</groupId>
+          <artifactId>datasketches-hive</artifactId>
+          <scope>runtime</scope>
+        </dependency>
+        <dependency>
+          <groupId>${hive.group}</groupId>
+          <artifactId>hive-llap-common</artifactId>
+          <scope>runtime</scope>
+        </dependency>
       </dependencies>
     </profile>
     <profile>
@@ -306,6 +339,7 @@
       <properties>
         <hive.deps.scope>provided</hive.deps.scope>
         <hive.llap.scope>provided</hive.llap.scope>
+        <calcite.deps.scope>provided</calcite.deps.scope>
       </properties>
     </profile>
     <profile>

dev/test-dependencies.sh

@@ -34,7 +34,7 @@ HADOOP_MODULE_PROFILES="-Phive-thriftserver -Pkubernetes -Pyarn -Phive \
     -Pspark-ganglia-lgpl -Pkinesis-asl -Phadoop-cloud -Pjvm-profiler"
 MVN="build/mvn"
 HADOOP_HIVE_PROFILES=(
-    hadoop-3-hive-2.3
+    hadoop-3-hive-4.0
 )
 
 MVN_EXEC_PLUGIN_VERSION=$(build/mvn help:evaluate \
@@ -85,7 +85,7 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de
 
 # Generate manifests for each Hadoop profile:
 for HADOOP_HIVE_PROFILE in "${HADOOP_HIVE_PROFILES[@]}"; do
-  if [[ $HADOOP_HIVE_PROFILE == **hadoop-3-hive-2.3** ]]; then
+  if [[ $HADOOP_HIVE_PROFILE == **hadoop-3-hive-4.0** ]]; then
     HADOOP_PROFILE=hadoop-3
   fi
   echo "Performing Maven install for $HADOOP_HIVE_PROFILE"

dev/test-jars.txt

@@ -10,7 +10,6 @@ sql/connect/common/src/test/resources/artifact-tests/smallJar.jar
 sql/core/src/test/resources/SPARK-33084.jar
 sql/core/src/test/resources/artifact-tests/udf_noA.jar
 sql/hive-thriftserver/src/test/resources/TestUDTF.jar
-sql/hive/src/test/noclasspath/hive-test-udfs.jar
 sql/hive/src/test/resources/SPARK-21101-1.0.jar
 sql/hive/src/test/resources/TestUDTF.jar
 sql/hive/src/test/resources/data/files/TestSerDe.jar

pom.xml

@@ -134,7 +134,8 @@
     <hive.group>org.apache.hive</hive.group>
     <hive.classifier>core</hive.classifier>
     <!-- Version used in Maven Hive dependency -->
-    <hive.version>2.3.10</hive.version>
+    <hive.version>4.1.0</hive.version>
+    <hive.storage.version>${hive.version}</hive.storage.version>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
     <kafka.version>3.9.1</kafka.version>
     <!-- After 10.17.1.0, the minimum required version is JDK19 -->
@@ -196,7 +197,7 @@
     <commons-lang3.version>3.18.0</commons-lang3.version>
     <!-- org.apache.commons/commons-pool2/-->
     <commons-pool2.version>2.12.1</commons-pool2.version>
-    <datanucleus-core.version>4.1.17</datanucleus-core.version>
+    <datanucleus-core.version>6.0.11</datanucleus-core.version>
     <guava.version>33.4.0-jre</guava.version>
     <gson.version>2.11.0</gson.version>
     <janino.version>3.1.9</janino.version>
@@ -205,7 +206,7 @@
     <jsr305.version>3.0.0</jsr305.version>
     <jaxb.version>2.2.11</jaxb.version>
     <libthrift.version>0.16.0</libthrift.version>
-    <antlr4.version>4.13.1</antlr4.version>
+    <antlr4.version>4.9.3</antlr4.version>
     <jpam.version>1.1</jpam.version>
     <selenium.version>4.21.0</selenium.version>
     <htmlunit3-driver.version>4.21.0</htmlunit3-driver.version>
@@ -276,9 +277,9 @@
       needing Hadoop classes in the classpath to compile).
     -->
     <derby.deps.scope>compile</derby.deps.scope>
+    <calcite.deps.scope>compile</calcite.deps.scope>
     <hadoop.deps.scope>compile</hadoop.deps.scope>
     <hive.deps.scope>compile</hive.deps.scope>
-    <hive.storage.version>2.8.1</hive.storage.version>
     <hive.storage.scope>compile</hive.storage.scope>
     <hive.common.scope>compile</hive.common.scope>
     <hive.llap.scope>test</hive.llap.scope>
@@ -335,6 +336,7 @@
       -Djdk.reflect.useDirectMethodHandle=false
       -Dio.netty.tryReflectionSetAccessible=true
       --enable-native-access=ALL-UNNAMED
+      -Dmvn.executable=${maven.multiModuleProjectDirectory}/build/mvn
     </extraJavaTestArgs>
     <mariadb.java.client.version>2.7.12</mariadb.java.client.version>
     <mysql.connector.version>9.2.0</mysql.connector.version>
@@ -830,6 +832,11 @@
         <artifactId>log4j-slf4j2-impl</artifactId>
         <version>${log4j.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-slf4j-impl</artifactId>
+        <version>${log4j.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.logging.log4j</groupId>
         <artifactId>log4j-api</artifactId>
@@ -1647,6 +1654,10 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-metastore</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-standalone-metastore-server</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
@@ -1679,6 +1690,10 @@
             <groupId>commons-logging</groupId>
             <artifactId>commons-logging</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1743,6 +1758,10 @@
         <version>${hive.version}</version>
         <scope>${hive.deps.scope}</scope>
         <exclusions>
+          <exclusion>
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-standalone-metastore-common</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1807,6 +1826,14 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-storage-api</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-common</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs-client</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
 
@@ -1852,8 +1879,8 @@
             <artifactId>ant</artifactId>
           </exclusion>
           <exclusion>
-            <groupId>com.esotericsoftware.kryo</groupId>
-            <artifactId>kryo</artifactId>
+            <groupId>org.datanucleus</groupId>
+            <artifactId>datanucleus-core</artifactId>
           </exclusion>
           <exclusion>
             <groupId>commons-codec</groupId>
@@ -1863,11 +1890,6 @@
             <groupId>org.apache.avro</groupId>
             <artifactId>avro-mapred</artifactId>
           </exclusion>
-          <!--  Do not need Calcite because we disabled hive.cbo.enable -->
-          <exclusion>
-            <groupId>org.apache.calcite</groupId>
-            <artifactId>calcite-core</artifactId>
-          </exclusion>
           <exclusion>
             <groupId>org.apache.calcite</groupId>
             <artifactId>calcite-avatica</artifactId>
@@ -1959,6 +1981,18 @@
             <groupId>net.hydromatic</groupId>
             <artifactId>aggdesigner-algorithm</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.atlas</groupId>
+            <artifactId>atlas-intg</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-registry</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.jayway.jsonpath</groupId>
+            <artifactId>json-path</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -2084,6 +2118,10 @@
             <groupId>com.jolbox</groupId>
             <artifactId>bonecp</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.orc</groupId>
+            <artifactId>orc-core</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
 
@@ -2168,7 +2206,7 @@
       <dependency>
         <groupId>${hive.group}</groupId>
         <artifactId>hive-service-rpc</artifactId>
-        <version>4.0.0</version>
+        <version>${hive.version}</version>
         <exclusions>
           <exclusion>
             <groupId>*</groupId>
@@ -2262,6 +2300,10 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-serde</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -2305,6 +2347,57 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <dependency>
+        <groupId>${hive.group}</groupId>
+        <artifactId>hive-udf</artifactId>
+        <version>${hive.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>${hive.group}</groupId>
+        <artifactId>hive-standalone-metastore-server</artifactId>
+        <version>${hive.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.antlr</groupId>
+            <artifactId>antlr-runtime</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.orc</groupId>
+            <artifactId>orc-core</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.datasketches</groupId>
+            <artifactId>datasketches-hive</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.github.joshelser</groupId>
+            <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.datasketches</groupId>
+        <artifactId>datasketches-hive</artifactId>
+        <version>2.0.0</version>
+        <exclusions>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>${hive.group}</groupId>
+        <artifactId>hive-parser</artifactId>
+        <version>${hive.version}</version>
+        <scope>runtime</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.calcite</groupId>
+        <artifactId>calcite-core</artifactId>
+        <version>1.38.0</version>
+        <scope>runtime</scope>
+      </dependency>
 
       <dependency>
         <groupId>org.apache.orc</groupId>
@@ -2457,6 +2550,11 @@
         <artifactId>datanucleus-core</artifactId>
         <version>${datanucleus-core.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.datanucleus</groupId>
+        <artifactId>datanucleus-api-jdo</artifactId>
+        <version>6.0.5</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.thrift</groupId>
         <artifactId>libthrift</artifactId>
@@ -2606,6 +2704,12 @@
         <artifactId>javax.servlet-api</artifactId>
         <version>${javaxservlet.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.maven.shared</groupId>
+        <artifactId>maven-invoker</artifactId>
+        <version>3.3.0</version>
+        <scope>test</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
@@ -2783,7 +2887,7 @@
             <jvmArgs>
               <jvmArg>-Xss128m</jvmArg>
               <jvmArg>-Xms4g</jvmArg>
-              <jvmArg>-Xmx4g</jvmArg>
+              <jvmArg>-Xmx8g</jvmArg>
               <jvmArg>-XX:MaxMetaspaceSize=2g</jvmArg>
               <jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
               <jvmArg>--enable-native-access=ALL-UNNAMED</jvmArg>