Merge pull request #187 from Kotlin/version-bumps

Jolanrensen · web-flow · commit f309ed97d775 · 2022-12-01T14:47:19.000+01:00
including spark 3.2.3, 3.3.1, 3.2.2, scala bumps, small fix in Encoding
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -10,24 +10,24 @@ jobs:
   build-all-versions:
     strategy:
       matrix:
-        scala: [ "2.12.16", "2.13.8" ]
-        spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+        scala: [ "2.12.17", "2.13.10" ]
+        spark: [ "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
         exclude:
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.3"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.2"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.1"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.0"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.3"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.2"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.1"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.0"
     runs-on: ubuntu-latest
 
diff --git a/.github/workflows/publish_dev_version.yml b/.github/workflows/publish_dev_version.yml
@@ -9,24 +9,24 @@ jobs:
   build-and-deploy:
     strategy:
       matrix:
-        scala: [ "2.12.16", "2.13.8" ]
-        spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+        scala: [ "2.12.17", "2.13.10" ]
+        spark: [ "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
         exclude:
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.3"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.2"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.1"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.0"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.3"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.2"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.1"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.0"
     runs-on: ubuntu-latest
     permissions:
@@ -72,7 +72,7 @@ jobs:
           ./gradlew 
           -Pspark=${{ matrix.spark }} 
           -Pscala=${{ matrix.scala }} 
-          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.10' && matrix.spark == '3.2.0') }} 
           clean 
           publishMavenPublicationToGitHubPackagesRepository 
           --scan
diff --git a/.github/workflows/publish_release_version.yml b/.github/workflows/publish_release_version.yml
@@ -1,34 +1,31 @@
 name: Deploy release version to Maven Central
 
 on:
-#  release: TODO temporary, revert to this after 1.2.1 release
-#    types: [created]
-  push:
-    branches:
-      - "release"
+  release:
+    types: [created]
 
 jobs:
   build-and-deploy-mvn-central:
     strategy:
       matrix:
-        scala: [ "2.12.16", "2.13.8" ]
-        spark: [ "3.3.0", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
+        scala: [ "2.12.17", "2.13.10" ]
+        spark: [ "3.3.1", "3.3.0", "3.2.3", "3.2.2", "3.2.1", "3.2.0", "3.1.3", "3.1.2", "3.1.1", "3.1.0", "3.0.3", "3.0.2", "3.0.1", "3.0.0" ]
         exclude:
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.3"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.2"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.1"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.1.0"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.3"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.2"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.1"
-          - scala: "2.13.8"
+          - scala: "2.13.10"
             spark: "3.0.0"
     runs-on: ubuntu-latest
     permissions:
@@ -77,7 +74,7 @@ jobs:
           ./gradlew 
           -Pspark=${{ matrix.spark }} 
           -Pscala=${{ matrix.scala }} 
-          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.8' && matrix.spark == '3.2.0') }} 
+          -PskipScalaTuplesInKotlin=${{ !(matrix.spark == '3.0.0' || matrix.scala == '2.13.10' && matrix.spark == '3.2.0') }} 
           clean 
           publishMavenPublicationToMavenCentralRepository
           --scan
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![Kotlin Stable](https://kotl.in/badges/stable.svg)](https://kotlinlang.org/docs/components-stability.html)
 [![JetBrains official project](http://jb.gg/badges/official.svg)](https://confluence.jetbrains.com/display/ALL/JetBrains+on+GitHub)
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.1_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.1_2.13")
 [![Join the chat at https://gitter.im/JetBrains/kotlin-spark-api](https://badges.gitter.im/JetBrains/kotlin-spark-api.svg)](https://gitter.im/JetBrains/kotlin-spark-api?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Your next API to work with  [Apache Spark](https://spark.apache.org/). 
@@ -38,8 +38,14 @@ We have opened a Spark Project Improvement Proposal: [Kotlin support for Apache
 
 | Apache Spark | Scala |       Kotlin for Apache Spark       |
 |:------------:|:-----:|:-----------------------------------:|
+|    3.3.1     | 2.13  | kotlin-spark-api_3.3.1_2.13:VERSION |
+|              | 2.12  | kotlin-spark-api_3.3.1_2.12:VERSION |
 |    3.3.0     | 2.13  | kotlin-spark-api_3.3.0_2.13:VERSION |
 |              | 2.12  | kotlin-spark-api_3.3.0_2.12:VERSION |
+|    3.2.3     | 2.13  | kotlin-spark-api_3.2.3_2.13:VERSION |
+|              | 2.12  | kotlin-spark-api_3.2.3_2.12:VERSION |
+|    3.2.2     | 2.13  | kotlin-spark-api_3.2.2_2.13:VERSION |
+|              | 2.12  | kotlin-spark-api_3.2.2_2.12:VERSION |
 |    3.2.1     | 2.13  | kotlin-spark-api_3.2.1_2.13:VERSION |
 |              | 2.12  | kotlin-spark-api_3.2.1_2.12:VERSION |
 |    3.2.0     | 2.13  | kotlin-spark-api_3.2.0_2.13:VERSION |
@@ -67,7 +73,7 @@ The Kotlin for Spark artifacts adhere to the following convention:
 The only exception to this is `scala-tuples-in-kotlin_[Scala core version]:[Kotlin for Apache Spark API version]`, which is 
 independent of Spark.
 
-[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.0_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.0_2.13")
+[![Maven Central](https://img.shields.io/maven-central/v/org.jetbrains.kotlinx.spark/kotlin-spark-api_3.3.1_2.13.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:"org.jetbrains.kotlinx.spark"%20AND%20a:"kotlin-spark-api_3.3.1_2.13")
 
 ## How to configure Kotlin for Apache Spark in your project
 
@@ -78,7 +84,7 @@ Here's an example `pom.xml`:
 ```xml
 <dependency>
   <groupId>org.jetbrains.kotlinx.spark</groupId>
-  <artifactId>kotlin-spark-api_3.3.0_2.13</artifactId>
+  <artifactId>kotlin-spark-api_3.3.1_2.13</artifactId>
   <version>${kotlin-spark-api.version}</version>
 </dependency>
 <dependency>
@@ -111,7 +117,7 @@ To it, simply add
 to the top of your notebook. This will get the latest version of the API, together with the latest version of Spark.
 To define a certain version of Spark or the API itself, simply add it like this:
 ```jupyterpython
-%use spark(spark=3.3.0, scala=2.13, v=1.2.0)
+%use spark(spark=3.3.1, scala=2.13, v=1.2.2)
 ```
 
 Inside the notebook a Spark session will be initiated automatically. This can be accessed via the `spark` value.
diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt
@@ -20,6 +20,7 @@ object Dependencies {
     inline val kotlinTest get() = "org.jetbrains.kotlin:kotlin-test:${Versions.kotlin}"
     inline val kotlinScriptingCommon get() = "org.jetbrains.kotlin:kotlin-scripting-common"
     inline val kotlinScriptingJvm get() = "org.jetbrains.kotlin:kotlin-scripting-jvm"
+    inline val jacksonDatabind get() = "com.fasterxml.jackson.core:jackson-databind:${Versions.jacksonDatabind}"
 }
 
 
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
@@ -1,7 +1,7 @@
 object Versions {
     const val project = "1.2.2-SNAPSHOT"
     const val groupID = "org.jetbrains.kotlinx.spark"
-    const val kotlin = "1.7.10"
+    const val kotlin = "1.7.20"
     const val jvmTarget = "11"
 
     inline val spark get() = System.getProperty("spark") as String
@@ -21,6 +21,7 @@ object Versions {
     const val hadoop = "3.3.1"
     const val kotlinxHtml = "0.7.5"
     const val klaxon = "5.5"
+    const val jacksonDatabind = "2.13.4.2"
 
     inline val versionMap
         get() = mapOf(
diff --git a/core/build.gradle.kts b/core/build.gradle.kts
@@ -26,6 +26,9 @@ dependencies {
             reflect,
         )
 
+        // https://github.com/FasterXML/jackson-bom/issues/52
+        if (Versions.spark == "3.3.1") implementation(jacksonDatabind)
+
         implementation(
             sparkSql,
         )
@@ -35,14 +38,18 @@ dependencies {
 
 java {
     toolchain {
-        languageVersion.set(
-            JavaLanguageVersion.of(Versions.jvmTarget)
-        )
+        if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8
+            languageVersion.set(
+                JavaLanguageVersion.of(Versions.jvmTarget)
+            )
+        }
     }
 }
 
 tasks.withType<ScalaCompile> {
-    targetCompatibility = Versions.jvmTarget
+    if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8
+        targetCompatibility = Versions.jvmTarget
+    }
 }
 
 val scalaMainSources = sourceSets.main.get().scala.sourceDirectories
diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md
@@ -72,13 +72,13 @@ Here's what the `pom.xml` looks like for this example:
         </dependency>
         <dependency> <!-- Kotlin Spark API dependency -->
             <groupId>org.jetbrains.kotlinx.spark</groupId>
-            <artifactId>kotlin-spark-api-3.0.0_2.12</artifactId>
-            <version>1.0.1</version>
+            <artifactId>kotlin-spark-api_3.3.1_2.13</artifactId>
+            <version>1.2.2</version>
         </dependency>
         <dependency> <!-- Spark dependency -->
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_2.12</artifactId>
-            <version>3.0.0</version>
+            <version>3.3.1</version>
         </dependency>
     </dependencies>
 
@@ -163,8 +163,8 @@ dependencies {
   // Kotlin stdlib
   implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.4.0'
   // Kotlin Spark API
-  implementation 'org.jetbrains.kotlinx.spark:kotlin-spark-api-3.0.0_2.12:1.0.1'  // Apache Spark
-  compileOnly 'org.apache.spark:spark-sql_2.12:3.0.0'
+  implementation 'org.jetbrains.kotlinx.spark:kotlin-spark-api_3.3.1_2.13:1.2.2'  // Apache Spark
+  compileOnly 'org.apache.spark:spark-sql_2.12:3.3.1'
 }
 
 compileKotlin {
@@ -198,9 +198,9 @@ dependencies {
   // Kotlin stdlib
   implementation ("org.jetbrains.kotlin:kotlin-stdlib:1.4.0")
   // Kotlin Spark API
-  implementation ("org.jetbrains.kotlinx.spark:kotlin-spark-api-3.0.0_2.12:1.0.1")
+  implementation ("org.jetbrains.kotlinx.spark:kotlin-spark-api_3.3.1_2.13:1.2.2")
   // Apache Spark
-  compileOnly ("org.apache.spark:spark-sql_2.12:3.0.0")
+  compileOnly ("org.apache.spark:spark-sql_2.12:3.3.1")
 }
 
 compileKotlin.kotlinOptions.jvmTarget = "1.8"
diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
@@ -21,6 +21,10 @@ dependencies {
     }
 
     with(Dependencies) {
+
+        // https://github.com/FasterXML/jackson-bom/issues/52
+        if (Versions.spark == "3.3.1") implementation(jacksonDatabind)
+
         implementation(
             sparkSql,
             sparkMl,
diff --git a/gradle.properties b/gradle.properties
@@ -7,9 +7,9 @@ GROUP=org.jetbrains.kotlinx.spark
 
 # Controls the spark and scala version for the entire project
 # can also be defined like ./gradlew -Pspark=X.X.X -Pscala=X.X.X build
-spark=3.3.0
-scala=2.13.8
-#scala=2.12.16
+spark=3.3.1
+scala=2.13.10
+# scala=2.12.17
 skipScalaTuplesInKotlin=false
 
 org.gradle.caching=true
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts
@@ -43,6 +43,10 @@ dependencies {
     }
 
     with(Dependencies) {
+
+        // https://github.com/FasterXML/jackson-bom/issues/52
+        if (Versions.spark == "3.3.1") implementation(jacksonDatabind)
+
         api(
             kotlinxHtml,
             sparkSql,
diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts
@@ -35,6 +35,10 @@ dependencies {
     }
 
     with(Dependencies) {
+
+        // https://github.com/FasterXML/jackson-bom/issues/52
+        if (Versions.spark == "3.3.1") implementation(jacksonDatabind)
+
         implementation(
             kotlinStdLib,
             reflect,
diff --git a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
@@ -236,7 +236,7 @@ fun schema(type: KType, map: Map<String, KType> = mapOf()): DataType {
             )
         }
 
-        klass.isSubclassOf(Map::class) -> {
+        klass == Map::class -> {
             val mapKeyParam = types.getValue(klass.typeParameters[0].name)
             val mapValueParam = types.getValue(klass.typeParameters[1].name)
 

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@ object Dependencies {`
`20`	`20`	`inline val kotlinTest get() = "org.jetbrains.kotlin:kotlin-test:${Versions.kotlin}"`
`21`	`21`	`inline val kotlinScriptingCommon get() = "org.jetbrains.kotlin:kotlin-scripting-common"`
`22`	`22`	`inline val kotlinScriptingJvm get() = "org.jetbrains.kotlin:kotlin-scripting-jvm"`
	`23`	`+ inline val jacksonDatabind get() = "com.fasterxml.jackson.core:jackson-databind:${Versions.jacksonDatabind}"`
`23`	`24`	`}`
`24`	`25`
`25`	`26`
Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,9 @@ dependencies {`
`26`	`26`	`reflect,`
`27`	`27`	`)`
`28`	`28`
	`29`	`+ // https://github.com/FasterXML/jackson-bom/issues/52`
	`30`	`+ if (Versions.spark == "3.3.1") implementation(jacksonDatabind)`
	`31`	`+`
`29`	`32`	`implementation(`
`30`	`33`	`sparkSql,`
`31`	`34`	`)`
`@@ -35,14 +38,18 @@ dependencies {`
`35`	`38`
`36`	`39`	`java {`
`37`	`40`	`toolchain {`
`38`		`- languageVersion.set(`
`39`		`- JavaLanguageVersion.of(Versions.jvmTarget)`
`40`		`- )`
	`41`	`+ if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8`
	`42`	`+ languageVersion.set(`
	`43`	`+ JavaLanguageVersion.of(Versions.jvmTarget)`
	`44`	`+ )`
	`45`	`+ }`
`41`	`46`	`}`
`42`	`47`	`}`
`43`	`48`
`44`	`49`	`tasks.withType<ScalaCompile> {`
`45`		`- targetCompatibility = Versions.jvmTarget`
	`50`	`+ if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8`
	`51`	`+ targetCompatibility = Versions.jvmTarget`
	`52`	`+ }`
`46`	`53`	`}`
`47`	`54`
`48`	`55`	`val scalaMainSources = sourceSets.main.get().scala.sourceDirectories`