HERE Data SDK for Java & Scala Release 2.20.0

Author: heremaps-bot

README.md

@@ -28,11 +28,12 @@ For more information, see a use case illustration of [keeping a client map up to
 
 | Name | Description | Source | Labels / Topics |
 | ---- | ----------- | ------ | --------------- |
-| Geometry Lifter | An application that takes level 14 partitions of road topology and geometry and aggregates them to higher-level (i.e. bigger) partitions. | [Java](data-processing/java/geometry-lifter) / [Scala](data-processing/scala/geometry-lifter) | Data Processing Library, Spark, Batch, Protobuf, HERE Map Content |
+| Geometry Lifter | An application that takes level 12 partitions of road topology and geometry and aggregates them to higher-level (i.e. bigger) partitions. | [Java](data-processing/java/geometry-lifter) / [Scala](data-processing/scala/geometry-lifter) | Data Processing Library, Spark, Batch, Protobuf, HERE Map Content |
 | Pedestrian Topologies Extraction to GeoJSON | Topologies, accessible by pedestrians, are selected based on the segment attributes and then are transformed into GeoJSON file format and stored in a new catalog. | [Java](data-processing/java/pedestrian-topologies-extraction-geojson) / [Scala](data-processing/scala/pedestrian-topologies-extraction-geojson) | Data Processing Library, Spark, Batch, GeoJSON, HERE Map Content |
 | Pedestrian Topologies Extraction to Protobuf | Topologies, accessible by pedestrians, are selected based on the segment attributes and then are transformed to a newly created proto schema format and stored in a new catalog layer that follows that schema. | [Java](data-processing/java/pedestrian-topologies-extraction-protobuf) / [Scala](data-processing/scala/pedestrian-topologies-extraction-protobuf) | Data Processing Library, Spark, Batch, Protobuf, HERE Map Content |
 | Statistics creation across multiple processing runs with stateful processing | The application counts how often the node cardinality of the topology changes in each partition. | [Java](data-processing/java/stateful-nodecardinality-extraction) / [Scala](data-processing/scala/stateful-nodecardinality-extraction) | Data Processing Library, Spark, Batch, JSON, HERE Map Content |
 | Here Map Content Diff-Tool | An application to compare the content of two different versions of an input catalog. | [Java](data-processing/java/heremapcontent-difftool) / [Scala](data-processing/scala/heremapcontent-difftool) | Data Processing Library, Spark, Batch, JSON, HERE Map Content  |
+| Here Map Content Validation | An application to validate road topology and geometry content against a set of acceptance criteria using [scalatest](www.scalatest.org). | [Scala](data-processing/scala/heremapcontent-validation) | Data Processing Library, Spark, Batch, JSON, HERE Map Content  |
 | Data Validation and Testing Part 1: Content Testing | The test considers input partitions containing an octagon as "PASS" tiles and partitions containing a straight line as "FAIL". It records the test result in a new output catalog. | [Java](data-validation/java/quick-start/testing) / [Scala](data-validation/scala/quick-start/testing) | Data Validation Library, Spark, Batch, Protobuf |
 | Data Validation and Testing Part 2: Metrics | Reads the output of the testing component, assigns an error severity of CRITICAL for the "FAIL" tiles in the test results, aggregates the failed tile IDs and records the metrics in a new output catalog. | [Java](data-validation/java/quick-start/metrics) / [Scala](data-validation/scala/quick-start/metrics) | Data Validation Library, Spark, Batch, Protobuf |
 | Data Validation and Testing Part 3: Assessment | Reads the output of the metrics component, gives a final value of FAIL, if more than 10% of the candidate tiles have CRITICAL errors, and writes the result to a new catalog. | [Java](data-validation/java/quick-start/assessment) / [Scala](data-validation/scala/quick-start/assessment) | Data Validation Library, Spark, Batch, Protobuf |

data-archive/java/avro-example/pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>com.here.platform.data.archive</groupId>
     <artifactId>data-archive-avro-example</artifactId>
-    <version>0.0.565</version>
+    <version>0.0.582</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section start -->
@@ -23,7 +23,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
 
-        <sdk-bom.version>2.19.10</sdk-bom.version>
+        <sdk-bom.version>2.20.11</sdk-bom.version>
         <scala.compat.version>2.11</scala.compat.version>
 
         <avro.version>1.10.0</avro.version>

data-archive/java/parquet-example/pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>com.here.platform.data.archive</groupId>
     <artifactId>data-archive-parquet-example</artifactId>
-    <version>0.0.565</version>
+    <version>0.0.582</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section start -->
@@ -23,7 +23,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
 
-        <sdk-bom.version>2.19.10</sdk-bom.version>
+        <sdk-bom.version>2.20.11</sdk-bom.version>
         <scala.compat.version>2.11</scala.compat.version>
 
         <hadoop-client.version>2.7.3</hadoop-client.version>

data-archive/java/protobuf-example/pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>com.here.platform.data.archive</groupId>
     <artifactId>data-archive-protobuf-example</artifactId>
-    <version>0.0.565</version>
+    <version>0.0.582</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section start -->
@@ -23,7 +23,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
 
-        <sdk-bom.version>2.19.10</sdk-bom.version>
+        <sdk-bom.version>2.20.11</sdk-bom.version>
         <scala.compat.version>2.11</scala.compat.version>
 
         <commons-logging.version>1.2</commons-logging.version>

data-archive/java/sensoris-parquet-example/pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>com.here.platform.examples</groupId>
     <artifactId>data-archive-sensoris-parquet-example</artifactId>
-    <version>0.0.565</version>
+    <version>0.0.582</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section -->
@@ -23,7 +23,7 @@
         <java.version>1.8</java.version>
         <artifact.wagon.version>1.6.0</artifact.wagon.version>
 
-        <sdk-bom.version>2.19.10</sdk-bom.version>
+        <sdk-bom.version>2.20.11</sdk-bom.version>
         <scala.compat.version>2.11</scala.compat.version>
         <hadoop-client.version>2.7.3</hadoop-client.version>
         <parquet-protobuf.version>1.11.0</parquet-protobuf.version>

data-archive/java/sensoris-protobuf-example/pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>com.here.platform.examples</groupId>
     <artifactId>data-archive-sensoris-protobuf-example</artifactId>
-    <version>0.0.565</version>
+    <version>0.0.582</version>
     <packaging>jar</packaging>
 
     <!-- Meta information section -->
@@ -25,7 +25,7 @@
         <java.version>1.8</java.version>
         <artifact.wagon.version>1.6.0</artifact.wagon.version>
 
-        <sdk-bom.version>2.19.10</sdk-bom.version>
+        <sdk-bom.version>2.20.11</sdk-bom.version>
         <scala.compat.version>2.11</scala.compat.version>
 
         <parquet-protobuf.version>1.10.1</parquet-protobuf.version>

data-processing/java/geometry-lifter/README.md

@@ -148,7 +148,7 @@ To run the compiler locally, you will need to run the entry point to the compile
 
 - `com.here.platform.data.processing.example.java.geometry.lifter.Main`
 
-As _arguments_, you must provide the `--master` _parameter_ with the address of the Spark server
+As _arguments_, you must provide the `-Dspark.master` _parameter_ with the address of the Spark server
 master to connect to, and any configuration parameters you want to override. Alternatively, you
 can add those parameters to the `application.conf` file.
 
@@ -165,6 +165,9 @@ partition of HERE Map Content. Make sure you update the layer coverage to reflec
 geographical region. In order to use this configuration file, you need to use the `-Dconfig.file`
 parameter.
 
+Setup the environment variable `$PATH_TO_CONFIG_FOLDER` to `./config/here`, 
+for the HERE platform China environment use the files in the `./config/here-china` directory.
+
 ### Run the Compiler from the Command Line
 
 Combine all of this together and run the following command line in the `geometry-lifter`
@@ -175,22 +178,10 @@ For the HERE platform environment:
 ```bash
 mvn exec:java \
 -Dexec.mainClass=com.here.platform.data.processing.example.java.geometry.lifter.Main \
--Dpipeline-config.file=./config/here/pipeline-config.conf \
+-Dpipeline-config.file=$PATH_TO_CONFIG_FOLDER/pipeline-config.conf \
 -Dpipeline-job.file=./config/here/pipeline-job.conf \
 -Dconfig.file=./config/here/local-application.conf \
--Dexec.args="--master local[*]" \
--Dhere.platform.data-client.request-signer.credentials.here-account.here-token-scope=$PROJECT_HRN
-```
-
-For the HERE platform China environment:
-
-```bash
-mvn exec:java \
--Dexec.mainClass=com.here.platform.data.processing.example.java.geometry.lifter.Main \
--Dpipeline-config.file=./config/here-china/pipeline-config.conf \
--Dpipeline-job.file=./config/here-china/pipeline-job.conf \
--Dconfig.file=./config/here-china/local-application.conf \
--Dexec.args="--master local[*]" \
+-Dspark.master=local[*] \
 -Dhere.platform.data-client.request-signer.credentials.here-account.here-token-scope=$PROJECT_HRN
 ```
 

data-processing/java/geometry-lifter/pom.xml

@@ -3,7 +3,7 @@
 
   <groupId>com.here.platform.data.processing.example.java.geometry.lifter</groupId>
   <artifactId>geometry-lifter</artifactId>
-  <version>0.0.565</version>
+  <version>0.0.582</version>
   <packaging>jar</packaging>
 
   <name>Batch Processor compiler for Geometry Lifter in Java</name>
@@ -22,7 +22,7 @@
     <java.version>1.8</java.version>
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
-    <sdk-bom.version>2.19.10</sdk-bom.version>
+    <sdk-bom.version>2.20.11</sdk-bom.version>
     <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
     <maven-shade-plugin.version>3.0.0</maven-shade-plugin.version>
     <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>

data-processing/java/heremapcontent-difftool/README.md

@@ -173,7 +173,7 @@ To run the compiler locally, you will need to run the entry point to the compile
 
 - `com.here.platform.data.processing.example.java.difftool.processor.Main`
 
-As _arguments_, you must provide the `--master` _parameter_ with the address of the Spark server
+As _arguments_, you must provide the `-Dspark.master` _parameter_ with the address of the Spark server
 master to connect to, and any configuration parameters you want to override. Alternatively, you can
 add those parameters to the `application.conf` file.
 
@@ -190,6 +190,9 @@ partition of HERE Map Content. Make sure you update the layer coverage to reflec
 geographical region. In order to use this configuration file, you need to use the `-Dconfig.file`
 parameter.
 
+Setup the environment variable `$PATH_TO_CONFIG_FOLDER` to `./config/here`, 
+for the HERE platform China environment use the files in the `./config/here-china` directory.
+
 ### Run the Compiler from the Command Line
 
 The first run of the pipeline will use the job configuration `pipeline-job-first.conf`. As
@@ -203,10 +206,10 @@ For the HERE platform environment:
 ```bash
 mvn exec:java \
 -Dexec.mainClass=com.here.platform.data.processing.example.java.difftool.processor.Main \
--Dpipeline-config.file=./config/here/pipeline-config.conf \
+-Dpipeline-config.file=$PATH_TO_CONFIG_FOLDER/pipeline-config.conf \
 -Dpipeline-job.file=./config/here/pipeline-job-first.conf \
 -Dconfig.file=./config/here/local-application.conf \
--Dexec.args="--master local[*]" \
+-Dspark.master=local[*] \
 -Dhere.platform.data-client.request-signer.credentials.here-account.here-token-scope=$PROJECT_HRN
 ```
 
@@ -217,32 +220,10 @@ the version specified in `pipeline-job-second.conf`. Run the following command l
 ```bash
 mvn exec:java \
 -Dexec.mainClass=com.here.platform.data.processing.example.java.difftool.processor.Main \
--Dpipeline-config.file=./config/here/pipeline-config.conf \
+-Dpipeline-config.file=$PATH_TO_CONFIG_FOLDER/pipeline-config.conf \
 -Dpipeline-job.file=./config/here/pipeline-job-second.conf \
 -Dconfig.file=./config/here/local-application.conf \
--Dexec.args="--master local[*]" \
--Dhere.platform.data-client.request-signer.credentials.here-account.here-token-scope=$PROJECT_HRN
-```
-
-For the HERE platform China environment, instead, use the files in the `config/here-chine` directory:
-
-```bash
-mvn exec:java \
--Dexec.mainClass=com.here.platform.data.processing.example.java.difftool.processor.Main \
--Dpipeline-config.file=./config/here-china/pipeline-config.conf \
--Dpipeline-job.file=./config/here-china/pipeline-job-first.conf \
--Dconfig.file=./config/here-china/local-application.conf \
--Dexec.args="--master local[*]" \
--Dhere.platform.data-client.request-signer.credentials.here-account.here-token-scope=$PROJECT_HRN
-```
-
-```bash
-mvn exec:java \
--Dexec.mainClass=com.here.platform.data.processing.example.java.difftool.processor.Main \
--Dpipeline-config.file=./config/here-china/pipeline-config.conf \
--Dpipeline-job.file=./config/here-china/pipeline-job-second.conf \
--Dconfig.file=./config/here-china/local-application.conf \
--Dexec.args="--master local[*]" \
+-Dspark.master=local[*] \
 -Dhere.platform.data-client.request-signer.credentials.here-account.here-token-scope=$PROJECT_HRN
 ```
 

data-processing/java/heremapcontent-difftool/pom.xml

@@ -3,7 +3,7 @@
 
   <groupId>com.here.platform.data.processing.example.java</groupId>
   <artifactId>heremapcontent-difftool</artifactId>
-  <version>0.0.565</version>
+  <version>0.0.582</version>
   <packaging>jar</packaging>
 
   <name>HERE Map Content Differences example for Java</name>
@@ -22,7 +22,7 @@
     <java.version>1.8</java.version>
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
-    <sdk-bom.version>2.19.10</sdk-bom.version>
+    <sdk-bom.version>2.20.11</sdk-bom.version>
     <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
     <maven-shade-plugin.version>3.0.0</maven-shade-plugin.version>
     <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>