demo with ssl (#62)

Author: rashtao

.circleci/config.yml

@@ -287,6 +287,7 @@ jobs:
       - setup_remote_docker
       - start-db:
           topology: 'cluster'
+          ssl: 'true'
       - load_cache
       - run:
           name: mvn version

arangodb-spark-commons/src/main/scala/org/apache/spark/sql/arangodb/commons/ArangoDBConf.scala

@@ -507,7 +507,6 @@ class ArangoDBDriverConf(opts: Map[String, String]) extends ArangoDBConf(opts) {
 
   val sslAlgorithm: String = getConf(sslAlgorithmConf)
 
-  // FIXME: merge with sslTrustStoreType
   val sslKeystoreType: String = getConf(sslKeystoreTypeConf)
 
   val sslProtocol: String = getConf(sslProtocolConf)

demo/README.md

@@ -27,16 +27,16 @@ For the python demo, you will also need
 Set environment variables:
 
 ```shell
-export ARANGO_SPARK_VERSION=1.7.0
+export ARANGO_SPARK_VERSION=1.8.0-SNAPSHOT
 ```
 
 Start ArangoDB cluster with docker:
 
 ```shell
-STARTER_MODE=cluster ./docker/start_db.sh
+SSL=true STARTER_MODE=cluster ./docker/start_db.sh
 ```
 
-The deployed cluster will be accessible at [http://172.28.0.1:8529](http://172.28.0.1:8529) with username `root` and
+The deployed cluster will be accessible at [https://172.28.0.1:8529](http://172.28.0.1:8529) with username `root` and
 password `test`.
 
 Start Spark cluster:
@@ -58,16 +58,18 @@ mvn -f ../pom.xml install -Dmaven.test.skip=true -Dgpg.skip=true -Dmaven.javadoc
 Test the Spark application in embedded mode:
 
 ```shell
-mvn test -Pscala-2.12 -Pspark-3.5
+mvn \
+  -Pscala-2.12 -Pspark-3.5 \
+  test
 ```
 
 Test the Spark application against ArangoDB Oasis deployment:
 
 ```shell
 mvn \
+  -Pscala-2.12 -Pspark-3.5 \
   -Dpassword=<root-password> \
   -Dendpoints=<endpoint> \
-  -Dssl.enabled=true \
   -Dssl.cert.value=<base64-encoded-cert> \
   test
 ```
@@ -91,7 +93,7 @@ docker run -it --rm \
   docker.io/bitnami/spark:3.5.2 \
   ./bin/spark-submit --master spark://spark-master:7077 \
     --packages="com.arangodb:arangodb-spark-datasource-3.5_2.12:$ARANGO_SPARK_VERSION" \
-    --class Demo /demo/target/demo-$ARANGO_SPARK_VERSION.jar
+    --class Demo /demo/target/demo-$ARANGO_SPARK_VERSION.jar    
 ```
 
 ## Python(PySpark) Demo
@@ -105,6 +107,7 @@ pip install -r ./python-demo/requirements.txt
 To run the PySpark demo, run 
 ```shell
 python ./python-demo/demo.py \
+  --ssl-enabled=true \
   --endpoints=172.28.0.1:8529,172.28.0.1:8539,172.28.0.1:8549
 ```
 

demo/docker/.ivy2/.gitignore renamed to demo/docker/.ivy2/cache/.gitignore

@@ -0,0 +1,3 @@
+*
+*/
+!.gitignore

demo/docker/.ivy2/jars/.gitignore

@@ -30,6 +30,7 @@ def create_base_arangodb_datasource_opts(password: str, endpoints: str, ssl_enab
         "endpoints": endpoints,
         "ssl.enabled": ssl_enabled,
         "ssl.cert.value": ssl_cert_value,
+        "ssl.verifyHost": "false"
     }
 
 
@@ -39,7 +40,7 @@ def main():
     parser.add_argument("--password", default="test")
     parser.add_argument("--endpoints", default="localhost:8529")
     parser.add_argument("--ssl-enabled", default="false")
-    parser.add_argument("--ssl-cert-value", default="")
+    parser.add_argument("--ssl-cert-value", default="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURlekNDQW1PZ0F3SUJBZ0lFZURDelh6QU5CZ2txaGtpRzl3MEJBUXNGQURCdU1SQXdEZ1lEVlFRR0V3ZFYKYm10dWIzZHVNUkF3RGdZRFZRUUlFd2RWYm10dWIzZHVNUkF3RGdZRFZRUUhFd2RWYm10dWIzZHVNUkF3RGdZRApWUVFLRXdkVmJtdHViM2R1TVJBd0RnWURWUVFMRXdkVmJtdHViM2R1TVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXdIaGNOTWpBeE1UQXhNVGcxTVRFNVdoY05NekF4TURNd01UZzFNVEU1V2pCdU1SQXdEZ1lEVlFRR0V3ZFYKYm10dWIzZHVNUkF3RGdZRFZRUUlFd2RWYm10dWIzZHVNUkF3RGdZRFZRUUhFd2RWYm10dWIzZHVNUkF3RGdZRApWUVFLRXdkVmJtdHViM2R1TVJBd0RnWURWUVFMRXdkVmJtdHViM2R1TVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDMVdpRG5kNCt1Q21NRzUzOVoKTlpCOE53STBSWkYzc1VTUUdQeDNsa3FhRlRaVkV6TVpMNzZIWXZkYzlRZzdkaWZ5S3lRMDlSTFNwTUFMWDlldQpTc2VEN2JaR25mUUg1MkJuS2NUMDllUTN3aDdhVlE1c04yb215Z2RITEM3WDl1c250eEFmdjdOem12ZG9nTlhvCkpReVkvaFNaZmY3UklxV0g4Tm5BVUtranFPZTZCZjVMRGJ4SEtFU21yRkJ4T0NPbmhjcHZaV2V0d3BpUmRKVlAKd1VuNVA4MkNBWnpmaUJmbUJabkI3RDBsKy82Q3Y0ak11SDI2dUFJY2l4blZla0JRemwxUmd3Y3p1aVpmMk1HTwo2NHZETU1KSldFOUNsWkYxdVF1UXJ3WEY2cXdodVAxSG5raWk2d05iVHRQV2xHU2txZXV0cjAwNCtIemJmOEtuClJZNFBBZ01CQUFHaklUQWZNQjBHQTFVZERnUVdCQlRCcnY5QXd5bnQzQzVJYmFDTnlPVzV2NEROa1RBTkJna3EKaGtpRzl3MEJBUXNGQUFPQ0FRRUFJbTlyUHZEa1lwbXpwU0loUjNWWEc5WTcxZ3hSRHJxa0VlTHNNb0V5cUdudwovengxYkRDTmVHZzJQbmNMbFc2elRJaXBFQm9vaXhJRTlVN0t4SGdaeEJ5MEV0NkVFV3ZJVW1ucjZGNEYrZGJUCkQwNTBHSGxjWjdlT2VxWVRQWWVRQzUwMkcxRm80dGROaTRsRFA5TDlYWnBmN1ExUWltUkgycWFMUzAzWkZaYTIKdFk3YWgvUlFxWkw4RGt4eDgvemMyNXNnVEhWcHhvSzg1M2dsQlZCcy9FTk1peUdKV21BWFFheWV3WTNFUHQvOQp3R3dWNEttVTNkUERsZVFlWFNVR1BVSVNlUXhGankrakN3MjFwWXZpV1ZKVE5CQTlsNW55M0doRW1jbk9UL2dRCkhDdlZSTHlHTE1iYU1aNEpyUHdiK2FBdEJncmdlaUs0eGVTTU12cmJodz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K")
     args = parser.parse_args()
 
     if args.import_path is None:

demo/python-demo/demo.py

@@ -4,8 +4,8 @@ object Demo {
   val importPath: String = System.getProperty("importPath", "/demo/docker/import")
   val password: String = System.getProperty("password", "test")
   val endpoints: String = System.getProperty("endpoints", "172.28.0.1:8529,172.28.0.1:8539,172.28.0.1:8549")
-  val sslEnabled: String = System.getProperty("ssl.enabled", "false")
-  val sslCertValue: String = System.getProperty("ssl.cert.value", "")
+  val sslEnabled: String = System.getProperty("ssl.enabled", "true")
+  val sslCertValue: String = System.getProperty("ssl.cert.value", "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURlekNDQW1PZ0F3SUJBZ0lFZURDelh6QU5CZ2txaGtpRzl3MEJBUXNGQURCdU1SQXdEZ1lEVlFRR0V3ZFYKYm10dWIzZHVNUkF3RGdZRFZRUUlFd2RWYm10dWIzZHVNUkF3RGdZRFZRUUhFd2RWYm10dWIzZHVNUkF3RGdZRApWUVFLRXdkVmJtdHViM2R1TVJBd0RnWURWUVFMRXdkVmJtdHViM2R1TVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXdIaGNOTWpBeE1UQXhNVGcxTVRFNVdoY05NekF4TURNd01UZzFNVEU1V2pCdU1SQXdEZ1lEVlFRR0V3ZFYKYm10dWIzZHVNUkF3RGdZRFZRUUlFd2RWYm10dWIzZHVNUkF3RGdZRFZRUUhFd2RWYm10dWIzZHVNUkF3RGdZRApWUVFLRXdkVmJtdHViM2R1TVJBd0RnWURWUVFMRXdkVmJtdHViM2R1TVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDMVdpRG5kNCt1Q21NRzUzOVoKTlpCOE53STBSWkYzc1VTUUdQeDNsa3FhRlRaVkV6TVpMNzZIWXZkYzlRZzdkaWZ5S3lRMDlSTFNwTUFMWDlldQpTc2VEN2JaR25mUUg1MkJuS2NUMDllUTN3aDdhVlE1c04yb215Z2RITEM3WDl1c250eEFmdjdOem12ZG9nTlhvCkpReVkvaFNaZmY3UklxV0g4Tm5BVUtranFPZTZCZjVMRGJ4SEtFU21yRkJ4T0NPbmhjcHZaV2V0d3BpUmRKVlAKd1VuNVA4MkNBWnpmaUJmbUJabkI3RDBsKy82Q3Y0ak11SDI2dUFJY2l4blZla0JRemwxUmd3Y3p1aVpmMk1HTwo2NHZETU1KSldFOUNsWkYxdVF1UXJ3WEY2cXdodVAxSG5raWk2d05iVHRQV2xHU2txZXV0cjAwNCtIemJmOEtuClJZNFBBZ01CQUFHaklUQWZNQjBHQTFVZERnUVdCQlRCcnY5QXd5bnQzQzVJYmFDTnlPVzV2NEROa1RBTkJna3EKaGtpRzl3MEJBUXNGQUFPQ0FRRUFJbTlyUHZEa1lwbXpwU0loUjNWWEc5WTcxZ3hSRHJxa0VlTHNNb0V5cUdudwovengxYkRDTmVHZzJQbmNMbFc2elRJaXBFQm9vaXhJRTlVN0t4SGdaeEJ5MEV0NkVFV3ZJVW1ucjZGNEYrZGJUCkQwNTBHSGxjWjdlT2VxWVRQWWVRQzUwMkcxRm80dGROaTRsRFA5TDlYWnBmN1ExUWltUkgycWFMUzAzWkZaYTIKdFk3YWgvUlFxWkw4RGt4eDgvemMyNXNnVEhWcHhvSzg1M2dsQlZCcy9FTk1peUdKV21BWFFheWV3WTNFUHQvOQp3R3dWNEttVTNkUERsZVFlWFNVR1BVSVNlUXhGankrakN3MjFwWXZpV1ZKVE5CQTlsNW55M0doRW1jbk9UL2dRCkhDdlZSTHlHTE1iYU1aNEpyUHdiK2FBdEJncmdlaUs0eGVTTU12cmJodz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K")
 
   val spark: SparkSession = SparkSession.builder
     .appName("arangodb-demo")
@@ -16,7 +16,8 @@ object Demo {
     "password" -> password,
     "endpoints" -> endpoints,
     "ssl.enabled" -> sslEnabled,
-    "ssl.cert.value" -> sslCertValue
+    "ssl.cert.value" -> sslCertValue,
+    "ssl.verifyHost" -> "false"
   )
 
   def main(args: Array[String]): Unit = {