"spark_run.sh exec" now works, add docker commands for "spark_run.sh exec" and "spark_run.sh driver", upload s3mdbseq_keys.txt from a standard path

Author: scality-gdoumergue

entrypoint.sh

@@ -32,6 +32,6 @@ in
    ;;
    exec)
     shift
-    $*
+    "$@"
    ;;
 esac

scripts/S3_FSCK/upload_metadata_backup_keys_to_s3.sh

@@ -28,13 +28,6 @@ if [ ! -f "${CONFIG_FILE}" ];then
 	exit 1
 fi
 
-export BUCKETD_HOSTPORT="$(shyaml get-value bucketd.url < ${CONFIG_FILE} | awk -F / '{print $NF}')"
-if [ -z "${BUCKETD_HOSTPORT}" ];then
-
-        echo "Provide bucketd URL in ${CONFIG_FILE}."
-        exit 1
-fi
-
 S3_ENDPOINT="$(shyaml get-value s3.endpoint < ${CONFIG_FILE})"
 export AWS_ACCESS_KEY_ID="$(shyaml get-value s3.access_key < ${CONFIG_FILE})"
 export AWS_SECRET_ACCESS_KEY="$(shyaml get-value s3.secret_key < ${CONFIG_FILE})"
@@ -44,7 +37,7 @@ RING="$(shyaml get-value ring < ${CONFIG_FILE})"
 # We use an exotic path for WORKDIR to make sure the container
 # was started with the documented command line.
 
-WORKDIR=/mnt/var_tmp
+WORKDIR=/opt/spark/tmp
 
 if [ ! -d "${WORKDIR}" ];then
         echo "ERROR ${WORKDIR} must be mounted by docker/ctr."

spark_run.sh

@@ -71,20 +71,34 @@ in
     shift
     case $container_command
         in
+	docker)
+	$container_command run --rm --net=host --name=EXEC \
+	-v "${appsdir}:/opt/spark/apps:rw" \
+	-v "${datadir}:/opt/spark/tmp:rw" \
+	-v "${logsdir}:/opt/spark/spark-events:rw" \
+	/opt/spark/entrypoint.sh exec "$@"
+	;;
 	ctr)
-	ctr run --net-host --rm --mount="type=bind,src=${appsdir},dst=/opt/spark/apps,options=rbind:rw" \
+	$container_command run --net-host --rm --mount="type=bind,src=${appsdir},dst=/opt/spark/apps,options=rbind:rw" \
 		--mount="type=bind,src=${datadir},dst=/opt/spark/tmp,options=rbind:rw" \
-		--mount="type=bind,src=${logdir},dst=/opt/spark/spark-events,options=rbind:rw" \
-		"${IMAGE_NAME}:${VERSION}" EXEC ./entrypoint.sh exec $*
+		--mount="type=bind,src=${logsdir},dst=/opt/spark/spark-events,options=rbind:rw" \
+		"${IMAGE_NAME}:${VERSION}" EXEC /opt/spark/entrypoint.sh exec "$@"
     esac
     ;;
     driver)
     case $container_command
         in
+	docker)
+	$container_command run --rm --net=host --name=SPARK-DRIVER \
+	-v "${appsdir}:/opt/spark/apps:rw" \
+	-v "${datadir}:/opt/spark/tmp:rw" \
+	-v "${logsdir}:/opt/spark/spark-events:rw" \
+	/opt/spark/entrypoint.sh driver
+	;;
 	ctr)
-	ctr run --net-host --rm -cwd /opt/spark/apps --mount="type=bind,src=${appsdir},dst=/opt/spark/apps,options=rbind:rw" \
+	$container_command run --net-host --rm -cwd /opt/spark/apps --mount="type=bind,src=${appsdir},dst=/opt/spark/apps,options=rbind:rw" \
 		--mount="type=bind,src=${datadir},dst=/opt/spark/tmp,options=rbind:rw" \
-		--mount="type=bind,src=${logdir},dst=/opt/spark/spark-events,options=rbind:rw" -t \
+		--mount="type=bind,src=${logsdir},dst=/opt/spark/spark-events,options=rbind:rw" -t \
 		"${IMAGE_NAME}:${VERSION}" SPARK-DRIVER /opt/spark/entrypoint.sh driver 2> /dev/null
     esac
     ;;
@@ -108,7 +122,7 @@ in
                     $container_command run --rm -dit --net=host --name spark-master \
                     --hostname=spark-master \
                     --add-host=spark-master:$master \
-                    --volume "${logdir}:/opt/spark/spark-events:rw" \
+                    --volume "${logsdir}:/opt/spark/spark-events:rw" \
                     --volume "${datadir}:/opt/spark/tmp:rw" \
                     "$host_storage" \
 		    ${spark_image_full} \
@@ -121,7 +135,7 @@ in
                     --hostname=spark-worker \
                     --add-host=spark-master:$master \
                     --add-host=spark-worker:"$local_worker" \
-                    --volume "${logdir}:/opt/spark/spark-events:rw" \
+                    --volume "${logsdir}:/opt/spark/spark-events:rw" \
                     --volume "${datadir}:/opt/spark/tmp:rw" \
                     "$host_storage" \
 		    ${spark_image_full} \
@@ -162,7 +176,7 @@ in
                     # start
                     $container_command run -d --net-host \
                     --env='SPARK_NO_DAEMONIZE=true' \
-                    --mount='type=bind,src='${logdir}',dst=/opt/spark/spark-events,options=rbind:rw' \
+                    --mount='type=bind,src='${logsdir}',dst=/opt/spark/spark-events,options=rbind:rw' \
                     --mount='type=bind,src='${datadir}',dst=/opt/spark/tmp,options=rbind:rw' \
 		    ${spark_image_full} spark-master \
                     ./entrypoint.sh master 2> /dev/null