Permit the addition of extra ip:hostname pairs within containers

Author: scality-gdoumergue

spark_run.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
 
-[ -z "$1" ] && echo "Usage: $0 <start|stop|status|env>" && exit 1
+[ -z "$1" ] && echo "Usage: $0 <start|stop|driver|exec|env>" && exit 1
 
 # Version of the Spark image and tooling. Export it.
 export VERSION=3.5.2-12
 export IMAGE_NAME=registry.scality.com/spark/spark-container
 
 # Please change the IPs according to your architecture
 # without tuning the entrypoint.sh, a worker cannot have a master running on the same host
-master="10.160.172.70"
-workers="10.160.169.6 10.160.175.70 10.160.168.250 10.160.170.227 10.160.169.48"
+master="10.160.169.162"
+workers="10.160.174.4 10.160.168.21 10.160.171.166 10.160.172.100 10.160.169.238"
+
 
 # Please change the directory where Spark workers will write temporary data
 
@@ -19,6 +20,14 @@ datadir="/scality/ssd01/spark"
 appsdir="/root/spark-apps"
 logsdir="/root/spark-logs"
 
+# Extra hosts you need containers to be able to resolve.
+# format: "ip1:host1 ip2:host2 ..."
+# Example:
+# extrahosts="10.2.4.1:s3.scality.com
+#             19.10.3.1:microsoft.com"
+
+extrahosts=""
+
 [ "${1}" = "env" ] && return 0
 
 ########### END of tunable variables
@@ -46,8 +55,38 @@ test -z "$container_command" && echo "docker or CTR not found!" && exit 1
 echo "Checking the Spark image with $container_command"
 case $container_command
     in
-    docker) version_check_result=$($container_command images --format '{{.Repository}}:{{.Tag}}' | grep "${spark_image_full}") ;;
-    ctr) version_check_result=$($container_command images list |  grep "${spark_image_full}" 2> /dev/null);;
+    docker) version_check_result=$($container_command images --format '{{.Repository}}:{{.Tag}}' | grep "${spark_image_full}")
+	    # extrahosts update
+	    add_hosts=""
+	    for i in $extrahosts ; do add_hosts+=" --add-host=$(echo $i)"; done
+    ;;
+    ctr) version_check_result=$($container_command images list |  grep "${spark_image_full}" 2> /dev/null)
+         # hosts update
+         host_storage=""
+         count=01
+         localname=
+         for i in $workers ; do
+             n=spark-worker-$(printf "%02d" ${count})
+             ((count++))
+             if echo "$local_ips" |grep -Fqw "$i" ; then n="${n} `hostname -s`" ; fi
+             test "$(grep -qw "$n" /etc/hosts ; echo $?)" == 0 && continue
+             host_storage+="$i $n\n"
+         done
+	 for i in $extrahosts ; do
+             test "$(grep -qw "$i" /etc/hosts ; echo $?)" == 0 && continue
+             host_storage+="$(echo $i | sed 's#:# #') # Part of Spark extrahosts $i\n"
+         done
+         test -n "$host_storage" && echo -e "# Host file updated for Spark\n$host_storage" >> /etc/hosts
+
+         # Manage when a role has changed in the same node
+         # spark-master should be known on all nodes
+         if ! grep -q "$master spark-master # set-by-spark_run.sh" /etc/hosts;
+         then
+             grep -q "spark-master # set-by-spark_run.sh" /etc/hosts && \
+                 sed -i '/spark-master # set-by-spark_run.sh/d' /etc/hosts
+             echo "$master spark-master # set-by-spark_run.sh" >> /etc/hosts
+         fi
+    ;;
 esac
 
 if [ -n "${version_check_result}" ];then
@@ -72,10 +111,12 @@ in
     case $container_command
         in
 	docker)
+
 	$container_command run --rm --net=host --name=EXEC \
 	-v "${appsdir}:/opt/spark/apps:rw" \
 	-v "${datadir}:/opt/spark/tmp:rw" \
 	-v "${logsdir}:/opt/spark/spark-events:rw" \
+	${add_hosts} \
 	"${spark_image_full}" \
 	exec "$@"
 	;;
@@ -96,6 +137,7 @@ in
 	-v "${logsdir}:/opt/spark/spark-events:rw" \
         --add-host="spark-master:${master}" \
         --add-host="$(hostname -s):$(echo ${workers} | awk '{print $1}')" \
+	${add_hosts} \
 	--workdir=/opt/spark/apps \
 	"${spark_image_full}" \
 	driver
@@ -119,7 +161,7 @@ in
             docker) echo "Running $container_command"
 
 	        # hosts update
-	        host_storage=""
+	        host_storage="${add_hosts}"
 	        count=01
 	        for i in $workers ; do host_storage+=" --add-host=spark-worker-$(printf "%02d" ${count}):$i"; ((count++)) ; done
 
@@ -153,27 +195,6 @@ in
                 fi
             ;;
             ctr) echo "Running $container_command"
-                # hosts update
-                host_storage=""
-                count=01
-                localname=
-                for i in $workers ; do
-                    n=spark-worker-$(printf "%02d" ${count})
-                    ((count++))
-                    if echo "$local_ips" |grep -Fqw "$i" ; then n="${n} `hostname -s`" ; fi
-                    test "$(grep -qw "$n" /etc/hosts ; echo $?)" == 0 && continue
-                    host_storage+="$i $n\n"
-                done
-                test -n "$host_storage" && echo -e "# Host file updated for Spark\n$host_storage" >> /etc/hosts
-
-                # Manage when a role has changed in the same node
-                # spark-master should be known on all nodes
-                if ! grep -q "$master spark-master # set-by-spark_run.sh" /etc/hosts;
-                then
-                    grep -q "spark-master # set-by-spark_run.sh" /etc/hosts && \
-                        sed -i '/spark-master # set-by-spark_run.sh/d' /etc/hosts
-                    echo "$master spark-master # set-by-spark_run.sh" >> /etc/hosts
-                fi
 
                 if [ -n "$local_master" ] ; then
                     echo "Running master here"