diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3dc4c38 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +openrc.sh diff --git a/README.md b/README.md index c92ce61..2945b71 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,9 @@ To build your own Virtual cluster, starting on your localhost: the NodeName and PartitionName line. * If you'd like to change the default node size, the ```node_size=```line in ```slurm_resume.sh``` must be changed. + This should take values corresponding to instance sizes in Jetstream, like + "m1.small" or "m1.large". Be sure to edit the ```slurm.conf``` file to + reflect the number of CPUs available. * If you'd like to enable any specific software, you should edit ```compute_build_base_img.yml```. The task named "install basic packages" can be easily extended to install anything available from a yum diff --git a/compute_build_base_img.yml b/compute_build_base_img.yml index 3fe0ae3..f63d8e9 100644 --- a/compute_build_base_img.yml +++ b/compute_build_base_img.yml @@ -3,12 +3,12 @@ - hosts: localhost vars: - compute_base_image: "JS-API-Featured-CentOS7-May-20-2019" - sec_group_global: "{{ clouds.tacc.auth.username }}-global-ssh" - sec_group_internal: "{{ clouds.tacc.auth.username }}-cluster-internal" + compute_base_image: "JS-API-Featured-CentOS7-Sep-09-2019" + sec_group_global: "{{ clouds.tacc.cluster_name }}-global-ssh" + sec_group_internal: "{{ clouds.tacc.cluster_name }}-cluster-internal" compute_base_size: "m1.small" - network_name: "{{ clouds.tacc.auth.username }}-elastic-net" - JS_ssh_keyname: "{{ clouds.tacc.auth.username }}-{{ clouds.tacc.auth.project_name }}-slurm-key" + network_name: "{{ clouds.tacc.cluster_name }}-elastic-net" + JS_ssh_keyname: "{{ clouds.tacc.cluster_name }}-{{ clouds.tacc.auth.project_name }}-slurm-key" vars_files: - clouds.yaml @@ -19,7 +19,7 @@ os_server: timeout: 300 state: present - name: "compute-{{ clouds.tacc.auth.username }}-base-instance" + name: "compute-{{ clouds.tacc.cluster_name }}-base-instance" cloud: "tacc" image: "{{ compute_base_image }}" key_name: "{{ JS_ssh_keyname }}" @@ -63,6 +63,13 @@ - "openmpi-gnu-ohpc" - "ohpc-slurm-client" - "lmod-ohpc" + - "git" + - "bind-utils" + - "gcc" + - "wget" + - "jq" + - "nodejs" + - "singularity" tasks: @@ -96,6 +103,12 @@ state: present lock_timeout: 300 + - name: enable hostfs mount on singularity + replace: + path: /etc/singularity/singularity.conf + regexp: "^(mount hostfs = no)$" + replace: "mount hostfs = yes" + - name: fix slurm user uid user: name: slurm @@ -191,6 +204,10 @@ - name: enable slurmd service: name=slurmd enabled=yes + - name: create local user called "user" + user: + name: user + #cat /etc/systemd/system/multi-user.target.wants/slurmd.service #[Unit] #Description=Slurm node daemon @@ -230,6 +247,9 @@ tasks: - name: create compute instance snapshot +<<<<<<< HEAD + script: ./compute_take_snapshot.sh "compute-{{ clouds.tacc.cluster_name }}-base-instance" +======= expect: command: ./compute_take_snapshot.sh responses: @@ -237,10 +257,13 @@ timeout: null # Need to check if this refers to the whole command or just the expect part? no_log: true # to avoid putting OS_PASSWORD in logs - uncomment and re-run if you run into errors! +>>>>>>> 7b201c00234ded4ae207a94673916057e7a63d39 - name: remove compute instance os_server: timeout: 200 state: absent - name: "compute-{{ clouds.tacc.auth.username }}-base-instance" + name: "compute-{{ clouds.tacc.cluster_name }}-base-instance" cloud: "tacc" + + diff --git a/compute_take_snapshot.sh b/compute_take_snapshot.sh index 454a833..3de4f36 100755 --- a/compute_take_snapshot.sh +++ b/compute_take_snapshot.sh @@ -2,8 +2,11 @@ source openrc.sh -compute_image="${OS_USERNAME}-compute-image-$(date +%m-%d-%Y)" -compute_instance="compute-${OS_USERNAME}-base-instance" +#compute_instance="compute-${cluster_name}-base-instance" +cluster_name=$(hostname -s) + +compute_instance=$1 +compute_image="${cluster_name}-compute-image-$(date +%m-%d-%Y)" openstack server stop $compute_instance diff --git a/headnode_create.sh b/headnode_create.sh index 97360d4..551e6b0 100755 --- a/headnode_create.sh +++ b/headnode_create.sh @@ -9,6 +9,7 @@ if [[ -z "$1" ]]; then echo "NO SERVER NAME GIVEN! Please re-run with ./headnode_create.sh " exit fi +server_name=$1 if [[ ! -e ${HOME}/.ssh/id_rsa.pub ]]; then #This may be temporary... but seems fairly reasonable. @@ -16,30 +17,30 @@ if [[ ! -e ${HOME}/.ssh/id_rsa.pub ]]; then exit fi -server_name=$1 source ./openrc.sh # Defining a function here to check for quotas, and exit if this script will cause problems! # also, storing 'quotas' in a global var, so we're not calling it every single time quotas=$(openstack quota show) -quota_check () -{ -quota_name=$1 -type_name=$2 #the name for a quota and the name for the thing itself are not the same -number_created=$3 #number of the thing that we'll create here. +quota_check () { + quota_name=$1 + type_name=$2 #the name for a quota and the name for the thing itself are not the same + number_created=$3 #number of the thing that we'll create here. -current_num=$(openstack ${type_name} list -f value | wc -l) + current_num=$(openstack ${type_name} list -f value | wc -l) -max_types=$(echo "${quotas}" | awk -v quota=${quota_name} '$0 ~ quota {print $4}') + max_types=$(echo "${quotas}" | awk -v quota=${quota_name} '$0 ~ quota {print $4}') -#echo "checking quota for ${quota_name} of ${type_name} to create ${number_created} - want ${current_num} to be less than ${max_types}" + #echo "checking quota for ${quota_name} of ${type_name} to create ${number_created} - want ${current_num} to be less than ${max_types}" -if [[ "${current_num}" -lt "$((max_types + number_created))" ]]; then - return 0 -fi -return 1 + if [[ "${current_num}" -lt "$((max_types + number_created))" ]]; then + return 0 + fi + return 1 } +set -x #show use which commands are executed +set -e #terminate as soon as any command fails quota_check "secgroups" "security group" 1 quota_check "networks" "network" 1 @@ -49,28 +50,27 @@ quota_check "key-pairs" "keypair" 1 quota_check "instances" "server" 1 # Ensure that the correct private network/router/subnet exists -if [[ -z "$(openstack network list | grep ${OS_USERNAME}-elastic-net)" ]]; then - openstack network create ${OS_USERNAME}-elastic-net - openstack subnet create --network ${OS_USERNAME}-elastic-net --subnet-range 10.0.0.0/24 ${OS_USERNAME}-elastic-subnet1 +if [[ -z "$(openstack network list | grep ${server_name}-elastic-net)" ]]; then + openstack network create ${server_name}-elastic-net + openstack subnet create --network ${server_name}-elastic-net --subnet-range 10.0.0.0/24 ${server_name}-elastic-subnet1 fi ##openstack subnet list -if [[ -z "$(openstack router list | grep ${OS_USERNAME}-elastic-router)" ]]; then - openstack router create ${OS_USERNAME}-elastic-router - openstack router add subnet ${OS_USERNAME}-elastic-router ${OS_USERNAME}-elastic-subnet1 - openstack router set --external-gateway public ${OS_USERNAME}-elastic-router +if [[ -z "$(openstack router list | grep ${server_name}-elastic-router)" ]]; then + openstack router create ${server_name}-elastic-router + openstack router add subnet ${server_name}-elastic-router ${server_name}-elastic-subnet1 + openstack router set --external-gateway public ${server_name}-elastic-router fi -#openstack router show ${OS_USERNAME}-api-router security_groups=$(openstack security group list -f value) -if [[ ! ("${security_groups}" =~ "${OS_USERNAME}-global-ssh") ]]; then - openstack security group create --description "ssh \& icmp enabled" ${OS_USERNAME}-global-ssh - openstack security group rule create --protocol tcp --dst-port 22:22 --remote-ip 0.0.0.0/0 ${OS_USERNAME}-global-ssh - openstack security group rule create --protocol icmp ${OS_USERNAME}-global-ssh +if [[ ! ("$security_groups" =~ "${server_name}-global-ssh") ]]; then + openstack security group create --description "ssh \& icmp enabled" $server_name-global-ssh + openstack security group rule create --protocol tcp --dst-port 22:22 --remote-ip 0.0.0.0/0 $server_name-global-ssh + openstack security group rule create --protocol icmp $server_name-global-ssh fi -if [[ ! ("${security_groups}" =~ "${OS_USERNAME}-cluster-internal") ]]; then - openstack security group create --description "internal group for cluster" ${OS_USERNAME}-cluster-internal - openstack security group rule create --protocol tcp --dst-port 1:65535 --remote-ip 10.0.0.0/0 ${OS_USERNAME}-cluster-internal - openstack security group rule create --protocol icmp ${OS_USERNAME}-cluster-internal +if [[ ! ("$security_groups" =~ "${server_name}-cluster-internal") ]]; then + openstack security group create --description "internal group for cluster" $server_name-cluster-internal + openstack security group rule create --protocol tcp --dst-port 1:65535 --remote-ip 10.0.0.0/0 $server_name-cluster-internal + openstack security group rule create --protocol icmp $server_name-cluster-internal fi #Check if ${HOME}/.ssh/id_rsa.pub exists in JS @@ -79,33 +79,19 @@ if [[ -e ${HOME}/.ssh/id_rsa.pub ]]; then fi openstack_keys=$(openstack keypair list -f value) -home_key_in_OS=$(echo "${openstack_keys}" | awk -v mykey=${home_key_fingerprint} '$2 ~ mykey {print $1}') +home_key_in_OS=$(echo "${openstack_keys}" | awk -v mykey="${home_key_fingerprint}" '$2 ~ mykey {print $1}') if [[ -n "${home_key_in_OS}" ]]; then OS_keyname=${home_key_in_OS} -elif [[ -n $(echo "${openstack_keys}" | grep ${OS_USERNAME}-elastic-key) ]]; then - openstack keypair delete ${OS_USERNAME}-elastic-key -# This doesn't need to depend on the OS_PROJECT_NAME, as the slurm-key does, in install.sh and slurm_resume - openstack keypair create --public-key ${HOME}/.ssh/id_rsa.pub ${OS_USERNAME}-elastic-key - OS_keyname=${OS_USERNAME}-elastic-key -else +elif [[ -n $(echo "${openstack_keys}" | grep ${server_name}-elastic-key) ]]; then + openstack keypair delete ${server_name}-elastic-key # This doesn't need to depend on the OS_PROJECT_NAME, as the slurm-key does, in install.sh and slurm_resume - openstack keypair create --public-key ${HOME}/.ssh/id_rsa.pub ${OS_USERNAME}-elastic-key - OS_keyname=${OS_USERNAME}-elastic-key + openstack keypair create --public-key ${HOME}/.ssh/id_rsa.pub ${server_name}-elastic-key + OS_keyname=${server_name}-elastic-key fi centos_base_image=$(openstack image list --status active | grep -iE "API-Featured-centos7-[[:alpha:]]{3,4}-[0-9]{2}-[0-9]{4}" | awk '{print $4}' | tail -n 1) -echo -e "openstack server create\ - --user-data prevent-updates.ci \ - --flavor m1.small \ - --image ${centos_base_image} \ - --key-name ${OS_keyname} \ - --security-group ${OS_USERNAME}-global-ssh \ - --security-group ${OS_USERNAME}-cluster-internal \ - --nic net-id=${OS_USERNAME}-elastic-net \ - ${server_name}" - openstack server create \ --user-data prevent-updates.ci \ --flavor m1.small \ @@ -117,6 +103,7 @@ openstack server create \ ${server_name} public_ip=$(openstack floating ip create public | awk '/floating_ip_address/ {print $4}') + #For some reason there's a time issue here - adding a sleep command to allow network to become ready sleep 10 openstack server add floating ip ${server_name} ${public_ip} diff --git a/install.sh b/install.sh index 6fd5333..62fea9c 100755 --- a/install.sh +++ b/install.sh @@ -10,11 +10,30 @@ if [[ $EUID -ne 0 ]]; then exit 1 fi +set -e +set -x + +yum -y install \ + ohpc-slurm-server \ + vim ansible \ + mailx \ + lmod-ohpc \ + bash-completion \ + gnu-compilers-ohpc \ + openmpi-gnu-ohpc \ + lmod-defaults-gnu-openmpi-ohpc \ + moreutils \ + bind-utils \ + nodejs \ + jq \ + git \ + singularity \ + python-openstackclient + #do this early, allow the user to leave while the rest runs! source ./openrc.sh -yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm \ - centos-release-openstack-rocky +yum -y install https://github.com/openhpc/ohpc/releases/download/v1.3.GA/ohpc-release-1.3-1.el7.x86_64.rpm centos-release-openstack-rocky yum -y install \ ohpc-slurm-server \ @@ -29,8 +48,12 @@ yum -y install \ lmod-defaults-gnu-openmpi-ohpc \ moreutils \ bind-utils \ + nodejs \ + jq \ + git \ + singularity \ python2-openstackclient \ - python2-pexpect + python2-pexpect yum -y update # until the base python2-openstackclient install works out of the box! @@ -43,6 +66,8 @@ yum -y update # until the base python2-openstackclient install works out of the [ ! -f /home/centos/.ssh/id_rsa ] && su centos - -c 'ssh-keygen -t rsa -b 2048 -P "" -f /home/centos/.ssh/id_rsa && cat /home/centos/.ssh/id_rsa.pub >> /home/centos/.ssh/authorized_keys' +cluster_name=$(hostname -s) + #create clouds.yaml file from contents of openrc echo -e "clouds: tacc: @@ -51,7 +76,9 @@ echo -e "clouds: auth_url: ${OS_AUTH_URL} project_name: ${OS_PROJECT_NAME} password: ${OS_PASSWORD} + cluster_name: $cluster_name user_domain_name: ${OS_USER_DOMAIN_NAME} + project_domain_id: ${OS_PROJECT_DOMAIN_ID} identity_api_version: 3" > clouds.yaml # There are different versions of openrc floating around between the js wiki and auto-generated openrc files. @@ -93,24 +120,24 @@ fi #quota_check "instances" "server" 1 -if [[ -n $(openstack keypair list | grep ${OS_USERNAME}-${OS_PROJECT_NAME}-slurm-key) ]]; then - openstack keypair delete ${OS_USERNAME}-${OS_PROJECT_NAME}-slurm-key - openstack keypair create --public-key slurm-key.pub ${OS_USERNAME}-${OS_PROJECT_NAME}-slurm-key +if [[ -n $(openstack keypair list | grep ${cluster_name}-${OS_PROJECT_NAME}-slurm-key) ]]; then + openstack keypair delete ${cluster_name}-${OS_PROJECT_NAME}-slurm-key + openstack keypair create --public-key slurm-key.pub ${cluster_name}-${OS_PROJECT_NAME}-slurm-key else - openstack keypair create --public-key slurm-key.pub ${OS_USERNAME}-${OS_PROJECT_NAME}-slurm-key + openstack keypair create --public-key slurm-key.pub ${cluser_name}-${OS_PROJECT_NAME}-slurm-key fi #make sure security groups exist... this could cause issues. if [[ ! ("$security_groups" =~ "global-ssh") ]]; then - openstack security group create --description "ssh \& icmp enabled" ${OS_USERNAME}-global-ssh - openstack security group rule create --protocol tcp --dst-port 22:22 --remote-ip 0.0.0.0/0 ${OS_USERNAME}-global-ssh - openstack security group rule create --protocol icmp ${OS_USERNAME}-global-ssh + openstack security group create --description "ssh \& icmp enabled" ${cluster_name}-global-ssh + openstack security group rule create --protocol tcp --dst-port 22:22 --remote-ip 0.0.0.0/0 ${cluster_name}-global-ssh + openstack security group rule create --protocol icmp ${cluster_name}-global-ssh fi if [[ ! ("$security_groups" =~ "cluster-internal") ]]; then - openstack security group create --description "internal 10.0.0.0/24 network allowed" ${OS_USERNAME}-cluster-internal - openstack security group rule create --protocol tcp --dst-port 1:65535 --remote-ip 10.0.0.0/24 ${OS_USERNAME}-cluster-internal - openstack security group rule create --protocol udp --dst-port 1:65535 --remote-ip 10.0.0.0/24 ${OS_USERNAME}-cluster-internal - openstack security group rule create --protocol icmp ${OS_USERNAME}-cluster-internal + openstack security group create --description "internal 10.0.0.0/24 network allowed" ${cluster_name}-cluster-internal + openstack security group rule create --protocol tcp --dst-port 1:65535 --remote-ip 10.0.0.0/24 ${cluster_name}-cluster-internal + openstack security group rule create --protocol udp --dst-port 1:65535 --remote-ip 10.0.0.0/24 ${cluster_name}-cluster-internal + openstack security group rule create --protocol icmp ${cluster_name}-cluster-internal fi #TACC-specific changes: @@ -126,8 +153,8 @@ fi #sed -i "s/network_name=.*/network_name=$headnode_os_subnet/" ./slurm_resume.sh #Set compute node names to $OS_USERNAME-compute- -sed -i "s/=compute-*/=${OS_USERNAME}-compute-/" ./slurm.conf -sed -i "s/Host compute-*/Host ${OS_USERNAME}-compute-/" ./ssh.cfg +sed -i "s/=compute-*/=${cluster_name}-compute-/" ./slurm.conf +sed -i "s/Host compute-*/Host ${cluster_name}-compute-/" ./ssh.cfg # Deal with files required by slurm - better way to encapsulate this section? @@ -145,9 +172,16 @@ setfacl -m u:slurm:rwx /etc/ chmod +t /etc -#How to generate a working openrc in the cloud-init script for this? Bash vars available? -# Gonna be tough, since openrc requires a password... -cp openrc.sh /etc/slurm/ +#Possible to handle this at the cloud-init level? From a machine w/ +# pre-loaded openrc, possible via user-data and write_files, yes. +echo -e "export OS_PROJECT_DOMAIN_NAME=tacc +export OS_USER_DOMAIN_NAME=tacc +export OS_PROJECT_NAME=${OS_PROJECT_NAME} +export OS_USERNAME=${OS_PROJECT_NAME} +export OS_PASSWORD=${OS_PASSWORD} +export OS_AUTH_URL=${OS_AUTH_URL} +export OS_IDENTITY_API_VERSION=3" > /etc/slurm/openrc.sh + chown slurm:slurm /etc/slurm/openrc.sh @@ -170,6 +204,7 @@ setfacl -m u:slurm:rw /etc/ansible/hosts setfacl -m u:slurm:rwx /etc/ansible/ cp slurm_*.sh /usr/local/sbin/ +#sed -i "s/node_size=.*/node_size=m1.xlarge/" /usr/local/sbin/slurm_resume.sh cp cron-node-check.sh /usr/local/sbin/ cp clean-os-error.sh /usr/local/sbin/ @@ -221,3 +256,5 @@ systemctl enable slurmctld munge nfs-server nfs-lock nfs rpcbind nfs-idmap systemctl start munge slurmctld nfs-server nfs-lock nfs rpcbind nfs-idmap echo -e "If you wish to enable an email when node state is drain or down, please uncomment \nthe cron-node-check.sh job in /etc/crontab, and place your email of choice in the 'email_addr' variable \nat the beginning of /usr/local/sbin/cron-node-check.sh" + + diff --git a/slurm.conf b/slurm.conf index 7c7a271..5e00dad 100644 --- a/slurm.conf +++ b/slurm.conf @@ -8,8 +8,8 @@ # # See the slurm.conf man page for more information. # -ClusterName=js-slurm-elastic -ControlMachine=slurm-example +ClusterName=js-slurm6-elastic +ControlMachine=slurm6 #ControlAddr= #BackupController= #BackupAddr= @@ -68,7 +68,8 @@ SchedulerType=sched/backfill #SchedulerRootFilter= #SelectType=select/linear SelectType=select/cons_res -SelectTypeParameters=CR_CPU +#SelectTypeParameters=CR_CPU +SelectTypeParameters=CR_CPU_Memory FastSchedule=0 #PriorityType=priority/multifactor #PriorityDecayHalfLife=14-0 @@ -98,18 +99,20 @@ AccountingStorageLoc=/var/log/slurm/slurm_jobacct.log #AccountingStorageUser= # #GENERAL RESOURCE -GresTypes="" +#GresTypes="" # #CLOUD CONFIGURATION PrivateData=cloud ResumeProgram=/usr/local/sbin/slurm_resume.sh SuspendProgram=/usr/local/sbin/slurm_suspend.sh -ResumeRate=0 #number of nodes per minute that can be created; 0 means no limit +ResumeRate=1 #number of nodes per minute that can be created; 0 means no limit ResumeTimeout=900 #max time in seconds between ResumeProgram running and when the node is ready for use -SuspendRate=0 #number of nodes per minute that can be suspended/destroyed -SuspendTime=60 #time in seconds before an idle node is suspended -SuspendTimeout=30 #time between running SuspendProgram and the node being completely down -#COMPUTE NODES -NodeName=compute-[0-1] State=CLOUD CPUs=2 -#PARTITIONS -PartitionName=cloud Nodes=compute-[0-1] Default=YES MaxTime=INFINITE State=UP +SuspendRate=1 #number of nodes per minute that can be suspended/destroyed +SuspendTime=300 #time in seconds before an idle node is suspended +SuspendTimeout=300 #time between running SuspendProgram and the node being completely down + +NodeName=slurm6-compute-[0-10] State=CLOUD CPUs=24 RealMemory=60388 +#NodeName=hayashis-compute-[1-10] State=CLOUD Sockets=24 CoresPerSocket=1 ThreadsPerCore=1 RealMemory=60388 +PartitionName=cloud LLN=YES Nodes=slurm6-compute-[1-10] Default=YES MaxTime=48:00:00 State=UP Shared=YES + + diff --git a/slurm_resume.sh b/slurm_resume.sh index 3da1508..d9c9783 100755 --- a/slurm_resume.sh +++ b/slurm_resume.sh @@ -2,10 +2,11 @@ source /etc/slurm/openrc.sh +cluster_name=$(hostname -s) node_size="m1.small" -node_image=$(openstack image list -f value | grep -i ${OS_USERNAME}-compute-image- | cut -f 2 -d' '| tail -n 1) -key_name="${OS_USERNAME}-${OS_PROJECT_NAME}-slurm-key" -network_name=${OS_USERNAME}-elastic-net +node_image=$(openstack image list -f value | grep -i ${cluster_name}-compute-image- | cut -f 2 -d' '| tail -n 1) +key_name="${cluster_name}-${OS_PROJECT_NAME}-slurm-key" +network_name=${cluster_name}-elastic-net log_loc=/var/log/slurm/slurm_elastic.log echo "Node resume invoked: $0 $*" >> $log_loc @@ -15,22 +16,23 @@ for host in $(scontrol show hostname $1) do #Launch compute nodes and check for new ip address in same subprocess - with 2s delay between Openstack requests + #--user-data <(cat /etc/slurm/prevent-updates.ci && echo -e "hostname: $host \npreserve_hostname: true\ndebug:") \ + # the current --user-data pulls in the slurm.conf as well, to avoid rebuilding node images + # when adding / changing partitions (echo "creating $host" >> $log_loc; openstack server create $host \ - --flavor $node_size \ - --image $node_image \ - --key-name $key_name \ - --user-data <(cat /etc/slurm/prevent-updates.ci && echo -e "hostname: $host \npreserve_hostname: true\ndebug:") \ - --security-group ${OS_USERNAME}-global-ssh --security-group ${OS_USERNAME}-cluster-internal \ - --nic net-id=$network_name 2>&1 \ + --flavor $node_size \ + --image $node_image \ + --key-name $key_name \ + --user-data <(cat /etc/slurm/prevent-updates.ci && echo -e "hostname: $host \npreserve_hostname: true\ndebug:" && echo -e "write_files:\n - encoding: b64\n owner: slurm\n path: /etc/slurm/slurm.conf\n permissions: 0644\n content: |\n$(cat /etc/slurm/slurm.conf | base64 | sed 's/^/ /')\n") \ + --security-group ${cluster_name}-global-ssh --security-group ${cluster_name}-cluster-internal \ + --nic net-id=$network_name 2>&1 \ | tee -a $log_loc | awk '/status/ {print $4}' >> $log_loc 2>&1; node_status="UNKOWN"; until [[ $node_status == "ACTIVE" ]]; do node_state=$(openstack server show $host 2>&1); node_status=$(echo -e "${node_state}" | awk '/status/ {print $4}'); -# echo "$host status is: $node_status" >> $log_loc; -# echo "$host ip is: $node_ip" >> $log_loc; sleep 3; done; node_ip=$(echo -e "${node_state}" | awk '/addresses/ {print gensub(/^.*=/,"","g",$4)}');