Skip to content

Commit 042c074

Browse files
zzhou1zzhou1
committed
Dev: run-functional-tests: "-n" or "-n X..Y" can grow more nodes
1 parent 1205b7c commit 042c074

File tree

1 file changed

+123
-59
lines changed

1 file changed

+123
-59
lines changed

‎test/run-functional-tests‎

Lines changed: 123 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ HA_NETWORK_V6_ARRAY[1]="2001:db8:20::/64"
1616
BEHAVE_CASE_DIR="$(dirname $0)/features/"
1717
BEHAVE_CASE_EXCLUDE="sbd|ocfs2"
1818

19+
declare -a hanode_list_to_form_cluster
20+
declare -a hanode_list_new_members
21+
declare -a hanode_list_current_cluster
22+
1923
read -r -d '' SSHD_CONFIG_AZURE << EOM
2024
PermitRootLogin no
2125
AuthorizedKeysFile .ssh/authorized_keys
@@ -103,6 +107,13 @@ is_number() {
103107
test ! -z "$num" && test "$num" -eq "$num" 2> /dev/null && test "$num" -gt 0 2> /dev/null
104108
}
105109

110+
is_num_or_seq__c_style() {
111+
echo $1|grep -q -e '^[0-9]\+$'
112+
[ $? -eq 0 ] && [ "$1" -gt 0 ] && return 1
113+
echo $1|grep -q -e '^[0-9]\+\.\.[0-9]\+$'
114+
[ $? -eq 0 ] && return 2
115+
return 0 # return 0 means error here
116+
}
106117

107118
check_docker_env() {
108119
# check if docker available
@@ -126,7 +137,7 @@ check_docker_env() {
126137
for network in ${HA_NETWORK_ARRAY[@]};do
127138
docker network ls|grep -q "$network"
128139
if [ "$?" -eq 0 ];then
129-
fatal "HA specific network \"$network\" already exists"
140+
warning "HA specific network \"$network\" already exists"
130141
fi
131142
done
132143
}
@@ -160,23 +171,33 @@ The container image is based on Tumbleweed with preinstalled packages of the clu
160171
Users can make the code change under crmsh.git including test cases. This tool will pick up the code change and "make install" to all running containers.
161172
162173
OPTIONS:
163-
-h, --help Show this help message and exit
164-
-l List existing functional test cases and exit
165-
-n NUM Only setup a cluster with NUM nodes(containers)
174+
-h, --help Show this help message and exit
175+
-l List existing functional test cases and exit
176+
-n <NUM|X..Y> NUM of nodes(containers) from hanode1 to hanode$NUM, or hanode$X to hanode$Y
166177
-x Don't config corosync on containers(with -n option)
167178
-d Cleanup the cluster containers
168179
-u Create normal users, and Azure like ssh environment
169180
-q Create a qnetd node(with -n and -x option)
170181
171182
EXAMPLES:
183+
172184
To launch 2 nodes with the running cluster with the very basic corosync.conf
173185
# crmsh.git/test/run-functional-tests -n 2
174186
187+
To grow more nodes with a bigger number than '2' in the above example, and skip nodes if exist
188+
# crmsh.git/test/run-functional-tests -n 5
189+
190+
To grow more nodes with the given sequence, even a leap, and skip nodes if exist
191+
# crmsh.git/test/run-functional-tests -n 10..12
192+
175193
To launch 2 nodes without the cluster stack running to play with "crm cluster init/join"
176-
# crmsh.git/run-functional-tests -n 2 -x
194+
# crmsh.git/test/run-functional-tests -n 2 -x
195+
196+
To grow more nodes without configure the cluster stack
197+
# crmsh.git/test/run-functional-tests -n 3..5 -x
177198
178199
To launch 2 nodes without the cluster stack running, and a qnetd node(named 'qnetd-node')
179-
# crmsh.git/run-functional-tests -n 2 -x -q
200+
# crmsh.git/test/run-functional-tests -n 2 -x -q
180201
181202
To list the existing test cases. Users could add his own new test cases.
182203
# crmsh.git/test/run-functional-tests -l
@@ -233,8 +254,12 @@ deploy_ha_node() {
233254

234255
info "Deploying \"$node_name\"..."
235256
docker run --restart always $docker_options $DOCKER_IMAGE &> /dev/null
257+
if [ $? -ne 0 ]; then
258+
warning Likely $node_name already exists.
259+
return
260+
fi
236261
for network in ${HA_NETWORK_ARRAY[@]};do
237-
docker network connect $network $node_name
262+
docker network connect $network $node_name &> /dev/null
238263
done
239264

240265
if [ "$node_name" != "qnetd-node" ];then
@@ -245,29 +270,29 @@ deploy_ha_node() {
245270
docker_exec $node_name "echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config"
246271

247272
if [ "$node_name" != "qnetd-node" ];then
248-
docker cp $PROJECT_PATH $node_name:/opt/crmsh
249-
info "Building crmsh on \"$node_name\"..."
250-
docker_exec $node_name "$make_cmd" 1> /dev/null || \
273+
docker cp $PROJECT_PATH $node_name:/opt/crmsh
274+
info "Building crmsh on \"$node_name\"..."
275+
docker_exec $node_name "$make_cmd" 1> /dev/null || \
251276
fatal "Building failed on $node_name!"
252-
docker_exec $node_name "chown hacluster:haclient -R /var/log/crmsh"
253-
docker_exec $node_name "chmod g+w -R /var/log/crmsh"
254-
create_alice_bob_carol
255-
if [ "$NORMAL_USER_FLAG" -eq 1 ];then
256-
set_sshd_config_like_in_azure $node_name
257-
fi
277+
docker_exec $node_name "chown hacluster:haclient -R /var/log/crmsh"
278+
docker_exec $node_name "chmod g+w -R /var/log/crmsh"
279+
create_alice_bob_carol
280+
if [ "$NORMAL_USER_FLAG" -eq 1 ];then
281+
set_sshd_config_like_in_azure $node_name
282+
fi
258283
else
259-
docker_exec $node_name "useradd -m -s /bin/bash alice 2>/dev/null"
260-
docker_exec $node_name "echo \"alice ALL=(ALL) NOPASSWD:ALL\" > /etc/sudoers.d/alice"
261-
docker_exec $node_name "cp -r /root/.ssh ~alice/ && chown alice:users -R ~alice/.ssh"
262-
info "Create user 'alice' on $node_name"
263-
[ "$NORMAL_USER_FLAG" -eq 1 ] && set_sshd_config_like_in_azure $node_name
284+
docker_exec $node_name "useradd -m -s /bin/bash alice 2>/dev/null"
285+
docker_exec $node_name "echo \"alice ALL=(ALL) NOPASSWD:ALL\" > /etc/sudoers.d/alice"
286+
docker_exec $node_name "cp -r /root/.ssh ~alice/ && chown alice:users -R ~alice/.ssh"
287+
info "Create user 'alice' on $node_name"
288+
[ "$NORMAL_USER_FLAG" -eq 1 ] && set_sshd_config_like_in_azure $node_name
264289
fi
265290
}
266291

267292

268293
create_node() {
269294
info "Loading docker image $DOCKER_IMAGE..."
270-
docker pull $DOCKER_IMAGE &> /dev/null
295+
docker pull $DOCKER_IMAGE &> /dev/null
271296

272297
for index in ${!HA_NETWORK_ARRAY[@]};do
273298
network=${HA_NETWORK_ARRAY[$index]}
@@ -281,40 +306,63 @@ create_node() {
281306
wait
282307
}
283308

309+
get_cluster_new_nodes() {
310+
hanode_list_to_form_cluster=($(docker ps -a --format '{{.Names}}'|grep hanode|sort -n -k1.7|tr '\r' ' '))
311+
hanode_list_current_cluster=($(docker_exec hanode1 "crm node server 2>/dev/null" 2>/dev/null|sort -n -k1.7|tr '\r' ' '))
312+
hanode_list_new_members=()
313+
for element in "${hanode_list_to_form_cluster[@]}"; do
314+
if ! [[ " ${hanode_list_current_cluster[@]} " =~ " $element " ]]; then
315+
hanode_list_new_members+=("$element")
316+
fi
317+
done
318+
#echo hanode_list_to_form_cluster= ${hanode_list_to_form_cluster[@]}
319+
#echo hanode_list_current_cluster = ${hanode_list_current_cluster[@]}
320+
#echo ${#hanode_list_new_members[@]} new nodes "'${hanode_list_new_members[@]}'"
321+
}
284322

285323
config_cluster() {
286-
node_num=$#
287-
insert_str=""
288-
container_ip_array=(`docker network inspect $HA_NETWORK_ARRAY -f '{{range .Containers}}{{printf "%s " .IPv4Address}}{{end}}'`)
324+
get_cluster_new_nodes
325+
info ${#hanode_list_new_members[@]} new node\(s\) "'${hanode_list_new_members[@]}'"
289326

290-
for i in $(seq $node_num -1 1);do
291-
ip=`echo ${container_ip_array[$((i-1))]}|awk -F/ '{print $1}'`
327+
if [ ${#hanode_list_new_members[@]} -eq 0 ]; then
328+
return
329+
fi
330+
331+
insert_str=""
332+
for i in $(seq 1 ${#hanode_list_to_form_cluster[@]});do
333+
node=${hanode_list_to_form_cluster[$((i-1))]}
334+
ip=$(docker network inspect "$HA_NETWORK_ARRAY" --format '{{range .Containers}}{{if eq .Name "'"${node}"'"}}{{.IPv4Address}}{{end}}{{end}}'|awk -F/ '{print $1}')
292335
insert_str+="\\n\\tnode {\n\t\tring0_addr: $ip\n\t\tnodeid: $i\n\t}"
293336
done
337+
294338
corosync_conf_str=$(sed "/nodelist/a \\${insert_str}" <(echo "$COROSYNC_CONF_TEMPLATE"))
295-
if [ $node_num -eq 2 ];then
339+
docker_exec "hanode1" "echo \"$corosync_conf_str\" > $COROSYNC_CONF"
340+
if [ ${#hanode_list_to_form_cluster[@]} -eq 2 ];then
296341
corosync_conf_str=$(sed "/corosync_votequorum/a \\\\ttwo_node: 1" <(echo "$corosync_conf_str"))
297342
fi
298343

299-
info "Copy corosync.conf to $*"
300-
for node in $*;do
301-
if [ $node == $1 ];then
302-
docker_exec $1 "echo \"$corosync_conf_str\" >> $COROSYNC_CONF"
303-
docker_exec $1 "corosync-keygen -l -k $COROSYNC_AUTH &> /dev/null"
344+
info "Copy corosync.conf to ${hanode_list_to_form_cluster[@]}"
345+
for node in ${hanode_list_to_form_cluster[@]};do
346+
if [ $node == "hanode1" ];then
347+
docker_exec "hanode1" "corosync-keygen -l -k $COROSYNC_AUTH &> /dev/null"
304348
else
305349
while :
306350
do
307-
docker_exec $1 "ssh -T -o Batchmode=yes $node true &> /dev/null" && break
351+
docker_exec "hanode1" "ssh -T -o Batchmode=yes $node true &> /dev/null" && break
308352
sleep 1
309353
done
310-
docker_exec $1 "scp -p $COROSYNC_CONF $COROSYNC_AUTH $node:/etc/corosync &> /dev/null"
354+
docker_exec "hanode1" "scp -p $COROSYNC_CONF $COROSYNC_AUTH $node:/etc/corosync &> /dev/null"
311355
fi
312356
done
313357
}
314358

315-
316359
start_cluster() {
317-
for node in $*;do
360+
if [ ${#hanode_list_current_cluster[@]} -ne 0 ] && [ ${#hanode_list_new_members[@]} -ne 0 ]; then
361+
docker_exec hanode1 "corosync-cfgtool -R > /dev/null"
362+
info Cluster exists at 'hanode1': Reloading corosync.conf... Done
363+
fi
364+
365+
for node in ${hanode_list_new_members[@]};do
318366
docker_exec $node "crm cluster enable && crm cluster start" 1> /dev/null
319367
if [ "$?" -eq 0 ];then
320368
info "Cluster service started on \"$node\""
@@ -324,35 +372,51 @@ start_cluster() {
324372
done
325373
}
326374

327-
328-
container_already_exists() {
329-
docker ps -a|grep -q "$1"
330-
if [ "$?" -eq 0 ];then
331-
fatal "Container \"$1\" already running"
332-
fi
375+
is_container_existing() {
376+
docker ps -a --format '{{.Names}}' | grep -q "^$1$"
333377
}
334378

335-
336379
setup_cluster() {
337-
hanodes_arry=()
338-
is_number $1
339-
if [ "$?" -eq 0 ];then
340-
for i in $(seq 1 $1);do
341-
hanodes_arry+=("hanode$i")
380+
get_cluster_new_nodes
381+
382+
hanodes_array=()
383+
is_num_or_seq__c_style $1
384+
ret=$?
385+
if [ $ret -eq 1 ];then
386+
# add more nodes after the last node, ordered by the node name
387+
if [ ${#hanode_list_to_form_cluster[@]} -gt 0 ]; then
388+
last_node_num="${hanode_list_to_form_cluster[-1]:6}"
389+
else
390+
last_node_num=0
391+
fi
392+
num_of_new_nodes=$(( $1 - ${#hanode_list_to_form_cluster[@]} ))
393+
if [ "$num_of_new_nodes" -gt 0 ]; then
394+
for i in $(seq $(( last_node_num + 1 )) $(( last_node_num + num_of_new_nodes )) ); do
395+
hanodes_array+=("hanode$i")
396+
done
397+
else
398+
info 0 new node
399+
return
400+
fi
401+
elif [ $ret -eq 2 ]; then
402+
is_container_existing "hanode1" || hanodes_array=("hanode1")
403+
from_to=($(echo $1 | tr ".." " "))
404+
for i in $(seq ${from_to[0]} ${from_to[1]});do
405+
hanodes_array+=("hanode$i")
342406
done
343407
else
344-
hanodes_arry=($*)
408+
hanodes_array=($*)
345409
fi
346410

347411
if [ $WITH_QNETD_NODE -eq 1 ];then
348-
create_node ${hanodes_arry[@]} "qnetd-node"
412+
create_node ${hanodes_array[@]} "qnetd-node"
349413
else
350-
create_node ${hanodes_arry[@]}
414+
create_node ${hanodes_array[@]}
351415
fi
352416

353417
[ "$CONFIG_COROSYNC_FLAG" -eq 0 ] && return
354-
config_cluster ${hanodes_arry[@]}
355-
start_cluster ${hanodes_arry[@]}
418+
config_cluster
419+
start_cluster
356420
docker_exec "hanode1" "crm configure property stonith-enabled=false"
357421
}
358422

@@ -431,7 +495,7 @@ run_origin_regression_test() {
431495

432496
prepare_coverage_env() {
433497
for node in $*; do
434-
docker exec -t $node /bin/sh -c 'sed -i '\''1a\import coverage\nimport atexit\ncov=coverage.Coverage(config_file="/opt/crmsh/test/features/coveragerc")\natexit.register(lambda:(cov.stop(),cov.save()))\ncov.start()'\'' /usr/sbin/crm'
498+
docker exec -t $node /bin/sh -c 'sed -i '\''1a\import coverage\nimport atexit\ncov=coverage.Coverage(config_file="/opt/crmsh/test/features/coveragerc")\natexit.register(lambda:(cov.stop(),cov.save()))\ncov.start()'\'' /usr/sbin/crm'
435499
done
436500
}
437501

@@ -479,7 +543,7 @@ case $1 in
479543
-n)
480544
check_docker_env
481545
shift
482-
is_number $1 || fatal "-n option need a number larger than 0"
546+
is_num_or_seq__c_style $1 && fatal "-n option need a number larger than 0, or a sequence like 5..10"
483547
SETUP_N_NODES_CLUSTER=$1
484548
shift
485549
;;
@@ -490,7 +554,7 @@ case $1 in
490554
esac
491555
done
492556

493-
if [ $SETUP_N_NODES_CLUSTER -ge 1 ];then
557+
if [ "$SETUP_N_NODES_CLUSTER" != "0" ];then
494558
setup_cluster $SETUP_N_NODES_CLUSTER
495559
exit $?
496560
fi
@@ -538,13 +602,13 @@ for case_num in $*;do
538602
setup_cluster ${node_arry[@]}
539603
adjust_test_case ${node_arry[0]} $case_file_in_container
540604
echo
541-
prepare_coverage_env "${node_arry[@]}"
605+
prepare_coverage_env "${node_arry[@]}"
542606
if [ "$NORMAL_USER_FLAG" -eq 0 ];then
543607
info "Running \"$case_file_in_container\" under 'root'..."
544608
docker_exec ${node_arry[0]} "behave --no-logcapture $case_file_in_container || exit 1" || exit 1
545609
else
546610
info "Running \"$case_file_in_container\" under normal user 'alice'..."
547-
docker_exec ${node_arry[0]} "su - alice -c 'sudo behave --no-logcapture $case_file_in_container || exit 1'" || exit 1
611+
docker_exec ${node_arry[0]} "su - alice -c 'sudo behave --no-logcapture $case_file_in_container || exit 1'" || exit 1
548612
fi
549613
fetch_coverage_report "${node_arry[@]}"
550614
echo

0 commit comments

Comments
 (0)