OCPEDGE-1932: feat: support abi with arbiter node (#1773)

eggfoobar · web-flow · commit 5c33bf97edf9 · 2025-07-23T23:48:17.000Z
* feat: support abi with arbiter node

Signed-off-by: ehila &lt;ehila@redhat.com&gt;

* feat: update to use num arbiters

removed ENABLE_ARBITER and added NUM_ARBITERS for creating arbiter nodes

Signed-off-by: ehila &lt;ehila@redhat.com&gt;

---------

Signed-off-by: ehila &lt;ehila@redhat.com&gt;
diff --git a/agent/05_agent_configure.sh b/agent/05_agent_configure.sh
@@ -59,13 +59,17 @@ function configure_node() {
   local cluster_name="${CLUSTER_NAME}_${node_type}_${node_num}"
   local hostname="$(printf "${MASTER_HOSTNAME_FORMAT}" "${node_num}")"
   local ip=$((base_ip + node_num))
+  if [[ "$node_type" == "arbiter" ]]; then
+    local hostname="$(printf "${ARBITER_HOSTNAME_FORMAT}" "${node_num}")"
+    local ip=$((base_ip + ${NUM_MASTERS} + node_num))
+  fi
   if [[ "$node_type" == "worker" ]]; then
     local hostname="$(printf "${WORKER_HOSTNAME_FORMAT}" "${node_num}")"
-    local ip=$((base_ip + ${NUM_MASTERS} + node_num))
+    local ip=$((base_ip + ${NUM_MASTERS} + ${NUM_ARBITERS} + node_num))
   fi
   if [[ "$node_type" == "extraworker" ]]; then
     local hostname="$(printf "${EXTRA_WORKER_HOSTNAME_FORMAT}" "${node_num}")"
-    local ip=$((base_ip + ${NUM_MASTERS} + ${NUM_WORKERS} + node_num))
+    local ip=$((base_ip + ${NUM_MASTERS} + ${NUM_ARBITERS} + ${NUM_WORKERS} + node_num))
   fi
 
   if [[ "$node_type" != "extraworker" ]]; then
@@ -137,6 +141,11 @@ function get_static_ips_and_macs() {
         configure_node "master" "$i"
       done
 
+      for (( i=0; i<${NUM_ARBITERS}; i++ ))
+      do
+        configure_node "arbiter" "$i"
+      done
+
       for (( i=0; i<${NUM_WORKERS}; i++ ))
       do
         configure_node "worker" "$i"
diff --git a/agent/06_agent_create_cluster.sh b/agent/06_agent_create_cluster.sh
@@ -564,6 +564,7 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
 
     attach_agent_iso master $NUM_MASTERS
     attach_agent_iso worker $NUM_WORKERS
+    attach_agent_iso arbiter $NUM_ARBITERS
 
     ;;
 
@@ -573,6 +574,7 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
 
     agent_pxe_boot master $NUM_MASTERS
     agent_pxe_boot worker $NUM_WORKERS
+    agent_pxe_boot arbiter $NUM_ARBITERS
     ;;
 
   "ISCSI" )
@@ -583,10 +585,12 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
 
     agent_iscsi_targets master $NUM_MASTERS
     agent_iscsi_targets worker $NUM_WORKERS
+    agent_iscsi_targets arbiter $NUM_ARBITERS
 
     # Update the nodes and restart
     agent_iscsi_update_nodes master $NUM_MASTERS
     agent_iscsi_update_nodes worker $NUM_WORKERS
+    agent_iscsi_update_nodes arbiter $NUM_ARBITERS
     ;;
 
   "DISKIMAGE" )
@@ -601,6 +605,7 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
     # Attach the diskimage to nodes
     attach_appliance_diskimage master $NUM_MASTERS
     attach_appliance_diskimage worker $NUM_WORKERS
+    attach_appliance_diskimage arbiter $NUM_ARBITERS
 
     # Delete the unused appliance.raw file and cache/temp directories
     # (to avoid storage overconsumption on the CI machine)
@@ -624,11 +629,13 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in
 
     attach_agent_iso_no_registry master $NUM_MASTERS
     attach_agent_iso_no_registry worker $NUM_WORKERS
+    attach_agent_iso_no_registry arbiter $NUM_ARBITERS
 
     echo "Waiting for 2 mins to arrive at agent-tui screen"
     sleep 120
     automate_rendezvousIP_selection master $NUM_MASTERS
     automate_rendezvousIP_selection worker $NUM_WORKERS
+    automate_rendezvousIP_selection arbiter $NUM_ARBITERS
 
     check_assisted_install_UI
 
diff --git a/agent/roles/manifests/templates/install-config_baremetal_yaml.j2 b/agent/roles/manifests/templates/install-config_baremetal_yaml.j2
@@ -6,6 +6,16 @@ compute:
   name: worker
   replicas: {{ num_workers }} 
   architecture: {{ goCPUArchitecture }}
+{% if num_arbiters != "0" %}
+arbiter:
+  architecture: {{ goCPUArchitecture }}
+  hyperthreading: Enabled
+  name: arbiter
+  replicas: {{ num_arbiters }}
+{% endif %}
+{% if feature_set %}
+featureSet: {{ feature_set }}
+{% endif %}
 controlPlane:
   architecture: {{ goCPUArchitecture }}
   hyperthreading: Enabled
diff --git a/agent/roles/manifests/templates/install-config_vsphere_yaml.j2 b/agent/roles/manifests/templates/install-config_vsphere_yaml.j2
@@ -5,6 +5,16 @@ compute:
   name: worker
   replicas: {{ num_workers }} 
   architecture: {{ goCPUArchitecture }}
+{% if num_arbiters != "0" %}
+arbiter:
+  architecture: {{ goCPUArchitecture }}
+  hyperthreading: Enabled
+  name: arbiter
+  replicas: {{ num_arbiters }}
+{% endif %}
+{% if feature_set %}
+featureSet: {{ feature_set }}
+{% endif %}
 controlPlane:
   architecture: {{ goCPUArchitecture }}
   hyperthreading: Enabled
diff --git a/agent/roles/manifests/templates/install-config_yaml.j2 b/agent/roles/manifests/templates/install-config_yaml.j2
@@ -5,6 +5,16 @@ compute:
   name: worker
   replicas: {{ num_workers }} 
   architecture: {{ goCPUArchitecture }}
+{% if num_arbiters != "0" %}
+arbiter:
+  architecture: {{ goCPUArchitecture }}
+  hyperthreading: Enabled
+  name: arbiter
+  replicas: {{ num_arbiters }}
+{% endif %}
+{% if feature_set %}
+featureSet: {{ feature_set }}
+{% endif %}
 controlPlane:
   architecture: {{ goCPUArchitecture }}
   hyperthreading: Enabled
diff --git a/agent/roles/manifests/vars/main.yml b/agent/roles/manifests/vars/main.yml
@@ -51,6 +51,8 @@ networking_mode: "{{ lookup('env', 'NETWORKING_MODE') }}"
 network_type: "{{ lookup('env', 'NETWORK_TYPE') }}"
 num_masters: "{{ lookup('env', 'NUM_MASTERS') }}"
 num_workers: "{{ lookup('env', 'NUM_WORKERS') }}"
+num_arbiters: "{{ lookup('env', 'NUM_ARBITERS') }}"
+feature_set: "{{ lookup('env', 'FEATURE_SET') }}"
 num_extra_workers: "{{ lookup('env', 'NUM_EXTRA_WORKERS', default='0') }}"
 no_proxy: "{{ lookup('env', 'AGENT_NO_PROXY') }}"
 platform_type: "{{ lookup('env', 'AGENT_PLATFORM_TYPE') }}"
diff --git a/common.sh b/common.sh
@@ -274,14 +274,15 @@ export CONTAINER_RUNTIME="podman"
 
 export NUM_MASTERS=${NUM_MASTERS:-"3"}
 export NUM_WORKERS=${NUM_WORKERS:-"2"}
-export ENABLE_ARBITER=${ENABLE_ARBITER:-}
+export NUM_ARBITERS=${NUM_ARBITERS:-"0"}
 export NUM_EXTRA_WORKERS=${NUM_EXTRA_WORKERS:-"0"}
 export EXTRA_WORKERS_ONLINE_STATUS=${EXTRA_WORKERS_ONLINE_STATUS:-"true"}
 export EXTRA_WORKERS_NAMESPACE=${EXTRA_WORKERS_NAMESPACE:-"openshift-machine-api"}
 export VM_EXTRADISKS=${VM_EXTRADISKS:-"false"}
 export VM_EXTRADISKS_LIST=${VM_EXTRADISKS_LIST:-"vdb"}
 export VM_EXTRADISKS_SIZE=${VM_EXTRADISKS_SIZE:-"8G"}
 export MASTER_HOSTNAME_FORMAT=${MASTER_HOSTNAME_FORMAT:-"master-%d"}
+export ARBITER_HOSTNAME_FORMAT=${ARBITER_HOSTNAME_FORMAT:-"arbiter-%d"}
 export WORKER_HOSTNAME_FORMAT=${WORKER_HOSTNAME_FORMAT:-"worker-%d"}
 export EXTRA_WORKER_HOSTNAME_FORMAT=${EXTRA_WORKER_HOSTNAME_FORMAT:-"extraworker-%d"}
 
@@ -310,7 +311,7 @@ export IRONIC_IMAGES_DIR="${IRONIC_DATA_DIR}/html/images"
 export VBMC_IMAGE=${VBMC_IMAGE:-"quay.io/metal3-io/vbmc"}
 export SUSHY_TOOLS_IMAGE=${SUSHY_TOOLS_IMAGE:-"quay.io/metal3-io/sushy-tools"}
 export VBMC_BASE_PORT=${VBMC_BASE_PORT:-"6230"}
-export VBMC_MAX_PORT=$((VBMC_BASE_PORT + NUM_MASTERS + NUM_WORKERS + NUM_EXTRA_WORKERS - 1))
+export VBMC_MAX_PORT=$((VBMC_BASE_PORT + NUM_MASTERS + NUM_ARBITERS + NUM_WORKERS + NUM_EXTRA_WORKERS - 1))
 export REDFISH_EMULATOR_IGNORE_BOOT_DEVICE="${REDFISH_EMULATOR_IGNORE_BOOT_DEVICE:-False}"
 
 # Which docker registry image should we use?
@@ -358,6 +359,16 @@ if [ ! -d "$IRONIC_IMAGES_DIR" ]; then
   sudo mkdir -p "$IRONIC_IMAGES_DIR"
 fi
 
+if [[ ${NUM_ARBITERS} -gt 1 ]]; then
+  error "Creating a cluster with more than 1 arbiter is currently not supported"
+  exit 1
+fi
+
+if [[ ${NUM_ARBITERS} -eq 1 ]] && [[ ${NUM_MASTERS} -ne 2 ]]; then
+  error "Creating a cluster with 1 arbiter and ${NUM_MASTERS} masters is not supported, please use 2 masters"
+  exit 1
+fi
+
 # Previously the directory was owned by root, we need to alter
 # permissions to be owned by the user running dev-scripts.
 if [ ! -f "$IRONIC_IMAGES_DIR/.permissions" ]; then
@@ -395,7 +406,7 @@ export AGENT_WAIT_FOR_INSTALL_COMPLETE=${AGENT_WAIT_FOR_INSTALL_COMPLETE:-true}
 # Agent specific configuration 
 
 function invalidAgentValue() {
-  printf "Found invalid value \"$AGENT_E2E_TEST_SCENARIO\" for AGENT_E2E_TEST_SCENARIO. Supported values: 'COMPACT_IPXX', 'HA_IPXX', 'SNO_IPXX', '4CONTROL_IPXX', or '5CONTROL_IPXX', where XX is 'V4', 'V6', or 'V4V6'"
+  printf "Found invalid value \"$AGENT_E2E_TEST_SCENARIO\" for AGENT_E2E_TEST_SCENARIO. Supported values: 'COMPACT_IPXX', 'HA_IPXX', 'SNO_IPXX', 'TNA_IPXX', '4CONTROL_IPXX', or '5CONTROL_IPXX', where XX is 'V4', 'V6', or 'V4V6'"
   exit 1
 }
 
@@ -457,6 +468,17 @@ if [[ ! -z ${AGENT_E2E_TEST_SCENARIO} ]]; then
           export MASTER_MEMORY=32768
           export NUM_WORKERS=0
           ;;
+      "TNA" )
+          export NUM_MASTERS=2
+          export MASTER_VCPU=4
+          export MASTER_DISK=100
+          export MASTER_MEMORY=32768
+          export NUM_ARBITERS=1
+          export ARBITER_VCPU=2
+          export ARBITER_MEMORY=16384
+          export ARBITER_DISK=100
+          export NUM_WORKERS=0
+          ;;
       "HA" )
           export NUM_MASTERS=3
           export MASTER_VCPU=4
diff --git a/config_example.sh b/config_example.sh
@@ -465,16 +465,6 @@ set -x
 #
 #export NODES_PLATFORM=baremetal
 
-# ENABLE_ARBITER -
-# Set to any non zero length string value to enable the creation of an arbiter node.
-# Arbiter nodes take the place of a master node but run only critical containers to maintain HA for the cluster,
-# it is a TechPreview feature so `export FEATURE_SET="TechPreviewNoUpgrade"` must be set.
-#
-# Furthermore, since an Arbiter node takes the place of a master node the NUM_MASTERS count
-# must be set to `export NUM_MASTERS=2` and not the default `3` to avoid even number etcd members.
-#
-#export ENABLE_ARBITER=1
-
 # ENABLE_WORKLOAD_PARTITIONING -
 # Set to any non zero length string value to enable workload partitioning in the install config.
 #
@@ -531,6 +521,26 @@ set -x
 #export MASTER_DISK=40
 #export MASTER_VCPU=8
 
+# ARBITER_HOSTNAME_FORMAT -
+# Set a custom hostname format for arbiters. This is a format string that should
+# include one %d field, which will be replaced with the number of the node.
+# Default: "arbiter-%d"
+#
+#export ARBITER_HOSTNAME_FORMAT=arbiter-%d
+
+# NUM_ARBITERS -
+# Indicate number of arbiter nodes to deploy.
+# Arbiter nodes take the place of a master node but run only critical containers to maintain HA for the cluster,
+# it is a TechPreview for OCP 4.19, so `export FEATURE_SET="TechPreviewNoUpgrade"` must be set.
+# Default: 0
+#
+# Furthermore, since an Arbiter node takes the place of a master node the NUM_MASTERS count
+# must be set to `export NUM_MASTERS=2` and not the default `3` to avoid even number etcd members.
+#
+# This script will currently only support 1 arbiter node and 2 masters.
+#
+#export NUM_ARBITERS=0
+
 # ARBITER_MEMORY, ARBITER_DISK, ARBITER_VCPU -
 # Change VM resources for arbiters
 ## Defaults:
@@ -794,6 +804,10 @@ set -x
 # - COMPACT_IPV6
 # - COMPACT_IPV4_DHCP
 # - COMPACT_IPV6_DHCP
+# - TNA_IPV4
+# - TNA_IPV6
+# - TNA_IPV4_DHCP
+# - TNA_IPV6_DHCP
 # - HA_IPV4
 # - HA_IPV6
 # - HA_IPV4_DHCP
diff --git a/ocp_install_env.sh b/ocp_install_env.sh
@@ -207,11 +207,11 @@ EOF
 }
 
 function arbiter_stanza() {
-    if [[ -n "${ENABLE_ARBITER:-}" ]]; then
+    if [[ ${NUM_ARBITERS} -gt 0 ]]; then
 cat <<EOF
 arbiter:
   name: arbiter
-  replicas: 1
+  replicas: ${NUM_ARBITERS}
   hyperthreading: Enabled
   architecture: $(get_arch install_config)
   platform:
@@ -384,9 +384,11 @@ EOF
     cat >> "${outdir}/install-config.yaml" << EOF
 $(node_map_to_install_config_hosts $NUM_MASTERS 0 master)
 $(node_map_to_install_config_hosts $NUM_WORKERS $NUM_MASTERS worker)
+$(node_map_to_install_config_hosts $NUM_ARBITERS $(( NUM_MASTERS + NUM_WORKERS )) arbiter)
 EOF
   else
     cat >> "${outdir}/install-config.yaml" << EOF
+$(node_map_to_install_config_hosts $NUM_ARBITERS $(( NUM_MASTERS + NUM_WORKERS )) arbiter)
 $(node_map_to_install_config_hosts $NUM_WORKERS $NUM_MASTERS worker)
 $(node_map_to_install_config_hosts $NUM_MASTERS 0 master)
 EOF
diff --git a/utils.sh b/utils.sh
@@ -224,22 +224,10 @@ function node_map_to_install_config_hosts() {
     start_idx="$2"
     role="$3"
 
-    # If arbiter is enabled, an arbiter node will be created so we increase the number of hosts by 1
-    # when the role is for master to capture the arbiter position.
-    # If the role is for a worker, we increment the index since the worker position has moved by 1.
-    if [[ ! -z "${ENABLE_ARBITER:-}" && "$role" == "master" ]]; then
-      num_hosts=$((num_hosts + 1))
-    elif [[ ! -z "${ENABLE_ARBITER:-}" && "$role" == "worker" ]]; then
-      start_idx=$((start_idx + 1))
-    fi
-
-    for ((idx=$start_idx;idx<$(($num_hosts + $start_idx));idx++)); do
+    for ((idx=$start_idx;idx<$(($1 + $start_idx));idx++)); do
       name=$(node_val ${idx} "name")
       mac=$(node_val ${idx} "ports[0].address")
       local node_role=$role
-      if [[ ! -z "${ENABLE_ARBITER:-}" && $idx -eq $(($num_hosts + $start_idx - 1)) && "$role" == "master" ]]; then
-        node_role=arbiter
-      fi
 
       driver=$(node_val ${idx} "driver")
       if [ $driver == "ipmi" ] ; then
@@ -262,7 +250,7 @@ function node_map_to_install_config_hosts() {
 
       cat << EOF
       - name: ${name}
-        role: ${node_role}
+        role: ${role}
         bootMACAddress: ${mac}
         bootMode: ${boot_mode}
         bmc:
@@ -324,7 +312,7 @@ function node_map_to_install_config_fencing_credentials() {
     return 0
   fi
 
-  if  [[ -z "${ENABLE_ARBITER:-}" ]] && [[ "${NUM_MASTERS}" -eq 2 ]]; then
+  if  [[ ${NUM_ARBITERS} -eq 0 ]] && [[ "${NUM_MASTERS}" -eq 2 ]]; then
 	cat <<EOF
   fencing:
     credentials:
diff --git a/vm_setup_vars.yml b/vm_setup_vars.yml
@@ -5,13 +5,11 @@
 # openshift-installer terraform templates
 ironic_prefix: "{{ ironic_prefix }}"
 
-arbiter_enabled: "{{ lookup('env', 'ENABLE_ARBITER') | default(false) }}"
-
 # We enable more memory and masters in dev-scripts compared to the minimal setup
 # in metal3-dev-env
 num_masters: 3
 num_workers: 1
-num_arbiters: "{{ 1 if arbiter_enabled else 0 }}"
+num_arbiters: "{{ lookup('env', 'NUM_ARBITERS') | default(0) }}"
 num_extraworkers: 0
 extradisks_list: "{{ lookup('env', 'VM_EXTRADISKS_LIST').split(' ') | default(['vdb']) }}"
 extradisks_size: "{{ lookup('env', 'VM_EXTRADISKS_SIZE') | default(['8G']) }}"
