From d1abd86d394844234034e2baf45516e1bbf043c6 Mon Sep 17 00:00:00 2001 From: ASRAF KHAN NAZAR Date: Wed, 11 Mar 2026 12:08:20 +0000 Subject: [PATCH 1/5] Added check for CFD - CSCwr66848 --- aci-preupgrade-validation-script.py | 68 +++++ docs/docs/validations.md | 23 +- .../fabricNode_no_spine.json | 32 ++ .../fabricNode_with_fixed_spine.json | 47 +++ .../fabricNode_with_modular_spine.json | 47 +++ .../test_modular_spine_bootscript_check.py | 273 ++++++++++++++++++ 6 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json create mode 100644 tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json create mode 100644 tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json create mode 100644 tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index f29c66b..b97124a 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6053,6 +6053,73 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title="Multi-Pod modular spine bootscript check") +def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, password, **kwargs): + result = PASS + headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present", "Bootstrap file Present"] + data = [] + recommended_action = "Bootscript is missing, delete bootstrap.xml from bootflash" + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multipod-modular-spine-bootscript-check" + + pod_count_resp = icurl('class', 'fabricSetupP.json?query-target=self&rsp-subtree-include=count') + if (int(pod_count_resp[0]['moCount']['attributes']['count'])) < 2: + return Result(result=NA, msg="Multi-Pod is not enabled in fabric.") + + if not (tversion.same_as("6.1(4h)") or tversion.same_as("6.1(5e)")): + return Result(result=NA, msg="Target version is not affected") + + modular_spine_models = {"N9K-C9408", "N9K-C9504", "N9K-C9508", "N9K-C9516"} + found_modular_spine = any( + node["fabricNode"]["attributes"].get("role") == "spine" and + node["fabricNode"]["attributes"].get("model") in modular_spine_models + for node in fabric_nodes + ) + if not found_modular_spine: + return Result(result=NA, msg="No modular spine (N9K-C9408, N9K-C9504, N9K-C9508, N9K-C9516) found in fabric.") + + has_error = False + for node in fabric_nodes: + attr = node["fabricNode"]["attributes"] + if attr.get("role") != "spine": + continue + + node_id = attr.get("id") + node_name = attr.get("name") + dn = re.search(node_regex, attr.get("dn", "")) + pod_id = dn.group("pod") if dn else "Unknown" + mgmt_ip = attr.get("address") + model = attr.get("model") + + c = Connection(mgmt_ip) + c.username = username + c.password = password + c.log = LOG_FILE + try: + c.connect() + c.cmd("ls -l bootflash/ | grep boots") + bootscript_present = "Yes" if "bootscript" in c.output else "No" + bootstrap_present = "Yes" if "bootstrap.xml" in c.output else "No" + except Exception as e: + ssh_error = f"SSH ERROR: {e}" + data.append([pod_id, node_id, node_name, model, ssh_error, ssh_error]) + has_error = True + continue + data.append([pod_id, node_id, node_name, model, bootscript_present, bootstrap_present]) + + if has_error: + result = ERROR + elif data: + bootscript_missing = any(row[4] == "No" for row in data) + bootstrap_missing = any(row[5] == "No" for row in data) + if bootscript_missing and not bootstrap_missing: + result = FAIL_O + elif bootscript_missing and bootstrap_missing: + result = FAIL_UF + recommended_action = "bootscript and bootstrap.xml files are not found, Move to Fix version." + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + # ---- Script Execution ---- @@ -6216,6 +6283,7 @@ class CheckManager: isis_database_byte_check, configpush_shard_check, auto_firmware_update_on_switch_check, + multipod_modular_spine_bootscript_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f46e03d..8eb8ee6 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -194,6 +194,7 @@ Items | Defect | This Script [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | :no_entry_sign: [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941 | :white_check_mark: | :no_entry_sign: +[Multi-Pod modular spine bootscript check][d30] | CSCwr66848 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -224,6 +225,7 @@ Items | Defect | This Script [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash [d29]: #auto-firmware-update-on-switch-discovery +[d30]: #multipod-modular-spine-bootscript-check ## General Check Details @@ -2667,6 +2669,24 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to !!! note This issue occurs because older switch firmware versions are not compatible with switch images 6.0(3) or newer. The APIC version is not a factor. +### Multi-Pod Modular Spine Bootscript + +Due to [CSCwr66848][64], in a Multi-Pod fabric, modular spine switches have `bootscript` file present in their bootflash from the initial bootstrap process. When upgrading to 6.1(4h) or 6.1(5e), if `bootscript` file is missing that can cause traffic loss across pods until the spine in this condition is clean reloaded. + +To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h) or 6.1(5e). If not found, the `bootstrap.xml` file must be removed before proceeding with the upgrade. + +!!! tip + You can manually check for the presence of `bootscript` on a spine switch by logging into the switch CLI and running the following command: + ``` + spine1# ls -l bootflash/ | grep boots + -rw-rw-rw- 1 root admin 152 Jan 5 11:51 bootscript + -rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml + ``` + If `bootscript` is not present, delete the `bootstrap.xml` file from bootflash to resolve the issue: + ``` + spine1# delete bootflash/bootstrap.xml + ``` + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html @@ -2731,4 +2751,5 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression [62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe83941 -[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html \ No newline at end of file +[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html +[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwr66848 \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json b/tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json new file mode 100644 index 0000000..8f9ca66 --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json b/tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json new file mode 100644 index 0000000..f68b992 --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json @@ -0,0 +1,47 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9316D-GX2", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.202", + "dn": "topology/pod-2/node-202", + "fabricSt": "active", + "id": "202", + "model": "N9K-C9316D-GX", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine202", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json b/tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json new file mode 100644 index 0000000..a7261a4 --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json @@ -0,0 +1,47 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.202", + "dn": "topology/pod-2/node-202", + "fabricSt": "active", + "id": "202", + "model": "N9K-C9508", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine202", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py new file mode 100644 index 0000000..9d2eece --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py @@ -0,0 +1,273 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) +test_function = "multipod_modular_spine_bootscript_check" +# Test data +fabric_nodes_with_modular_spine = read_data(dir, "fabricNode_with_modular_spine.json") +fabric_nodes_no_spine = read_data(dir, "fabricNode_no_spine.json") +fabric_nodes_with_fixed_spine = read_data(dir, "fabricNode_with_fixed_spine.json") +# API query +fabric_setup_count = 'fabricSetupP.json?query-target=self&rsp-subtree-include=count' +# icurl response data +not_multipod_setup = [{"moCount": {"attributes": {"count": "1"}}}] +multipod_setup = [{"moCount": {"attributes": {"count": "2"}}}] +# SSH command +bootscript_cmd = "ls -l bootflash/ | grep boots" +# SSH command outputs +bootscript_found = """\ +ls -l bootflash/ | grep boots +-rw-rw-rw- 1 root admin 152 Jan 5 11:51 bootscript +-rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml +ifav42-spine1# +""" +bootscript_not_found = """\ +ls -l bootflash/ | grep boots +-rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml +ifav42-spine1# +""" +bootstript_and_bootstrap_not_found = """\ +ls -l bootflash/ | grep boots +ifav42-spine1# +""" + +@pytest.mark.parametrize( + "icurl_outputs, tversion, fabric_nodes, conn_failure, conn_cmds, expected_result, expected_data", + [ + # Test 1: NA - Not a Multi-Pod setup (fabricSetupP count < 2) + ( + {fabric_setup_count: not_multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + {}, + script.NA, + [], + ), + # Test 2: NA - tversion not 6.1(4h) or 6.1(5e) + ( + {fabric_setup_count: multipod_setup}, + "6.1(3a)", + fabric_nodes_with_modular_spine, + False, + {}, + script.NA, + [], + ), + # Test 3: NA - No modular spine found in fabric + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_no_spine, + False, + {}, + script.NA, + [], + ), + # Test 4: PASS - bootscript present on all spine nodes (tversion 6.1(4h)) + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + }, + script.PASS, + [ + ["1", "201", "Spine1", "N9K-C9504", "Yes", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "Yes", "Yes"], + ], + ), + # Test 5: PASS - bootscript present on all spine nodes (tversion 6.1(5e)) + ( + {fabric_setup_count: multipod_setup}, + "6.1(5e)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + }, + script.PASS, + [ + ["1", "201", "Spine1", "N9K-C9504", "Yes", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "Yes", "Yes"], + ], + ), + # Test 6: FAIL_O - bootscript missing on all spine nodes + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_not_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_not_found, + "exception": None, + } + ], + }, + script.FAIL_O, + [ + ["1", "201", "Spine1", "N9K-C9504", "No", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "No", "Yes"], + ], + ), + # Test 7: FAIL_O - bootscript missing on one spine node + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_not_found, + "exception": None, + } + ], + }, + script.FAIL_O, + [ + ["1", "201", "Spine1", "N9K-C9504", "Yes", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "No", "Yes"], + ], + ), + # Test 8: FAIL_UF - bootscript and bootstrap.xml missing on all spine nodes + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootstript_and_bootstrap_not_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootstript_and_bootstrap_not_found, + "exception": None, + } + ], + }, + script.FAIL_UF, + [ + ["1", "201", "Spine1", "N9K-C9504", "No", "No"], + ["2", "202", "Spine2", "N9K-C9508", "No", "No"], + ], + ), + # Test 9: ERROR - SSH connection exception on spine node + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": "", + "exception": Exception("SSH failed"), + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": "", + "exception": Exception("SSH failed"), + } + ], + }, + script.ERROR, + [ + ["1", "201", "Spine1", "N9K-C9504", "SSH ERROR: SSH failed", "SSH ERROR: SSH failed"], + ["2", "202", "Spine2", "N9K-C9508", "SSH ERROR: SSH failed", "SSH ERROR: SSH failed"], + ], + ), + # Test 10: ERROR - SSH connection failure (login failed) + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + True, + {}, + script.ERROR, + [], + ), + # Test 11: PASS - bootscript present on fixed spine (check all spines) + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_fixed_spine, + False, + { + "10.0.0.2": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + }, + script.NA, + [], + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, fabric_nodes, mock_conn, mock_run_cmd, expected_result, expected_data): + result = run_check(tversion=script.AciVersion(tversion) if tversion else None,username="fake_username",password="fake_password",fabric_nodes=fabric_nodes,) + assert result.result == expected_result \ No newline at end of file From 270096a5d576571cd996eac362fc8abae79c8640 Mon Sep 17 00:00:00 2001 From: ASRAF KHAN NAZAR Date: Fri, 13 Mar 2026 06:39:00 +0000 Subject: [PATCH 2/5] Updated recommended_action message --- aci-preupgrade-validation-script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index b97124a..171a11c 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6059,7 +6059,7 @@ def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, pa result = PASS headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present", "Bootstrap file Present"] data = [] - recommended_action = "Bootscript is missing, delete bootstrap.xml from bootflash" + recommended_action = "Bootscript is missing, delete bootstrap.xml from /bootflash folder and do clean reboot" doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multipod-modular-spine-bootscript-check" pod_count_resp = icurl('class', 'fabricSetupP.json?query-target=self&rsp-subtree-include=count') From 6e741caa58f7e14c65ba16e3ebe9cc739d2ea28e Mon Sep 17 00:00:00 2001 From: ASRAF KHAN NAZAR Date: Mon, 16 Mar 2026 12:32:02 +0000 Subject: [PATCH 3/5] Updated recommended_action message and validation.md file description to remove 6.1.5 version --- aci-preupgrade-validation-script.py | 16 ++++++---------- docs/docs/validations.md | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 171a11c..44e8e13 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6059,24 +6059,20 @@ def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, pa result = PASS headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present", "Bootstrap file Present"] data = [] - recommended_action = "Bootscript is missing, delete bootstrap.xml from /bootflash folder and do clean reboot" + recommended_action = "Delete bootstrap.xml from /bootflash folder and do clean reboot" doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multipod-modular-spine-bootscript-check" pod_count_resp = icurl('class', 'fabricSetupP.json?query-target=self&rsp-subtree-include=count') if (int(pod_count_resp[0]['moCount']['attributes']['count'])) < 2: - return Result(result=NA, msg="Multi-Pod is not enabled in fabric.") + return Result(result=NA, msg="Not MultiPod Fabric.") - if not (tversion.same_as("6.1(4h)") or tversion.same_as("6.1(5e)")): + if not (tversion.same_as("6.1(4h)")): return Result(result=NA, msg="Target version is not affected") modular_spine_models = {"N9K-C9408", "N9K-C9504", "N9K-C9508", "N9K-C9516"} - found_modular_spine = any( - node["fabricNode"]["attributes"].get("role") == "spine" and - node["fabricNode"]["attributes"].get("model") in modular_spine_models - for node in fabric_nodes - ) + found_modular_spine = any(node["fabricNode"]["attributes"].get("model") in modular_spine_models for node in fabric_nodes) if not found_modular_spine: - return Result(result=NA, msg="No modular spine (N9K-C9408, N9K-C9504, N9K-C9508, N9K-C9516) found in fabric.") + return Result(result=NA, msg="No modular spine found in fabric.") has_error = False for node in fabric_nodes: @@ -6116,7 +6112,7 @@ def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, pa result = FAIL_O elif bootscript_missing and bootstrap_missing: result = FAIL_UF - recommended_action = "bootscript and bootstrap.xml files are not found, Move to Fix version." + recommended_action = "Move to Fix version." return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 8eb8ee6..b9b27f8 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2673,7 +2673,7 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to Due to [CSCwr66848][64], in a Multi-Pod fabric, modular spine switches have `bootscript` file present in their bootflash from the initial bootstrap process. When upgrading to 6.1(4h) or 6.1(5e), if `bootscript` file is missing that can cause traffic loss across pods until the spine in this condition is clean reloaded. -To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h) or 6.1(5e). If not found, the `bootstrap.xml` file must be removed before proceeding with the upgrade. +To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h). If not found, the `bootstrap.xml` file must be removed before proceeding with the upgrade. !!! tip You can manually check for the presence of `bootscript` on a spine switch by logging into the switch CLI and running the following command: From 56f96bfba6e3f1d37fc40ff6e4c14641f755dd4c Mon Sep 17 00:00:00 2001 From: ASRAF KHAN NAZAR Date: Tue, 17 Mar 2026 04:47:30 +0000 Subject: [PATCH 4/5] changed workaround on removing bootstrap file --- aci-preupgrade-validation-script.py | 15 ++-- docs/docs/validations.md | 16 +---- .../test_modular_spine_bootscript_check.py | 68 ++----------------- 3 files changed, 13 insertions(+), 86 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 44e8e13..016f9f4 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6057,9 +6057,9 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs): @check_wrapper(check_title="Multi-Pod modular spine bootscript check") def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, password, **kwargs): result = PASS - headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present", "Bootstrap file Present"] + headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present"] data = [] - recommended_action = "Delete bootstrap.xml from /bootflash folder and do clean reboot" + recommended_action = "clean reboot on impacted spine" doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multipod-modular-spine-bootscript-check" pod_count_resp = icurl('class', 'fabricSetupP.json?query-target=self&rsp-subtree-include=count') @@ -6095,24 +6095,19 @@ def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, pa c.connect() c.cmd("ls -l bootflash/ | grep boots") bootscript_present = "Yes" if "bootscript" in c.output else "No" - bootstrap_present = "Yes" if "bootstrap.xml" in c.output else "No" except Exception as e: ssh_error = f"SSH ERROR: {e}" - data.append([pod_id, node_id, node_name, model, ssh_error, ssh_error]) + data.append([pod_id, node_id, node_name, model, ssh_error]) has_error = True continue - data.append([pod_id, node_id, node_name, model, bootscript_present, bootstrap_present]) + data.append([pod_id, node_id, node_name, model, bootscript_present]) if has_error: result = ERROR elif data: bootscript_missing = any(row[4] == "No" for row in data) - bootstrap_missing = any(row[5] == "No" for row in data) - if bootscript_missing and not bootstrap_missing: + if bootscript_missing: result = FAIL_O - elif bootscript_missing and bootstrap_missing: - result = FAIL_UF - recommended_action = "Move to Fix version." return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index b9b27f8..baa88be 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2671,21 +2671,9 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to ### Multi-Pod Modular Spine Bootscript -Due to [CSCwr66848][64], in a Multi-Pod fabric, modular spine switches have `bootscript` file present in their bootflash from the initial bootstrap process. When upgrading to 6.1(4h) or 6.1(5e), if `bootscript` file is missing that can cause traffic loss across pods until the spine in this condition is clean reloaded. +Due to [CSCwr66848][64], in a Multi-Pod fabric, modular spine switches have `bootscript` file present in their bootflash from the initial bootstrap process. When upgrading to 6.1(4h), if `bootscript` file is missing that can cause traffic loss across pods until the spine in this condition is clean reloaded. -To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h). If not found, the `bootstrap.xml` file must be removed before proceeding with the upgrade. - -!!! tip - You can manually check for the presence of `bootscript` on a spine switch by logging into the switch CLI and running the following command: - ``` - spine1# ls -l bootflash/ | grep boots - -rw-rw-rw- 1 root admin 152 Jan 5 11:51 bootscript - -rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml - ``` - If `bootscript` is not present, delete the `bootstrap.xml` file from bootflash to resolve the issue: - ``` - spine1# delete bootflash/bootstrap.xml - ``` +To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h). If not found, we have to do clean reboot on impacted spine. [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script diff --git a/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py index 9d2eece..3244025 100644 --- a/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py +++ b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py @@ -94,36 +94,8 @@ }, script.PASS, [ - ["1", "201", "Spine1", "N9K-C9504", "Yes", "Yes"], - ["2", "202", "Spine2", "N9K-C9508", "Yes", "Yes"], - ], - ), - # Test 5: PASS - bootscript present on all spine nodes (tversion 6.1(5e)) - ( - {fabric_setup_count: multipod_setup}, - "6.1(5e)", - fabric_nodes_with_modular_spine, - False, - { - "10.0.0.201": [ - { - "cmd": bootscript_cmd, - "output": bootscript_found, - "exception": None, - } - ], - "10.0.0.202": [ - { - "cmd": bootscript_cmd, - "output": bootscript_found, - "exception": None, - } - ], - }, - script.PASS, - [ - ["1", "201", "Spine1", "N9K-C9504", "Yes", "Yes"], - ["2", "202", "Spine2", "N9K-C9508", "Yes", "Yes"], + ["1", "201", "Spine1", "N9K-C9504", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "Yes"], ], ), # Test 6: FAIL_O - bootscript missing on all spine nodes @@ -150,8 +122,8 @@ }, script.FAIL_O, [ - ["1", "201", "Spine1", "N9K-C9504", "No", "Yes"], - ["2", "202", "Spine2", "N9K-C9508", "No", "Yes"], + ["1", "201", "Spine1", "N9K-C9504", "No"], + ["2", "202", "Spine2", "N9K-C9508", "No"], ], ), # Test 7: FAIL_O - bootscript missing on one spine node @@ -178,36 +150,8 @@ }, script.FAIL_O, [ - ["1", "201", "Spine1", "N9K-C9504", "Yes", "Yes"], - ["2", "202", "Spine2", "N9K-C9508", "No", "Yes"], - ], - ), - # Test 8: FAIL_UF - bootscript and bootstrap.xml missing on all spine nodes - ( - {fabric_setup_count: multipod_setup}, - "6.1(4h)", - fabric_nodes_with_modular_spine, - False, - { - "10.0.0.201": [ - { - "cmd": bootscript_cmd, - "output": bootstript_and_bootstrap_not_found, - "exception": None, - } - ], - "10.0.0.202": [ - { - "cmd": bootscript_cmd, - "output": bootstript_and_bootstrap_not_found, - "exception": None, - } - ], - }, - script.FAIL_UF, - [ - ["1", "201", "Spine1", "N9K-C9504", "No", "No"], - ["2", "202", "Spine2", "N9K-C9508", "No", "No"], + ["1", "201", "Spine1", "N9K-C9504", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "No"], ], ), # Test 9: ERROR - SSH connection exception on spine node From 524dbc81fc7eb14f0d0fec94d6c5299bc7e65dec Mon Sep 17 00:00:00 2001 From: ASRAF KHAN NAZAR Date: Tue, 17 Mar 2026 06:08:43 +0000 Subject: [PATCH 5/5] updated pytest from bootstrap validation --- .../test_modular_spine_bootscript_check.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py index 3244025..098878a 100644 --- a/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py +++ b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py @@ -32,10 +32,6 @@ -rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml ifav42-spine1# """ -bootstript_and_bootstrap_not_found = """\ -ls -l bootflash/ | grep boots -ifav42-spine1# -""" @pytest.mark.parametrize( "icurl_outputs, tversion, fabric_nodes, conn_failure, conn_cmds, expected_result, expected_data",