diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index f29c66b..016f9f4 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6053,6 +6053,64 @@ def auto_firmware_update_on_switch_check(cversion, tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + +@check_wrapper(check_title="Multi-Pod modular spine bootscript check") +def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, password, **kwargs): + result = PASS + headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present"] + data = [] + recommended_action = "clean reboot on impacted spine" + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multipod-modular-spine-bootscript-check" + + pod_count_resp = icurl('class', 'fabricSetupP.json?query-target=self&rsp-subtree-include=count') + if (int(pod_count_resp[0]['moCount']['attributes']['count'])) < 2: + return Result(result=NA, msg="Not MultiPod Fabric.") + + if not (tversion.same_as("6.1(4h)")): + return Result(result=NA, msg="Target version is not affected") + + modular_spine_models = {"N9K-C9408", "N9K-C9504", "N9K-C9508", "N9K-C9516"} + found_modular_spine = any(node["fabricNode"]["attributes"].get("model") in modular_spine_models for node in fabric_nodes) + if not found_modular_spine: + return Result(result=NA, msg="No modular spine found in fabric.") + + has_error = False + for node in fabric_nodes: + attr = node["fabricNode"]["attributes"] + if attr.get("role") != "spine": + continue + + node_id = attr.get("id") + node_name = attr.get("name") + dn = re.search(node_regex, attr.get("dn", "")) + pod_id = dn.group("pod") if dn else "Unknown" + mgmt_ip = attr.get("address") + model = attr.get("model") + + c = Connection(mgmt_ip) + c.username = username + c.password = password + c.log = LOG_FILE + try: + c.connect() + c.cmd("ls -l bootflash/ | grep boots") + bootscript_present = "Yes" if "bootscript" in c.output else "No" + except Exception as e: + ssh_error = f"SSH ERROR: {e}" + data.append([pod_id, node_id, node_name, model, ssh_error]) + has_error = True + continue + data.append([pod_id, node_id, node_name, model, bootscript_present]) + + if has_error: + result = ERROR + elif data: + bootscript_missing = any(row[4] == "No" for row in data) + if bootscript_missing: + result = FAIL_O + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + # ---- Script Execution ---- @@ -6216,6 +6274,7 @@ class CheckManager: isis_database_byte_check, configpush_shard_check, auto_firmware_update_on_switch_check, + multipod_modular_spine_bootscript_check, ] ssh_checks = [ diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f46e03d..baa88be 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -194,6 +194,7 @@ Items | Defect | This Script [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | :no_entry_sign: [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941 | :white_check_mark: | :no_entry_sign: +[Multi-Pod modular spine bootscript check][d30] | CSCwr66848 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -224,6 +225,7 @@ Items | Defect | This Script [d27]: #isis-dteps-byte-size [d28]: #policydist-configpushshardcont-crash [d29]: #auto-firmware-update-on-switch-discovery +[d30]: #multipod-modular-spine-bootscript-check ## General Check Details @@ -2667,6 +2669,12 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to !!! note This issue occurs because older switch firmware versions are not compatible with switch images 6.0(3) or newer. The APIC version is not a factor. +### Multi-Pod Modular Spine Bootscript + +Due to [CSCwr66848][64], in a Multi-Pod fabric, modular spine switches have `bootscript` file present in their bootflash from the initial bootstrap process. When upgrading to 6.1(4h), if `bootscript` file is missing that can cause traffic loss across pods until the spine in this condition is clean reloaded. + +To avoid this issue, verify that `bootscript` file exists in the bootflash of each spine switch prior to upgrading to 6.1(4h). If not found, we have to do clean reboot on impacted spine. + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html @@ -2731,4 +2739,5 @@ To avoid this risk, consider disabling Auto Firmware Update before upgrading to [60]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#Inter [61]: https://www.cisco.com/c/en/us/solutions/collateral/data-center-virtualization/application-centric-infrastructure/white-paper-c11-743951.html#EnablePolicyCompression [62]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe83941 -[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html \ No newline at end of file +[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html +[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwr66848 \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json b/tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json new file mode 100644 index 0000000..8f9ca66 --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/fabricNode_no_spine.json @@ -0,0 +1,32 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.101", + "dn": "topology/pod-1/node-101", + "fabricSt": "active", + "id": "101", + "model": "N9K-C93180YC-FX", + "monPolDn": "uni/fabric/monfab-default", + "name": "leaf101", + "nodeType": "unspecified", + "role": "leaf" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json b/tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json new file mode 100644 index 0000000..f68b992 --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json @@ -0,0 +1,47 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9316D-GX2", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.202", + "dn": "topology/pod-2/node-202", + "fabricSt": "active", + "id": "202", + "model": "N9K-C9316D-GX", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine202", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json b/tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json new file mode 100644 index 0000000..a7261a4 --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json @@ -0,0 +1,47 @@ +[ + { + "fabricNode": { + "attributes": { + "address": "10.0.0.1", + "dn": "topology/pod-1/node-1", + "fabricSt": "commissioned", + "id": "1", + "model": "APIC-SERVER-L2", + "monPolDn": "uni/fabric/monfab-default", + "name": "apic1", + "nodeType": "unspecified", + "role": "controller" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.201", + "dn": "topology/pod-1/node-201", + "fabricSt": "active", + "id": "201", + "model": "N9K-C9504", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine201", + "nodeType": "unspecified", + "role": "spine" + } + } + }, + { + "fabricNode": { + "attributes": { + "address": "10.0.0.202", + "dn": "topology/pod-2/node-202", + "fabricSt": "active", + "id": "202", + "model": "N9K-C9508", + "monPolDn": "uni/fabric/monfab-default", + "name": "spine202", + "nodeType": "unspecified", + "role": "spine" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py new file mode 100644 index 0000000..098878a --- /dev/null +++ b/tests/checks/modular_spine_bootscript_check/test_modular_spine_bootscript_check.py @@ -0,0 +1,213 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) +test_function = "multipod_modular_spine_bootscript_check" +# Test data +fabric_nodes_with_modular_spine = read_data(dir, "fabricNode_with_modular_spine.json") +fabric_nodes_no_spine = read_data(dir, "fabricNode_no_spine.json") +fabric_nodes_with_fixed_spine = read_data(dir, "fabricNode_with_fixed_spine.json") +# API query +fabric_setup_count = 'fabricSetupP.json?query-target=self&rsp-subtree-include=count' +# icurl response data +not_multipod_setup = [{"moCount": {"attributes": {"count": "1"}}}] +multipod_setup = [{"moCount": {"attributes": {"count": "2"}}}] +# SSH command +bootscript_cmd = "ls -l bootflash/ | grep boots" +# SSH command outputs +bootscript_found = """\ +ls -l bootflash/ | grep boots +-rw-rw-rw- 1 root admin 152 Jan 5 11:51 bootscript +-rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml +ifav42-spine1# +""" +bootscript_not_found = """\ +ls -l bootflash/ | grep boots +-rw-r--r-- 1 600 admin 14119 Jan 5 11:51 bootstrap.xml +ifav42-spine1# +""" + +@pytest.mark.parametrize( + "icurl_outputs, tversion, fabric_nodes, conn_failure, conn_cmds, expected_result, expected_data", + [ + # Test 1: NA - Not a Multi-Pod setup (fabricSetupP count < 2) + ( + {fabric_setup_count: not_multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + {}, + script.NA, + [], + ), + # Test 2: NA - tversion not 6.1(4h) or 6.1(5e) + ( + {fabric_setup_count: multipod_setup}, + "6.1(3a)", + fabric_nodes_with_modular_spine, + False, + {}, + script.NA, + [], + ), + # Test 3: NA - No modular spine found in fabric + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_no_spine, + False, + {}, + script.NA, + [], + ), + # Test 4: PASS - bootscript present on all spine nodes (tversion 6.1(4h)) + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + }, + script.PASS, + [ + ["1", "201", "Spine1", "N9K-C9504", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "Yes"], + ], + ), + # Test 6: FAIL_O - bootscript missing on all spine nodes + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_not_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_not_found, + "exception": None, + } + ], + }, + script.FAIL_O, + [ + ["1", "201", "Spine1", "N9K-C9504", "No"], + ["2", "202", "Spine2", "N9K-C9508", "No"], + ], + ), + # Test 7: FAIL_O - bootscript missing on one spine node + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": bootscript_not_found, + "exception": None, + } + ], + }, + script.FAIL_O, + [ + ["1", "201", "Spine1", "N9K-C9504", "Yes"], + ["2", "202", "Spine2", "N9K-C9508", "No"], + ], + ), + # Test 9: ERROR - SSH connection exception on spine node + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + False, + { + "10.0.0.201": [ + { + "cmd": bootscript_cmd, + "output": "", + "exception": Exception("SSH failed"), + } + ], + "10.0.0.202": [ + { + "cmd": bootscript_cmd, + "output": "", + "exception": Exception("SSH failed"), + } + ], + }, + script.ERROR, + [ + ["1", "201", "Spine1", "N9K-C9504", "SSH ERROR: SSH failed", "SSH ERROR: SSH failed"], + ["2", "202", "Spine2", "N9K-C9508", "SSH ERROR: SSH failed", "SSH ERROR: SSH failed"], + ], + ), + # Test 10: ERROR - SSH connection failure (login failed) + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_modular_spine, + True, + {}, + script.ERROR, + [], + ), + # Test 11: PASS - bootscript present on fixed spine (check all spines) + ( + {fabric_setup_count: multipod_setup}, + "6.1(4h)", + fabric_nodes_with_fixed_spine, + False, + { + "10.0.0.2": [ + { + "cmd": bootscript_cmd, + "output": bootscript_found, + "exception": None, + } + ], + }, + script.NA, + [], + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, fabric_nodes, mock_conn, mock_run_cmd, expected_result, expected_data): + result = run_check(tversion=script.AciVersion(tversion) if tversion else None,username="fake_username",password="fake_password",fabric_nodes=fabric_nodes,) + assert result.result == expected_result \ No newline at end of file