diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index f29c66b..a215e81 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -2982,17 +2982,19 @@ def scalability_faults_check(**kwargs): @check_wrapper(check_title="APIC Disk Space Usage (F1527, F1528, F1529 equipment-full)") -def apic_disk_space_faults_check(cversion, **kwargs): +def apic_disk_space_faults_check(cversion, tversion, **kwargs): result = FAIL_UF headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Current Usage %', 'Recommended Action'] data = [] unformatted_headers = ['Fault', 'Fault DN', 'Recommended Action'] unformatted_data = [] doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-disk-space-usage" + # we are checking /tmp utilization because high usage can lead to snaphsot corruption during an upgrade. After the fix version, snapshot storage location moved to /data. recommended_action = { '/firmware': 'Remove unneeded images', '/techsupport': 'Remove unneeded techsupports/cores', - '/data/log': 'Remove unneeded logs in var/log/dme/log' + '/data/log': 'Remove unneeded logs in var/log/dme/log', + '/tmp': 'Remove unneeded logs in /tmp directory' } default_action = 'Contact Cisco TAC.' if cversion.same_as('4.0(1h)') or cversion.older_than('3.2(6i)'): @@ -3001,6 +3003,8 @@ def apic_disk_space_faults_check(cversion, **kwargs): dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' desc_regex = r'is (?P\d{2,3}%) full' + tmp_faults_skip_versions = ["6.0(9f)", "6.1(4h)", "6.2(1g)"] + tmp_faults_skipped = False # Track if we skip /tmp faults for CSCwo96334 versions faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F1527"),eq(faultInst.code,"F1528"),eq(faultInst.code,"F1529"))') for faultInst in faultInsts: @@ -3010,14 +3014,24 @@ def apic_disk_space_faults_check(cversion, **kwargs): fc = faultInst['faultInst']['attributes']['code'] dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn']) desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr']) - if dn and desc: - data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), + if dn: + mountpoint = dn.group('mountpoint') + # CSCwo96334: Skip /tmp faults when target is >= 6.1(4h) or any unaffected versions + if mountpoint == '/tmp' and (not tversion.older_than("6.1(4h)") or any(tversion.same_as(version) for version in tmp_faults_skip_versions)): + tmp_faults_skipped = True + continue + if desc: + data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), desc.group('usage'), recommended_action.get(dn.group('mountpoint'), default_action)]) - else: - unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], default_action]) + else: + unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], default_action]) if not data and not unformatted_data: - result = PASS + # If we only found /tmp faults that were skipped (CSCwo96334 fixed target versions), return NA + if tmp_faults_skipped: + result = NA + else: + result = PASS return Result( result=result, headers=headers, diff --git a/tests/checks/apic_disk_space_faults_check/Fault_combination.json b/tests/checks/apic_disk_space_faults_check/Fault_combination.json index b6312cb..7d1453e 100644 --- a/tests/checks/apic_disk_space_faults_check/Fault_combination.json +++ b/tests/checks/apic_disk_space_faults_check/Fault_combination.json @@ -88,5 +88,95 @@ "type": "operational" } } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1529", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "critical", + "prevSeverity": "critical", + "rule": "eqpt-storage-full-critical", + "severity": "critical", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1528", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528", + "domain": "infra", + "highestSeverity": "major", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "major", + "prevSeverity": "major", + "rule": "eqpt-storage-full-major", + "severity": "major", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 1501496, New: 240908), capUtilized (Old: 79, New: 82), inodesFree (Old: 12148991, New: 12148990), inodesUsed (Old: 721, New: 722), used (Old: 595656, New: 1856244)", + "childAction": "", + "code": "F1527", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /data/log on Node 1 with hostname fab3-apic1 mounted at /data/log is 82% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/data/log]-f-[tmpfs]/fault-F1527", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-full-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } } ] \ No newline at end of file diff --git a/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json b/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json index 76ee8b5..79b77fe 100644 --- a/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json +++ b/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json @@ -87,5 +87,65 @@ "type": "operational" } } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1529", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "retaining", + "occur": "1", + "origSeverity": "critical", + "prevSeverity": "critical", + "rule": "eqpt-storage-full-critical", + "severity": "cleared", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1528", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528", + "domain": "infra", + "highestSeverity": "major", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "soaking-clearing", + "occur": "1", + "origSeverity": "major", + "prevSeverity": "major", + "rule": "eqpt-storage-full-major", + "severity": "major", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } } ] \ No newline at end of file diff --git a/tests/checks/apic_disk_space_faults_check/Fault_raised.json b/tests/checks/apic_disk_space_faults_check/Fault_raised.json index 37284ba..58747a2 100644 --- a/tests/checks/apic_disk_space_faults_check/Fault_raised.json +++ b/tests/checks/apic_disk_space_faults_check/Fault_raised.json @@ -87,5 +87,95 @@ "type": "operational" } } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1529", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "critical", + "prevSeverity": "critical", + "rule": "eqpt-storage-full-critical", + "severity": "critical", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1528", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528", + "domain": "infra", + "highestSeverity": "major", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "major", + "prevSeverity": "major", + "rule": "eqpt-storage-full-major", + "severity": "major", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1527", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 82% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1527", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-full-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } } ] \ No newline at end of file diff --git a/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py b/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py index 40ff8b1..a593ab4 100644 --- a/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py +++ b/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py @@ -16,30 +16,36 @@ @pytest.mark.parametrize( - "icurl_outputs, cversion, expected_result, expected_data", + "icurl_outputs, cversion, tversion, expected_result, expected_data", [ # PASS - No raised faults ( {faultInst: []}, "4.2(1h)", + "4.2(1h)", script.PASS, [], ), - # FAIL - Raised faults with /firmware,/techsupport,/data/log mount points + # FAIL - Raised faults with /firmware,/techsupport,/data/log, /tmp mount points ( {faultInst: read_data(dir, "Fault_raised.json")}, "4.2(1h)", + "4.2(1h)", script.FAIL_UF, [ ["F1528", "1", "1", "/data/log", "89%", "Remove unneeded logs in var/log/dme/log"], ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], ["F1528", "1", "1", "/techsupport", "89%", "Remove unneeded techsupports/cores"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/tmp", "82%", "Remove unneeded logs in /tmp directory"], ], ), # PASS - Faults exist but not raised nor soaking (cleared) ( {faultInst: read_data(dir, "Fault_exists_not_raised.json")}, "4.2(1h)", + "4.2(1h)", script.PASS, [], ), @@ -47,16 +53,69 @@ ( {faultInst: read_data(dir, "Fault_combination.json")}, "4.2(1h)", + "4.2(1h)", + script.FAIL_UF, + [ + ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], + ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], + ], + ), + # FAIL - /tmp included when tversion is below 6.1(4h) + ( + {faultInst: read_data(dir, "Fault_combination.json")}, + "4.2(1h)", + "6.1(2f)", + script.FAIL_UF, + [ + ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], + ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], + ], + ), + # FAIL - /tmp included when tversion is not one of CSCwo96334 fixed target versions + ( + {faultInst: read_data(dir, "Fault_combination.json")}, + "4.2(1h)", + "6.1(2g)", script.FAIL_UF, [ ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], ], ), + # FAIL - /tmp skipped when tversion is one of CSCwo96334 fixed target versions + ( + {faultInst: read_data(dir, "Fault_combination.json")}, + "4.2(1h)", + "6.0(9f)", + script.FAIL_UF, + [ + ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], + ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], + ], + ), + # NA - only /tmp faults and tversion is one of CSCwo96334 fixed target versions + ( + {faultInst: read_data(dir, "Fault_combination.json")[3:5]}, + "4.2(1h)", + "6.1(4h)", + script.NA, + [], + ), # FAIL - Raised faults with unknown mount point (unformatted data) ( {faultInst: read_data(dir, "Fault_unformatted_data.json")}, "4.2(1h)", + "4.2(1h)", script.FAIL_UF, [ ["F1528", "1", "1", "/unknown", "88%", "Contact Cisco TAC."], @@ -66,6 +125,7 @@ ( {faultInst: read_data(dir, "Fault_unformatted_data.json")}, "4.0(1h)", + "4.0(1h)", script.FAIL_UF, [ ["F1528", "1", "1", "/unknown", "88%", "Contact Cisco TAC. A typical issue is CSCvn13119."], @@ -75,12 +135,16 @@ ( {faultInst: []}, "4.0(1h)", + "4.0(1h)", script.PASS, [], ), ], ) -def test_logic(run_check, mock_icurl, cversion, expected_result, expected_data): - result = run_check(cversion=script.AciVersion(cversion)) +def test_logic(run_check, mock_icurl, cversion, tversion, expected_result, expected_data): + result = run_check( + cversion=script.AciVersion(cversion), + tversion=script.AciVersion(tversion), + ) assert result.result == expected_result assert result.data == expected_data