diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 698b0e3..2c0386e 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -4361,6 +4361,195 @@ def cimc_cscwo74485_advisory_check(cversion, **kwargs): return Result(result=NA, doc_url=doc_url) +def cimc_cscwe61589_is_affected(cimc_version): + """ + Return True/False whether the CIMC version is affected by CSCwe61589, or None if the version + string cannot be parsed. + + CIMC versions come in two formats depending on the release train: + - `.()`, e.g. 4.2(3g) + - `.(.)`, e.g. 4.3(2.230207) + + Fixed versions: 4.2(3g), 4.3(2.230207), 6.0(1.250127) and later. 4.0 and 4.1 have no fixed release. + """ + m = re.match(r'^(\d+)\.(\d+)\((\d+)(?:\.(\d+)|([a-z]*))\)$', (cimc_version or '').strip()) + if not m: + return None + major, minor, patch = int(m.group(1)), int(m.group(2)), int(m.group(3)) + build = int(m.group(4)) if m.group(4) else None + letter = m.group(5) or '' + + if (major, minor) < (4, 0): + return False + if (major, minor) < (4, 2): + return True + if (major, minor) == (4, 2): + return (patch, letter) < (3, 'g') + if (major, minor) == (4, 3): + if patch < 2: + return True + if patch == 2 and build is not None: + return build < 230207 + return False + if (major, minor) == (6, 0) and patch == 1 and build is not None: + return build < 250127 + return False + + +def parse_acidiag_show_cimc(output): + """ + Parse `acidiag show cimc` output (available in 6.1(3) and later) and return a dict. + + ==================== CIMC DETAILS ==================== + PID : APIC-SERVER-M4 + Serial Number : WZP00000001 + Product Name : APIC-SERVER-M4 + ... + """ + fields = {} + for line in (output or '').splitlines(): + m = re.match(r'^\s*(PID|Serial Number|Product Name|CIMC Version)\s*:\s*(.*?)\s*$', line) + if m: + fields[m.group(1)] = m.group(2) + return fields + + +def cimc_cscwe61589_identity_findings(fields, inventory_serial=None): + """ + Return (definitive, review) lists of findings for the APIC identity reported by `acidiag show cimc`. + + CSCwe61589 makes the APIC read an incorrect product name/serial number at boot, e.g. the UCS PID + `UCSC-C220-M5` or the default `123456789` as the serial number. Those are definitive findings. + A serial number that merely differs from the APIC inventory (eqptCh) can happen for other reasons + and is reported for review only. + """ + definitive = [] + review = [] + pid = fields.get('PID', '') + serial = fields.get('Serial Number', '') + product = fields.get('Product Name', '') + default_values = ('123456789', 'default string') + + for label, value in (('PID', pid), ('Serial Number', serial), ('Product Name', product)): + if not value: + definitive.append('{} is empty'.format(label)) + if serial and (serial.lower() in default_values or re.match(r'^(UCS[CB]?-|APIC-)', serial, re.I)): + definitive.append('Serial Number is not a serial number ({})'.format(serial)) + if product and not product.upper().startswith('APIC-SERVER'): + definitive.append('Product Name is not an APIC model ({})'.format(product)) + if pid and product and pid != product: + definitive.append('PID ({}) differs from Product Name ({})'.format(pid, product)) + if not definitive and serial and inventory_serial and serial != inventory_serial: + review.append('Serial Number ({}) differs from APIC inventory ({})'.format(serial, inventory_serial)) + return definitive, review + + +def cimc_cscwe61589_get_identity(apic_addr, username, password): + """ + Run `acidiag show cimc` on the APIC and return (fields, error). + - (dict, None): identity retrieved + - (None, None): the command does not exist on this APIC release + - (None, str): the identity could not be retrieved + """ + if not apic_addr: + return None, "no APIC address" + try: + c = Connection(apic_addr) + c.username = username + c.password = password + c.log = LOG_FILE + c.connect() + c.cmd("acidiag show cimc") + except Exception as e: + return None, str(e) + fields = parse_acidiag_show_cimc(c.output) + if fields: + return fields, None + if re.search(r'invalid choice|unrecognized arguments|usage:', c.output, re.I): + return None, None + return None, "unexpected `acidiag show cimc` output" + + +# Connection Based Check +@check_wrapper(check_title="CIMC IPMI Stall and APIC Identity (CSCwe61589)") +def cimc_cscwe61589_check(cversion, username, password, fabric_nodes, **kwargs): + result = PASS + headers = ["Node ID", "APIC Name", "Model", "CIMC Version", "Finding"] + data = [] + msg = "" + recommended_action = ( + 'Upgrade CIMC to a version that includes the fix for CSCwe61589 before upgrading the APIC. ' + 'Until then, restart the CIMC before rebooting any APIC if it shows symptoms of CSCwe61589.' + ) + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#cimc-ipmi-stall-and-apic-identity-cscwe61589' + + apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"] + if not apics: + return Result(result=ERROR, msg="No APIC records found.", doc_url=doc_url) + + chassis = {} + for eqptCh in icurl('class', 'eqptCh.json?query-target-filter=wcard(eqptCh.descr,"APIC")'): + attr = eqptCh.get('eqptCh', {}).get('attributes', {}) + node = re.search(r'/node-(\d+)/', attr.get('dn', '')) + if node: + chassis[node.group(1)] = attr + + fail_found = False + identity_unsupported = 0 + for apic in apics: + attr = apic["fabricNode"]["attributes"] + apic_id = attr["id"] + apic_name = attr["name"] + apic_addr = attr.get("address") + ch = chassis.get(apic_id, {}) + apic_model = ch.get("descr", "") + current_cimc = ch.get("cimcVersion", "") + row = [apic_id, apic_name, apic_model or "-", current_cimc or "-"] + + findings = [] + + # Part A: CIMC version. APIC M3/L3 (UCS C220 M5) and M4/L4 (UCS C225 M6) can be affected. + # APIC M2/L2 (UCS M4) are outside the mechanism of the defect. + cimc_affected = False + if apic_model.upper().endswith(("-M3", "-L3", "-M4", "-L4")): + affected = cimc_cscwe61589_is_affected(current_cimc) + if affected is None: + findings.append("Cannot verify the CIMC version") + elif affected: + cimc_affected = True + findings.append("CIMC version is affected (fixed in 4.2(3g), 4.3(2.230207), or later)") + + # Part B: APIC identity reported by `acidiag show cimc` (not available in older releases). + fields, error = cimc_cscwe61589_get_identity(apic_addr, username, password) + if error: + findings.append("Cannot verify the APIC identity: " + error) + elif fields is None: + identity_unsupported += 1 + else: + incorrect, review = cimc_cscwe61589_identity_findings(fields, ch.get("ser")) + if incorrect: + findings.append("Incorrect APIC identity: " + "; ".join(incorrect)) + if cimc_affected: + # An incorrect identity together with an affected CIMC is the CSCwe61589 condition. + fail_found = True + else: + findings.append("CIMC version is not known to be affected by CSCwe61589") + else: + findings.extend(review) + + if findings: + data.append(row + ["; ".join(findings)]) + + if fail_found: + result = FAIL_UF + elif data: + result = MANUAL + elif identity_unsupported: + msg = "`acidiag show cimc` is not available on this APIC release. Only the CIMC version was checked." + return Result(result=result, msg=msg, headers=headers, data=data, + recommended_action=recommended_action, doc_url=doc_url) + + # Subprocess Check - icurl @check_wrapper(check_title="Intersight Device Connector upgrade status") def intersight_upgrade_status_check(**kwargs): @@ -9690,6 +9879,7 @@ class CheckManager: # Bugs observer_db_size_check, multipod_modular_spine_bootscript_check, + cimc_cscwe61589_check, ] cli_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 749851e..e167b4a 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -231,6 +231,7 @@ Items | Defect | This Script [APIC Connected Port VLAN Override][d44] | CSCwn64461 | :white_check_mark: | :no_entry_sign: [NX-OS IPN Multicast RPF Defect][d45] | CSCwt59437 | :white_check_mark: | :no_entry_sign: [vzAny Service Graph on Stretched VRF][d46] | CSCwt14573 | :white_check_mark: | :no_entry_sign: +[CIMC IPMI Stall and APIC Identity][d47] | CSCwe61589 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -278,6 +279,7 @@ Items | Defect | This Script [d44]: #apic-connected-port-vlan-override [d45]: #nx-os-ipn-multicast-rpf-defect-cscwt59437 [d46]: #vzany-service-graph-on-stretched-vrf +[d47]: #cimc-ipmi-stall-and-apic-identity-cscwe61589 ## General Check Details @@ -3093,6 +3095,24 @@ The graph instance may be scoped to a VRF, a tenant, or globally (`uni`). Tenant Recommended action: Migrate the vzAny service graph configuration to NDO before upgrade using brownfield import. NDO 4.2(3e) or later is required for vzAny PBR support on stretched VRFs. This is tracked under [CSCwt14573][91]. +### CIMC IPMI Stall and APIC Identity (CSCwe61589) + +Due to [CSCwe61589][95], the IPMI stack of an APIC's CIMC can get into a restart loop (`I2C Controller 5 software FIFO is full`, `Restarting IPMI Stack->Reason: VKCS UnResponsive`). While this condition persists, the APIC may read an incorrect product name and serial number from the hardware at boot, for example the UCS PID `UCSC-C220-M5` or the default `123456789`. An APIC that boots in this state during an upgrade generates its certificate from the incorrect identity and cannot join the cluster, so the cluster upgrade gets stuck. Cisco TAC support is then required to work around the problem and complete the upgrade. + +The check evaluates each APIC for two independent conditions: + +1. **CIMC version.** On APIC M3/L3 (UCS C220 M5) and M4/L4 (UCS C225 M6), the CIMC version is compared with the releases that include the fix: 4.2(3g), 4.3(2.230207) and later. CIMC 4.0 and 4.1 have no fixed release. An affected CIMC version alone returns `MANUAL CHECK REQUIRED`, because it does not mean the condition is occurring now. APIC M2/L2 (UCS M4) are not evaluated for the CIMC version. +2. **APIC identity.** The PID, serial number and product name from `acidiag show cimc` are checked on every APIC model. The identity is incorrect when any of them is empty, when the serial number is a default value or a product ID, when the product name is not an APIC model, or when the PID and the product name differ. + +An incorrect identity together with an affected CIMC version is the CSCwe61589 condition, and returns `FAIL - UPGRADE FAILURE!!`. An incorrect identity without an affected CIMC version returns `MANUAL CHECK REQUIRED` because it is not attributed to this defect. A valid-looking serial number that differs from the APIC inventory also returns `MANUAL CHECK REQUIRED`, because that can have other causes. If the identity cannot be retrieved (SSH failure or unexpected output), the check returns `MANUAL CHECK REQUIRED` instead of passing. + +!!! note + * `acidiag show cimc` is not available in older APIC releases. When the command is not available, only the CIMC version is checked and the check does not report a failure for the unavailable command. + * The check connects to every APIC with SSH, so it is not run with the API-only option. + * `acidiag show cimc` reflects what the APIC read at boot. A correct result does not guarantee that the next reboot reads the identity correctly while the CIMC is still affected. A CIMC that is currently in the restart loop cannot be detected from the APIC with the `admin` user, because `ipmitool` and `dmidecode` require root. + +Recommended action: Upgrade CIMC to a release that includes the fix for [CSCwe61589][95] before upgrading the APIC. Until then, restart the CIMC before rebooting any APIC if it shows symptoms of CSCwe61589. To confirm whether the condition is occurring, collect the CIMC technical support from the CIMC itself and search the `obfl/obfl-log` files for `I2C Controller 5 software FIFO is full` together with `Restarting IPMI Stack->Reason: VKCS UnResponsive`. + ### App Center deprecation when crossing 6.1(2) Starting with APIC release 6.1(2), App Center and its applications can no longer be hosted on APIC because the APIC App Infrastructure is removed. When an upgrade crosses from a release before 6.1(2) to 6.1(2) or later, this check inventories all installed App Center applications, including inactive applications, and returns `MANUAL CHECK REQUIRED` when any are found. @@ -3201,3 +3221,4 @@ This validation is not applicable when the current APIC release is already 6.1(2 [92]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwn95571 [93]: https://dcappcenter.cisco.com/elam-assistant.html [94]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/elam-assistant-user-guide/elam-assistant-user-guide/m-overview.html +[95]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe61589 diff --git a/tests/checks/cimc_cscwe61589_check/test_cimc_cscwe61589_check.py b/tests/checks/cimc_cscwe61589_check/test_cimc_cscwe61589_check.py new file mode 100644 index 0000000..ce28729 --- /dev/null +++ b/tests/checks/cimc_cscwe61589_check/test_cimc_cscwe61589_check.py @@ -0,0 +1,224 @@ +import importlib + +import pytest + +script = importlib.import_module("aci-preupgrade-validation-script") + +test_function = "cimc_cscwe61589_check" +eqptCh_api = 'eqptCh.json?query-target-filter=wcard(eqptCh.descr,"APIC")' +CMD = "acidiag show cimc" + + +def fabric_node(node_id, ip): + return {"fabricNode": {"attributes": { + "id": node_id, "name": "apic" + node_id, "address": ip, "role": "controller", + "dn": "topology/pod-1/node-" + node_id, + }}} + + +def eqpt_ch(node_id, model="L3", cimc="4.3(2.250016)", serial="WZP00000001"): + return {"eqptCh": {"attributes": { + "descr": "APIC-SERVER-" + model, "cimcVersion": cimc, "ser": serial, + "dn": "topology/pod-1/node-{}/sys/ch".format(node_id), + }}} + + +def acidiag_output(pid="APIC-SERVER-L3", serial="WZP00000001", product=None): + return "\n".join([ + CMD, + "==================== CIMC DETAILS ====================", + "PID : " + pid, + "Serial Number : " + serial, + "Product Name : " + (product if product is not None else pid), + "BIOS Version : C220M5.4.2.2b.0.0613220203", + "CIMC Version : 4.3(2.250016)", + "==================== END of CIMC DETAILS!! ====================", + "apic1#", + ]) + + +# Real output of `acidiag show cimc` on APIC 6.0(3d), where the command does not exist (choices shortened). +UNSUPPORTED_OUTPUT = "\n".join([ + CMD, + "usage: acidiag [-h] [-v]", + " {avread,fnvread,version,platform,hwcheck,verifyapic,reboot,dmecore} ...", + "acidiag: error: argument : invalid choice: 'show' (choose from 'avread', 'fnvread', " + "'version', 'platform', 'hwcheck', 'verifyapic', 'reboot', 'dmecore')", + "apic1#", +]) + + +BAD_IDENTITY_OUTPUT = acidiag_output(serial="UCSC-C220-M5", product="UCSC-C220-M5") + + +def conn_output(ip, output=None, exception=None): + return {ip: [{"cmd": CMD, "output": output, "exception": exception}]} + + +@pytest.mark.parametrize("version, expected", [ + ("3.0(4d)", False), + ("4.0(2g)", True), + ("4.1(3m)", True), + ("4.2(2a)", True), + ("4.2(3b)", True), + ("4.2(3e)", True), + ("4.2(3g)", False), + ("4.2(3h)", False), + ("4.3(1.220001)", True), + ("4.3(2.220001)", True), + ("4.3(2.230206)", True), + ("4.3(2.230207)", False), + ("4.3(2.250016)", False), + ("4.3(4.241063)", False), + ("6.0(1.250126)", True), + ("6.0(1.250127)", False), + ("6.0(2.260044)", False), + ("", None), + (None, None), + ("garbage", None), +]) +def test_is_affected(version, expected): + assert script.cimc_cscwe61589_is_affected(version) is expected + + +@pytest.mark.parametrize("kwargs, definitive_count, review_count", [ + ({}, 0, 0), + ({"product": "UCSC-C220-M5", "serial": "UCSC-C220-M5"}, 3, 0), + ({"serial": "123456789"}, 1, 0), + ({"product": "UCSC-C220-M5"}, 2, 0), + ({"pid": "APIC-SERVER-L3", "product": "APIC-SERVER-M3"}, 1, 0), +]) +def test_identity_findings(kwargs, definitive_count, review_count): + fields = script.parse_acidiag_show_cimc(acidiag_output(**kwargs)) + definitive, review = script.cimc_cscwe61589_identity_findings(fields, "WZP00000001") + assert (len(definitive), len(review)) == (definitive_count, review_count) + + +def test_identity_findings_empty_and_serial_mismatch(): + definitive, review = script.cimc_cscwe61589_identity_findings({}, None) + assert len(definitive) == 3 + fields = script.parse_acidiag_show_cimc(acidiag_output(serial="WZP00000002")) + definitive, review = script.cimc_cscwe61589_identity_findings(fields, "WZP00000001") + assert definitive == [] + assert len(review) == 1 + + +IP1 = "10.0.0.1" +NODES = [fabric_node("1", IP1)] + + +@pytest.mark.parametrize( + "icurl_outputs, conn_failure, conn_cmds, fabric_nodes, expected_result, expected_nodes", + [ + # Fixed CIMC, healthy identity + ( + {eqptCh_api: [eqpt_ch("1")]}, False, conn_output(IP1, acidiag_output()), NODES, + script.PASS, [], + ), + # Affected CIMC on APIC M3/L3, healthy identity -> MANUAL + ( + {eqptCh_api: [eqpt_ch("1", cimc="4.2(3b)")]}, False, conn_output(IP1, acidiag_output()), NODES, + script.MANUAL, ["1"], + ), + # APIC M4/L4 (UCS C225 M6) can be affected too + ( + {eqptCh_api: [eqpt_ch("1", model="M4", cimc="4.2(3b)")]}, False, + conn_output(IP1, acidiag_output(pid="APIC-SERVER-M4")), NODES, + script.MANUAL, ["1"], + ), + # Affected CIMC and `acidiag show cimc` not supported -> MANUAL (CIMC version only) + ( + {eqptCh_api: [eqpt_ch("1", cimc="4.2(3b)")]}, False, conn_output(IP1, UNSUPPORTED_OUTPUT), NODES, + script.MANUAL, ["1"], + ), + # Fixed CIMC on APIC M4/L4 + ( + {eqptCh_api: [eqpt_ch("1", model="L4", cimc="4.3(4.241063)")]}, False, + conn_output(IP1, acidiag_output(pid="APIC-SERVER-L4")), NODES, + script.PASS, [], + ), + # Old CIMC on APIC M2/L2 (UCS M4) is out of scope for the version check + ( + {eqptCh_api: [eqpt_ch("1", model="M2", cimc="4.1(2m)")]}, False, + conn_output(IP1, acidiag_output(pid="APIC-SERVER-M2")), NODES, + script.PASS, [], + ), + # CIMC version unparseable on M3/L3 + ( + {eqptCh_api: [eqpt_ch("1", cimc="")]}, False, conn_output(IP1, acidiag_output()), NODES, + script.MANUAL, ["1"], + ), + # Bad identity and affected CIMC -> FAIL_UF (the CSCwe61589 condition) + ( + {eqptCh_api: [eqpt_ch("1", cimc="4.2(3b)")]}, False, conn_output(IP1, BAD_IDENTITY_OUTPUT), NODES, + script.FAIL_UF, ["1"], + ), + # Bad identity with a CIMC that is not affected -> MANUAL (not attributed to CSCwe61589) + ( + {eqptCh_api: [eqpt_ch("1")]}, False, conn_output(IP1, BAD_IDENTITY_OUTPUT), NODES, + script.MANUAL, ["1"], + ), + # Bad identity on APIC M2/L2 where the CIMC version is not evaluated -> MANUAL + ( + {eqptCh_api: [eqpt_ch("1", model="M2", cimc="4.1(2m)")]}, False, + conn_output(IP1, BAD_IDENTITY_OUTPUT), NODES, + script.MANUAL, ["1"], + ), + # Valid-looking serial that differs from inventory -> MANUAL only + ( + {eqptCh_api: [eqpt_ch("1")]}, False, conn_output(IP1, acidiag_output(serial="WZP00000002")), NODES, + script.MANUAL, ["1"], + ), + # `acidiag show cimc` not supported (older release) -> PASS with message + ( + {eqptCh_api: [eqpt_ch("1")]}, False, + conn_output(IP1, UNSUPPORTED_OUTPUT), NODES, + script.PASS, [], + ), + # `acidiag show cimc` output unexpected -> cannot verify + ( + {eqptCh_api: [eqpt_ch("1")]}, False, conn_output(IP1, CMD + "\nCIMC query failed\napic1#"), NODES, + script.MANUAL, ["1"], + ), + # Command raised an exception + ( + {eqptCh_api: [eqpt_ch("1")]}, False, conn_output(IP1, "", Exception("Simulated exception")), NODES, + script.MANUAL, ["1"], + ), + # Connection failure + ( + {eqptCh_api: [eqpt_ch("1")]}, True, {}, NODES, + script.MANUAL, ["1"], + ), + ], +) +def test_logic(run_check, mock_icurl, mock_conn, fabric_nodes, expected_result, expected_nodes): + result = run_check( + cversion=script.AciVersion("6.1(6g)"), + username="fake_username", + password="fake_password", + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result + assert [row[0] for row in result.data] == expected_nodes + + +@pytest.mark.parametrize("icurl_outputs, conn_failure, conn_cmds", [({eqptCh_api: []}, False, {})]) +def test_no_apic(run_check, mock_icurl, mock_conn): + result = run_check( + cversion=script.AciVersion("6.1(6g)"), username="u", password="p", fabric_nodes=[], + ) + assert result.result == script.ERROR + + +@pytest.mark.parametrize("icurl_outputs, conn_failure, conn_cmds", [ + ({eqptCh_api: [eqpt_ch("1", cimc="4.2(3b)")]}, False, conn_output(IP1, acidiag_output())), +]) +def test_serialized_result(run_check, mock_icurl, mock_conn): + result = run_check( + cversion=script.AciVersion("6.1(6g)"), username="u", password="p", fabric_nodes=NODES, + ) + aci_result = script.AciResult(test_function, "CIMC IPMI Stall and APIC Identity (CSCwe61589)", result) + assert aci_result.ruleStatus == script.AciResult.FAIL + assert aci_result.failureDetails["failType"] == script.MANUAL + assert aci_result.failureDetails["data"][0]["Node ID"] == "1"