Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 190 additions & 0 deletions aci-preupgrade-validation-script.py
Original file line number Diff line number Diff line change
Expand Up @@ -4361,6 +4361,195 @@ def cimc_cscwo74485_advisory_check(cversion, **kwargs):
return Result(result=NA, doc_url=doc_url)


def cimc_cscwe61589_is_affected(cimc_version):
"""
Return True/False whether the CIMC version is affected by CSCwe61589, or None if the version
string cannot be parsed.

CIMC versions come in two formats depending on the release train:
- `<major>.<minor>(<patch><letter>)`, e.g. 4.2(3g)
- `<major>.<minor>(<patch>.<build>)`, e.g. 4.3(2.230207)

Fixed versions: 4.2(3g), 4.3(2.230207), 6.0(1.250127) and later. 4.0 and 4.1 have no fixed release.
"""
m = re.match(r'^(\d+)\.(\d+)\((\d+)(?:\.(\d+)|([a-z]*))\)$', (cimc_version or '').strip())
if not m:
return None
major, minor, patch = int(m.group(1)), int(m.group(2)), int(m.group(3))
build = int(m.group(4)) if m.group(4) else None
letter = m.group(5) or ''

if (major, minor) < (4, 0):
return False
if (major, minor) < (4, 2):
return True
if (major, minor) == (4, 2):
return (patch, letter) < (3, 'g')
if (major, minor) == (4, 3):
if patch < 2:
return True
if patch == 2 and build is not None:
return build < 230207
return False
if (major, minor) == (6, 0) and patch == 1 and build is not None:
return build < 250127
return False


def parse_acidiag_show_cimc(output):
"""
Parse `acidiag show cimc` output (available in 6.1(3) and later) and return a dict.

==================== CIMC DETAILS ====================
PID : APIC-SERVER-M4
Serial Number : WZP00000001
Product Name : APIC-SERVER-M4
...
"""
fields = {}
for line in (output or '').splitlines():
m = re.match(r'^\s*(PID|Serial Number|Product Name|CIMC Version)\s*:\s*(.*?)\s*$', line)
if m:
fields[m.group(1)] = m.group(2)
return fields


def cimc_cscwe61589_identity_findings(fields, inventory_serial=None):
"""
Return (definitive, review) lists of findings for the APIC identity reported by `acidiag show cimc`.

CSCwe61589 makes the APIC read an incorrect product name/serial number at boot, e.g. the UCS PID
`UCSC-C220-M5` or the default `123456789` as the serial number. Those are definitive findings.
A serial number that merely differs from the APIC inventory (eqptCh) can happen for other reasons
and is reported for review only.
"""
definitive = []
review = []
pid = fields.get('PID', '')
serial = fields.get('Serial Number', '')
product = fields.get('Product Name', '')
default_values = ('123456789', 'default string')

for label, value in (('PID', pid), ('Serial Number', serial), ('Product Name', product)):
if not value:
definitive.append('{} is empty'.format(label))
if serial and (serial.lower() in default_values or re.match(r'^(UCS[CB]?-|APIC-)', serial, re.I)):
definitive.append('Serial Number is not a serial number ({})'.format(serial))
if product and not product.upper().startswith('APIC-SERVER'):
definitive.append('Product Name is not an APIC model ({})'.format(product))
if pid and product and pid != product:
definitive.append('PID ({}) differs from Product Name ({})'.format(pid, product))
if not definitive and serial and inventory_serial and serial != inventory_serial:
review.append('Serial Number ({}) differs from APIC inventory ({})'.format(serial, inventory_serial))
return definitive, review


def cimc_cscwe61589_get_identity(apic_addr, username, password):
"""
Run `acidiag show cimc` on the APIC and return (fields, error).
- (dict, None): identity retrieved
- (None, None): the command does not exist on this APIC release
- (None, str): the identity could not be retrieved
"""
if not apic_addr:
return None, "no APIC address"
try:
c = Connection(apic_addr)
c.username = username
c.password = password
c.log = LOG_FILE
c.connect()
c.cmd("acidiag show cimc")
except Exception as e:
return None, str(e)
fields = parse_acidiag_show_cimc(c.output)
if fields:
return fields, None
if re.search(r'invalid choice|unrecognized arguments|usage:', c.output, re.I):
return None, None
return None, "unexpected `acidiag show cimc` output"


# Connection Based Check
@check_wrapper(check_title="CIMC IPMI Stall and APIC Identity (CSCwe61589)")
def cimc_cscwe61589_check(cversion, username, password, fabric_nodes, **kwargs):
result = PASS
headers = ["Node ID", "APIC Name", "Model", "CIMC Version", "Finding"]
data = []
msg = ""
recommended_action = (
'Upgrade CIMC to a version that includes the fix for CSCwe61589 before upgrading the APIC. '
'Until then, restart the CIMC before rebooting any APIC if it shows symptoms of CSCwe61589.'
)
doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#cimc-ipmi-stall-and-apic-identity-cscwe61589'

apics = [node for node in fabric_nodes if node["fabricNode"]["attributes"]["role"] == "controller"]
if not apics:
return Result(result=ERROR, msg="No APIC records found.", doc_url=doc_url)

chassis = {}
for eqptCh in icurl('class', 'eqptCh.json?query-target-filter=wcard(eqptCh.descr,"APIC")'):
attr = eqptCh.get('eqptCh', {}).get('attributes', {})
node = re.search(r'/node-(\d+)/', attr.get('dn', ''))
if node:
chassis[node.group(1)] = attr

fail_found = False
identity_unsupported = 0
for apic in apics:
attr = apic["fabricNode"]["attributes"]
apic_id = attr["id"]
apic_name = attr["name"]
apic_addr = attr.get("address")
ch = chassis.get(apic_id, {})
apic_model = ch.get("descr", "")
current_cimc = ch.get("cimcVersion", "")
row = [apic_id, apic_name, apic_model or "-", current_cimc or "-"]

findings = []

# Part A: CIMC version. APIC M3/L3 (UCS C220 M5) and M4/L4 (UCS C225 M6) can be affected.
# APIC M2/L2 (UCS M4) are outside the mechanism of the defect.
cimc_affected = False
if apic_model.upper().endswith(("-M3", "-L3", "-M4", "-L4")):
affected = cimc_cscwe61589_is_affected(current_cimc)
if affected is None:
findings.append("Cannot verify the CIMC version")
elif affected:
cimc_affected = True
findings.append("CIMC version is affected (fixed in 4.2(3g), 4.3(2.230207), or later)")

# Part B: APIC identity reported by `acidiag show cimc` (not available in older releases).
fields, error = cimc_cscwe61589_get_identity(apic_addr, username, password)
if error:
findings.append("Cannot verify the APIC identity: " + error)
elif fields is None:
identity_unsupported += 1
else:
incorrect, review = cimc_cscwe61589_identity_findings(fields, ch.get("ser"))
if incorrect:
findings.append("Incorrect APIC identity: " + "; ".join(incorrect))
if cimc_affected:
# An incorrect identity together with an affected CIMC is the CSCwe61589 condition.
fail_found = True
else:
findings.append("CIMC version is not known to be affected by CSCwe61589")
else:
findings.extend(review)

if findings:
data.append(row + ["; ".join(findings)])

if fail_found:
result = FAIL_UF
elif data:
result = MANUAL
elif identity_unsupported:
msg = "`acidiag show cimc` is not available on this APIC release. Only the CIMC version was checked."
return Result(result=result, msg=msg, headers=headers, data=data,
recommended_action=recommended_action, doc_url=doc_url)


# Subprocess Check - icurl
@check_wrapper(check_title="Intersight Device Connector upgrade status")
def intersight_upgrade_status_check(**kwargs):
Expand Down Expand Up @@ -9690,6 +9879,7 @@ class CheckManager:
# Bugs
observer_db_size_check,
multipod_modular_spine_bootscript_check,
cimc_cscwe61589_check,
]
cli_checks = [
# General
Expand Down
21 changes: 21 additions & 0 deletions docs/docs/validations.md
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ Items | Defect | This Script
[APIC Connected Port VLAN Override][d44] | CSCwn64461 | :white_check_mark: | :no_entry_sign:
[NX-OS IPN Multicast RPF Defect][d45] | CSCwt59437 | :white_check_mark: | :no_entry_sign:
[vzAny Service Graph on Stretched VRF][d46] | CSCwt14573 | :white_check_mark: | :no_entry_sign:
[CIMC IPMI Stall and APIC Identity][d47] | CSCwe61589 | :white_check_mark: | :no_entry_sign:

[d1]: #ep-announce-compatibility
[d2]: #eventmgr-db-size-defect-susceptibility
Expand Down Expand Up @@ -278,6 +279,7 @@ Items | Defect | This Script
[d44]: #apic-connected-port-vlan-override
[d45]: #nx-os-ipn-multicast-rpf-defect-cscwt59437
[d46]: #vzany-service-graph-on-stretched-vrf
[d47]: #cimc-ipmi-stall-and-apic-identity-cscwe61589

## General Check Details

Expand Down Expand Up @@ -3093,6 +3095,24 @@ The graph instance may be scoped to a VRF, a tenant, or globally (`uni`). Tenant

Recommended action: Migrate the vzAny service graph configuration to NDO before upgrade using brownfield import. NDO 4.2(3e) or later is required for vzAny PBR support on stretched VRFs. This is tracked under [CSCwt14573][91].

### CIMC IPMI Stall and APIC Identity (CSCwe61589)

Due to [CSCwe61589][95], the IPMI stack of an APIC's CIMC can get into a restart loop (`I2C Controller 5 software FIFO is full`, `Restarting IPMI Stack->Reason: VKCS UnResponsive`). While this condition persists, the APIC may read an incorrect product name and serial number from the hardware at boot, for example the UCS PID `UCSC-C220-M5` or the default `123456789`. An APIC that boots in this state during an upgrade generates its certificate from the incorrect identity and cannot join the cluster, so the cluster upgrade gets stuck. Cisco TAC support is then required to work around the problem and complete the upgrade.

The check evaluates each APIC for two independent conditions:

1. **CIMC version.** On APIC M3/L3 (UCS C220 M5) and M4/L4 (UCS C225 M6), the CIMC version is compared with the releases that include the fix: 4.2(3g), 4.3(2.230207) and later. CIMC 4.0 and 4.1 have no fixed release. An affected CIMC version alone returns `MANUAL CHECK REQUIRED`, because it does not mean the condition is occurring now. APIC M2/L2 (UCS M4) are not evaluated for the CIMC version.
2. **APIC identity.** The PID, serial number and product name from `acidiag show cimc` are checked on every APIC model. The identity is incorrect when any of them is empty, when the serial number is a default value or a product ID, when the product name is not an APIC model, or when the PID and the product name differ.

An incorrect identity together with an affected CIMC version is the CSCwe61589 condition, and returns `FAIL - UPGRADE FAILURE!!`. An incorrect identity without an affected CIMC version returns `MANUAL CHECK REQUIRED` because it is not attributed to this defect. A valid-looking serial number that differs from the APIC inventory also returns `MANUAL CHECK REQUIRED`, because that can have other causes. If the identity cannot be retrieved (SSH failure or unexpected output), the check returns `MANUAL CHECK REQUIRED` instead of passing.

!!! note
* `acidiag show cimc` is not available in older APIC releases. When the command is not available, only the CIMC version is checked and the check does not report a failure for the unavailable command.
* The check connects to every APIC with SSH, so it is not run with the API-only option.
* `acidiag show cimc` reflects what the APIC read at boot. A correct result does not guarantee that the next reboot reads the identity correctly while the CIMC is still affected. A CIMC that is currently in the restart loop cannot be detected from the APIC with the `admin` user, because `ipmitool` and `dmidecode` require root.

Recommended action: Upgrade CIMC to a release that includes the fix for [CSCwe61589][95] before upgrading the APIC. Until then, restart the CIMC before rebooting any APIC if it shows symptoms of CSCwe61589. To confirm whether the condition is occurring, collect the CIMC technical support from the CIMC itself and search the `obfl/obfl-log` files for `I2C Controller 5 software FIFO is full` together with `Restarting IPMI Stack->Reason: VKCS UnResponsive`.

### App Center deprecation when crossing 6.1(2)

Starting with APIC release 6.1(2), App Center and its applications can no longer be hosted on APIC because the APIC App Infrastructure is removed. When an upgrade crosses from a release before 6.1(2) to 6.1(2) or later, this check inventories all installed App Center applications, including inactive applications, and returns `MANUAL CHECK REQUIRED` when any are found.
Expand Down Expand Up @@ -3201,3 +3221,4 @@ This validation is not applicable when the current APIC release is already 6.1(2
[92]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwn95571
[93]: https://dcappcenter.cisco.com/elam-assistant.html
[94]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/elam-assistant-user-guide/elam-assistant-user-guide/m-overview.html
[95]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwe61589
Loading