Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 65 additions & 30 deletions aci-preupgrade-validation-script.py
Original file line number Diff line number Diff line change
Expand Up @@ -2435,7 +2435,7 @@ def switch_group_guideline_check(fabric_nodes, **kwargs):


@check_wrapper(check_title="Switch Node /bootflash usage")
def switch_bootflash_usage_check(sw_cversion, tversion, **kwargs):
def switch_bootflash_usage_check(sw_cversion, tversion, fabric_nodes, **kwargs):
result = FAIL_UF
msg = ''
headers = ["Pod-ID", "Node-ID", "Avail (MB)", "Required (MB)"]
Expand Down Expand Up @@ -2488,7 +2488,7 @@ def missing_images(image_rows):
for maintUpgJob in download_sts:
dn = re.search(node_regex, maintUpgJob['maintUpgJob']['attributes']['dn'])
if dn:
predownloaded_nodes.append(dn.group("node"))
predownloaded_nodes.append((dn.group("pod"), dn.group("node")))

# Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit
# isos (`-cs_64` suffix for 64-bit). Below that, only a single 32-bit iso exists.
Expand All @@ -2506,11 +2506,23 @@ def missing_images(image_rows):
target_size_32 = fw_sizes.get(switch_target_version)
target_size_64 = fw_sizes.get(switch_target_version_64)

# sw_cversion (lowest switch version), not the APIC cversion, drives the image split
# decision: the upgrade guide has APICs reach 6.0(2a)+ before the switches, so the
# switches can still be pre-split while the APIC cluster is in the split-image era.
target_is_pre_split = tversion.older_than(boundary_version)
current_is_pre_split = sw_cversion.older_than(boundary_version)
node_versions = {}
node_images = {}
if not target_is_pre_split:
for fabric_node in fabric_nodes:
attrs = fabric_node['fabricNode']['attributes']
dn = re.search(node_regex, attrs.get('dn', ''))
if dn and attrs.get('role') != 'controller':
try:
node_versions[(dn.group('pod'), dn.group('node'))] = AciVersion(attrs.get('version') or '')
except ValueError:
pass
for running in icurl('class', 'firmwareRunning.json'):
attrs = running['firmwareRunning']['attributes']
dn = re.search(node_regex, attrs.get('dn', ''))
if dn:
node_images[(dn.group('pod'), dn.group('node'))] = attrs.get('ksFile') or ''

if target_is_pre_split:
# Only the 32-bit image is ever used for a pre-6.0(2a) target.
Expand All @@ -2520,8 +2532,8 @@ def missing_images(image_rows):
downloaded_required_space = target_size_32

else:
# The larger image is used as a conservative estimate, so both sizes must be
# known; a missing one can't be assumed to be the smaller (or zero-byte) one.
# Keep the existing repository requirement for both target architectures;
# crossing nodes need both sizes for the boundary calculation.
if target_size_32 is None and target_size_64 is None:
return missing_images([
['Target', '32-bit', switch_target_version],
Expand All @@ -2532,44 +2544,67 @@ def missing_images(image_rows):
elif target_size_64 is None:
return missing_images([['Target', '64-bit', switch_target_version_64]])

if current_is_pre_split:
# Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a
# 32-bit image, so its size is freed once removed during the upgrade.
switch_current_version = "aci-n9000-dk9.1{}.bin".format(sw_cversion.dot_version)
current_size = fw_sizes.get(switch_current_version)
if current_size is None:
return missing_images([['Current', '32-bit', switch_current_version]])
if target_size_32 > current_size:
required_space = 2 * (target_size_32 + target_size_64 - current_size)
else:
required_space = 2 * max(target_size_32, target_size_64)
# A downloaded 32-bit target is already reflected in the current `avail`.
downloaded_required_space = required_space - target_size_32
else:
required_space = 2 * max(target_size_32, target_size_64)
downloaded_required_space = max(target_size_32, target_size_64)

required_space_kb = required_space / 1024.0 # eqptcapacityFSPartition avail/used are in KB
downloaded_required_space_kb = downloaded_required_space / 1024.0
manual_nodes = []

for eqptcapacityFSPartition in partitions:
dn = re.search(node_regex, eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['dn'])
pod = dn.group("pod")
node = dn.group("node")
avail = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['avail'])

if not target_is_pre_split:
node_version = node_versions.get((pod, node))
if node_version is None:
manual_nodes.append([pod, node, 'Current version unavailable'])
continue
if node_version.older_than(boundary_version):
# Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a
# 32-bit image, so its size is freed once removed during the upgrade.
switch_current_version = "aci-n9000-dk9.1{}.bin".format(node_version.dot_version)
current_size = fw_sizes.get(switch_current_version)
if current_size is None:
return missing_images([['Current', '32-bit', switch_current_version]])
if target_size_32 > current_size:
required_space = 2 * (target_size_32 + target_size_64 - current_size)
else:
required_space = 2 * max(target_size_32, target_size_64)
# A downloaded 32-bit target is already reflected in the current `avail`.
downloaded_required_space = required_space - target_size_32
else:
image = node_images.get((pod, node), '')
if not image.endswith('.bin'):
manual_nodes.append([pod, node, 'Current architecture unavailable'])
continue
target_size = target_size_64 if image.endswith('-cs_64.bin') else target_size_32
required_space = 2 * target_size
downloaded_required_space = target_size

required_space_kb = required_space / 1024.0 # partition availability is in KB
downloaded_required_space_kb = downloaded_required_space / 1024.0

# dnldStatus == downloaded only proves the image was delivered, not that
# extraction (which still consumes bootflash) has completed, so a downloaded
# node is still checked, just against the smaller extraction-only requirement.
node_required_space_kb = downloaded_required_space_kb if node in predownloaded_nodes else required_space_kb
node_required_space_kb = downloaded_required_space_kb if (pod, node) in predownloaded_nodes else required_space_kb

if avail < node_required_space_kb:
data.append([pod, node, round(avail / 1024.0, 2), round(node_required_space_kb / 1024.0, 2)])

if not data:
if manual_nodes:
recommended_action += (
" Verify the current version and running image for the listed unevaluated nodes, "
"then rerun this check."
)
if not data:
result = MANUAL
elif not data:
result = PASS
msg = 'Sufficient bootflash space.'
return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
return Result(
result=result, msg=msg, headers=headers, data=data,
unformatted_headers=["Pod-ID", "Node-ID", "Not evaluated"] if manual_nodes else [],
unformatted_data=manual_nodes, recommended_action=recommended_action, doc_url=doc_url,
)


@check_wrapper(check_title="L3Out MTU")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,28 @@

test_function = "switch_bootflash_usage_check"

@pytest.fixture
def run_check(run_check, icurl_outputs):
"""Provide per-node inventory for the existing homogeneous-version cases."""
def run(**kwargs):
if 'fabric_nodes' not in kwargs:
nodes = []
running = []
for partition in icurl_outputs.get(partitions, []):
dn = partition['eqptcapacityFSPartition']['attributes']['dn'].split('/sys/')[0]
nodes.append({'fabricNode': {'attributes': {
'dn': dn, 'role': 'leaf', 'version': str(kwargs.get('sw_cversion') or ''),
}}})
running.append({'firmwareRunning': {'attributes': {
'dn': dn + '/sys/fwstatuscont/running',
'ksFile': 'aci-n9000-dk9.16.0.3a-cs_64.bin',
}}})
kwargs['fabric_nodes'] = nodes
icurl_outputs.setdefault('firmwareRunning.json', running)
return run_check(**kwargs)
return run


# icurl queries
partitions = "eqptcapacityFSPartition.json"
partitions += '?query-target-filter=eq(eqptcapacityFSPartition.path,"/bootflash")'
Expand Down Expand Up @@ -432,3 +454,86 @@ def test_logic(run_check, mock_icurl, sw_cversion, tversion, expected_result):
tversion=script.AciVersion(tversion) if tversion else None,
)
assert result.result == expected_result


def node_inventory(pod, node, version, image):
dn = 'topology/pod-{}/node-{}'.format(pod, node)
return (
{'fabricNode': {'attributes': {'dn': dn, 'role': 'leaf', 'version': version}}},
{'firmwareRunning': {'attributes': {'dn': dn + '/sys/fwstatuscont/running', 'ksFile': image}}},
)


def bootflash(pod, node, available_mb):
return {'eqptcapacityFSPartition': {'attributes': {
'dn': 'topology/pod-{}/node-{}/sys/eqptcapacity/fspartition-bootflash'.format(pod, node),
'avail': str(int(available_mb * 1024)), 'used': '0',
}}}


@pytest.mark.parametrize('icurl_outputs', [{}])
@pytest.mark.parametrize('insufficient', [False, True])
def test_issue_493_mixed_versions_current_image_absent(run_check, mock_icurl, icurl_outputs, insufficient):
inventory = [node_inventory(1, node, '16.0(9c)', 'aci-n9000-dk9.16.0.9c-cs_64.bin')
for node in range(1109, 1115)]
icurl_outputs.update({
partitions: [bootflash(1, node, 1 if insufficient else available) for node, available in zip(
range(1109, 1115), [6490.13, 6491.29, 6489.39, 6501.6, 6489.37, 6489.61])],
'maintUpgJob.json?query-target-filter=and(eq(maintUpgJob.dnldStatus,"downloaded"),eq(maintUpgJob.desiredVersion,"n9000-16.1(6g)"))': [],
firmware: [
{'firmwareFirmware': {'attributes': {'isoname': name, 'size': str(int(size * 1024 ** 2))}}}
for name, size in [('aci-n9000-dk9.15.3.2e.bin', 1995.57),
('aci-n9000-dk9.16.1.6g.bin', 2560.78),
('aci-n9000-dk9.16.1.6g-cs_64.bin', 2694.73)]
],
'firmwareRunning.json': [running for _, running in inventory],
})
result = run_check(sw_cversion=script.AciVersion('5.3(2e)'),
tversion=script.AciVersion('6.1(6g)'),
fabric_nodes=[node for node, _ in inventory])
assert result.result == (script.FAIL_UF if insufficient else script.PASS)
assert result.data == ([["1", str(node), "1.0", "5389.46"] for node in range(1109, 1115)]
if insufficient else [])


@pytest.mark.parametrize('missing', [None, 'version', 'architecture', 'architecture_with_failure'])
@pytest.mark.parametrize('icurl_outputs', [{}])
def test_per_node_architecture_and_missing_inventory(run_check, mock_icurl, icurl_outputs, missing):
# Repeated node IDs across pods must retain their own version/architecture.
node32, running32 = node_inventory(1, 101, '16.0(2a)', 'aci-n9000-dk9.16.0.2a.bin')
node64, running64 = node_inventory(2, 101, '16.0(9c)', 'aci-n9000-dk9.16.0.9c-cs_64.bin')
if missing == 'version':
node64['fabricNode']['attributes']['version'] = None
elif missing in ('architecture', 'architecture_with_failure'):
running64['firmwareRunning']['attributes']['ksFile'] = None
icurl_outputs.update({
partitions: [bootflash(1, 101, 1 if missing == 'architecture_with_failure' else 4000),
bootflash(2, 101, 4000)],
download_sts_602: [], firmware: firmware_dual_602,
'firmwareRunning.json': [running32, running64],
})
result = run_check(sw_cversion=script.AciVersion('5.3(2e)'),
tversion=script.AciVersion('6.0(2h)'), fabric_nodes=[node32, node64])
if missing:
assert result.result == (script.FAIL_UF if missing == 'architecture_with_failure' else script.MANUAL)
assert result.data == ([['1', '101', '1.0', '3814.7']] if missing == 'architecture_with_failure' else [])
assert [row[:2] for row in result.unformatted_data] == [['2', '101']]
else:
assert result.result == script.FAIL_UF
assert result.data == [['2', '101', '4000.0', '5722.05']]


@pytest.mark.parametrize('icurl_outputs', [{}])
def test_architecture_specific_requirement_and_predownload_pod_join(run_check, mock_icurl, icurl_outputs):
inventory = [node_inventory(pod, 101, '16.0(9c)', image) for pod, image in [
(1, 'aci-n9000-dk9.16.0.9c.bin'), (2, 'aci-n9000-dk9.16.0.9c-cs_64.bin')]]
icurl_outputs.update({
partitions: [bootflash(1, 101, 1), bootflash(2, 101, 1)],
download_sts_602: maintUpgJob_node_101_downloaded,
firmware: firmware_dual_602,
'firmwareRunning.json': [running for _, running in inventory],
})
result = run_check(sw_cversion=script.AciVersion('6.0(9c)'),
tversion=script.AciVersion('6.0(2h)'), fabric_nodes=[node for node, _ in inventory])
assert result.result == script.FAIL_UF
assert result.data == [['1', '101', '1.0', '1907.35'], ['2', '101', '1.0', '5722.05']]
9 changes: 7 additions & 2 deletions tests/checks/test_result_details.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,19 @@ def icurl(_, query):
if query.startswith('eqptcapacityFSPartition.'):
return [{'eqptcapacityFSPartition': {'attributes': {
'dn': 'topology/pod-1/node-101/sys/eqptcapacity/fspartition-bootflash', 'avail': '1'}}}]
if query.startswith('maintUpgJob.'):
if query.startswith(('maintUpgJob.', 'firmwareRunning.')):
return []
assert query.startswith('firmwareFirmware.')
return [{'firmwareFirmware': {'attributes': {'isoname': name, 'size': '2000000000'}}}
for name in available]

monkeypatch.setattr(script, 'icurl', icurl)
name = 'switch_bootflash_usage_check'
result = run_check(name, sw_cversion=script.AciVersion(current), tversion=script.AciVersion(target))
result = run_check(
name, sw_cversion=script.AciVersion(current), tversion=script.AciVersion(target),
fabric_nodes=[{'fabricNode': {'attributes': {
'dn': 'topology/pod-1/node-101', 'role': 'leaf', 'version': current,
}}}],
)
assert result.result == script.MANUAL
assert result.data == missing
Loading