diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ccdb55c..698b0e3 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -2435,7 +2435,7 @@ def switch_group_guideline_check(fabric_nodes, **kwargs): @check_wrapper(check_title="Switch Node /bootflash usage") -def switch_bootflash_usage_check(sw_cversion, tversion, **kwargs): +def switch_bootflash_usage_check(sw_cversion, tversion, fabric_nodes, **kwargs): result = FAIL_UF msg = '' headers = ["Pod-ID", "Node-ID", "Avail (MB)", "Required (MB)"] @@ -2488,7 +2488,7 @@ def missing_images(image_rows): for maintUpgJob in download_sts: dn = re.search(node_regex, maintUpgJob['maintUpgJob']['attributes']['dn']) if dn: - predownloaded_nodes.append(dn.group("node")) + predownloaded_nodes.append((dn.group("pod"), dn.group("node"))) # Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit # isos (`-cs_64` suffix for 64-bit). Below that, only a single 32-bit iso exists. @@ -2506,11 +2506,23 @@ def missing_images(image_rows): target_size_32 = fw_sizes.get(switch_target_version) target_size_64 = fw_sizes.get(switch_target_version_64) - # sw_cversion (lowest switch version), not the APIC cversion, drives the image split - # decision: the upgrade guide has APICs reach 6.0(2a)+ before the switches, so the - # switches can still be pre-split while the APIC cluster is in the split-image era. target_is_pre_split = tversion.older_than(boundary_version) - current_is_pre_split = sw_cversion.older_than(boundary_version) + node_versions = {} + node_images = {} + if not target_is_pre_split: + for fabric_node in fabric_nodes: + attrs = fabric_node['fabricNode']['attributes'] + dn = re.search(node_regex, attrs.get('dn', '')) + if dn and attrs.get('role') != 'controller': + try: + node_versions[(dn.group('pod'), dn.group('node'))] = AciVersion(attrs.get('version') or '') + except ValueError: + pass + for running in icurl('class', 'firmwareRunning.json'): + attrs = running['firmwareRunning']['attributes'] + dn = re.search(node_regex, attrs.get('dn', '')) + if dn: + node_images[(dn.group('pod'), dn.group('node'))] = attrs.get('ksFile') or '' if target_is_pre_split: # Only the 32-bit image is ever used for a pre-6.0(2a) target. @@ -2520,8 +2532,8 @@ def missing_images(image_rows): downloaded_required_space = target_size_32 else: - # The larger image is used as a conservative estimate, so both sizes must be - # known; a missing one can't be assumed to be the smaller (or zero-byte) one. + # Keep the existing repository requirement for both target architectures; + # crossing nodes need both sizes for the boundary calculation. if target_size_32 is None and target_size_64 is None: return missing_images([ ['Target', '32-bit', switch_target_version], @@ -2532,25 +2544,7 @@ def missing_images(image_rows): elif target_size_64 is None: return missing_images([['Target', '64-bit', switch_target_version_64]]) - if current_is_pre_split: - # Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a - # 32-bit image, so its size is freed once removed during the upgrade. - switch_current_version = "aci-n9000-dk9.1{}.bin".format(sw_cversion.dot_version) - current_size = fw_sizes.get(switch_current_version) - if current_size is None: - return missing_images([['Current', '32-bit', switch_current_version]]) - if target_size_32 > current_size: - required_space = 2 * (target_size_32 + target_size_64 - current_size) - else: - required_space = 2 * max(target_size_32, target_size_64) - # A downloaded 32-bit target is already reflected in the current `avail`. - downloaded_required_space = required_space - target_size_32 - else: - required_space = 2 * max(target_size_32, target_size_64) - downloaded_required_space = max(target_size_32, target_size_64) - - required_space_kb = required_space / 1024.0 # eqptcapacityFSPartition avail/used are in KB - downloaded_required_space_kb = downloaded_required_space / 1024.0 + manual_nodes = [] for eqptcapacityFSPartition in partitions: dn = re.search(node_regex, eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['dn']) @@ -2558,18 +2552,59 @@ def missing_images(image_rows): node = dn.group("node") avail = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['avail']) + if not target_is_pre_split: + node_version = node_versions.get((pod, node)) + if node_version is None: + manual_nodes.append([pod, node, 'Current version unavailable']) + continue + if node_version.older_than(boundary_version): + # Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a + # 32-bit image, so its size is freed once removed during the upgrade. + switch_current_version = "aci-n9000-dk9.1{}.bin".format(node_version.dot_version) + current_size = fw_sizes.get(switch_current_version) + if current_size is None: + return missing_images([['Current', '32-bit', switch_current_version]]) + if target_size_32 > current_size: + required_space = 2 * (target_size_32 + target_size_64 - current_size) + else: + required_space = 2 * max(target_size_32, target_size_64) + # A downloaded 32-bit target is already reflected in the current `avail`. + downloaded_required_space = required_space - target_size_32 + else: + image = node_images.get((pod, node), '') + if not image.endswith('.bin'): + manual_nodes.append([pod, node, 'Current architecture unavailable']) + continue + target_size = target_size_64 if image.endswith('-cs_64.bin') else target_size_32 + required_space = 2 * target_size + downloaded_required_space = target_size + + required_space_kb = required_space / 1024.0 # partition availability is in KB + downloaded_required_space_kb = downloaded_required_space / 1024.0 + # dnldStatus == downloaded only proves the image was delivered, not that # extraction (which still consumes bootflash) has completed, so a downloaded # node is still checked, just against the smaller extraction-only requirement. - node_required_space_kb = downloaded_required_space_kb if node in predownloaded_nodes else required_space_kb + node_required_space_kb = downloaded_required_space_kb if (pod, node) in predownloaded_nodes else required_space_kb if avail < node_required_space_kb: data.append([pod, node, round(avail / 1024.0, 2), round(node_required_space_kb / 1024.0, 2)]) - if not data: + if manual_nodes: + recommended_action += ( + " Verify the current version and running image for the listed unevaluated nodes, " + "then rerun this check." + ) + if not data: + result = MANUAL + elif not data: result = PASS msg = 'Sufficient bootflash space.' - return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + return Result( + result=result, msg=msg, headers=headers, data=data, + unformatted_headers=["Pod-ID", "Node-ID", "Not evaluated"] if manual_nodes else [], + unformatted_data=manual_nodes, recommended_action=recommended_action, doc_url=doc_url, + ) @check_wrapper(check_title="L3Out MTU") diff --git a/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py b/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py index 386c918..80b48fc 100644 --- a/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py +++ b/tests/checks/switch_bootflash_usage_check/test_switch_bootflash_usage_check.py @@ -11,6 +11,28 @@ test_function = "switch_bootflash_usage_check" +@pytest.fixture +def run_check(run_check, icurl_outputs): + """Provide per-node inventory for the existing homogeneous-version cases.""" + def run(**kwargs): + if 'fabric_nodes' not in kwargs: + nodes = [] + running = [] + for partition in icurl_outputs.get(partitions, []): + dn = partition['eqptcapacityFSPartition']['attributes']['dn'].split('/sys/')[0] + nodes.append({'fabricNode': {'attributes': { + 'dn': dn, 'role': 'leaf', 'version': str(kwargs.get('sw_cversion') or ''), + }}}) + running.append({'firmwareRunning': {'attributes': { + 'dn': dn + '/sys/fwstatuscont/running', + 'ksFile': 'aci-n9000-dk9.16.0.3a-cs_64.bin', + }}}) + kwargs['fabric_nodes'] = nodes + icurl_outputs.setdefault('firmwareRunning.json', running) + return run_check(**kwargs) + return run + + # icurl queries partitions = "eqptcapacityFSPartition.json" partitions += '?query-target-filter=eq(eqptcapacityFSPartition.path,"/bootflash")' @@ -432,3 +454,86 @@ def test_logic(run_check, mock_icurl, sw_cversion, tversion, expected_result): tversion=script.AciVersion(tversion) if tversion else None, ) assert result.result == expected_result + + +def node_inventory(pod, node, version, image): + dn = 'topology/pod-{}/node-{}'.format(pod, node) + return ( + {'fabricNode': {'attributes': {'dn': dn, 'role': 'leaf', 'version': version}}}, + {'firmwareRunning': {'attributes': {'dn': dn + '/sys/fwstatuscont/running', 'ksFile': image}}}, + ) + + +def bootflash(pod, node, available_mb): + return {'eqptcapacityFSPartition': {'attributes': { + 'dn': 'topology/pod-{}/node-{}/sys/eqptcapacity/fspartition-bootflash'.format(pod, node), + 'avail': str(int(available_mb * 1024)), 'used': '0', + }}} + + +@pytest.mark.parametrize('icurl_outputs', [{}]) +@pytest.mark.parametrize('insufficient', [False, True]) +def test_issue_493_mixed_versions_current_image_absent(run_check, mock_icurl, icurl_outputs, insufficient): + inventory = [node_inventory(1, node, '16.0(9c)', 'aci-n9000-dk9.16.0.9c-cs_64.bin') + for node in range(1109, 1115)] + icurl_outputs.update({ + partitions: [bootflash(1, node, 1 if insufficient else available) for node, available in zip( + range(1109, 1115), [6490.13, 6491.29, 6489.39, 6501.6, 6489.37, 6489.61])], + 'maintUpgJob.json?query-target-filter=and(eq(maintUpgJob.dnldStatus,"downloaded"),eq(maintUpgJob.desiredVersion,"n9000-16.1(6g)"))': [], + firmware: [ + {'firmwareFirmware': {'attributes': {'isoname': name, 'size': str(int(size * 1024 ** 2))}}} + for name, size in [('aci-n9000-dk9.15.3.2e.bin', 1995.57), + ('aci-n9000-dk9.16.1.6g.bin', 2560.78), + ('aci-n9000-dk9.16.1.6g-cs_64.bin', 2694.73)] + ], + 'firmwareRunning.json': [running for _, running in inventory], + }) + result = run_check(sw_cversion=script.AciVersion('5.3(2e)'), + tversion=script.AciVersion('6.1(6g)'), + fabric_nodes=[node for node, _ in inventory]) + assert result.result == (script.FAIL_UF if insufficient else script.PASS) + assert result.data == ([["1", str(node), "1.0", "5389.46"] for node in range(1109, 1115)] + if insufficient else []) + + +@pytest.mark.parametrize('missing', [None, 'version', 'architecture', 'architecture_with_failure']) +@pytest.mark.parametrize('icurl_outputs', [{}]) +def test_per_node_architecture_and_missing_inventory(run_check, mock_icurl, icurl_outputs, missing): + # Repeated node IDs across pods must retain their own version/architecture. + node32, running32 = node_inventory(1, 101, '16.0(2a)', 'aci-n9000-dk9.16.0.2a.bin') + node64, running64 = node_inventory(2, 101, '16.0(9c)', 'aci-n9000-dk9.16.0.9c-cs_64.bin') + if missing == 'version': + node64['fabricNode']['attributes']['version'] = None + elif missing in ('architecture', 'architecture_with_failure'): + running64['firmwareRunning']['attributes']['ksFile'] = None + icurl_outputs.update({ + partitions: [bootflash(1, 101, 1 if missing == 'architecture_with_failure' else 4000), + bootflash(2, 101, 4000)], + download_sts_602: [], firmware: firmware_dual_602, + 'firmwareRunning.json': [running32, running64], + }) + result = run_check(sw_cversion=script.AciVersion('5.3(2e)'), + tversion=script.AciVersion('6.0(2h)'), fabric_nodes=[node32, node64]) + if missing: + assert result.result == (script.FAIL_UF if missing == 'architecture_with_failure' else script.MANUAL) + assert result.data == ([['1', '101', '1.0', '3814.7']] if missing == 'architecture_with_failure' else []) + assert [row[:2] for row in result.unformatted_data] == [['2', '101']] + else: + assert result.result == script.FAIL_UF + assert result.data == [['2', '101', '4000.0', '5722.05']] + + +@pytest.mark.parametrize('icurl_outputs', [{}]) +def test_architecture_specific_requirement_and_predownload_pod_join(run_check, mock_icurl, icurl_outputs): + inventory = [node_inventory(pod, 101, '16.0(9c)', image) for pod, image in [ + (1, 'aci-n9000-dk9.16.0.9c.bin'), (2, 'aci-n9000-dk9.16.0.9c-cs_64.bin')]] + icurl_outputs.update({ + partitions: [bootflash(1, 101, 1), bootflash(2, 101, 1)], + download_sts_602: maintUpgJob_node_101_downloaded, + firmware: firmware_dual_602, + 'firmwareRunning.json': [running for _, running in inventory], + }) + result = run_check(sw_cversion=script.AciVersion('6.0(9c)'), + tversion=script.AciVersion('6.0(2h)'), fabric_nodes=[node for node, _ in inventory]) + assert result.result == script.FAIL_UF + assert result.data == [['1', '101', '1.0', '1907.35'], ['2', '101', '1.0', '5722.05']] diff --git a/tests/checks/test_result_details.py b/tests/checks/test_result_details.py index 6027801..0d22803 100644 --- a/tests/checks/test_result_details.py +++ b/tests/checks/test_result_details.py @@ -121,7 +121,7 @@ def icurl(_, query): if query.startswith('eqptcapacityFSPartition.'): return [{'eqptcapacityFSPartition': {'attributes': { 'dn': 'topology/pod-1/node-101/sys/eqptcapacity/fspartition-bootflash', 'avail': '1'}}}] - if query.startswith('maintUpgJob.'): + if query.startswith(('maintUpgJob.', 'firmwareRunning.')): return [] assert query.startswith('firmwareFirmware.') return [{'firmwareFirmware': {'attributes': {'isoname': name, 'size': '2000000000'}}} @@ -129,6 +129,11 @@ def icurl(_, query): monkeypatch.setattr(script, 'icurl', icurl) name = 'switch_bootflash_usage_check' - result = run_check(name, sw_cversion=script.AciVersion(current), tversion=script.AciVersion(target)) + result = run_check( + name, sw_cversion=script.AciVersion(current), tversion=script.AciVersion(target), + fabric_nodes=[{'fabricNode': {'attributes': { + 'dn': 'topology/pod-1/node-101', 'role': 'leaf', 'version': current, + }}}], + ) assert result.result == script.MANUAL assert result.data == missing