diff --git a/test/integration/plugins/ontap/README.md b/test/integration/plugins/ontap/README.md index 6e0d0e7d6be5..bc7550de889e 100644 --- a/test/integration/plugins/ontap/README.md +++ b/test/integration/plugins/ontap/README.md @@ -32,7 +32,7 @@ CI wiring: test/integration/plugins/ontap/ ├── ontap.cfg # Environment config (IPs, credentials, zone info) ├── ontap_test_base.py # Shared base class and ONTAP REST client -├── TEST_CASES.md # Full test case reference table (62 tests) +├── TEST_CASES.md # Full test case reference table (68 tests) ├── README.md # This file │ ├── nfs3/ @@ -296,7 +296,9 @@ self.assertEqual(result.state, "Maintenance") | `get_igroup(svm_name, name)` | iSCSI igroup existence and initiator list | iSCSI suites | | `list_luns_in_volume(svm_name, vol_name)` | LUNs present in a FlexVol | iSCSI volume/instance suites | | `list_lun_maps_for_volume(svm_name, vol_name)` | Active LUN-maps for a volume | iSCSI instance suite | -| `list_files_in_volume(svm_name, vol_name)` | Files inside a FlexVol | NFS3 instance suite | +| `list_files_in_volume(vol_name)` | Files inside a FlexVol | NFS3 instance suite | +| `write_file_in_volume(vol_name, filename, size_bytes)` | Write incompressible data into a FlexVol file (no VM) | NFS3/iSCSI pool lifecycle suites | +| `delete_file_in_volume(vol_name, filename)` | Remove a FlexVol file | NFS3/iSCSI pool lifecycle suites | --- @@ -304,12 +306,12 @@ self.assertEqual(result.state, "Maintenance") | Suite | File | Tests | What it covers | |-------|------|-------|---------------| -| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete | +| NFS3 Pool Lifecycle | `nfs3/pool/test_pool_lifecycle.py` | 11 | Create, grow, safe shrink, disable, enable, maintenance, delete, then re-create with a volume, reject shrink below used capacity, cleanup | | NFS3 Pool with Volumes | `nfs3/pool/test_pool_with_volumes.py` | 7 | Same + live volume present; negative delete guard | | NFS3 Zone-Scoped Pool | `nfs3/pool/test_zone_scoped_pool.py` | 4 | Zone scope — all hosts connected via `attachZone` | | NFS3 Volume Lifecycle | `nfs3/volume/test_volume_lifecycle.py` | 5 | Volume is metadata-only; FlexVol unchanged on delete | | NFS3 VM + Volume Attach | `nfs3/instance/test_vm_volume_attach.py` | 8 | Full VM lifecycle with hot-plug/detach | -| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 8 | Create, disable, enable, maintenance, delete + igroups | +| iSCSI Pool Lifecycle | `iscsi/pool/test_pool_lifecycle.py` | 11 | Create, grow, safe shrink, disable, enable, maintenance, delete + igroups, then re-create with a volume (LUN), reject shrink below used capacity, cleanup | | iSCSI Pool with Volumes | `iscsi/pool/test_pool_with_volumes.py` | 7 | Same + live LUN present; negative delete guard | | iSCSI Zone-Scoped Pool | `iscsi/pool/test_zone_scoped_pool.py` | 4 | Zone scope | | iSCSI Volume Lifecycle | `iscsi/volume/test_volume_lifecycle.py` | 5 | LUN created per CS volume; LUN removed on delete | diff --git a/test/integration/plugins/ontap/TEST_CASES.md b/test/integration/plugins/ontap/TEST_CASES.md index 73dc1990a5b6..4b3d3cc4a168 100644 --- a/test/integration/plugins/ontap/TEST_CASES.md +++ b/test/integration/plugins/ontap/TEST_CASES.md @@ -19,7 +19,7 @@ # ONTAP Integration Test Cases -Complete reference for all 62 test cases across 10 test suites. +Complete reference for all 68 test cases across 10 test suites. Each suite is sequential — tests must run in numbered order; each step builds on state created by the previous step. --- @@ -42,18 +42,21 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `nfs3/pool/test_pool_lifecycle.py` **Class:** `TestOntapNFS3PrimaryStorageWorkflow` **Tag:** `nfs3_workflow` -**Total:** 8 tests | **Scope:** cluster-scoped NFS3 pool, no volumes for tests 01–06 +**Total:** 11 tests | **Scope:** cluster-scoped NFS3 pool, no volumes for tests 01–08 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| | 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped NFS3 primary storage pool | setUpClass (zone, cluster, account) | `pool.state == "Up"`, `pool.type == "NetworkFilesystem"`, `nfsmountopts` contains `vers=3` | FlexVol exists and `state == "online"`, export policy exists with each cluster host IP as a rule, at least one NFS data LIF present on SVM | positive | -| 02 | `test_02_disable_storage_pool` | Disable the pool (admin operation) | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | -| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | -| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_04 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | -| 06 | `test_06_delete_pool_from_maintenance` | Enter maintenance then permanently delete the pool | test_05 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol deleted (not found by `GET /api/storage/volumes?name=`); export policy deleted | positive | -| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume on it | test_06 (pool deleted; creates new pool) | New `pool.state == "Up"`; `createVolume` returns non-None volume object | FlexVol `online` after volume allocation; export policy present | positive | -| 08 | `test_08_delete_volume_and_pool` | Delete the volume then force-delete the pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | positive | +| 02 | `test_02_grow_storage_pool` | Increase the original empty pool capacity | test_01 | `capacitybytes` reaches the requested size; pool stays `Up` | FlexVol `space.size` reaches the requested size; remains `online`; export policy unchanged | positive | +| 03 | `test_03_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_02 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol `space.size` reaches the safe target; remains `online`; export policy unchanged | positive | +| 04 | `test_04_disable_storage_pool` | Disable the pool (admin operation) | test_03 | `pool.state == "Disabled"` | FlexVol still `online`; export policy still present | positive | +| 05 | `test_05_enable_storage_pool` | Re-enable the pool | test_04 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 06 | `test_06_enter_maintenance_mode` | Put pool into maintenance (drains new volume allocations) | test_05 | `pool.state == "Maintenance"` | FlexVol still `online`; export policy still present (maintenance is CS-only state) | positive | +| 07 | `test_07_cancel_maintenance_mode` | Cancel maintenance, return pool to service | test_06 | `pool.state == "Up"` | FlexVol still `online`; export policy still present | positive | +| 08 | `test_08_delete_pool_from_maintenance` | Enter maintenance then permanently delete the original pool | test_07 | Pool no longer returned by `listStoragePools` (CS 431 error expected on ID lookup) | FlexVol and export policy deleted | positive | +| 09 | `test_09_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack data volume | test_08 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; export policy present | positive | +| 10 | `test_10_reject_shrink_below_used_capacity` | Reject shrink below ONTAP used space, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API if used space is still below that minimum (no VM) and removes it before returning | test_09 (`pool`, `volume`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and export policy unchanged; used space stays above the FlexVol minimum | negative | +| 11 | `test_11_delete_volume_and_pool` | Delete the volume then force-delete the pool | test_10 | Volume and pool no longer listed | FlexVol and export policy deleted | cleanup | --- @@ -124,7 +127,7 @@ Each suite is sequential — tests must run in numbered order; each step builds | 04 | `test_04_attach_volume_to_vm` | Attach the ONTAP data volume to the running VM (hot-plug) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id`; `attachVolume` job succeeds | FlexVol `online`; after attach, a data file matching volume UUID present in FlexVol (`list_files_in_volume`) | positive | | 05 | `test_05_stop_vm_export_retained` | Stop the running VM with volume attached | test_04 | `vm.state == "Stopped"` | FlexVol still `online`; NFS export policy still present | positive | | 06 | `test_06_start_vm_volume_accessible` | Start the stopped VM | test_05 | `vm.state == "Running"` | FlexVol still `online` | positive | -| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM (TDS Detach NFS3) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the ONTAP volume from the running VM | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared; `volume.state == "Ready"` | FlexVol still `online`; data file **still present** (NFS3: file persists until `deleteVolume`, not on detach) | positive | | 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM no longer listed; volume no longer listed; pool no longer listed | FlexVol deleted; export policy deleted | cleanup | --- @@ -134,18 +137,21 @@ Each suite is sequential — tests must run in numbered order; each step builds **File:** `iscsi/pool/test_pool_lifecycle.py` **Class:** `TestOntapISCSIPoolLifecycle` **Tag:** `iscsi_workflow` -**Total:** 8 tests | **Scope:** cluster-scoped iSCSI pool, no volumes for tests 01–06 +**Total:** 11 tests | **Scope:** cluster-scoped iSCSI pool, no volumes for tests 01–08 | # | Test method | Goal | Depends on | CloudStack success criteria | ONTAP success criteria | Type | |---|-------------|------|------------|-----------------------------|------------------------|------| -| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | -| 02 | `test_02_disable_storage_pool` | Disable the pool | test_01 (`pool`) | `pool.state == "Disabled"` | FlexVol still `online` | positive | -| 03 | `test_03_enable_storage_pool` | Re-enable the pool | test_02 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 04 | `test_04_enter_maintenance_mode` | Put pool into maintenance | test_03 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | -| 05 | `test_05_cancel_maintenance_mode` | Cancel maintenance | test_04 | `pool.state == "Up"` | FlexVol still `online` | positive | -| 06 | `test_06_enter_maintenance_and_delete_pool` | Enter maintenance then force-delete the pool | test_05 | Pool no longer listed | FlexVol deleted; all igroups for cluster hosts deleted | positive | -| 07 | `test_07_create_volume_on_pool` | Create a second fresh pool and allocate a CloudStack data volume (creates a LUN) | test_06 (new pool) | New `pool.state == "Up"`; volume object non-None | FlexVol `online`; ≥1 LUN present inside FlexVol (`list_luns_in_volume`) | positive | -| 08 | `test_08_delete_volume_and_pool` | Delete the volume (removes LUN), enter maintenance, force-delete pool | test_07 (`pool`, `volume`) | Volume no longer listed; pool no longer listed | LUN no longer in FlexVol; FlexVol deleted; igroups deleted | positive | +| 01 | `test_01_create_primary_storage_pool` | Create a cluster-scoped iSCSI primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "Iscsi"` | FlexVol `online`; one igroup per cluster host (named `cs_{svmName}_{hostShortName}`) with host IQN as initiator | positive | +| 02 | `test_02_grow_storage_pool` | Increase the original empty pool capacity | test_01 | `capacitybytes` reaches the requested size; pool stays `Up` | FlexVol reaches the requested size and remains `online`; igroups remain present | positive | +| 03 | `test_03_shrink_storage_pool` | Safely shrink the original empty pool to its initial capacity | test_02 | `capacitybytes` reaches the safe target; pool stays `Up` | FlexVol reaches the safe target and remains `online`; igroups remain present | positive | +| 04 | `test_04_disable_storage_pool` | Disable the pool | test_03 | `pool.state == "Disabled"` | FlexVol still `online` | positive | +| 05 | `test_05_enable_storage_pool` | Re-enable the pool | test_04 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 06 | `test_06_enter_maintenance_mode` | Put pool into maintenance | test_05 | `pool.state == "Maintenance"` | FlexVol still `online`; igroups unchanged | positive | +| 07 | `test_07_cancel_maintenance_mode` | Cancel maintenance | test_06 | `pool.state == "Up"` | FlexVol still `online` | positive | +| 08 | `test_08_enter_maintenance_and_delete_pool` | Enter maintenance then delete the original pool | test_07 | Pool no longer listed | FlexVol and all host igroups deleted | positive | +| 09 | `test_09_create_volume_on_pool` | Create a fresh pool and allocate a CloudStack volume | test_08 | New pool is `Up`; `createVolume` returns a volume | FlexVol `online`; at least one LUN present | positive | +| 10 | `test_10_reject_shrink_below_used_capacity` | Reject shrink below ONTAP used space, with the target held above the ONTAP FlexVol minimum. Writes incompressible data through the ONTAP files API only if the LUN has not already pushed used space above that minimum (no VM) and removes it before returning | test_09 (`pool`, `volume`) | `CloudstackAPIException`; volume remains listed with unchanged `id`/`state`/`size`/`poolid`; pool capacity unchanged | FlexVol size and LUN uuid/name list unchanged; used space stays above the FlexVol minimum | negative | +| 11 | `test_11_delete_volume_and_pool` | Delete the volume and force-delete the pool | test_10 | Volume and pool no longer listed | LUN, FlexVol, and igroups deleted | cleanup | --- @@ -162,7 +168,7 @@ Each suite is sequential — tests must run in numbered order; each step builds | 02 | `test_02_disable_pool_volume_survives` | Disable pool with volume present | test_01 (`pool`, `volume`) | `pool.state == "Disabled"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 03 | `test_03_enable_pool_volume_intact` | Re-enable pool with volume | test_02 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 04 | `test_04_enter_maintenance_volume_present` | Enter maintenance with volume | test_03 | `pool.state == "Maintenance"`; volume still listed | FlexVol still `online`; LUN still present | positive | -| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume (TDS iSCSI cancel maintenance) | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | +| 05 | `test_05_cancel_maintenance_volume_present` | Cancel maintenance with volume | test_04 | `pool.state == "Up"`; volume still listed | FlexVol still `online`; LUN still present | positive | | 06 | `test_06_forced_false_delete_rejected` | Attempt `deleteStoragePool(forced=False)` with LUN-backed volume present — must be rejected | test_05 | `CloudstackAPIException` raised; pool still in `Maintenance` | No ONTAP objects removed | negative | | 07 | `test_07_delete_volume_and_force_delete_pool` | Delete volume (LUN removed) then force-delete pool | test_06 (`pool`, `volume`) | Volume gone; pool gone | LUN removed; FlexVol deleted; igroups deleted | cleanup | @@ -213,13 +219,13 @@ Each suite is sequential — tests must run in numbered order; each step builds | 01 | `test_01_create_iscsi_pool` | Create iSCSI ONTAP primary storage pool | setUpClass | `pool.state == "Up"`, `pool.type == "OntapiSCSI"` | FlexVol `online`; igroup per cluster host with host IQN | positive | | 02 | `test_02_create_ontap_data_volume` | Allocate a CloudStack data volume (creates a LUN in the FlexVol) | test_01 (`pool`) | Volume non-None | ≥1 LUN in FlexVol | positive | | 03 | `test_03_deploy_vm` | Deploy VM using first ready KVM template; verify 0 LUN-maps exist before attach | test_02 (`volume`) | `vm.state == "Running"`; 0 LUN-maps on ONTAP | 0 LUN-maps (`list_lun_maps_for_volume` returns empty) | positive | -| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created (TDS SN 27) | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | -| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed (TDS VM Stop iSCSI) | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | -| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created (TDS VM Start iSCSI) | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | -| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM (TDS Detach iSCSI) | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | +| 04 | `test_04_attach_volume_to_vm` | Hot-attach the ONTAP iSCSI volume to the running VM — a LUN-map is created | test_03 (`vm`, `volume`) | `volume.virtualmachineid == vm.id` | ≥1 LUN-map linking the LUN to the host's igroup | positive | +| 05 | `test_05_stop_vm_lun_unmapped` | Stop VM — LUN-maps must be removed | test_04 | `vm.state == "Stopped"` | 0 LUN-maps; LUN itself **still present** in FlexVol | positive | +| 06 | `test_06_start_vm_lun_remapped` | Start VM — LUN-maps must be re-created | test_05 | `vm.state == "Running"` | ≥1 LUN-map re-created | positive | +| 07 | `test_07_detach_volume_from_vm` | Hot-detach the iSCSI volume from the running VM | test_06 (`vm`, `volume`) | `volume.virtualmachineid` cleared | 0 LUN-maps; LUN still in FlexVol | positive ⚠️ | | 08 | `test_08_destroy_vm_and_cleanup` | Destroy VM (expunge), delete volume, enter maintenance, delete pool | test_07 | VM gone; volume gone; pool gone | FlexVol deleted; all LUNs and igroups deleted | cleanup | -> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. All other 61 tests pass. +> ⚠️ **test_07 known status:** iSCSI hot-detach from a running VM relies on the KVM guest acknowledging the SCSI device removal. On this environment the guest does not acknowledge in time, causing CloudStack error 530. This is a KVM-host-level or guest-template limitation, not a test code defect. --- @@ -227,14 +233,14 @@ Each suite is sequential — tests must run in numbered order; each step builds | Suite | Protocol | Scope | Tests | Status | |-------|---------|-------|-------|--------| -| NFS3 Pool Lifecycle | NFS3 | Cluster | 8 | ✅ | +| NFS3 Pool Lifecycle | NFS3 | Cluster | 11 | ⚠️ restructured resize flow not run | | NFS3 Pool with Volumes | NFS3 | Cluster | 7 | ✅ | | NFS3 Zone-Scoped Pool | NFS3 | Zone | 4 | ✅ | | NFS3 Volume Lifecycle | NFS3 | Cluster | 5 | ✅ | | NFS3 VM + Volume Attach | NFS3 | Cluster | 8 | ✅ | -| iSCSI Pool Lifecycle | iSCSI | Cluster | 8 | ✅ | +| iSCSI Pool Lifecycle | iSCSI | Cluster | 11 | ⚠️ restructured resize flow not run | | iSCSI Pool with Volumes | iSCSI | Cluster | 7 | ✅ | | iSCSI Zone-Scoped Pool | iSCSI | Zone | 4 | ✅ | | iSCSI Volume Lifecycle | iSCSI | Cluster | 5 | ✅ | | iSCSI VM + Volume Attach | iSCSI | Cluster | 8 | ⚠️ 7/8 | -| **Total** | | | **62** | **61 passing** | +| **Total** | | | **68** | **Restructured resize flows not run; 1 known environment failure** | diff --git a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py index cc87bacf0e76..83c239abb6e2 100644 --- a/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/iscsi/pool/test_pool_lifecycle.py @@ -19,18 +19,21 @@ Sequential workflow integration tests for NetApp ONTAP iSCSI primary storage pool lifecycle (no volumes). -Tests are numbered test_01 ... test_08 and must run in that order. Each step +Tests are numbered test_01 ... test_11 and must run in that order. Each step builds on the shared state established by the previous step. Workflow: 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Enter maintenance mode and delete the storage pool - 07 Create a new pool and allocate a CloudStack data volume (LUN created) - 08 Delete the volume (LUN removed), enter maintenance, force-delete pool + 02 Increase storage pool capacity + 03 Safely shrink storage pool capacity + 04 Disable storage pool + 05 Enable storage pool + 06 Enter maintenance mode + 07 Cancel maintenance mode + 08 Enter maintenance mode and delete the storage pool + 09 Create a new pool and allocate a CloudStack data volume (LUN created) + 10 Reject shrink below ONTAP used capacity + 11 Delete the volume (LUN removed), enter maintenance, force-delete pool Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -146,6 +149,7 @@ class TestOntapISCSIPoolLifecycle(OntapTestBase): # ---- iSCSI-specific state (set/cleared by individual tests) -------- _vol_name_prefix = "OntapISCSIVol" + resize_original_size = None @classmethod def setUpClass(cls): @@ -327,11 +331,104 @@ def test_01_create_primary_storage_pool(self): self._assert_pool_capacity(pool, "pool-created") # ------------------------------------------------------------------ - # Step 02 - Disable storage pool + # Step 02 - Increase storage pool capacity # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): + def test_02_grow_storage_pool(self): + """Increase the original pool and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after increase" % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 03 - Safely shrink storage pool capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_03_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_02 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent - test_02 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + + for host in self.cluster_hosts: + iqn = ( + getattr(host, "storageurl", None) + or getattr(host, "StorageUrl", None) + ) + if iqn and iqn.startswith("iqn."): + igroup_name = _igroup_name(self.svm_name, host.name) + self.assertIsNotNone( + self.ontap.get_igroup(self.svm_name, igroup_name), + "ONTAP igroup '%s' disappeared after safe shrink" + % igroup_name, + ) + + # ------------------------------------------------------------------ + # Step 04 - Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_04_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled @@ -357,11 +454,11 @@ def test_02_disable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 03 - Enable storage pool + # Step 05 - Enable storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): + def test_05_enable_storage_pool(self): """ Re-enable the pool and verify: - CloudStack reports Up @@ -387,11 +484,11 @@ def test_03_enable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 04 - Enter maintenance mode + # Step 06 - Enter maintenance mode # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): + def test_06_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance @@ -416,11 +513,11 @@ def test_04_enter_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 05 - Cancel maintenance mode + # Step 07 - Cancel maintenance mode # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): + def test_07_cancel_maintenance_mode(self): """ Cancel maintenance and verify: - CloudStack reports Up @@ -444,11 +541,11 @@ def test_05_cancel_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 06 - Enter maintenance mode and delete the storage pool + # Step 08 - Enter maintenance mode and delete the storage pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_06_enter_maintenance_and_delete_pool(self): + def test_08_enter_maintenance_and_delete_pool(self): """ Enter maintenance mode then delete the pool. Verifies the pool is removed from CloudStack and the backing ONTAP @@ -465,6 +562,7 @@ def test_06_enter_maintenance_and_delete_pool(self): self._delete_pool(pool.id) self.__class__.pool = None + self.__class__.resize_original_size = None # CloudStack: pool must be gone try: @@ -493,11 +591,11 @@ def test_06_enter_maintenance_and_delete_pool(self): ) # ------------------------------------------------------------------ - # Step 07 - Create fresh pool and allocate a CloudStack volume (LUN) + # Step 09 - Create fresh pool and allocate a CloudStack volume (LUN) # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_07_create_volume_on_pool(self): + def test_09_create_volume_on_pool(self): """ Create a new iSCSI pool and allocate a CloudStack data volume. For iSCSI, createAsync creates a LUN inside the pool's ONTAP FlexVol. @@ -511,7 +609,7 @@ def test_07_create_volume_on_pool(self): self.__class__.pool = pool log_progress( logger, "info", - "test_07: created storage pool name='%s' id=%s state=%s type=%s", + "test_09: created storage pool name='%s' id=%s state=%s type=%s", pool.name, pool.id, pool.state, pool.type, ) @@ -529,7 +627,7 @@ def test_07_create_volume_on_pool(self): self.assertIsNotNone(vol, "createVolume returned None") log_progress( logger, "info", - "test_07: created CloudStack volume name='%s' id=%s state=%s " + "test_09: created CloudStack volume name='%s' id=%s state=%s " "on pool='%s' (id=%s) account='%s' domain='%s' — " "switch to this account in the UI to see the volume", getattr(vol, "name", "?"), getattr(vol, "id", "?"), @@ -559,13 +657,98 @@ def test_07_create_volume_on_pool(self): self._assert_pool_capacity(pool, "volume-allocated") # ------------------------------------------------------------------ - # Step 08 - Delete volume (LUN) then force-delete the pool + # Step 10 - Reject shrink below ONTAP used capacity + # ------------------------------------------------------------------ + + @attr(tags=["iscsi_workflow"], required_hardware=True) + def test_10_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the new pool below ONTAP used space and verify the + request is rejected while the volume and ONTAP LUNs remain unchanged. + + The LUN allocated by test_09 usually already pushes used space above + the ONTAP FlexVol minimum. If it does not, incompressible data is + written through the ONTAP files API (no VM) so the shrink target + stays above that minimum, and it is removed before this test returns. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent - test_09 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent - test_09 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + luns = self.ontap.list_luns_in_volume(self.svm_name, pool.name) + self.assertTrue( + len(luns) > 0, + "No LUNs in FlexVol '%s' before rejected shrink" % pool.name, + ) + luns_before_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in luns + ) + + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking iSCSI pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + expected_error=( + "insufficient", "used", "too small", "cannot reduce", + "Cannot reduce", "smaller", "minimum safe", + "too small to hold", "current volume data", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + luns_after_used = sorted( + (lun.get("uuid") or "", lun.get("name") or "") + for lun in self.ontap.list_luns_in_volume( + self.svm_name, pool.name + ) + ) + self.assertEqual( + luns_after_used, luns_before_used, + "LUN list in FlexVol '%s' changed after rejected " + "used-capacity shrink" % pool.name, + ) + self.assertTrue( + len(luns_after_used) > 0, + "LUN disappeared from FlexVol '%s' after rejected shrink" + % pool.name, + ) + finally: + self._delete_filler_file(pool.name, filler_name) + + # ------------------------------------------------------------------ + # Step 11 - Delete volume (LUN) then force-delete the pool # ------------------------------------------------------------------ @attr(tags=["iscsi_workflow"], required_hardware=True) - def test_08_delete_volume_and_pool(self): + def test_11_delete_volume_and_pool(self): """ - Delete the volume from test_07, enter maintenance, then force-delete + Delete the volume from test_09, enter maintenance, then force-delete the pool. Verifies: - deleteVolume removes the LUN from ONTAP @@ -574,8 +757,8 @@ def test_08_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: igroups for all cluster hosts deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_09 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_09 must pass first") pool = self.__class__.pool pool_name = pool.name diff --git a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py index 5d1812cdad4f..62b2746832bd 100644 --- a/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py +++ b/test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py @@ -18,18 +18,21 @@ """ Sequential workflow integration tests for NetApp ONTAP NFS3 primary storage pool. -Tests are numbered test_01 ... test_08 and must run in that order. Each step +Tests are numbered test_01 ... test_11 and must run in that order. Each step builds on the shared state established by the previous step. Workflow: 01 Create primary storage pool - 02 Disable storage pool - 03 Enable storage pool - 04 Enter maintenance mode - 05 Cancel maintenance mode - 06 Delete the storage pool - 07 Create fresh pool and allocate a CloudStack volume - 08 Delete volume then force-delete the pool + 02 Increase storage pool capacity + 03 Safely shrink storage pool capacity + 04 Disable storage pool + 05 Enable storage pool + 06 Enter maintenance mode + 07 Cancel maintenance mode + 08 Delete the storage pool + 09 Create fresh pool and allocate a CloudStack volume + 10 Reject shrink below ONTAP used capacity + 11 Delete volume then force-delete the pool Prerequisites: - CloudStack management server with the NetApp ONTAP plugin deployed @@ -42,7 +45,8 @@ --marvin-config=test/integration/plugins/ontap/ontap.cfg \\ test/integration/plugins/ontap/nfs3/pool/test_pool_lifecycle.py -v -Note: Tests 01-06 share class-level state (sequential). Running a single test +Note: Tests 01-08 share class-level state, as do tests 09-11 (sequential). +Running a single test with -m "test_NN" will invoke setUpClass but the guard assertion will fail immediately if earlier steps have not yet run. Always run the full suite. """ @@ -145,6 +149,7 @@ class TestOntapNFS3PrimaryStorageWorkflow(OntapTestBase): pool_ep_name = None # NFS export policy name for pool pool2_ep_name = None # export policy for pool stashed from test_01-04 cluster_host_ips = None + resize_original_size = None _vol_name_prefix = "OntapNFS3Vol" @@ -458,11 +463,89 @@ def test_01_create_primary_storage_pool(self): self._assert_pool_capacity(pool, "pool-created") # ------------------------------------------------------------------ - # Step 02 — Disable storage pool + # Step 02 - Increase storage pool capacity # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_02_disable_storage_pool(self): + def test_02_grow_storage_pool(self): + """Increase the original pool and verify CloudStack and ONTAP converge.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_01 must pass first" + ) + pool = self.__class__.pool + listed = list_storage_pools(self.apiClient, id=pool.id) + self.assertTrue(listed, "Pool missing before capacity increase") + ontap_vol = self.ontap.get_volume(pool.name) + self.assertIsNotNone(ontap_vol, "ONTAP FlexVol missing before increase") + + original_size = max( + int(getattr(listed[0], "capacitybytes", 0) or 0), + int(ontap_vol.get("space", {}).get("size", 0) or 0), + int(self.testdata[TestData.primaryStorage]["capacitybytes"]), + ) + requested_size = original_size + TestData.ONTAP_MIN_VOLUME_SIZE + self.__class__.resize_original_size = original_size + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = requested_size + self.apiClient.updateStoragePool(cmd) + + resized_pool = self._poll_pool_capacity( + pool.id, requested_size, timeout=120 + ) + self.assertEqual(resized_pool.state, "Up") + resized_ontap_vol = self._poll_ontap_volume_size( + pool.name, requested_size, timeout=120 + ) + self.assertEqual(resized_ontap_vol.get("state"), "online") + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after capacity increase", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + # ------------------------------------------------------------------ + # Step 03 - Safely shrink storage pool capacity + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_03_shrink_storage_pool(self): + """Shrink the original pool back to its initial safe capacity.""" + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_02 must pass first" + ) + target_size = self.__class__.resize_original_size + self.assertIsNotNone( + target_size, "Original size absent — test_02 must pass first" + ) + pool = self.__class__.pool + + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = pool.id + cmd.capacitybytes = target_size + self.apiClient.updateStoragePool(cmd) + + shrunk_pool = self._poll_pool_capacity( + pool.id, target_size, timeout=120 + ) + self.assertEqual(shrunk_pool.state, "Up") + shrunk_ontap_vol = self._poll_ontap_volume_size( + pool.name, target_size, timeout=120 + ) + self.assertEqual(shrunk_ontap_vol.get("state"), "online") + self.assertIsNotNone( + self.ontap.get_export_policy(self.__class__.pool_ep_name), + "Export policy disappeared after safe shrink", + ) + self._assert_export_policy_has_host_ips(self.__class__.pool_ep_name) + + # ------------------------------------------------------------------ + # Step 04 — Disable storage pool + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_04_disable_storage_pool(self): """ Disable the pool and verify: - CloudStack reports Disabled @@ -494,11 +577,11 @@ def test_02_disable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 03 — Enable storage pool + # Step 05 — Enable storage pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_03_enable_storage_pool(self): + def test_05_enable_storage_pool(self): """ Re-enable the pool and verify: - CloudStack reports Up @@ -530,11 +613,11 @@ def test_03_enable_storage_pool(self): ) # ------------------------------------------------------------------ - # Step 04 — Enter maintenance mode + # Step 06 — Enter maintenance mode # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_04_enter_maintenance_mode(self): + def test_06_enter_maintenance_mode(self): """ Put the pool into maintenance mode and verify: - CloudStack reports Maintenance @@ -566,11 +649,11 @@ def test_04_enter_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 05 — Cancel maintenance mode + # Step 07 — Cancel maintenance mode # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_05_cancel_maintenance_mode(self): + def test_07_cancel_maintenance_mode(self): """ Cancel maintenance mode and verify the pool returns to Up. @@ -616,11 +699,11 @@ def test_05_cancel_maintenance_mode(self): ) # ------------------------------------------------------------------ - # Step 06 — Delete the storage pool + # Step 08 — Delete the storage pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_06_delete_pool_from_maintenance(self): + def test_08_delete_pool_from_maintenance(self): """ Enter maintenance mode then delete the storage pool. @@ -634,7 +717,7 @@ def test_06_delete_pool_from_maintenance(self): pool_name = pool.name ep_name = self.__class__.pool_ep_name - # Pool is Up after test_05 succeeded; must enter Maintenance before deletion. + # Pool is Up after test_07 succeeded; must enter Maintenance before deletion. maint_cmd = enableStorageMaintenance.enableStorageMaintenanceCmd() maint_cmd.id = pool.id self.apiClient.enableStorageMaintenance(maint_cmd) @@ -643,6 +726,7 @@ def test_06_delete_pool_from_maintenance(self): self._delete_pool(pool.id) self.__class__.pool = None self.__class__.pool_ep_name = None + self.__class__.resize_original_size = None # CloudStack: pool must be gone try: @@ -667,11 +751,11 @@ def test_06_delete_pool_from_maintenance(self): ) # ------------------------------------------------------------------ - # Step 07 - Create fresh pool and allocate a CloudStack volume + # Step 09 - Create fresh pool and allocate a CloudStack volume # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_07_create_volume_on_pool(self): + def test_09_create_volume_on_pool(self): """ Create a new NFS3 pool and allocate a CloudStack data volume. For NFS3, createAsync is a no-op on ONTAP (volume is a CloudStack record @@ -686,7 +770,7 @@ def test_07_create_volume_on_pool(self): self.__class__.pool = pool log_progress( logger, "info", - "test_07: created storage pool name='%s' id=%s state=%s", + "test_09: created storage pool name='%s' id=%s state=%s", pool.name, pool.id, pool.state, ) @@ -703,7 +787,7 @@ def test_07_create_volume_on_pool(self): self.assertIsNotNone(vol, "createVolume returned None") log_progress( logger, "info", - "test_07: created CloudStack volume name='%s' id=%s state=%s " + "test_09: created CloudStack volume name='%s' id=%s state=%s " "on pool='%s' (id=%s) account='%s' domain='%s' — " "switch to this account in the UI to see the volume", getattr(vol, "name", "?"), getattr(vol, "id", "?"), @@ -733,13 +817,101 @@ def test_07_create_volume_on_pool(self): self._assert_pool_capacity(pool, "volume-allocated") # ------------------------------------------------------------------ - # Step 08 - Delete volume then force-delete the pool + # Step 10 - Reject shrink below ONTAP used capacity + # ------------------------------------------------------------------ + + @attr(tags=["nfs3_workflow"], required_hardware=True) + def test_10_reject_shrink_below_used_capacity(self): + """ + Attempt to shrink the new pool below ONTAP used space and verify the + request is rejected while the volume, pool capacity, and NFS export + policy remain unchanged. + + An NFS3 CloudStack volume is a thin file, so the FlexVol is still + nearly empty after test_09. Incompressible data is written through + the ONTAP files API (no VM) so used space sits above the ONTAP + FlexVol minimum, and it is removed before this test returns. + """ + self.assertIsNotNone( + self.__class__.pool, "Pool absent — test_09 must pass first" + ) + self.assertIsNotNone( + self.__class__.volume, "Volume absent — test_09 must pass first" + ) + pool = self.__class__.pool + self.assertIsNotNone( + self.ontap.get_volume(pool.name), + "ONTAP FlexVol not found for pool '%s'" % pool.name, + ) + + export_policy_name = self.__class__.pool_ep_name + self.assertIsNotNone( + export_policy_name, + "Export policy name was not recorded by test_09", + ) + export_policy_before = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_before, + "Export policy '%s' missing before rejected used-capacity shrink" + % export_policy_name, + ) + + vol_before = self._cs_volume_snapshot(self.__class__.volume.id) + filler_name, used_bytes = self._fill_flexvol_above_minimum(pool.name) + self.__class__.filler_filename = filler_name + self.__class__.filler_flexvol = pool.name if filler_name else None + try: + below_used = self._shrink_target_below_used(used_bytes) + log_progress( + logger, "info", + "Shrinking NFS3 pool '%s' below ONTAP used capacity: " + "used=%d B, requested=%d B " + "(FlexVol minimum=%d B, expect reject)", + pool.name, used_bytes, below_used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + self._assert_capacity_update_rejected( + pool.id, below_used, "shrink-below-used", + pool.id, pool.name, + expected_error=( + "insufficient", "used", "too small", "cannot reduce", + "Cannot reduce", "smaller", "minimum safe", + "too small to hold", "current volume data", + ), + ) + self._assert_cs_volume_untouched(vol_before, "shrink-below-used") + self.assertGreater( + self._flexvol_used_bytes(pool.name), + self.ONTAP_MIN_FLEXVOL_SIZE, + "ONTAP used space dropped below the FlexVol minimum after " + "the rejected shrink", + ) + export_policy_after_used = self.ontap.get_export_policy( + export_policy_name + ) + self.assertIsNotNone( + export_policy_after_used, + "Export policy '%s' disappeared after rejected " + "used-capacity shrink" % export_policy_name, + ) + self.assertEqual( + export_policy_after_used.get("name"), + export_policy_before.get("name"), + "Export policy changed during rejected used-capacity shrink", + ) + self._assert_export_policy_has_host_ips(export_policy_name) + finally: + self._delete_filler_file(pool.name, filler_name) + + # ------------------------------------------------------------------ + # Step 11 - Delete volume then force-delete the pool # ------------------------------------------------------------------ @attr(tags=["nfs3_workflow"], required_hardware=True) - def test_08_delete_volume_and_pool(self): + def test_11_delete_volume_and_pool(self): """ - Delete the volume from test_07, enter maintenance, then force-delete + Delete the volume from test_09, enter maintenance, then force-delete the pool. Verifies: - deleteVolume completes (or expected NFS3 libvirt pool-not-found) @@ -748,8 +920,8 @@ def test_08_delete_volume_and_pool(self): - ONTAP: FlexVol deleted - ONTAP: export policy deleted """ - self.assertIsNotNone(self.__class__.pool, "Pool absent - test_07 must pass first") - self.assertIsNotNone(self.__class__.volume, "Volume absent - test_07 must pass first") + self.assertIsNotNone(self.__class__.pool, "Pool absent - test_09 must pass first") + self.assertIsNotNone(self.__class__.volume, "Volume absent - test_09 must pass first") pool = self.__class__.pool pool_name = pool.name diff --git a/test/integration/plugins/ontap/ontap_test_base.py b/test/integration/plugins/ontap/ontap_test_base.py index 4f60dbf9433f..176dcc0f1b29 100644 --- a/test/integration/plugins/ontap/ontap_test_base.py +++ b/test/integration/plugins/ontap/ontap_test_base.py @@ -26,12 +26,13 @@ """ import logging +import os import random import requests import sys import time import urllib3 -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from marvin.cloudstackAPI import ( cancelStorageMaintenance, @@ -42,11 +43,14 @@ updateStoragePool as updateStoragePoolAPI, ) from marvin.cloudstackAPI import listHosts as listHostsAPI +from marvin.cloudstackException import CloudstackAPIException from marvin.cloudstackTestCase import cloudstackTestCase from marvin.jsonHelper import jsonDump from marvin.lib.base import Account, DiskOffering from marvin.sshClient import SshClient -from marvin.lib.common import get_domain, get_zone, list_clusters, list_storage_pools +from marvin.lib.common import ( + get_domain, get_zone, list_clusters, list_storage_pools, +) from marvin.lib.utils import cleanup_resources urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -144,7 +148,44 @@ def _delete(self, path, params=None): url = self._base + path resp = requests.delete(url, auth=self._auth, params=params, verify=False, timeout=30) - resp.raise_for_status() + self._raise_http(resp) + + def _raise_http(self, resp): + if resp.ok: + return + body = "" + try: + body = resp.text + except Exception: + body = "" + raise requests.HTTPError( + "%s Client Error: %s for url: %s body: %s" + % (resp.status_code, resp.reason, resp.url, body), + response=resp, + ) + + def _post(self, path, params=None, data=None, json_body=None, timeout=60, + headers=None, files=None): + url = self._base + path + resp = requests.post( + url, auth=self._auth, params=params, data=data, json=json_body, + headers=headers, files=files, verify=False, timeout=timeout, + ) + self._raise_http(resp) + if not resp.content: + return None + try: + return resp.json() + except ValueError: + return None + + def _volume_uuid(self, vol_name): + vol = self.get_volume(vol_name) + return (vol or {}).get("uuid") + + def _files_path(self, vol_uuid, filename): + encoded = quote("/" + filename.lstrip("/"), safe="") + return "/storage/volumes/%s/files/%s" % (vol_uuid, encoded) def delete_volume(self, name): """Delete the ONTAP FlexVol with the given name. No-op if not found.""" @@ -262,7 +303,6 @@ def list_files_in_volume(self, vol_name, path="/"): if not vol_uuid: return [] # URL-encode the path component (/ → %2F) and embed it in the URL. - from urllib.parse import quote encoded_path = quote(path, safe="") try: resp = self._get( @@ -274,6 +314,47 @@ def list_files_in_volume(self, vol_name, path="/"): return [r.get("name", "") for r in resp.get("records", []) if r.get("name") not in (".", "..")] + def write_file_in_volume(self, vol_name, filename, size_bytes, + chunk_bytes=512 * 1024): + """Write *size_bytes* of incompressible data into a FlexVol file. + + Zeros compress to almost nothing on ONTAP, so the payload is random. + The files API requires ``multipart/form-data`` and rejects writes + larger than 1 MiB, so data is sent in chunks. No VM, NFS mount, or + CloudStack volume is required. + """ + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + raise RuntimeError("ONTAP FlexVol '%s' not found" % vol_name) + url_path = self._files_path(vol_uuid, filename) + written = 0 + size_bytes = int(size_bytes) + while written < size_bytes: + chunk = min(int(chunk_bytes), size_bytes - written) + files = { + "file": (filename, os.urandom(chunk), + "application/octet-stream"), + } + self._post( + url_path, + params={"byte_offset": written, "overwrite": "true"}, + files=files, + timeout=120, + ) + written += chunk + + def delete_file_in_volume(self, vol_name, filename): + """Delete a file from the FlexVol. No-op if the volume or file is gone.""" + vol_uuid = self._volume_uuid(vol_name) + if not vol_uuid: + return + try: + self._delete(self._files_path(vol_uuid, filename)) + except requests.HTTPError as exc: + status = getattr(exc.response, "status_code", None) + if status not in (404, 409): + raise + # --------------------------------------------------------------------------- # Base test class @@ -281,9 +362,21 @@ def list_files_in_volume(self, vol_name, path="/"): class OntapTestBase(cloudstackTestCase): + # ONTAP refuses to shrink a FlexVol below this; distinct from the + # plugin's 1.56 GiB create-time floor (ONTAP_MIN_VOLUME_SIZE). + ONTAP_MIN_FLEXVOL_SIZE = 20 * 1024 * 1024 + FILLER_FILENAME = "ontap-filler.bin" + FILLER_SIZE = 32 * 1024 * 1024 + # ---- shared state (set/cleared by individual tests) ---------------- pool = None volume = None + # Volumes a single test allocates on top of ``volume`` and deletes before + # it returns; tracked here only so a failed run still cleans them up. + extra_volumes = [] + # File written into the FlexVol by tests that must raise ONTAP used space. + filler_filename = None + filler_flexvol = None pool2 = None volume2 = None disk_offering_id = None @@ -441,9 +534,36 @@ def _cleanup_kvm_storage_pool_mounts(cls, pool_uuid): # ---- shared teardown ----------------------------------------------- + @classmethod + def _all_tracked_volumes(cls): + """Every volume the suite created, most recent first, without repeats.""" + seen = set() + ordered = [] + for vol in list(cls.extra_volumes or []) + [cls.volume2, cls.volume]: + vol_id = getattr(vol, "id", None) + if vol is None or vol_id in seen: + continue + seen.add(vol_id) + ordered.append(vol) + return ordered + @classmethod def tearDownClass(cls): """Best-effort cleanup of any resources left behind by a failed run.""" + if (getattr(cls, "ontap", None) is not None + and getattr(cls, "filler_filename", None) + and getattr(cls, "filler_flexvol", None)): + try: + cls.ontap.delete_file_in_volume( + cls.filler_flexvol, cls.filler_filename + ) + except Exception as fe: + logger.warning( + "tearDownClass: could not delete filler file %s in %s: %s" + % (cls.filler_filename, cls.filler_flexvol, fe) + ) + cls.filler_filename = None + cls.filler_flexvol = None for pool in [p for p in (cls.pool2, cls.pool) if p is not None]: try: # Step 1: Check current pool state @@ -477,7 +597,7 @@ def tearDownClass(cls): # state. For iSCSI this works even in Maintenance; for NFS3/KVM # it may fail with NPE ("storagePoolInformation is null") when # pool is in Maintenance — that exception is caught below. - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + for vol in cls._all_tracked_volumes(): try: cmd = deleteVolumeAPI.deleteVolumeCmd() cmd.id = vol.id @@ -539,7 +659,7 @@ def tearDownClass(cls): pass # Clean up volumes that may not have been handled with pool teardown - for vol in [v for v in (cls.volume2, cls.volume) if v is not None]: + for vol in cls._all_tracked_volumes(): try: cmd = deleteVolumeAPI.deleteVolumeCmd() cmd.id = vol.id @@ -601,6 +721,327 @@ def _poll_pool_state(self, pool_id, target_state, timeout=120, interval=5): % (pool_id, target_state, timeout, current_state) ) + def _poll_pool_capacity(self, pool_id, expected_bytes, timeout=120, + interval=5): + """Poll listStoragePools until capacitybytes equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for pool %s to report capacitybytes=%d " + "(timeout=%ds, poll every %ds).", + pool_id, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + pools = list_storage_pools(self.apiClient, id=pool_id) + if pools: + current = int(getattr(pools[0], "capacitybytes", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "Pool %s reported capacitybytes=%d after %ds (%d polls).", + pool_id, expected_bytes, elapsed, attempt, + ) + return pools[0] + log_progress( + logger, "info", + "Capacity poll #%d: pool %s capacitybytes=%d (want %d) " + "[elapsed %ds, ~%ds left]", + attempt, pool_id, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "Pool %s did not report capacitybytes=%d within %ds (last: %d).", + pool_id, expected_bytes, timeout, current, + ) + self.fail( + "Pool %s did not report capacitybytes=%d within %ds (last: %d)" + % (pool_id, expected_bytes, timeout, current) + ) + + def _poll_ontap_volume_size(self, volume_name, expected_bytes, + timeout=120, interval=5): + """Poll ONTAP until the FlexVol space.size equals expected_bytes.""" + start = time.time() + deadline = start + timeout + attempt = 0 + current = 0 + log_progress( + logger, "info", + "Waiting for ONTAP FlexVol '%s' to report space.size=%d " + "(timeout=%ds, poll every %ds).", + volume_name, expected_bytes, timeout, interval, + ) + while time.time() < deadline: + attempt += 1 + elapsed = int(time.time() - start) + remaining = max(0, int(deadline - time.time())) + volume = self.ontap.get_volume(volume_name) + if volume: + current = int(volume.get("space", {}).get("size", 0) or 0) + if current == expected_bytes: + log_progress( + logger, "info", + "ONTAP FlexVol '%s' reported space.size=%d after " + "%ds (%d polls).", + volume_name, expected_bytes, elapsed, attempt, + ) + return volume + log_progress( + logger, "info", + "ONTAP capacity poll #%d: FlexVol '%s' space.size=%d " + "(want %d) [elapsed %ds, ~%ds left]", + attempt, volume_name, current, expected_bytes, + elapsed, remaining, + ) + time.sleep(interval) + log_progress( + logger, "error", + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d).", + volume_name, expected_bytes, timeout, current, + ) + self.fail( + "ONTAP FlexVol '%s' did not report space.size=%d within %ds " + "(last: %d)" + % (volume_name, expected_bytes, timeout, current) + ) + + def _get_cs_volume(self, vol_id): + """Return the CloudStack volume object, or None if it is gone.""" + from marvin.cloudstackAPI import listVolumes as listVolumesAPI + cmd = listVolumesAPI.listVolumesCmd() + cmd.id = vol_id + cmd.listall = True + vols = self.apiClient.listVolumes(cmd) or [] + return vols[0] if vols else None + + def _volume_exists_in_cs(self, vol_id): + """Return True if the volume is still listed by CloudStack.""" + return self._get_cs_volume(vol_id) is not None + + def _cs_volume_snapshot(self, vol_id): + """Capture id, state, size, and pool so a later check can detect mutation.""" + vol = self._get_cs_volume(vol_id) + self.assertIsNotNone( + vol, "CloudStack volume %s is not listed in listVolumes" % vol_id + ) + pool_id = ( + getattr(vol, "storageid", None) + or getattr(vol, "poolid", None) + ) + return { + "id": getattr(vol, "id", None), + "state": getattr(vol, "state", None), + "size": int(getattr(vol, "size", 0) or 0), + "poolid": pool_id, + } + + def _assert_cs_volume_untouched(self, before, label): + """Assert listVolumes still returns the same id, state, size, and pool.""" + after = self._cs_volume_snapshot(before["id"]) + self.assertEqual( + after["id"], before["id"], + "[%s] CloudStack volume id changed (%s -> %s)" + % (label, before["id"], after["id"]), + ) + self.assertEqual( + after["state"], before["state"], + "[%s] CloudStack volume state changed (%s -> %s)" + % (label, before["state"], after["state"]), + ) + self.assertEqual( + after["size"], before["size"], + "[%s] CloudStack volume size changed (%s -> %s)" + % (label, before["size"], after["size"]), + ) + self.assertEqual( + after["poolid"], before["poolid"], + "[%s] CloudStack volume poolid changed (%s -> %s)" + % (label, before["poolid"], after["poolid"]), + ) + + def _align_flexvol_bytes(self, value): + """Round *value* down to the 4 KiB boundary ONTAP uses for FlexVol size.""" + return (int(value) // 4096) * 4096 + + def _flexvol_used_bytes(self, vol_name): + """ONTAP physical used bytes on the FlexVol (used + reserved).""" + ontap_vol = self.ontap.get_volume(vol_name) + self.assertIsNotNone( + ontap_vol, "ONTAP FlexVol '%s' not found" % vol_name + ) + space = ontap_vol.get("space") or {} + return int(space.get("used") or 0) + + def _fill_flexvol_above_minimum(self, vol_name, timeout=90): + """Write a FlexVol file until ONTAP used space exceeds the minimum. + + Returns ``(filler_filename_or_None, used_bytes)``. The caller must + delete any returned filename. No CloudStack volume or VM is created. + """ + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + log_progress( + logger, "info", + "FlexVol '%s' already has %d B used (ONTAP FlexVol minimum " + "%d B); no filler file needed", + vol_name, used, self.ONTAP_MIN_FLEXVOL_SIZE, + ) + return None, used + log_progress( + logger, "info", + "FlexVol '%s' has %d B used; writing %d B filler file '%s'", + vol_name, used, self.FILLER_SIZE, self.FILLER_FILENAME, + ) + self.ontap.write_file_in_volume( + vol_name, self.FILLER_FILENAME, self.FILLER_SIZE + ) + deadline = time.time() + timeout + while time.time() < deadline: + used = self._flexvol_used_bytes(vol_name) + if used > self.ONTAP_MIN_FLEXVOL_SIZE: + break + time.sleep(2) + self.assertGreater( + used, self.ONTAP_MIN_FLEXVOL_SIZE, + "FlexVol '%s' still has only %d B used after writing a %d B " + "filler file; cannot exceed the ONTAP FlexVol minimum of %d B" + % (vol_name, used, self.FILLER_SIZE, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + log_progress( + logger, "info", + "FlexVol '%s' has %d B used after filler file '%s'", + vol_name, used, self.FILLER_FILENAME, + ) + return self.FILLER_FILENAME, used + + def _delete_filler_file(self, vol_name, filename): + if not filename or not vol_name: + return + try: + self.ontap.delete_file_in_volume(vol_name, filename) + except Exception as exc: + logger.warning( + "could not delete filler file '%s' in FlexVol '%s': %s", + filename, vol_name, exc, + ) + if self.__class__.filler_filename == filename: + self.__class__.filler_filename = None + self.__class__.filler_flexvol = None + + def _shrink_target_below_used(self, used_bytes): + """4 KiB-aligned size below used_bytes but above the FlexVol min.""" + used_bytes = int(used_bytes) + self.assertGreater( + used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE, + "Used capacity %d B is not above the ONTAP FlexVol " + "minimum %d B; cannot distinguish a used-capacity reject " + "from a minimum-size reject" + % (used_bytes, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + target = self._align_flexvol_bytes(used_bytes - 4096) + if target <= self.ONTAP_MIN_FLEXVOL_SIZE: + target = self._align_flexvol_bytes( + (used_bytes + self.ONTAP_MIN_FLEXVOL_SIZE) // 2 + ) + self.assertGreater( + target, self.ONTAP_MIN_FLEXVOL_SIZE, + "Shrink target %d B is not above the ONTAP FlexVol minimum %d B" + % (target, self.ONTAP_MIN_FLEXVOL_SIZE), + ) + self.assertLess( + target, used_bytes, + "Shrink target %d B must be below used capacity %d B" + % (target, used_bytes), + ) + return target + + def _assert_capacity_update_rejected( + self, cmd_pool_id, capacitybytes, label, verify_pool_id, + volume_name, expected_error=None): + """Assert updateStoragePool(capacitybytes) fails and sizes stay put. + + CloudStack and ONTAP are compared against their own pre-request values + because the two do not have to agree: a FlexVol created with a snapshot + reserve reports a larger space.size than the usable capacity + CloudStack records. + """ + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] listStoragePools returned no result for pool %s" + % (label, verify_pool_id), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol '%s' not found" % (label, volume_name), + ) + before_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + before_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + log_progress( + logger, "info", + "Negative resize %s: pool_id=%s capacitybytes=%s " + "(expect reject; CS=%d B ONTAP=%d B)", + label, cmd_pool_id, capacitybytes, before_cs, before_ontap, + ) + cmd = updateStoragePoolAPI.updateStoragePoolCmd() + cmd.id = cmd_pool_id + cmd.capacitybytes = capacitybytes + with self.assertRaises(CloudstackAPIException) as caught: + self.apiClient.updateStoragePool(cmd) + error_text = str(caught.exception) + log_progress( + logger, "info", "Rejected resize %s: %s", label, error_text, + ) + if expected_error: + needles = ( + expected_error + if isinstance(expected_error, (list, tuple)) + else (expected_error,) + ) + self.assertTrue( + any(needle in error_text for needle in needles), + "[%s] expected the rejection to report one of %r, got: %s" + % (label, needles, error_text), + ) + + listed = list_storage_pools(self.apiClient, id=verify_pool_id) + self.assertTrue( + listed, + "[%s] pool disappeared after rejected resize" % label, + ) + after_cs = int(getattr(listed[0], "capacitybytes", 0) or 0) + self.assertEqual( + after_cs, before_cs, + "[%s] CloudStack capacity changed after rejected resize " + "(got %d, want %d)" % (label, after_cs, before_cs), + ) + self.assertEqual( + listed[0].state, "Up", + "[%s] pool should remain Up after rejected resize, got '%s'" + % (label, listed[0].state), + ) + ontap_vol = self.ontap.get_volume(volume_name) + self.assertIsNotNone( + ontap_vol, + "[%s] ONTAP FlexVol disappeared after rejected resize" % label, + ) + after_ontap = int(ontap_vol.get("space", {}).get("size", 0) or 0) + self.assertEqual( + after_ontap, before_ontap, + "[%s] ONTAP FlexVol size changed after rejected resize " + "(got %d, want %d)" % (label, after_ontap, before_ontap), + ) + def _create_volume(self, pool_id): """Create a data volume on the given pool; uses _vol_name_prefix.""" cmd = createVolumeAPI.createVolumeCmd()