From aa3121b0049f7c0e4029d147186953eda75a472e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 10 Jun 2024 20:19:06 +0530 Subject: [PATCH 1/4] VMware: Improve error messaging / logs when starting non-user VMs, and secondary storage not available or doesn't have enough capacity --- .../ImageStoreProviderManagerImpl.java | 2 +- .../vmware/manager/VmwareManagerImpl.java | 15 +++++++-------- .../vmware/resource/VmwareResource.java | 19 ++++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 5bb0d19be74b..27fb77660acb 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -201,7 +201,7 @@ public int compare(DataStore store1, DataStore store2) { // No store with space found s_logger.error(String.format("Can't find an image storage in zone with less than %d usage", - Math.round(_statsCollector.getImageStoreCapacityThreshold()*100))); + Math.round(_statsCollector.getImageStoreCapacityThreshold() * 100))); return null; } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index b5f4cf3a93f0..564c29427523 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -562,7 +562,6 @@ public List addHostToPodCluster(VmwareContext serviceCon @Override public Pair getSecondaryStorageStoreUrlAndId(long dcId) { - String secUrl = null; Long secId = null; DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dcId); @@ -572,18 +571,18 @@ public Pair getSecondaryStorageStoreUrlAndId(long dcId) { } if (secUrl == null) { - // we are using non-NFS image store, then use cache storage instead - s_logger.info("Secondary storage is not NFS, we need to use staging storage"); + // image stores doesn't have enough capacity or we are using non-NFS image store, then use cache storage instead + s_logger.info("Secondary storage is either not having free capacity or not NFS, we need to use staging storage"); DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId); if (cacheStore != null) { secUrl = cacheStore.getUri(); secId = cacheStore.getId(); } else { - s_logger.warn("No staging storage is found when non-NFS secondary storage is used"); + s_logger.warn("No staging storage is found when NFS secondary storage with free capacity not available or non-NFS secondary storage is used"); } } - return new Pair(secUrl, secId); + return new Pair<>(secUrl, secId); } @Override @@ -599,13 +598,13 @@ public List> getSecondaryStorageStoresUrlAndIdList(long dcId) } if (urlIdList.isEmpty()) { - // we are using non-NFS image store, then use cache storage instead - s_logger.info("Secondary storage is not NFS, we need to use staging storage"); + // image stores doesn't have enough capacity or we are using non-NFS image store, then use cache storage instead + s_logger.info("Secondary storage is either not having free capacity or not NFS, we need to use staging storage"); DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId); if (cacheStore != null) { urlIdList.add(new Pair<>(cacheStore.getUri(), cacheStore.getId())); } else { - s_logger.warn("No staging storage is found when non-NFS secondary storage is used"); + s_logger.warn("No staging storage is found when NFS secondary storage with free capacity not available or non-NFS secondary storage is used"); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 1b7a76e26cfb..e84b2d1cbcc3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -48,6 +48,7 @@ import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; +import com.cloud.capacity.CapacityManager; import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; import com.vmware.vim25.FileInfo; import com.vmware.vim25.FileQueryFlags; @@ -2279,15 +2280,15 @@ protected StartAnswer execute(StartCommand cmd) { // attach ISO (for patching of system VM) Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (less than %d usage) or not ready yet, or non-NFS secondary storage is used", + _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + String msg = "Failed to prepare secondary storage on host, NFS secondary or cache store url: " + secStoreUrl + " in dc "+ _dcId; throw new Exception(msg); } DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); @@ -4613,15 +4614,15 @@ protected Answer execute(PrepareForMigrationCommand cmd) { List> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId)); for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); if (secStoreUrl == null) { - String msg = String.format("Secondary storage for dc %s is not ready yet?", _dcId); + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (less than %d usage) or not ready yet, or non-NFS secondary storage is used", + _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + String msg = "Failed to prepare secondary storage on host, NFS secondary or cache store url: " + secStoreUrl + " in dc "+ _dcId; throw new Exception(msg); } } @@ -7342,14 +7343,14 @@ private List relocateVirtualMachine(final VmwareHypervisorHost h VmwareManager mgr = targetHyperHost.getContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (less than %d usage) or not ready yet, or non-NFS secondary storage is used", + _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, targetHyperHost); if (morSecDs == null) { - throw new Exception(String.format("Failed to prepare secondary storage on host, secondary store url: %s", secStoreUrl)); + throw new Exception(String.format("Failed to prepare secondary storage on host, NFS secondary or cache store url: %s in dc %s", secStoreUrl, _dcId)); } } From 6557da60494b19332c0637520c2cbe297bf29591 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 12 Jun 2024 14:39:35 +0530 Subject: [PATCH 2/4] Addressed review comments --- .../hypervisor/vmware/manager/VmwareManagerImpl.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 564c29427523..61a949f42d3e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -571,14 +571,13 @@ public Pair getSecondaryStorageStoreUrlAndId(long dcId) { } if (secUrl == null) { - // image stores doesn't have enough capacity or we are using non-NFS image store, then use cache storage instead - s_logger.info("Secondary storage is either not having free capacity or not NFS, we need to use staging storage"); + s_logger.info("Secondary storage is either not having free capacity or not NFS, then use cache/staging storage instead"); DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId); if (cacheStore != null) { secUrl = cacheStore.getUri(); secId = cacheStore.getId(); } else { - s_logger.warn("No staging storage is found when NFS secondary storage with free capacity not available or non-NFS secondary storage is used"); + s_logger.warn("No cache/staging storage found when NFS secondary storage with free capacity not available or non-NFS secondary storage is used"); } } @@ -598,13 +597,12 @@ public List> getSecondaryStorageStoresUrlAndIdList(long dcId) } if (urlIdList.isEmpty()) { - // image stores doesn't have enough capacity or we are using non-NFS image store, then use cache storage instead - s_logger.info("Secondary storage is either not having free capacity or not NFS, we need to use staging storage"); + s_logger.info("Secondary storage is either not having free capacity or not NFS, then use cache/staging storage instead"); DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId); if (cacheStore != null) { urlIdList.add(new Pair<>(cacheStore.getUri(), cacheStore.getId())); } else { - s_logger.warn("No staging storage is found when NFS secondary storage with free capacity not available or non-NFS secondary storage is used"); + s_logger.warn("No cache/staging storage found when NFS secondary storage with free capacity not available or non-NFS secondary storage is used"); } } From eddbce4babacc7db7bb5873b5fb09b468f64477e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 13 Jun 2024 18:18:57 +0530 Subject: [PATCH 3/4] Addressed review comments --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index e84b2d1cbcc3..7f4228e22a15 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2281,7 +2281,7 @@ protected StartAnswer execute(StartCommand cmd) { Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); String secStoreUrl = secStoreUrlAndId.first(); if (secStoreUrl == null) { - String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (less than %d usage) or not ready yet, or non-NFS secondary storage is used", + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d usage threshold) or not ready yet, or non-NFS secondary storage is used", _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } @@ -4615,7 +4615,7 @@ protected Answer execute(PrepareForMigrationCommand cmd) { for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { String secStoreUrl = secStoreUrlAndId.first(); if (secStoreUrl == null) { - String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (less than %d usage) or not ready yet, or non-NFS secondary storage is used", + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d usage threshold) or not ready yet, or non-NFS secondary storage is used", _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } @@ -7344,7 +7344,7 @@ private List relocateVirtualMachine(final VmwareHypervisorHost h Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); String secStoreUrl = secStoreUrlAndId.first(); if (secStoreUrl == null) { - String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (less than %d usage) or not ready yet, or non-NFS secondary storage is used", + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d usage threshold) or not ready yet, or non-NFS secondary storage is used", _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } From fd46dac6ebbd42c9bcf5150da6d40873fef0d8f4 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Tue, 25 Jun 2024 10:56:44 +0530 Subject: [PATCH 4/4] minor log update --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 7f4228e22a15..ca228bda0a6b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2281,7 +2281,7 @@ protected StartAnswer execute(StartCommand cmd) { Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); String secStoreUrl = secStoreUrlAndId.first(); if (secStoreUrl == null) { - String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d usage threshold) or not ready yet, or non-NFS secondary storage is used", + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d%% usage threshold) or not ready yet, or non-NFS secondary storage is used", _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } @@ -4615,7 +4615,7 @@ protected Answer execute(PrepareForMigrationCommand cmd) { for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { String secStoreUrl = secStoreUrlAndId.first(); if (secStoreUrl == null) { - String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d usage threshold) or not ready yet, or non-NFS secondary storage is used", + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d%% usage threshold) or not ready yet, or non-NFS secondary storage is used", _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); } @@ -7344,7 +7344,7 @@ private List relocateVirtualMachine(final VmwareHypervisorHost h Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); String secStoreUrl = secStoreUrlAndId.first(); if (secStoreUrl == null) { - String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d usage threshold) or not ready yet, or non-NFS secondary storage is used", + String msg = String.format("NFS secondary or cache storage of dc %s either doesn't have enough capacity (has reached %d%% usage threshold) or not ready yet, or non-NFS secondary storage is used", _dcId, Math.round(CapacityManager.SecondaryStorageCapacityThreshold.value() * 100)); throw new Exception(msg); }