This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new 3220eb442a5 PowerFlex/ScaleIO - MDM and host SDC connection
enhancements (#11047)
3220eb442a5 is described below
commit 3220eb442a5b2a1416b2c44a1112dc029dd35708
Author: Suresh Kumar Anaparti <sureshkumar.anapa...@gmail.com>
AuthorDate: Wed Jul 16 11:55:28 2025 +0530
PowerFlex/ScaleIO - MDM and host SDC connection enhancements (#11047)
* Cumulative enhancements fix for ScaleIO: MDM add/remove, Host
prepare/unprepare, validate Storage Pool can be created in Agent.
- Implemented validation to fail Host disconnect from Storage Pool if there
are Volumes attached and SDC client MDM removal requires scini service to be
restarted
- Implemented Storage Pool validation by checking whether MDM addresses
from configuration file and from memory (using CLI) matches, otherwise file
ModifyStoragePool command.
- Introduced configuration key to apply timeout after making MDM changes
for ScaleIO: powerflex.mdm.change.apply.timeout.ms (default 1000ms)
- Implemented logic to apply timeout after making MDM changes for ScaleIO
in prepare and unprepare logic
- Added detection of MDM removal support via CLI
- If MDM removal support via CLI supported then use CLI, fall back to edit
drv_cfg.txt and restart scini instead
Co-authored-by: Suresh Kumar Anaparti <suresh.anapa...@shapeblue.com>
Co-authored-by: mprokopchuk <mprokopc...@apple.com>
---
.../cloud/agent/properties/AgentProperties.java | 2 +-
.../LibvirtModifyStoragePoolCommandWrapper.java | 16 +-
.../kvm/storage/ScaleIOStorageAdaptor.java | 174 ++++++++--
.../kvm/storage/ScaleIOStorageAdaptorTest.java | 4 +
.../datastore/client/ScaleIOGatewayClient.java | 3 +
.../ScaleIOPrimaryDataStoreLifeCycle.java | 39 +--
.../datastore/manager/ScaleIOSDCManager.java | 37 +++
.../datastore/manager/ScaleIOSDCManagerImpl.java | 58 +++-
.../datastore/provider/ScaleIOHostListener.java | 12 +-
.../storage/datastore/util/ScaleIOUtil.java | 351 ++++++++++++++++++---
.../main/java/com/cloud/utils/script/Script.java | 20 ++
11 files changed, 611 insertions(+), 105 deletions(-)
diff --git
a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
index 69537621673..b6e90160c07 100644
--- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
+++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java
@@ -833,7 +833,7 @@ public class AgentProperties{
private T defaultValue;
private Class<T> typeClass;
- Property(String name, T value) {
+ public Property(String name, T value) {
init(name, value);
}
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
index 06883c708d9..990cefda8f3 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtModifyStoragePoolCommandWrapper.java
@@ -31,6 +31,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager;
import com.cloud.resource.CommandWrapper;
import com.cloud.resource.ResourceWrapper;
import com.cloud.storage.template.TemplateProp;
+import com.cloud.utils.exception.CloudRuntimeException;
@ResourceWrapper(handles = ModifyStoragePoolCommand.class)
public final class LibvirtModifyStoragePoolCommandWrapper extends
CommandWrapper<ModifyStoragePoolCommand, Answer, LibvirtComputingResource> {
@@ -49,11 +50,16 @@ public final class LibvirtModifyStoragePoolCommandWrapper
extends CommandWrapper
return answer;
}
- final KVMStoragePool storagepool =
- storagePoolMgr.createStoragePool(command.getPool().getUuid(),
command.getPool().getHost(), command.getPool().getPort(),
command.getPool().getPath(), command.getPool()
- .getUserInfo(), command.getPool().getType(),
command.getDetails());
- if (storagepool == null) {
- return new Answer(command, false, " Failed to create storage
pool");
+ final KVMStoragePool storagepool;
+ try {
+ storagepool =
+
storagePoolMgr.createStoragePool(command.getPool().getUuid(),
command.getPool().getHost(), command.getPool().getPort(),
command.getPool().getPath(), command.getPool()
+ .getUserInfo(), command.getPool().getType(),
command.getDetails());
+ if (storagepool == null) {
+ return new Answer(command, false, " Failed to create storage
pool");
+ }
+ } catch (CloudRuntimeException e) {
+ return new Answer(command, false, String.format("Failed to create
storage pool: %s", e.getLocalizedMessage()));
}
final Map<String, TemplateProp> tInfo = new HashMap<String,
TemplateProp>();
diff --git
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
index 29c152f934f..efa8024a34b 100644
---
a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
+++
b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java
@@ -22,11 +22,13 @@ import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import com.cloud.agent.api.PrepareStorageClientCommand;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager;
import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil;
@@ -38,6 +40,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg;
import org.apache.cloudstack.utils.qemu.QemuImgException;
import org.apache.cloudstack.utils.qemu.QemuImgFile;
import org.apache.cloudstack.utils.qemu.QemuObject;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -149,7 +152,7 @@ public class ScaleIOStorageAdaptor implements
StorageAdaptor {
@Override
public KVMStoragePool createStoragePool(String uuid, String host, int
port, String path, String userInfo, Storage.StoragePoolType type, Map<String,
String> details, boolean isPrimaryStorage) {
ScaleIOStoragePool storagePool = new ScaleIOStoragePool(uuid, host,
port, path, type, details, this);
- if (details != null &&
details.containsKey(ScaleIOSDCManager.ConnectOnDemand.key())) {
+ if (MapUtils.isNotEmpty(details) &&
details.containsKey(ScaleIOSDCManager.ConnectOnDemand.key())) {
String connectOnDemand =
details.get(ScaleIOSDCManager.ConnectOnDemand.key());
if (connectOnDemand != null &&
!Boolean.parseBoolean(connectOnDemand)) {
Ternary<Boolean, Map<String, String>, String>
prepareStorageClientStatus = prepareStorageClient(uuid, details);
@@ -158,10 +161,49 @@ public class ScaleIOStorageAdaptor implements
StorageAdaptor {
}
}
}
+
+ validateMdmState(details);
+
MapStorageUuidToStoragePool.put(uuid, storagePool);
return storagePool;
}
+ /**
+ * Validate Storage Pool state to ensure it healthy and can operate
requests.
+ * There is observed situation where ScaleIO configuration file has
different values than ScaleIO CLI.
+ * Validation compares values from both drv_cfg.txt and drv_cfg CLI and
throws exception if there is mismatch.
+ *
+ * @param details see {@link PrepareStorageClientCommand#getDetails()}
+ * and {@link @UnprepareStorageClientCommand#getDetails()},
expected to contain
+ * {@link ScaleIOSDCManager#ValidateMdmsOnConnect#key()}
+ * @throws CloudRuntimeException in case if Storage Pool is not
operate-able
+ */
+ private void validateMdmState(Map<String, String> details) {
+ String configKey = ScaleIOSDCManager.ValidateMdmsOnConnect.key();
+ if (MapUtils.isEmpty(details) || !details.containsKey(configKey)) {
+ logger.debug(String.format("Skipped PowerFlex MDMs validation as
property %s not sent by Management Server", configKey));
+ return;
+ }
+
+ String configValue = details.get(configKey);
+
+ // be as much verbose as possible, otherwise it will be difficult to
troubleshoot operational issue without logs
+ if (StringUtils.isEmpty(configValue)) {
+ logger.debug(String.format("Skipped PowerFlex MDMs validation as
property %s sent by Management Server is empty", configKey));
+ } else if (Boolean.valueOf(configValue).equals(Boolean.FALSE)) {
+ logger.debug(String.format("Skipped PowerFlex MDMs validation as
property %s received as %s", configKey, configValue));
+ } else {
+ Collection<String> mdmsFromConfigFile =
ScaleIOUtil.getMdmsFromConfigFile();
+ Collection<String> mdmsFromCliCmd =
ScaleIOUtil.getMdmsFromCliCmd();
+ if (!mdmsFromCliCmd.equals(mdmsFromConfigFile)) {
+ String msg = String.format("PowerFlex MDM addresses from CLI
and Configuration File doesn't match. " +
+ "CLI values: %s, Configuration File values: %s",
mdmsFromCliCmd, mdmsFromConfigFile);
+ logger.warn(msg);
+ throw new CloudRuntimeException(msg);
+ }
+ }
+ }
+
@Override
public boolean deleteStoragePool(String uuid) {
ScaleIOStoragePool storagePool = (ScaleIOStoragePool)
MapStorageUuidToStoragePool.get(uuid);
@@ -173,7 +215,7 @@ public class ScaleIOStorageAdaptor implements
StorageAdaptor {
@Override
public boolean deleteStoragePool(String uuid, Map<String, String> details)
{
- if (details != null &&
details.containsKey(ScaleIOSDCManager.ConnectOnDemand.key())) {
+ if (MapUtils.isNotEmpty(details) &&
details.containsKey(ScaleIOSDCManager.ConnectOnDemand.key())) {
String connectOnDemand =
details.get(ScaleIOSDCManager.ConnectOnDemand.key());
if (connectOnDemand != null &&
!Boolean.parseBoolean(connectOnDemand)) {
Pair<Boolean, String> unprepareStorageClientStatus =
unprepareStorageClient(uuid, details);
@@ -259,7 +301,7 @@ public class ScaleIOStorageAdaptor implements
StorageAdaptor {
volumePath = ScaleIOUtil.getVolumePath(volumePath);
int waitTimeInSec = DEFAULT_DISK_WAIT_TIME_IN_SECS;
- if (details != null &&
details.containsKey(StorageManager.STORAGE_POOL_DISK_WAIT.toString())) {
+ if (MapUtils.isNotEmpty(details) &&
details.containsKey(StorageManager.STORAGE_POOL_DISK_WAIT.toString())) {
String waitTime =
details.get(StorageManager.STORAGE_POOL_DISK_WAIT.toString());
if (StringUtils.isNotEmpty(waitTime)) {
waitTimeInSec = Integer.valueOf(waitTime).intValue();
@@ -607,28 +649,37 @@ public class ScaleIOStorageAdaptor implements
StorageAdaptor {
}
if (!ScaleIOUtil.isSDCServiceActive()) {
+ logger.debug("SDC service is not active on host, starting it");
if (!ScaleIOUtil.startSDCService()) {
return new Ternary<>(false, null, "Couldn't start SDC service
on host");
}
}
- if (details != null &&
details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
+ if (MapUtils.isNotEmpty(details) &&
details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
// Assuming SDC service is started, add mdms
String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
String[] mdmAddresses = mdms.split(",");
if (mdmAddresses.length > 0) {
- if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+ if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
return new Ternary<>(true, getSDCDetails(details), "MDM
added, no need to prepare the SDC client");
}
- ScaleIOUtil.addMdms(Arrays.asList(mdmAddresses));
- if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+ ScaleIOUtil.addMdms(mdmAddresses);
+ if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
return new Ternary<>(false, null, "Failed to add MDMs");
+ } else {
+ logger.debug(String.format("MDMs %s added to storage pool
%s", mdms, uuid));
+ applyMdmsChangeWaitTime(details);
}
}
}
- return new Ternary<>( true, getSDCDetails(details), "Prepared client
successfully");
+ Map<String, String> sdcDetails = getSDCDetails(details);
+ if (MapUtils.isEmpty(sdcDetails)) {
+ return new Ternary<>(false, null, "Couldn't get the SDC details on
the host");
+ }
+
+ return new Ternary<>(true, sdcDetails, "Prepared client successfully");
}
public Pair<Boolean, String> unprepareStorageClient(String uuid,
Map<String, String> details) {
@@ -642,40 +693,127 @@ public class ScaleIOStorageAdaptor implements
StorageAdaptor {
return new Pair<>(true, "SDC service not enabled on host, no need
to unprepare the SDC client");
}
- if (details != null &&
details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
+ if (MapUtils.isNotEmpty(details) &&
details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_MDMS)) {
String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS);
String[] mdmAddresses = mdms.split(",");
if (mdmAddresses.length > 0) {
- if (!ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+ if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
return new Pair<>(true, "MDM not added, no need to
unprepare the SDC client");
+ } else {
+ String configKey =
ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key();
+ String configValue = details.get(configKey);
+
+ if (StringUtils.isEmpty(configValue)) {
+ logger.debug(String.format("Configuration key %s not
provided", configKey));
+ } else {
+ logger.debug(String.format("Configuration key %s
provided as %s", configKey, configValue));
+ }
+ Boolean blockUnprepare = Boolean.valueOf(configValue);
+ if (!ScaleIOUtil.isRemoveMdmCliSupported() // scini
restart required when --remove_mdm is not supported
+ && !ScaleIOUtil.getVolumeIds().isEmpty()
+ && Boolean.TRUE.equals(blockUnprepare)) {
+ return new Pair<>(false, "Failed to remove MDMs, SDC
client requires service to be restarted, but there are Volumes attached to the
Host");
+ }
}
- ScaleIOUtil.removeMdms(Arrays.asList(mdmAddresses));
- if (ScaleIOUtil.mdmAdded(mdmAddresses[0])) {
+ // Immediate removal of MDMs after unmapping volume throws
Error: "Volume is mappedKernel module rejects removing MDM"
+ // Wait before removing MDMs for any volumes to get unmapped.
+ applyMdmsChangeWaitTime(details);
+ ScaleIOUtil.removeMdms(mdmAddresses);
+ if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) {
return new Pair<>(false, "Failed to remove MDMs, unable to
unprepare the SDC client");
+ } else {
+ logger.debug(String.format("MDMs %s removed from storage
pool %s", mdms, uuid));
+ applyMdmsChangeWaitTime(details);
}
}
}
+ /*
+ * TODO:
+ * 1. Verify on-demand is true
+ * 2. If on-demand is true check whether other MDM addresses are still
present
+ * 3. If there are no MDM addresses, then stop SDC service.
+ */
+
return new Pair<>(true, "Unprepared SDC client successfully");
}
+ /**
+ * Check whether details map has wait time configured and do "apply wait
time" pause before returning response
+ * (to have ScaleIO changes applied).
+ *
+ * @param details see {@link PrepareStorageClientCommand#getDetails()}
+ * and {@link @UnprepareStorageClientCommand#getDetails()},
expected to contain
+ * {@link ScaleIOSDCManager#MdmsChangeApplyWaitTime#key()}
+ */
+ private void applyMdmsChangeWaitTime(Map<String, String> details) {
+ String configKey = ScaleIOSDCManager.MdmsChangeApplyWaitTime.key();
+ if (MapUtils.isEmpty(details) || !details.containsKey(configKey)) {
+ logger.debug(String.format("Apply wait time property %s not sent
by Management Server, skip", configKey));
+ return;
+ }
+
+ String configValue = details.get(configKey);
+ if (StringUtils.isEmpty(configValue)) {
+ logger.debug(String.format("Apply wait time value not defined in
property %s, skip", configKey));
+ return;
+ }
+ long timeoutMs;
+ try {
+ timeoutMs = Long.parseLong(configValue);
+ } catch (NumberFormatException e) {
+ logger.warn(String.format("Invalid apply wait time value defined
in property %s, skip", configKey), e);
+ return;
+ }
+ if (timeoutMs < 1) {
+ logger.warn(String.format("Apply wait time value is too small (%s
ms), skipping", timeoutMs));
+ return;
+ }
+ try {
+ logger.debug(String.format("Waiting for %d ms as defined in
property %s", timeoutMs, configKey));
+ Thread.sleep(timeoutMs);
+ } catch (InterruptedException e) {
+ logger.warn(String.format("Waiting for %d ms interrupted",
timeoutMs), e);
+ }
+ }
+
private Map<String, String> getSDCDetails(Map<String, String> details) {
Map<String, String> sdcDetails = new HashMap<String, String>();
- if (details == null ||
!details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) {
+ if (MapUtils.isEmpty(details) ||
!details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) {
return sdcDetails;
}
String storageSystemId =
details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
- String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
- if (sdcId != null) {
- sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
- } else {
+ if (StringUtils.isEmpty(storageSystemId)) {
+ return sdcDetails;
+ }
+
+ int numberOfTries = 5;
+ int timeBetweenTries = 1000; // Try more frequently (every sec) and
return early when SDC Id or Guid found
+ int attempt = 1;
+ do {
+ logger.debug("Get SDC details, attempt #{}", attempt);
+ String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
+ if (sdcId != null) {
+ sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
+ return sdcDetails;
+ }
+
String sdcGuId = ScaleIOUtil.getSdcGuid();
if (sdcGuId != null) {
sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId);
+ return sdcDetails;
}
- }
+
+ try {
+ Thread.sleep(timeBetweenTries);
+ } catch (Exception ignore) {
+ }
+ numberOfTries--;
+ attempt++;
+ } while (numberOfTries > 0);
+
return sdcDetails;
}
diff --git
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
index 421fee09634..eddaa8f6499 100644
---
a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
+++
b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
@@ -194,8 +194,12 @@ public class ScaleIOStorageAdaptorTest {
details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl
status scini"))).thenReturn(3);
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl
is-enabled scini"))).thenReturn(0);
+ when(Script.executeCommand(Mockito.eq("sed -i '/1.1.1.1\\,/d'
/etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>(null, null));
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl
restart scini"))).thenReturn(0);
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg
--query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID
4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
+
when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new
Pair<>(null, null));
+
when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg
--query_vols"))).thenReturn(new Pair<>("", null));
+
Pair<Boolean, String> result =
scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
diff --git
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
index 2dc5acffcbc..77a8677e6b1 100644
---
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
+++
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClient.java
@@ -38,6 +38,9 @@ public interface ScaleIOGatewayClient {
String GATEWAY_API_PASSWORD = "powerflex.gw.password";
String STORAGE_POOL_NAME = "powerflex.storagepool.name";
String STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id";
+ /**
+ * Storage Pool Metadata Management (MDM) IP address(es).
+ */
String STORAGE_POOL_MDMS = "powerflex.storagepool.mdms";
String SDC_ID = "powerflex.sdc.id";
String SDC_GUID = "powerflex.sdc.guid";
diff --git
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
index 461992be102..5917e7895ca 100644
---
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
+++
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java
@@ -31,7 +31,6 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import com.cloud.host.HostVO;
-import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao;
import org.apache.cloudstack.api.ApiConstants;
import com.cloud.utils.StringUtils;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
@@ -105,8 +104,6 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends
BasePrimaryDataStoreLifeCy
@Inject
private AgentManager agentMgr;
private ScaleIOSDCManager sdcManager;
- @Inject
- private StoragePoolAndAccessGroupMapDao storagePoolAndAccessGroupMapDao;
public ScaleIOPrimaryDataStoreLifeCycle() {
sdcManager = new ScaleIOSDCManagerImpl();
@@ -306,14 +303,19 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends
BasePrimaryDataStoreLifeCy
@Override
public boolean maintain(DataStore store) {
- Map<String,String> details = new HashMap<>();
- StoragePoolDetailVO systemIdDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
- if (systemIdDetail != null) {
- details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID,
systemIdDetail.getValue());
- StoragePoolDetailVO mdmsDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_MDMS);
- if (mdmsDetail != null) {
- details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS,
mdmsDetail.getValue());
- details.put(ScaleIOSDCManager.ConnectOnDemand.key(), "false");
+ Map<String, String> details = new HashMap<>();
+ StoragePoolVO storagePoolVO =
primaryDataStoreDao.findById(store.getId());
+ if (storagePoolVO != null) {
+ sdcManager = ComponentContext.inject(sdcManager);
+ sdcManager.populateSdcSettings(details,
storagePoolVO.getDataCenterId());
+ StoragePoolDetailVO systemIdDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+ if (systemIdDetail != null) {
+ details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID,
systemIdDetail.getValue());
+ StoragePoolDetailVO mdmsDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_MDMS);
+ if (mdmsDetail != null) {
+ details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS,
mdmsDetail.getValue());
+ details.put(ScaleIOSDCManager.ConnectOnDemand.key(),
"false");
+ }
}
}
@@ -324,14 +326,15 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends
BasePrimaryDataStoreLifeCy
@Override
public boolean cancelMaintain(DataStore store) {
- Map<String,String> details = new HashMap<>();
- StoragePoolDetailVO systemIdDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
- if (systemIdDetail != null) {
- details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID,
systemIdDetail.getValue());
+ Map<String, String> details = new HashMap<>();
+ StoragePoolVO storagePoolVO =
primaryDataStoreDao.findById(store.getId());
+ if (storagePoolVO != null) {
sdcManager = ComponentContext.inject(sdcManager);
- if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) {
- StoragePoolVO storagePoolVO =
primaryDataStoreDao.findById(store.getId());
- if (storagePoolVO != null) {
+ sdcManager.populateSdcSettings(details,
storagePoolVO.getDataCenterId());
+ StoragePoolDetailVO systemIdDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
+ if (systemIdDetail != null) {
+ details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID,
systemIdDetail.getValue());
+ if (sdcManager.areSDCConnectionsWithinLimit(store.getId())) {
details.put(ScaleIOSDCManager.ConnectOnDemand.key(),
String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(storagePoolVO.getDataCenterId())));
StoragePoolDetailVO mdmsDetail =
storagePoolDetailsDao.findDetail(store.getId(),
ScaleIOGatewayClient.STORAGE_POOL_MDMS);
if (mdmsDetail != null) {
diff --git
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
index b02732051b5..8529d5dac73 100644
---
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
+++
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManager.java
@@ -22,6 +22,8 @@ import org.apache.cloudstack.framework.config.ConfigKey;
import com.cloud.host.Host;
+import java.util.Map;
+
public interface ScaleIOSDCManager {
ConfigKey<Boolean> ConnectOnDemand = new ConfigKey<>("Storage",
Boolean.class,
@@ -32,6 +34,34 @@ public interface ScaleIOSDCManager {
Boolean.TRUE,
ConfigKey.Scope.Zone);
+ /**
+ * Timeout for Host to wait after MDM changes made on Host until changes
will be applied.
+ * Needed to avoid cases when Storage Pool is not connected yet, but Agent
already starts to use Storage Pool.
+ */
+ ConfigKey<Integer> MdmsChangeApplyWaitTime = new ConfigKey<>("Storage",
+ Integer.class,
+ "powerflex.mdm.change.apply.wait",
+ "3000",
+ "Time (in ms) to wait after MDM addition, and before & after MDM
removal changes made on the Host, default value: 3000 ms",
+ Boolean.TRUE,
+ ConfigKey.Scope.Zone);
+
+ ConfigKey<Boolean> ValidateMdmsOnConnect = new ConfigKey<>("Storage",
+ Boolean.class,
+ "powerflex.mdm.validate.on.connect",
+ Boolean.FALSE.toString(),
+ "Flag to validate PowerFlex MDMs on the host, present in
Configuration File and in CLI during storage pool registration in agent,
default value: false",
+ Boolean.TRUE,
+ ConfigKey.Scope.Zone);
+
+ ConfigKey<Boolean> BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached =
new ConfigKey<>("Storage",
+ Boolean.class,
+ "powerflex.block.sdc.unprepare",
+ Boolean.FALSE.toString(),
+ "Block storage client un-preparation if SDC service restart needed
after PowerFlex MDM removal (i.e. no support for --remove_mdm in drv_cfg cmd)
when there are Volumes mapped to the Host",
+ Boolean.TRUE,
+ ConfigKey.Scope.Zone);
+
/**
* Checks SDC connections limit.
* @param storagePoolId the storage pool id
@@ -93,4 +123,11 @@ public interface ScaleIOSDCManager {
* @return Comma-separated list of MDM IPs of the pool
*/
String getMdms(long poolId);
+
+ /**
+ * Adds the SDC settings to the details map.
+ * @param details the details map to add the settings
+ * @param dataCenterId the datacenter id for the settings
+ */
+ void populateSdcSettings(Map<String, String> details, long dataCenterId);
}
diff --git
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
index 79f82a1e0df..c13ad61a6cd 100644
---
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
+++
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
@@ -183,12 +183,13 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
storagePoolHost.setLocalPath(sdcId);
storagePoolHostDao.update(storagePoolHost.getId(),
storagePoolHost);
}
- }
- int waitTimeInSecs = 15; // Wait for 15 secs (usual tests with SDC
service start took 10-15 secs)
- if (isHostSdcConnected(sdcId, dataStore, waitTimeInSecs)) {
- return sdcId;
+ int waitTimeInSecs = 15; // Wait for 15 secs (usual tests with
SDC service start took 10-15 secs)
+ if (isHostSdcConnected(sdcId, dataStore, waitTimeInSecs)) {
+ return sdcId;
+ }
}
+
return null;
} finally {
if (storageSystemIdLock != null) {
@@ -204,9 +205,10 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
private String prepareSDCOnHost(Host host, DataStore dataStore, String
systemId, String mdms) {
logger.debug("Preparing SDC on the host {}", host);
- Map<String,String> details = new HashMap<>();
+ Map<String, String> details = new HashMap<>();
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms);
+ populateSdcSettings(details, host.getDataCenterId());
PrepareStorageClientCommand cmd = new
PrepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(),
dataStore.getUuid(), details);
int timeoutSeconds = 60;
cmd.setWait(timeoutSeconds);
@@ -247,7 +249,7 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
}
if (StringUtils.isBlank(sdcId)) {
- logger.warn("Couldn't retrieve PowerFlex storage SDC details from
the host: {}, try (re)install SDC and restart agent", host);
+ logger.warn("Couldn't retrieve PowerFlex storage SDC details from
the host: {}, add MDMs if On-demand connect disabled or try (re)install SDC &
restart agent", host);
return null;
}
@@ -297,8 +299,7 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
}
if (!canUnprepareSDC(host, dataStore)) {
- logger.debug("Cannot unprepare SDC, there might be other
connected pools of same PowerFlex storage cluster," +
- "or some volumes mapped to the SDC that belongs to any
of the storage pools of the PowerFlex storage cluster");
+ logger.debug("Cannot unprepare SDC, there are other pools of
the same PowerFlex storage cluster with some volumes mapped to the host SDC");
return false;
}
@@ -324,6 +325,7 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
logger.debug(String.format("Unpreparing SDC on the host %s (%s)",
host.getId(), host.getName()));
Map<String,String> details = new HashMap<>();
details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, mdms);
+ populateSdcSettings(details, host.getDataCenterId());
UnprepareStorageClientCommand cmd = new
UnprepareStorageClientCommand(((PrimaryDataStore) dataStore).getPoolType(),
dataStore.getUuid(), details);
int timeoutSeconds = 60;
cmd.setWait(timeoutSeconds);
@@ -359,18 +361,28 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
return false;
}
- List<StoragePoolHostVO> poolHostVOsBySdc =
storagePoolHostDao.findByLocalPath(sdcId);
- if (CollectionUtils.isNotEmpty(poolHostVOsBySdc) &&
poolHostVOsBySdc.size() > 1) {
- logger.debug(String.format("There are other connected pools with
the same SDC of the host %s, shouldn't unprepare SDC", host));
+ try {
+ if (logger.isDebugEnabled()) {
+ List<StoragePoolHostVO> poolHostVOsBySdc =
storagePoolHostDao.findByLocalPath(sdcId);
+ if (CollectionUtils.isNotEmpty(poolHostVOsBySdc) &&
poolHostVOsBySdc.size() > 1) {
+ logger.debug(String.format("There are other connected
pools with the same SDC of the host %s", host));
+ }
+ }
+
+ return !areVolumesMappedToPoolSdc(dataStore.getId(), sdcId);
+ } catch (Exception e) {
+ logger.warn("Unable to check whether the SDC of the pool: " +
dataStore.getId() + " can be unprepared on the host: " + host.getId() + ", due
to " + e.getMessage(), e);
return false;
}
+ }
+ private boolean areVolumesMappedToPoolSdc(long storagePoolId, String
sdcId) throws Exception {
try {
- final ScaleIOGatewayClient client =
getScaleIOClient(dataStore.getId());
- return client.listVolumesMappedToSdc(sdcId).isEmpty();
+ final ScaleIOGatewayClient client =
getScaleIOClient(storagePoolId);
+ return
CollectionUtils.isNotEmpty(client.listVolumesMappedToSdc(sdcId));
} catch (Exception e) {
- logger.warn("Unable to check whether the SDC of the pool: " +
dataStore.getId() + " can be unprepared on the host: " + host.getId() + ", due
to " + e.getMessage(), e);
- return false;
+ logger.warn("Unable to check the volumes mapped to SDC of the
pool: " + storagePoolId + ", due to " + e.getMessage());
+ throw e;
}
}
@@ -467,9 +479,23 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
private ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId)
throws Exception {
StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId);
+ if (storagePool == null) {
+ throw new CloudRuntimeException("Unable to find the storage pool
with id " + storagePoolId);
+ }
return
ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePool,
storagePoolDetailsDao);
}
+ @Override
+ public void populateSdcSettings(Map<String, String> details, long
dataCenterId) {
+ if (details == null) {
+ details = new HashMap<>();
+ }
+
+ details.put(ScaleIOSDCManager.MdmsChangeApplyWaitTime.key(),
String.valueOf(ScaleIOSDCManager.MdmsChangeApplyWaitTime.valueIn(dataCenterId)));
+ details.put(ScaleIOSDCManager.ValidateMdmsOnConnect.key(),
String.valueOf(ScaleIOSDCManager.ValidateMdmsOnConnect.valueIn(dataCenterId)));
+
details.put(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.key(),
String.valueOf(ScaleIOSDCManager.BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached.valueIn(dataCenterId)));
+ }
+
@Override
public String getConfigComponentName() {
return ScaleIOSDCManager.class.getSimpleName();
@@ -477,6 +503,6 @@ public class ScaleIOSDCManagerImpl implements
ScaleIOSDCManager, Configurable {
@Override
public ConfigKey<?>[] getConfigKeys() {
- return new ConfigKey[]{ConnectOnDemand};
+ return new ConfigKey[]{ConnectOnDemand, MdmsChangeApplyWaitTime,
ValidateMdmsOnConnect, BlockSdcUnprepareIfRestartNeededAndVolumesAreAttached};
}
}
diff --git
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
index 0db329204e8..f169b581b2c 100644
---
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
+++
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java
@@ -27,7 +27,6 @@ import
org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import
org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.manager.ScaleIOSDCManager;
@@ -59,7 +58,6 @@ public class ScaleIOHostListener implements
HypervisorHostListener {
@Inject private DataStoreManager _dataStoreMgr;
@Inject private HostDao _hostDao;
@Inject private StoragePoolHostDao _storagePoolHostDao;
- @Inject private PrimaryDataStoreDao _primaryDataStoreDao;
@Inject private StoragePoolDetailsDao _storagePoolDetailsDao;
private ScaleIOSDCManager _sdcManager = new ScaleIOSDCManagerImpl();
@@ -109,9 +107,10 @@ public class ScaleIOHostListener implements
HypervisorHostListener {
if (systemId == null) {
throw new CloudRuntimeException("Failed to get the system id for
PowerFlex storage pool " + storagePool.getName());
}
- Map<String,String> details = new HashMap<>();
+ Map<String, String> details = new HashMap<>();
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
_sdcManager = ComponentContext.inject(_sdcManager);
+ _sdcManager.populateSdcSettings(details, host.getDataCenterId());
if (_sdcManager.areSDCConnectionsWithinLimit(poolId)) {
details.put(ScaleIOSDCManager.ConnectOnDemand.key(),
String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
String mdms = _sdcManager.getMdms(poolId);
@@ -120,7 +119,7 @@ public class ScaleIOHostListener implements
HypervisorHostListener {
ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true,
storagePool, storagePool.getPath(), details);
ModifyStoragePoolAnswer answer = sendModifyStoragePoolCommand(cmd,
storagePool, host);
- Map<String,String> poolDetails = answer.getPoolInfo().getDetails();
+ Map<String, String> poolDetails = answer.getPoolInfo().getDetails();
if (MapUtils.isEmpty(poolDetails)) {
String msg = String.format("PowerFlex storage SDC details not
found on the host: %s, (re)install SDC and restart agent", host);
logger.warn(msg);
@@ -137,7 +136,7 @@ public class ScaleIOHostListener implements
HypervisorHostListener {
}
if (StringUtils.isBlank(sdcId)) {
- String msg = String.format("Couldn't retrieve PowerFlex storage
SDC details from the host: %s, (re)install SDC and restart agent", host);
+ String msg = String.format("Couldn't retrieve PowerFlex storage
SDC details from the host: %s, add MDMs if On-demand connect disabled or try
(re)install SDC & restart agent", host);
logger.warn(msg);
_alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST,
host.getDataCenterId(), host.getPodId(), "SDC details not found on host: " +
host.getUuid(), msg);
return null;
@@ -201,9 +200,10 @@ public class ScaleIOHostListener implements
HypervisorHostListener {
if (systemId == null) {
throw new CloudRuntimeException("Failed to get the system id for
PowerFlex storage pool " + storagePool.getName());
}
- Map<String,String> details = new HashMap<>();
+ Map<String, String> details = new HashMap<>();
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
_sdcManager = ComponentContext.inject(_sdcManager);
+ _sdcManager.populateSdcSettings(details, host.getDataCenterId());
if (_sdcManager.canUnprepareSDC(host, dataStore)) {
details.put(ScaleIOSDCManager.ConnectOnDemand.key(),
String.valueOf(ScaleIOSDCManager.ConnectOnDemand.valueIn(host.getDataCenterId())));
String mdms = _sdcManager.getMdms(poolId);
diff --git
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
index a8c02a90bf9..70ea6f429e7 100644
---
a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
+++
b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
@@ -17,16 +17,27 @@
package org.apache.cloudstack.storage.datastore.util;
-import java.util.List;
-
-import org.apache.commons.collections.CollectionUtils;
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
+import com.cloud.utils.Pair;
import com.cloud.utils.UuidUtils;
import com.cloud.utils.script.Script;
import org.apache.commons.lang3.StringUtils;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
public class ScaleIOUtil {
protected static Logger LOGGER = LogManager.getLogger(ScaleIOUtil.class);
@@ -39,7 +50,7 @@ public class ScaleIOUtil {
public static final String VMSNAPSHOT_PREFIX = "vmsnap";
public static final int IDENTIFIER_LENGTH = 16;
- public static final Long MINIMUM_ALLOWED_IOPS_LIMIT = Long.valueOf(10);
+ public static final Long MINIMUM_ALLOWED_IOPS_LIMIT = 10L;
public static final String DISK_PATH = "/dev/disk/by-id";
public static final String DISK_NAME_PREFIX = "emc-vol-";
@@ -63,9 +74,17 @@ public class ScaleIOUtil {
private static final String SDC_SERVICE_ENABLE_CMD = "systemctl enable
scini";
public static final String CONNECTED_SDC_COUNT_STAT = "ConnectedSDCCount";
+
+ /**
+ * Time (in seconds) to wait after SDC service 'scini'
start/restart/stop.<br>
+ * Data type: Integer.<br>
+ * Default value: <code>3</code>
+ */
+ public static final AgentProperties.Property<Integer>
SDC_SERVICE_ACTION_WAIT = new
AgentProperties.Property<>("powerflex.sdc.service.wait", 3);
+
/**
* Cmd for querying volumes in SDC
- * Sample output for cmd: drv_cfg --query_vols:
+ * Sample output for cmd {@code drv_cfg --query_vols}:
* Retrieved 2 volume(s)
* VOL-ID 6c33633100000009 MDM-ID 218ce1797566a00f
* VOL-ID 6c3362a30000000a MDM-ID 218ce1797566a00f
@@ -74,14 +93,14 @@ public class ScaleIOUtil {
/**
* Cmd for querying guid in SDC
- * Sample output for cmd: drv_cfg --query_guid:
+ * Sample output for cmd {@code drv_cfg --query_guid}:
* B0E3BFB8-C20B-43BF-93C8-13339E85AA50
*/
private static final String QUERY_GUID_CMD = "drv_cfg --query_guid";
/**
* Cmd for querying MDMs in SDC
- * Sample output for cmd: drv_cfg --query_mdms:
+ * Sample output for cmd {@code drv_cfg --query_mdms}:
* Retrieved 2 mdm(s)
* MDM-ID 3ef46cbf2aaf5d0f SDC ID 6b18479c00000003 INSTALLATION ID
68ab55462cbb3ae4 IPs [0]-x.x.x.x [1]-x.x.x.x
* MDM-ID 2e706b2740ec200f SDC ID 301b852c00000003 INSTALLATION ID
33f8662e7a5c1e6c IPs [0]-x.x.x.x [1]-x.x.x.x
@@ -89,61 +108,251 @@ public class ScaleIOUtil {
private static final String QUERY_MDMS_CMD = "drv_cfg --query_mdms";
private static final String ADD_MDMS_CMD = "drv_cfg --add_mdm";
+
+ private static final String REMOVE_MDM_PARAMETER = "--remove_mdm";
+
+ /**
+ * Calls the kernel module to remove MDM.
+ */
+ private static final String REMOVE_MDMS_CMD = "drv_cfg " +
REMOVE_MDM_PARAMETER;
+
+ /**
+ * Command to get back "Usage" response. As of now it is just drv_cfg
without parameters.
+ */
+ private static final String USAGE_CMD = "drv_cfg";
+
private static final String DRV_CFG_FILE = "/etc/emc/scaleio/drv_cfg.txt";
- public static void addMdms(List<String> mdmAddresses) {
- if (CollectionUtils.isEmpty(mdmAddresses)) {
- return;
+ /**
+ * Sample Command - sed -i '/x.x.x.x\,/d' /etc/emc/scaleio/drv_cfg.txt
+ */
+ private static final String REMOVE_MDM_CMD_TEMPLATE = "sed -i '/%s\\,/d'
%s";
+
+ /**
+ * Patterns to parse {@link ScaleIOUtil#DRV_CFG_FILE} and {@code
--query_mdms} command output.
+ * The format is:
+ * MDM-ID {HEX_ID} SDC ID {HEX_ID} INSTALLATION ID {HEX_ID} IPs
[{DEC_INC_NUMBER}]-{IP_ADDRESS} [{DEC_INC_NUMBER}]-{IP_ADDRESS} ...
+ */
+ private static final Pattern NEW_LINE_PATTERN = Pattern.compile("\\r?\\n");
+ /**
+ * Pattern to find "IPs" substring in {@link ScaleIOUtil#QUERY_MDMS_CMD}
command output.
+ * The output format is:
+ * MDM-ID {HEX_ID} SDC ID {HEX_ID} INSTALLATION ID {HEX_ID} IPs
[{DEC_INC_NUMBER}]-{IP_ADDRESS} [{DEC_INC_NUMBER}]-{IP_ADDRESS} ...
+ */
+ private static final Pattern USAGE_IPS_LINE_PATTERN =
Pattern.compile("IPs\\s*(.*)$");
+ /**
+ * Pattern to find individual IP address in {@link
ScaleIOUtil#QUERY_MDMS_CMD} command output.
+ */
+ private static final Pattern USAGE_IP_TOKEN_PATTERN =
Pattern.compile("(\\s*\\[\\d\\]\\-)([^\\s$]+)");
+
+ /**
+ * Pattern to find Volume ID in {@link ScaleIOUtil#QUERY_VOLUMES_CMD}
+ */
+ private static final Pattern VOLUME_ID_TOKEN_PATTERN =
Pattern.compile("VOL-ID\\s*([^\\s$]+)");
+ /**
+ * Pattern to find MDM entries line in {@link ScaleIOUtil#DRV_CFG_FILE}.
+ */
+ private static final Pattern DRV_CFG_MDM_LINE_PATTERN =
Pattern.compile("^mdm\\s*([0-9A-F:\\.,]+)$", Pattern.CASE_INSENSITIVE);
+ /**
+ * Pattern to split comma separated string of IP addresses (space aware).
+ */
+ private static final Pattern DRV_CFG_MDM_IPS_PATTERN =
Pattern.compile("\\s*,\\s*");
+
+ public static boolean addMdms(String... mdmAddresses) {
+ if (mdmAddresses.length < 1) {
+ return false;
}
// Sample Cmd - /opt/emc/scaleio/sdc/bin/drv_cfg --add_mdm --ip
x.x.x.x,x.x.x.x --file /etc/emc/scaleio/drv_cfg.txt
- String addMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.ADD_MDMS_CMD;
- addMdmsCmd += " --ip " + String.join(",", mdmAddresses);
- addMdmsCmd += " --file " + DRV_CFG_FILE;
- String result = Script.runSimpleBashScript(addMdmsCmd);
- if (result == null) {
- LOGGER.warn("Failed to add mdms");
+ String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.ADD_MDMS_CMD;
+ command += " --ip " + String.join(",", mdmAddresses);
+ command += " --file " + DRV_CFG_FILE;
+ return runCmd(command);
+ }
+
+ /**
+ * Remove MDM via ScaleIO via CLI.
+ *
+ * @param mdmAddress MDM address to remove
+ * @return true if IP address successfully removed
+ */
+ private static boolean removeMdm(String mdmAddress) {
+ // Sample Cmd - /opt/emc/scaleio/sdc/bin/drv_cfg --remove_mdm --ip
x.x.x.x,x.x.x.x --file /etc/emc/scaleio/drv_cfg.txt
+ String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.REMOVE_MDMS_CMD;
+ command += " --ip " + String.join(",", mdmAddress);
+ command += " --file " + DRV_CFG_FILE;
+ return runCmd(command);
+ }
+
+ /**
+ * Run command, log command result and return {@link Boolean#TRUE} if
command succeeded.
+ * FIXME: may need to do refactoring and replace static method calls with
dynamic.
+ */
+ private static boolean runCmd(String command) {
+ Pair<String, String> result = Script.executeCommand(command);
+ String stdOut = result.first();
+ String stdErr = result.second();
+ boolean succeeded = StringUtils.isEmpty(stdErr);
+ if (succeeded) {
+ LOGGER.debug(String.format("Successfully executed command '%s':
%s", command, stdOut));
+ } else {
+ LOGGER.warn(String.format("Failed to execute command '%s': %s",
command, stdErr));
}
+ return succeeded;
}
- public static void removeMdms(List<String> mdmAddresses) {
- if (CollectionUtils.isEmpty(mdmAddresses)) {
- return;
+ /**
+ * Remove MDMs either via ScaleIO CLI (if supported) or by updating
configuration file.
+ *
+ * @param mdmAddresses MDM addresses
+ * @return returns {@link Boolean#TRUE} if changes were applied to the
configuration
+ */
+ public static boolean removeMdms(String... mdmAddresses) {
+ if (mdmAddresses.length < 1) {
+ return false;
}
- // (i) Remove MDMs from config file (ii) Restart scini
- // Sample Cmd - sed -i '/x.x.x.x\,/d' /etc/emc/scaleio/drv_cfg.txt
+
+ boolean changesApplied = false;
+ boolean removeMdmCliSupported = isRemoveMdmCliSupported();
boolean restartSDC = false;
- String removeMdmsCmdFormat = "sed -i '/%s\\,/d' %s";
for (String mdmAddress : mdmAddresses) {
- if (mdmAdded(mdmAddress)) {
- restartSDC = true;
+ // continue to next address if current MDM is not present in
configuration
+ if (!isMdmPresent(mdmAddress)) {
+ continue;
+ }
+ // remove MDM via CLI if it is supported
+ if (removeMdmCliSupported) {
+ if (removeMdm(mdmAddress)) {
+ changesApplied = true;
+ }
+ } else {
+ String command = String.format(REMOVE_MDM_CMD_TEMPLATE,
mdmAddress, DRV_CFG_FILE);
+ String stdErr = Script.executeCommand(command).second();
+ if(StringUtils.isEmpty(stdErr)) {
+ // restart SDC needed only if configuration file modified
manually (not by CLI)
+ restartSDC = true;
+ changesApplied = true;
+ } else {
+ LOGGER.error(String.format("Failed to remove MDM %s from
%s: %s", mdmAddress, DRV_CFG_FILE, stdErr));
+ }
}
- String removeMdmsCmd = String.format(removeMdmsCmdFormat,
mdmAddress, DRV_CFG_FILE);
- Script.runSimpleBashScript(removeMdmsCmd);
}
if (restartSDC) {
restartSDCService();
}
+ return changesApplied;
}
- public static boolean mdmAdded(String mdmAddress) {
+ /**
+ * Returns MDM entries from {@link ScaleIOUtil#DRV_CFG_FILE}.
+ */
+ public static Collection<String> getMdmsFromConfigFile() {
+ List<String> configFileLines;
+ try {
+ configFileLines = Files.readAllLines(Path.of(DRV_CFG_FILE));
+ } catch (IOException e) {
+ LOGGER.error(String.format("Failed to read MDMs from %s",
DRV_CFG_FILE), e);
+ return List.of();
+ }
+ Set<String> mdms = new LinkedHashSet<>();
+ for (String line : configFileLines) {
+ Matcher mdmLineMatcher = DRV_CFG_MDM_LINE_PATTERN.matcher(line);
+ if(mdmLineMatcher.find() && mdmLineMatcher.groupCount() > 0) {
+ String mdmLine = mdmLineMatcher.group(1);
+ String[] mdmValues = DRV_CFG_MDM_IPS_PATTERN.split(mdmLine);
+ mdms.addAll(Arrays.asList(mdmValues));
+ }
+ }
+ return mdms;
+ }
+ /**
+ * Returns Volume Ids from {@link ScaleIOUtil#DRV_CFG_FILE}.
+ */
+ public static Collection<String> getVolumeIds() {
+ String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.QUERY_VOLUMES_CMD;
+ Pair<String, String> result = Script.executeCommand(command);
+ String stdOut = result.first();
+
+ Set<String> volumeIds = new LinkedHashSet<>();
+ String[] stdOutLines = NEW_LINE_PATTERN.split(stdOut);
+ for (String line : stdOutLines) {
+ Matcher volumeIdMatcher = VOLUME_ID_TOKEN_PATTERN.matcher(line);
+ if (volumeIdMatcher.find() && volumeIdMatcher.groupCount() > 0) {
+ volumeIds.add(volumeIdMatcher.group(1));
+ }
+ }
+ return volumeIds;
+ }
+
+ /**
+ * Returns MDM entries from CLI Cmd using {@code --query_mdms}.
+ */
+ public static Collection<String> getMdmsFromCliCmd() {
+ String command = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.QUERY_MDMS_CMD;
+ Pair<String, String> result = Script.executeCommand(command);
+ String stdOut = result.first();
+
+ Set<String> mdms = new LinkedHashSet<>();
+ String[] stdOutLines = NEW_LINE_PATTERN.split(stdOut);
+ for (String line : stdOutLines) {
+ Matcher ipsLineMatcher = USAGE_IPS_LINE_PATTERN.matcher(line);
+ if (ipsLineMatcher.find() && ipsLineMatcher.groupCount() > 0) {
+ String ipToken = ipsLineMatcher.group(1);
+ Matcher ipMatcher = USAGE_IP_TOKEN_PATTERN.matcher(ipToken);
+ while (ipMatcher.find()) {
+ if (ipMatcher.groupCount() > 1) {
+ mdms.add(ipMatcher.group(2));
+ }
+ }
+ }
+ }
+ return mdms;
+ }
+
+ /**
+ * Returns {@link Boolean#TRUE} if ScaleIO CLI tool (drv_cfg) supports
MDMs removal.
+ */
+ public static boolean isRemoveMdmCliSupported() {
+ /*
+ * New version of drv_cfg supports remove mdm API.
+ * Instead of defining supported version and checking it, the logic is
to check drv_cfg "Usage" output
+ * and see whether remove_mdm command supported.
+ * The "Usage" returned if tool executed without parameters or with
invalid parameters.
+ */
+ String command = SDC_HOME_PATH + "/bin/" + USAGE_CMD;
+
+ Pair<String, String> result = Script.executeCommand(command);
+ String stdOut = result.first();
+ String stdErr = result.second();
+
+ /*
+ * Check whether stderr or stdout contains mdm removal "--remove_mdm"
parameter.
+ *
+ * Current version returns "Usage" in stderr, check stdout as well in
case this will be changed in the future,
+ * as returned "Usage" is not an error.
+ */
+ return (stdOut + stdErr).toLowerCase().contains(REMOVE_MDM_PARAMETER);
+ }
+
+ /**
+ * Returns true if provided MDM address is present in configuration.
+ */
+ public static boolean isMdmPresent(String mdmAddress) {
//query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id>
INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with
ID: <MDM-Id>
String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.QUERY_MDMS_CMD;
queryMdmsCmd += "|grep " + mdmAddress;
String result = Script.runSimpleBashScript(queryMdmsCmd);
- if (StringUtils.isNotBlank(result) && result.contains(mdmAddress)) {
- return true;
- }
- return false;
+
+ return StringUtils.isNotBlank(result) && result.contains(mdmAddress);
}
public static String getSdcHomePath() {
- String sdcHomePath = DEFAULT_SDC_HOME_PATH;
String sdcHomePropertyCmdFormat = "sed -n '/%s/p' '%s' 2>/dev/null |
sed 's/%s=//g' 2>/dev/null";
String sdcHomeCmd = String.format(sdcHomePropertyCmdFormat,
SDC_HOME_PARAMETER, AGENT_PROPERTIES_FILE, SDC_HOME_PARAMETER);
-
String result = Script.runSimpleBashScript(sdcHomeCmd);
+ String sdcHomePath;
if (result == null) {
- LOGGER.warn("Failed to get sdc home path from agent.properties,
fallback to default path");
+ sdcHomePath = DEFAULT_SDC_HOME_PATH;
+ LOGGER.warn(String.format("Failed to get sdc home path from
agent.properties, fallback to default path %s", sdcHomePath));
} else {
sdcHomePath = result;
}
@@ -151,7 +360,7 @@ public class ScaleIOUtil {
return sdcHomePath;
}
- public static final void rescanForNewVolumes() {
+ public static void rescanForNewVolumes() {
// Detecting new volumes
String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.RESCAN_CMD;
@@ -161,7 +370,7 @@ public class ScaleIOUtil {
}
}
- public static final String getSystemIdForVolume(String volumeId) {
+ public static String getSystemIdForVolume(String volumeId) {
//query_vols outputs "VOL-ID <VolumeID> MDM-ID <SystemID>" for a
volume with ID: <VolumeID>
String queryDiskCmd = SDC_HOME_PATH + "/bin/" +
ScaleIOUtil.QUERY_VOLUMES_CMD;
queryDiskCmd += "|grep " + volumeId + "|awk '{print $4}'";
@@ -225,7 +434,7 @@ public class ScaleIOUtil {
return result;
}
- public static final String getVolumePath(String volumePathWithName) {
+ public static String getVolumePath(String volumePathWithName) {
if (StringUtils.isEmpty(volumePathWithName)) {
return volumePathWithName;
}
@@ -237,7 +446,7 @@ public class ScaleIOUtil {
return volumePathWithName;
}
- public static final String updatedPathWithVolumeName(String volumePath,
String volumeName) {
+ public static String updatedPathWithVolumeName(String volumePath, String
volumeName) {
if (StringUtils.isAnyEmpty(volumePath, volumeName)) {
return volumePath;
}
@@ -267,16 +476,76 @@ public class ScaleIOUtil {
public static boolean startSDCService() {
int exitValue =
Script.runSimpleBashScriptForExitValue(SDC_SERVICE_START_CMD);
- return exitValue == 0;
+ if (exitValue != 0) {
+ return false;
+ }
+ waitForSdcServiceActionToComplete();
+ return true;
}
public static boolean stopSDCService() {
int exitValue =
Script.runSimpleBashScriptForExitValue(SDC_SERVICE_STOP_CMD);
- return exitValue == 0;
+ if (exitValue != 0) {
+ return false;
+ }
+ waitForSdcServiceActionToComplete();
+ return true;
}
public static boolean restartSDCService() {
int exitValue =
Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD);
- return exitValue == 0;
+ if (exitValue != 0) {
+ return false;
+ }
+ waitForSdcServiceActionToComplete();
+ return true;
+ }
+
+ private static void waitForSdcServiceActionToComplete() {
+ // Wait for the SDC service to settle after start/restart/stop and
reaches a stable state
+ int waitTimeInSecs =
AgentPropertiesFileHandler.getPropertyValue(SDC_SERVICE_ACTION_WAIT);
+ if (waitTimeInSecs < 0) {
+ waitTimeInSecs = SDC_SERVICE_ACTION_WAIT.getDefaultValue();
+ }
+ try {
+ LOGGER.debug(String.format("Waiting for %d secs after SDC service
action, to reach a stable state", waitTimeInSecs));
+ Thread.sleep(waitTimeInSecs * 1000L);
+ } catch (InterruptedException ignore) {
+ }
+ }
+
+ /**
+ * Represents {@link ScaleIOUtil#DRV_CFG_FILE} MDM entry (SDC and
Installation Ids are skipped).
+ */
+ public static class MdmEntry {
+ private String mdmId;
+ private Collection<String> ips;
+
+ /**
+ * MDM entry constructor.
+ *
+ * @param mdmId MDM Id
+ * @param ips IP Addresses
+ */
+ public MdmEntry(String mdmId, Collection<String> ips) {
+ this.mdmId = mdmId;
+ this.ips = ips;
+ }
+
+ public String getMdmId() {
+ return mdmId;
+ }
+
+ public void setMdmId(String mdmId) {
+ this.mdmId = mdmId;
+ }
+
+ public Collection<String> getIps() {
+ return ips;
+ }
+
+ public void setIps(Collection<String> ips) {
+ this.ips = ips;
+ }
}
}
diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java
b/utils/src/main/java/com/cloud/utils/script/Script.java
index a1104b37c27..7ba2bb1b4d9 100644
--- a/utils/src/main/java/com/cloud/utils/script/Script.java
+++ b/utils/src/main/java/com/cloud/utils/script/Script.java
@@ -671,6 +671,26 @@ public class Script implements Callable<String> {
return runScript(getScriptForCommandRun(command));
}
+ /**
+ * Execute command and return standard output and standard error.
+ *
+ * @param command OS command to be executed
+ * @return {@link Pair} with standard output as first and standard error
as second field
+ */
+ public static Pair<String, String> executeCommand(String command) {
+ // wrap command into bash
+ Script script = new Script("/bin/bash");
+ script.add("-c");
+ script.add(command);
+
+ // parse all lines from the output
+ OutputInterpreter.AllLinesParser parser = new
OutputInterpreter.AllLinesParser();
+ String stdErr = script.execute(parser);
+ String stdOut = parser.getLines();
+ LOGGER.debug(String.format("Command [%s] result - stdout: [%s],
stderr: [%s]", command, stdOut, stdErr));
+ return new Pair<>(stdOut, stdErr);
+ }
+
public static int executeCommandForExitValue(long timeout, String...
command) {
return runScriptForExitValue(getScriptForCommandRun(timeout, command));
}
