This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch 4.19
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit c779b1c616471b8fea833c4b090a7f3bc2dc6ab7
Merge: 91c7bc722f2 2339412f734
Author: Daan Hoogland <d...@onecht.net>
AuthorDate: Thu Jun 6 11:24:09 2024 +0200
Merge branch '4.18' into 4.19
.../kvm/storage/LinstorStorageAdaptor.java | 24 +-
.../driver/LinstorPrimaryDataStoreDriverImpl.java | 30 +-
.../storage/datastore/util/LinstorUtil.java | 33 +
.../com/cloud/sample/UserCloudAPIExecutor.java | 188 --
.../com/cloud/test/longrun/BuildGuestNetwork.java | 123 --
.../java/com/cloud/test/longrun/GuestNetwork.java | 107 -
.../com/cloud/test/longrun/PerformanceWithAPI.java | 190 --
.../main/java/com/cloud/test/longrun/User.java | 202 --
.../com/cloud/test/longrun/VirtualMachine.java | 95 -
.../java/com/cloud/test/regression/ApiCommand.java | 848 --------
.../java/com/cloud/test/regression/ConfigTest.java | 125 --
.../cloud/test/regression/DelegatedAdminTest.java | 129 --
.../java/com/cloud/test/regression/Deploy.java | 109 -
.../com/cloud/test/regression/EventsApiTest.java | 176 --
.../main/java/com/cloud/test/regression/HA.java | 80 -
.../cloud/test/regression/LoadBalancingTest.java | 142 --
.../cloud/test/regression/PortForwardingTest.java | 143 --
.../java/com/cloud/test/regression/SanityTest.java | 86 -
.../main/java/com/cloud/test/regression/Test.java | 88 -
.../com/cloud/test/regression/TestCaseEngine.java | 275 ---
.../java/com/cloud/test/regression/VMApiTest.java | 91 -
.../main/java/com/cloud/test/stress/SshTest.java | 90 -
.../cloud/test/stress/StressTestDirectAttach.java | 1353 ------------
.../com/cloud/test/stress/TestClientWithAPI.java | 2289 --------------------
.../main/java/com/cloud/test/stress/WgetTest.java | 150 --
.../cloud/test/ui/AbstractSeleniumTestCase.java | 55 -
.../java/com/cloud/test/ui/AddAndDeleteAISO.java | 127 --
.../com/cloud/test/ui/AddAndDeleteATemplate.java | 126 --
.../java/com/cloud/test/ui/UIScenarioTest.java | 86 -
.../java/com/cloud/test/utils/ConsoleProxy.java | 110 -
.../java/com/cloud/test/utils/IpSqlGenerator.java | 89 -
.../java/com/cloud/test/utils/ProxyLoadTemp.java | 112 -
.../main/java/com/cloud/test/utils/SignEC2.java | 143 --
.../java/com/cloud/test/utils/SignRequest.java | 112 -
.../com/cloud/test/utils/SqlDataGenerator.java | 49 -
.../main/java/com/cloud/test/utils/SubmitCert.java | 198 --
.../main/java/com/cloud/test/utils/TestClient.java | 385 ----
.../java/com/cloud/test/utils/UtilsForTest.java | 210 --
ui/public/locales/en.json | 2 +-
ui/src/components/page/GlobalFooter.vue | 2 +-
40 files changed, 46 insertions(+), 8926 deletions(-)
diff --cc
plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
index 9612781ee4c,8b9b768d2a4..bc5b9b6a0e2
---
a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
+++
b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java
@@@ -28,8 -28,6 +28,7 @@@ import com.linbit.linstor.api.model.Res
import com.linbit.linstor.api.model.ResourceDefinitionCreate;
import com.linbit.linstor.api.model.ResourceDefinitionModify;
import com.linbit.linstor.api.model.ResourceGroupSpawn;
+import com.linbit.linstor.api.model.ResourceMakeAvailable;
- import com.linbit.linstor.api.model.ResourceWithVolumes;
import com.linbit.linstor.api.model.Snapshot;
import com.linbit.linstor.api.model.SnapshotRestore;
import com.linbit.linstor.api.model.VolumeDefinition;
@@@ -441,32 -393,13 +421,32 @@@ public class LinstorPrimaryDataStoreDri
try
{
s_logger.info("Linstor: Spawn resource " + rscName);
- ApiCallRcList answers = linstorApi.resourceGroupSpawn(rscGrp,
rscGrpSpawn);
+ ApiCallRcList answers = api.resourceGroupSpawn(rscGrp,
rscGrpSpawn);
checkLinstorAnswersThrow(answers);
- applyAuxProps(linstorApi, rscName, vol.getName(),
vol.getAttachedVmName());
+ applyAuxProps(api, rscName, volName, vmName);
+
- return getDeviceName(api, rscName);
++ return LinstorUtil.getDevicePath(api, rscName);
+ } catch (ApiException apiEx)
+ {
+ s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
+ throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+ }
+ }
+
+ private String createResource(VolumeInfo vol, StoragePoolVO
storagePoolVO) {
+ DevelopersApi linstorApi =
LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
+ final String rscGrp = getRscGrp(storagePoolVO);
+
+ final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid();
+ String deviceName = createResourceBase(
+ rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(),
linstorApi, rscGrp);
+
+ try
+ {
applyQoSSettings(storagePoolVO, linstorApi, rscName,
vol.getMaxIops());
- return deviceName;
+ return LinstorUtil.getDevicePath(linstorApi, rscName);
} catch (ApiException apiEx)
{
s_logger.error("Linstor: ApiEx - " + apiEx.getMessage());
@@@ -833,267 -676,10 +813,267 @@@
}
@Override
- public void copyAsync(DataObject srcData, DataObject destData,
AsyncCompletionCallback<CopyCommandResult> callback)
+ public void copyAsync(DataObject srcData, DataObject dstData,
AsyncCompletionCallback<CopyCommandResult> callback)
{
- // as long as canCopy is false, this isn't called
- s_logger.debug("Linstor: copyAsync with srcdata: " +
srcData.getUuid());
+ s_logger.debug("LinstorPrimaryDataStoreDriverImpl.copyAsync: "
+ + srcData.getType() + " -> " + dstData.getType());
+
+ final CopyCommandResult res;
+ if (canCopySnapshotCond(srcData, dstData)) {
+ String errMsg = null;
+ Answer answer = copySnapshot(srcData, dstData);
+ if (answer != null && !answer.getResult()) {
+ errMsg = answer.getDetails();
+ } else {
+ // delete primary storage snapshot
+ SnapshotInfo sinfo = (SnapshotInfo) srcData;
+ VolumeInfo volume = sinfo.getBaseVolume();
+ deleteSnapshot(
+ srcData.getDataStore(),
+ LinstorUtil.RSC_PREFIX + volume.getUuid(),
+ LinstorUtil.RSC_PREFIX + sinfo.getUuid());
+ }
+ res = new CopyCommandResult(null, answer);
+ res.setResult(errMsg);
+ } else if (canCopyTemplateCond(srcData, dstData)) {
+ Answer answer = copyTemplate(srcData, dstData);
+ res = new CopyCommandResult(null, answer);
+ } else if (canCopyVolumeCond(srcData, dstData)) {
+ Answer answer = copyVolume(srcData, dstData);
+ res = new CopyCommandResult(null, answer);
+ } else {
+ Answer answer = new Answer(null, false, "noimpl");
+ res = new CopyCommandResult(null, answer);
+ res.setResult("Not implemented yet");
+ }
+ callback.complete(res);
+ }
+
+ /**
+ * Tries to get a Linstor cloudstack end point, that is at least diskless.
+ *
+ * @param api Linstor java api object
+ * @param rscName resource name to make available on node
+ * @return Optional RemoteHostEndPoint if one could get found.
+ * @throws ApiException
+ */
+ private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api,
String rscName) throws ApiException {
+ List<String> linstorNodeNames = LinstorUtil.getLinstorNodeNames(api);
+ Collections.shuffle(linstorNodeNames); // do not always pick the
first linstor node
+
+ Host host = null;
+ for (String nodeName : linstorNodeNames) {
+ host = _hostDao.findByName(nodeName);
+ if (host != null && host.getResourceState() ==
ResourceState.Enabled) {
+ s_logger.info(String.format("Linstor: Make resource %s
available on node %s ...", rscName, nodeName));
+ ApiCallRcList answers =
api.resourceMakeAvailableOnNode(rscName, nodeName, new ResourceMakeAvailable());
+ if (!answers.hasError()) {
+ break; // found working host
+ } else {
+ s_logger.error(
+ String.format("Linstor: Unable to make resource %s on
node %s available: %s",
+ rscName,
+ nodeName,
+ LinstorUtil.getBestErrorMessage(answers)));
+ }
+ }
+ }
+
+ if (host == null)
+ {
+ s_logger.error("Linstor: Couldn't create a resource on any
cloudstack host.");
+ return Optional.empty();
+ }
+ else
+ {
+ return
Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
+ }
+ }
+
+ private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api,
String rscName) throws ApiException {
+ List<com.linbit.linstor.api.model.StoragePool> linSPs =
LinstorUtil.getDiskfulStoragePools(api, rscName);
+ if (linSPs != null) {
+ for (com.linbit.linstor.api.model.StoragePool sp : linSPs) {
+ Host host = _hostDao.findByName(sp.getNodeName());
+ if (host != null && host.getResourceState() ==
ResourceState.Enabled) {
+ return
Optional.of(RemoteHostEndPoint.getHypervisorHostEndPoint(host));
+ }
+ }
+ }
+ s_logger.error("Linstor: No diskfull host found.");
+ return Optional.empty();
+ }
+
+ private String restoreResourceFromSnapshot(
+ DevelopersApi api,
+ StoragePoolVO storagePoolVO,
+ String rscName,
+ String snapshotName,
+ String restoredName) throws ApiException {
+ final String rscGrp = getRscGrp(storagePoolVO);
+ ResourceDefinitionCreate rdc =
createResourceDefinitionCreate(restoredName, rscGrp);
+ api.resourceDefinitionCreate(rdc);
+
+ SnapshotRestore sr = new SnapshotRestore();
+ sr.toResource(restoredName);
+ api.resourceSnapshotsRestoreVolumeDefinition(rscName, snapshotName,
sr);
+
+ api.resourceSnapshotRestore(rscName, snapshotName, sr);
+
- return getDeviceName(api, restoredName);
++ return LinstorUtil.getDevicePath(api, restoredName);
+ }
+
+ private Answer copyTemplate(DataObject srcData, DataObject dstData) {
+ TemplateInfo tInfo = (TemplateInfo) dstData;
+ final StoragePoolVO pool =
_storagePoolDao.findById(dstData.getDataStore().getId());
+ final DevelopersApi api =
LinstorUtil.getLinstorAPI(pool.getHostAddress());
+ final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid();
+ createResourceBase(
+ LinstorUtil.RSC_PREFIX + dstData.getUuid(),
+ tInfo.getSize(),
+ tInfo.getName(),
+ "",
+ api,
+ getRscGrp(pool));
+
+ int nMaxExecutionMinutes = NumbersUtil.parseInt(
+ _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()),
30);
+ CopyCommand cmd = new CopyCommand(
+ srcData.getTO(),
+ dstData.getTO(),
+ nMaxExecutionMinutes * 60 * 1000,
+ VirtualMachineManager.ExecuteInSequence.value());
+ Answer answer;
+
+ try {
+ Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+ if (optEP.isPresent()) {
+ answer = optEP.get().sendMessage(cmd);
+ }
+ else {
+ answer = new Answer(cmd, false, "Unable to get matching
Linstor endpoint.");
+ deleteResourceDefinition(pool, rscName);
+ }
+ } catch (ApiException exc) {
+ s_logger.error("copy template failed: ", exc);
+ deleteResourceDefinition(pool, rscName);
+ throw new CloudRuntimeException(exc.getBestMessage());
+ }
+ return answer;
+ }
+
+ private Answer copyVolume(DataObject srcData, DataObject dstData) {
+ VolumeInfo srcVolInfo = (VolumeInfo) srcData;
+ final StoragePoolVO pool =
_storagePoolDao.findById(srcVolInfo.getDataStore().getId());
+ final DevelopersApi api =
LinstorUtil.getLinstorAPI(pool.getHostAddress());
+ final String rscName = LinstorUtil.RSC_PREFIX + srcVolInfo.getUuid();
+
+ VolumeObjectTO to = (VolumeObjectTO) srcVolInfo.getTO();
+ // patch source format
+ // Linstor volumes are stored as RAW, but we can't set the correct
format as RAW (we use QCOW2)
+ // otherwise create template from snapshot won't work, because this
operation
+ // uses the format of the base volume and we backup snapshots as QCOW2
+ //
https://github.com/apache/cloudstack/pull/8802#issuecomment-2024019927
+ to.setFormat(Storage.ImageFormat.RAW);
+ int nMaxExecutionSeconds = NumbersUtil.parseInt(
+ _configDao.getValue(Config.CopyVolumeWait.key()), 10800);
+ CopyCommand cmd = new CopyCommand(
+ to,
+ dstData.getTO(),
+ nMaxExecutionSeconds,
+ VirtualMachineManager.ExecuteInSequence.value());
+ Answer answer;
+
+ try {
+ Optional<RemoteHostEndPoint> optEP = getLinstorEP(api, rscName);
+ if (optEP.isPresent()) {
+ answer = optEP.get().sendMessage(cmd);
+ }
+ else {
+ answer = new Answer(cmd, false, "Unable to get matching
Linstor endpoint.");
+ }
+ } catch (ApiException exc) {
+ s_logger.error("copy volume failed: ", exc);
+ throw new CloudRuntimeException(exc.getBestMessage());
+ }
+ return answer;
+ }
+
+ /**
+ * Create a temporary resource from the snapshot to backup, so we can
copy the data on a diskless agent
+ * @param api Linstor Developer api object
+ * @param pool StoragePool this resource resides on
+ * @param rscName rscName of the snapshotted resource
+ * @param snapshotInfo snapshot info of the snapshot
+ * @param origCmd original LinstorBackupSnapshotCommand that needs to
have a patched path
+ * @return answer from agent operation
+ * @throws ApiException if any Linstor api operation fails
+ */
+ private Answer copyFromTemporaryResource(
+ DevelopersApi api, StoragePoolVO pool, String rscName,
SnapshotInfo snapshotInfo, CopyCommand origCmd)
+ throws ApiException {
+ Answer answer;
+ String restoreName = rscName + "-rst";
+ String snapshotName = LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid();
+ String devName = restoreResourceFromSnapshot(api, pool, rscName,
snapshotName, restoreName);
+
+ Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api,
restoreName);
+ if (optEPAny.isPresent()) {
+ // patch the src device path to the temporary linstor resource
+ SnapshotObjectTO soTO = (SnapshotObjectTO)snapshotInfo.getTO();
+ soTO.setPath(devName);
+ origCmd.setSrcTO(soTO);
+ answer = optEPAny.get().sendMessage(origCmd);
+ } else{
+ answer = new Answer(origCmd, false, "Unable to get matching
Linstor endpoint.");
+ }
+ // delete the temporary resource, noop if already gone
+ api.resourceDefinitionDelete(restoreName);
+ return answer;
+ }
+
+ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
+ String value =
_configDao.getValue(Config.BackupSnapshotWait.toString());
+ int _backupsnapshotwait = NumbersUtil.parseInt(
+ value,
Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));
+
+ SnapshotInfo snapshotInfo = (SnapshotInfo)srcData;
+ Boolean snapshotFullBackup = snapshotInfo.getFullBackup();
+ final StoragePoolVO pool =
_storagePoolDao.findById(srcData.getDataStore().getId());
+ final DevelopersApi api =
LinstorUtil.getLinstorAPI(pool.getHostAddress());
+ boolean fullSnapshot = true;
+ if (snapshotFullBackup != null) {
+ fullSnapshot = snapshotFullBackup;
+ }
+ Map<String, String> options = new HashMap<>();
+ options.put("fullSnapshot", fullSnapshot + "");
+ options.put(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key(),
+
String.valueOf(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()));
+ options.put("volumeSize", snapshotInfo.getBaseVolume().getSize() +
"");
+
+ try {
+ CopyCommand cmd = new LinstorBackupSnapshotCommand(
+ srcData.getTO(),
+ destData.getTO(),
+ _backupsnapshotwait,
+ VirtualMachineManager.ExecuteInSequence.value());
+ cmd.setOptions(options);
+
+ String rscName = LinstorUtil.RSC_PREFIX +
snapshotInfo.getBaseVolume().getUuid();
+ Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, rscName);
+ Answer answer;
+ if (optEP.isPresent()) {
+ answer = optEP.get().sendMessage(cmd);
+ } else {
+ s_logger.debug("No diskfull endpoint found to copy image,
creating diskless endpoint");
+ answer = copyFromTemporaryResource(api, pool, rscName,
snapshotInfo, cmd);
+ }
+ return answer;
+ } catch (Exception e) {
+ s_logger.debug("copy snapshot failed: ", e);
+ throw new CloudRuntimeException(e.toString());
+ }
+
}
@Override
