andrijapanicsb commented on code in PR #14256: URL: https://github.com/apache/cloudstack/pull/14256#discussion_r4135937418
########## plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareCbtMigrationServiceImpl.java: ########## @@ -0,0 +1,587 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.manager; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.cloudstack.vm.VmwareCbtChangedBlockInfo; +import org.apache.cloudstack.vm.VmwareCbtChangedDiskInfo; +import org.apache.cloudstack.vm.VmwareCbtDiskInfo; +import org.apache.cloudstack.vm.VmwareCbtMigrationService; +import org.apache.cloudstack.vm.VmwareCbtPreflightDiskInfo; +import org.apache.cloudstack.vm.VmwareCbtPreflightInfo; +import org.apache.cloudstack.vm.VmwareCbtPreflightNicInfo; +import org.apache.cloudstack.vm.VmwareCbtSnapshotInfo; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.hypervisor.vmware.mo.DatacenterMO; +import com.cloud.hypervisor.vmware.mo.HostMO; +import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; +import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; +import com.cloud.hypervisor.vmware.resource.VmwareContextFactory; +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.cloud.utils.exception.CloudRuntimeException; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.VirtualDevice; +import com.vmware.vim25.VirtualDeviceBackingInfo; +import com.vmware.vim25.VirtualDisk; +import com.vmware.vim25.VirtualMachineCapability; +import com.vmware.vim25.VirtualMachineConfigInfo; +import com.vmware.vim25.VirtualMachineConfigSpec; +import com.vmware.vim25.VirtualMachineConfigSummary; +import com.vmware.vim25.VirtualMachinePowerState; +import com.vmware.vim25.VirtualMachineRuntimeInfo; +import com.vmware.vim25.VirtualMachineSnapshotInfo; +import com.vmware.vim25.VirtualMachineSnapshotTree; + +public class VmwareCbtMigrationServiceImpl implements VmwareCbtMigrationService { + + private static final Logger LOGGER = LogManager.getLogger(VmwareCbtMigrationServiceImpl.class); + + @Override + public VmwareCbtPreflightInfo getPreflightInfo(String vcenter, String datacenterName, String username, + String password, String sourceHost, String sourceVmName) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + VirtualMachineCapability capability = (VirtualMachineCapability)context.getVimClient() + .getDynamicProperty(lookup.vmMO.getMor(), "capability"); + VirtualMachineConfigInfo configInfo = lookup.vmMO.getConfigInfo(); + VirtualMachineConfigSummary configSummary = lookup.vmMO.getConfigSummary(); + VirtualMachineRuntimeInfo runtimeInfo = lookup.vmMO.getRuntimeInfo(); + VirtualMachineSnapshotInfo snapshotInfo = lookup.vmMO.getSnapshotInfo(); + UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(lookup.hyperHost, lookup.vmMO); + + VmwareCbtPreflightInfo preflightInfo = new VmwareCbtPreflightInfo(sourceVmName, getManagedObjectReferenceValue(lookup.vmMO.getMor()), + capability == null ? null : capability.isChangeTrackingSupported(), + configInfo == null ? null : configInfo.isChangeTrackingEnabled(), + runtimeInfo == null ? null : runtimeInfo.isConsolidationNeeded(), + countSnapshots(snapshotInfo), + toVmwareCbtPreflightDiskInfo(unmanagedInstance, collectDiskDeviceInfo(lookup.vmMO)), + toVmwareCbtPreflightNicInfo(unmanagedInstance), + configSummary == null ? null : configSummary.getNumCpu(), + configSummary == null ? null : configSummary.getCpuReservation(), + configSummary == null ? null : configSummary.getMemorySizeMB(), + unmanagedInstance == null ? null : unmanagedInstance.getOperatingSystemId(), + unmanagedInstance == null ? null : unmanagedInstance.getOperatingSystem()); + if (unmanagedInstance != null) { + preflightInfo.setBootType(unmanagedInstance.getBootType()); + preflightInfo.setBootMode(unmanagedInstance.getBootMode()); + } + return preflightInfo; + } catch (Exception e) { + String message = String.format("Unable to check VMware CBT prerequisites for VM %s in vCenter %s: %s", + sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public List<VmwareCbtDiskInfo> listSourceDisks(String vcenter, String datacenterName, String username, String password, + String sourceHost, String sourceVmName) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(lookup.hyperHost, lookup.vmMO); + return toVmwareCbtDiskInfo(unmanagedInstance, collectDiskDeviceInfo(lookup.vmMO)); + } catch (Exception e) { + String message = String.format("Unable to discover VMware CBT source disks for VM %s in vCenter %s: %s", + sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public List<VmwareCbtDiskInfo> listSnapshotDisks(String vcenter, String datacenterName, String username, String password, + String sourceHost, String sourceVmName, String snapshotMor) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(lookup.hyperHost, lookup.vmMO); + ManagedObjectReference snapshot = toManagedObjectReference("VirtualMachineSnapshot", snapshotMor); + List<VirtualDevice> snapshotDevices = context.getVimClient().getDynamicProperty(snapshot, + "config.hardware.device"); + return toVmwareCbtDiskInfo(unmanagedInstance, collectDiskDeviceInfo(snapshotDevices)); + } catch (Exception e) { + String message = String.format("Unable to discover VMware CBT snapshot disks for VM %s snapshot %s in vCenter %s: %s", + sourceVmName, snapshotMor, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public void ensureChangeTrackingEnabled(String vcenter, String datacenterName, String username, String password, + String sourceHost, String sourceVmName) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + ensureChangeTrackingEnabled(context, lookup.vmMO, sourceVmName); + } catch (Exception e) { + String message = String.format("Unable to enable VMware CBT for VM %s in vCenter %s: %s", + sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public VmwareCbtSnapshotInfo createSnapshot(String vcenter, String datacenterName, String username, String password, + String sourceHost, String sourceVmName, String snapshotName, + String snapshotDescription, boolean quiesce) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + ensureChangeTrackingEnabled(context, lookup.vmMO, sourceVmName); + ManagedObjectReference snapshot = lookup.vmMO.createSnapshotGetReference(snapshotName, + snapshotDescription, false, quiesce); + if (snapshot == null) { + throw new CloudRuntimeException(String.format("Unable to create VMware snapshot %s for VM %s", + snapshotName, sourceVmName)); + } + return new VmwareCbtSnapshotInfo(snapshotName, formatManagedObjectReference(snapshot), + getManagedObjectReferenceValue(lookup.vmMO.getMor())); + } catch (Exception e) { + String message = String.format("Unable to create VMware CBT snapshot for VM %s in vCenter %s: %s", + sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public List<VmwareCbtChangedDiskInfo> queryChangedDiskAreas(String vcenter, String datacenterName, String username, + String password, String sourceHost, String sourceVmName, + List<VmwareCbtDiskInfo> disks, String snapshotMor) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + ManagedObjectReference snapshot = toManagedObjectReference("VirtualMachineSnapshot", snapshotMor); + List<VmwareCbtChangedDiskInfo> changedDisks = new ArrayList<>(); + if (CollectionUtils.isEmpty(disks)) { + return changedDisks; + } + for (VmwareCbtDiskInfo disk : disks) { + changedDisks.add(queryChangedDiskAreas(context, lookup.vmMO, snapshot, disk)); + } + return changedDisks; + } catch (Exception e) { + String message = String.format("Unable to query VMware CBT changed areas for VM %s in vCenter %s: %s", + sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public UnmanagedInstanceTO.PowerState getPowerState(String vcenter, String datacenterName, String username, + String password, String sourceHost, String sourceVmName) { + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + VmwareVmLookup lookup = lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + return toUnmanagedPowerState(lookup.vmMO.getPowerState()); + } catch (Exception e) { + String message = String.format("Unable to check VMware power state for VM %s in vCenter %s: %s", + sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + @Override + public void removeSnapshot(String vcenter, String datacenterName, String username, String password, + String sourceHost, String sourceVmName, String snapshotMor) { + if (StringUtils.isBlank(snapshotMor)) { + return; + } + + try { + VmwareContext context = VmwareContextFactory.getContext(vcenter, username, password); + lookupVirtualMachine(context, vcenter, datacenterName, sourceHost, sourceVmName); + ManagedObjectReference snapshot = toManagedObjectReference("VirtualMachineSnapshot", snapshotMor); + ManagedObjectReference task = context.getService().removeSnapshotTask(snapshot, false, true); + if (!context.getVimClient().waitForTask(task)) { + throw new CloudRuntimeException(String.format("Unable to remove VMware snapshot %s for VM %s", + snapshotMor, sourceVmName)); + } + context.waitForTaskProgressDone(task); + } catch (Exception e) { + String message = String.format("Unable to remove VMware CBT snapshot %s for VM %s in vCenter %s: %s", + snapshotMor, sourceVmName, vcenter, e.getMessage()); + LOGGER.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + private VmwareVmLookup lookupVirtualMachine(VmwareContext context, String vcenter, String datacenterName, + String sourceHost, String sourceVmName) throws Exception { + DatacenterMO datacenterMO = new DatacenterMO(context, datacenterName); + if (datacenterMO.getMor() == null) { + throw new CloudRuntimeException(String.format("Unable to find VMware datacenter %s in vCenter %s", + datacenterName, vcenter)); + } + + VmwareHypervisorHost hyperHost; + VirtualMachineMO vmMO; + if (StringUtils.isNotBlank(sourceHost)) { + ManagedObjectReference hostMor = datacenterMO.findHost(sourceHost); + if (hostMor == null) { + throw new CloudRuntimeException(String.format("Unable to find VMware host %s in vCenter %s", + sourceHost, vcenter)); + } + HostMO hostMO = new HostMO(context, hostMor); + vmMO = hostMO.findVmOnHyperHost(sourceVmName); + hyperHost = hostMO; + } else { + vmMO = datacenterMO.findVm(sourceVmName); + hyperHost = vmMO != null ? vmMO.getRunningHost() : null; + } + + if (vmMO == null) { + throw new CloudRuntimeException(String.format("Unable to find VMware VM %s in datacenter %s", + sourceVmName, datacenterName)); + } + return new VmwareVmLookup(hyperHost, vmMO); + } + + private List<VmwareCbtDiskInfo> toVmwareCbtDiskInfo(UnmanagedInstanceTO unmanagedInstance, + Map<String, DiskDeviceInfo> diskDeviceInfo) { + List<VmwareCbtDiskInfo> disks = new ArrayList<>(); + if (unmanagedInstance == null || CollectionUtils.isEmpty(unmanagedInstance.getDisks())) { + return disks; + } + + for (UnmanagedInstanceTO.Disk disk : unmanagedInstance.getDisks()) { + String sourceDiskId = StringUtils.defaultIfBlank(disk.getDiskId(), disk.getLabel()); + String sourceDiskPath = StringUtils.defaultIfBlank(disk.getImagePath(), disk.getFileBaseName()); + DiskDeviceInfo deviceInfo = diskDeviceInfo.get(sourceDiskId); + if (deviceInfo == null) { + deviceInfo = diskDeviceInfo.get(sourceDiskPath); + } + if (deviceInfo == null) { + deviceInfo = diskDeviceInfo.get(disk.getLabel()); + } + disks.add(new VmwareCbtDiskInfo(sourceDiskId, deviceInfo != null ? deviceInfo.deviceKey : null, + disk.getLabel(), sourceDiskPath, disk.getDatastoreName(), disk.getCapacity(), + deviceInfo != null ? deviceInfo.changeId : null)); + } + return disks; + } + + private List<VmwareCbtPreflightDiskInfo> toVmwareCbtPreflightDiskInfo(UnmanagedInstanceTO unmanagedInstance, + Map<String, DiskDeviceInfo> diskDeviceInfo) { + List<VmwareCbtPreflightDiskInfo> disks = new ArrayList<>(); + if (unmanagedInstance == null || CollectionUtils.isEmpty(unmanagedInstance.getDisks())) { + return disks; + } + + for (UnmanagedInstanceTO.Disk disk : unmanagedInstance.getDisks()) { + String sourceDiskId = StringUtils.defaultIfBlank(disk.getDiskId(), disk.getLabel()); + String sourceDiskPath = StringUtils.defaultIfBlank(disk.getImagePath(), disk.getFileBaseName()); + DiskDeviceInfo deviceInfo = diskDeviceInfo.get(sourceDiskId); + if (deviceInfo == null) { + deviceInfo = diskDeviceInfo.get(sourceDiskPath); + } + if (deviceInfo == null) { + deviceInfo = diskDeviceInfo.get(disk.getLabel()); + } + disks.add(new VmwareCbtPreflightDiskInfo(sourceDiskId, deviceInfo != null ? deviceInfo.deviceKey : null, + disk.getLabel(), sourceDiskPath, disk.getDatastoreName(), disk.getCapacity(), + deviceInfo != null ? deviceInfo.changeId : null, + deviceInfo != null ? deviceInfo.backingType : null, + deviceInfo != null ? deviceInfo.diskMode : null, + deviceInfo != null ? deviceInfo.rdmCompatibilityMode : null, + deviceInfo != null && deviceInfo.independentDisk, + deviceInfo != null && deviceInfo.physicalRdm)); + } + return disks; + } + + private List<VmwareCbtPreflightNicInfo> toVmwareCbtPreflightNicInfo(UnmanagedInstanceTO unmanagedInstance) { + List<VmwareCbtPreflightNicInfo> nics = new ArrayList<>(); + if (unmanagedInstance == null || CollectionUtils.isEmpty(unmanagedInstance.getNics())) { + return nics; + } + for (UnmanagedInstanceTO.Nic nic : unmanagedInstance.getNics()) { + VmwareCbtPreflightNicInfo nicInfo = new VmwareCbtPreflightNicInfo(nic.getNicId(), nic.getAdapterType(), + nic.getMacAddress(), nic.getVlan()); + // Older or limited Tools versions report only the bare address list without + // ipConfig - carry those addresses too (without prefix), IP preservation only + // needs the address itself. + nicInfo.setIpv4Cidrs(CollectionUtils.isNotEmpty(nic.getIpv4Cidrs()) ? nic.getIpv4Cidrs() : nic.getIpAddress()); + nicInfo.setIpv4Gateway(nic.getIpv4Gateway()); + nicInfo.setDnsServers(nic.getDnsServers()); + nics.add(nicInfo); + } + return nics; + } + + private Map<String, DiskDeviceInfo> collectDiskDeviceInfo(VirtualMachineMO vmMO) throws Exception { + VirtualDisk[] disks = vmMO.getAllDiskDevice(); + return collectDiskDeviceInfo(disks == null ? null : Arrays.asList(disks)); + } + + private Map<String, DiskDeviceInfo> collectDiskDeviceInfo(List<? extends VirtualDevice> devices) { + Map<String, DiskDeviceInfo> diskDeviceInfo = new HashMap<>(); + if (devices == null) { + return diskDeviceInfo; + } + + for (VirtualDevice device : devices) { + if (!(device instanceof VirtualDisk)) { + continue; + } + VirtualDisk disk = (VirtualDisk) device; + VirtualDeviceBackingInfo backing = disk.getBacking(); + String diskMode = getBackingStringValue(backing, "getDiskMode"); + String rdmCompatibilityMode = getBackingStringValue(backing, "getCompatibilityMode"); + DiskDeviceInfo deviceInfo = new DiskDeviceInfo(disk.getKey(), getBackingStringValue(backing, "getChangeId"), + backing == null ? null : backing.getClass().getSimpleName(), diskMode, rdmCompatibilityMode, + StringUtils.containsIgnoreCase(diskMode, "independent"), + StringUtils.containsIgnoreCase(rdmCompatibilityMode, "physical")); + if (StringUtils.isNotBlank(disk.getDiskObjectId())) { + diskDeviceInfo.put(disk.getDiskObjectId(), deviceInfo); + } + diskDeviceInfo.put(String.valueOf(disk.getKey()), deviceInfo); + if (disk.getDeviceInfo() != null && StringUtils.isNotBlank(disk.getDeviceInfo().getLabel())) { + diskDeviceInfo.put(disk.getDeviceInfo().getLabel(), deviceInfo); + } + String fileName = getBackingStringValue(backing, "getFileName"); + if (StringUtils.isNotBlank(fileName)) { + diskDeviceInfo.put(fileName, deviceInfo); + } + } + return diskDeviceInfo; + } + + private int countSnapshots(VirtualMachineSnapshotInfo snapshotInfo) { + if (snapshotInfo == null || CollectionUtils.isEmpty(snapshotInfo.getRootSnapshotList())) { + return 0; + } + return countSnapshotTrees(snapshotInfo.getRootSnapshotList()); + } + + private int countSnapshotTrees(List<VirtualMachineSnapshotTree> snapshotTrees) { + int count = 0; + for (VirtualMachineSnapshotTree snapshotTree : snapshotTrees) { + count++; + if (CollectionUtils.isNotEmpty(snapshotTree.getChildSnapshotList())) { + count += countSnapshotTrees(snapshotTree.getChildSnapshotList()); + } + } + return count; + } + + private UnmanagedInstanceTO.PowerState toUnmanagedPowerState(VirtualMachinePowerState powerState) { + if (powerState == VirtualMachinePowerState.POWERED_OFF) { + return UnmanagedInstanceTO.PowerState.PowerOff; + } + if (powerState == VirtualMachinePowerState.POWERED_ON) { + return UnmanagedInstanceTO.PowerState.PowerOn; + } + return UnmanagedInstanceTO.PowerState.PowerUnknown; + } + + private VmwareCbtChangedDiskInfo queryChangedDiskAreas(VmwareContext context, VirtualMachineMO vmMO, + ManagedObjectReference snapshot, VmwareCbtDiskInfo disk) + throws ReflectiveOperationException { + if (disk.getSourceDiskDeviceKey() == null) { + throw new CloudRuntimeException(String.format("VMware disk device key is missing for source disk %s", + disk.getSourceDiskId())); + } + if (StringUtils.isBlank(disk.getChangeId())) { + throw new CloudRuntimeException(String.format("VMware CBT change ID is missing for source disk %s", + disk.getSourceDiskId())); + } + + List<VmwareCbtChangedBlockInfo> changedBlocks = new ArrayList<>(); + long startOffset = 0L; + long capacityBytes = disk.getCapacityBytes() == null ? 0L : disk.getCapacityBytes(); + String nextChangeId = null; + + do { + Object diskChangeInfo = invokeQueryChangedDiskAreas(context, vmMO, snapshot, disk, startOffset); + nextChangeId = StringUtils.defaultIfBlank(getObjectStringValue(diskChangeInfo, "getChangeId"), Review Comment: Thanks, you are right: DiskChangeInfo does not expose changeId. Commit a5578d6f53 now reads the next per-disk checkpoint from the matching disk backing in the cycle snapshot, and fails if that ID is missing. Focused unit tests cover matching and missing IDs. ########## plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtVmwareCbtSyncCommandWrapper.java: ########## @@ -0,0 +1,489 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.VmwareCbtMigrationAnswer; +import com.cloud.agent.api.VmwareCbtSyncCommand; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.agent.api.to.VmwareCbtChangedBlockRangeTO; +import com.cloud.agent.api.to.VmwareCbtDiskSyncResultTO; +import com.cloud.agent.api.to.VmwareCbtDiskTO; +import com.cloud.agent.api.to.VmwareCbtTargetStorageType; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; + +@ResourceWrapper(handles = VmwareCbtSyncCommand.class) +public class LibvirtVmwareCbtSyncCommandWrapper extends CommandWrapper<VmwareCbtSyncCommand, Answer, LibvirtComputingResource> { + + private static final long DEFAULT_MAX_COPY_CHUNK_BYTES = 256L * 1024L * 1024L; + private static final Pattern SHA1_FINGERPRINT_PATTERN = Pattern.compile("(?i)(?:SHA1\\s+)?Fingerprint\\s*=\\s*([0-9A-F:]+)"); + + @Override + public Answer execute(VmwareCbtSyncCommand cmd, LibvirtComputingResource serverResource) { + if (!serverResource.hostSupportsVddkBlockCopy(cmd.getVddkLibDir())) { + String msg = String.format("Cannot synchronize VMware CBT migration %s on host %s. VDDK, qemu-img, qemu-nbd and qemu-io are required.", + cmd.getMigrationUuid(), serverResource.getPrivateIp()); + logger.info(msg); + return new VmwareCbtMigrationAnswer(cmd, false, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + 0, 0, 0, false, null); + } + + long startTime = System.currentTimeMillis(); + List<VmwareCbtChangedBlockRangeTO> changedBlocks = cmd.getChangedBlocks(); + if (changedBlocks == null || changedBlocks.isEmpty()) { + long durationSeconds = Math.max(1L, (System.currentTimeMillis() - startTime) / 1000L); + String msg = String.format("VMware CBT cycle %s for migration %s completed with no changed blocks.", + cmd.getCycleNumber(), cmd.getMigrationUuid()); + logger.info(msg); + return new VmwareCbtMigrationAnswer(cmd, true, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + 0, 0, durationSeconds, true, null); + } + + VmwareCbtSyncPlan syncPlan = VmwareCbtSyncPlan.create(cmd.getDisks(), changedBlocks); + if (!syncPlan.isValid()) { + String msg = String.format("Cannot synchronize VMware CBT cycle %s for migration %s: %s", + cmd.getCycleNumber(), cmd.getMigrationUuid(), syncPlan.getValidationError()); + logger.info(msg); + return new VmwareCbtMigrationAnswer(cmd, false, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + 0, 0, 0, false, null); + } + + try { + validateSyncCommand(cmd); + String vddkLibDir = resolveVddkSetting(cmd.getVddkLibDir(), serverResource.getVddkLibDir()); + if (StringUtils.isBlank(vddkLibDir)) { + String msg = String.format("Cannot synchronize VMware CBT migration %s because no VDDK library directory is configured or detected.", + cmd.getMigrationUuid()); + logger.info(msg); + return new VmwareCbtMigrationAnswer(cmd, false, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + syncPlan.getChangedBytes(), 0, 0, false, null); + } + + RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); + String vddkThumbprint = resolveVddkSetting(cmd.getVddkThumbprint(), serverResource.getVddkThumbprint()); + if (StringUtils.isBlank(vddkThumbprint)) { + vddkThumbprint = getVcenterThumbprint(sourceInstance.getVcenterHost(), getTimeout(cmd), + sourceInstance.getInstanceName()); + } + if (StringUtils.isBlank(vddkThumbprint)) { + String msg = String.format("Cannot synchronize VMware CBT migration %s because the vCenter SSL thumbprint could not be determined.", + cmd.getMigrationUuid()); + logger.info(msg); + return new VmwareCbtMigrationAnswer(cmd, false, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + syncPlan.getChangedBytes(), 0, 0, false, null); + } + + KVMStoragePool targetPool = getTargetStoragePool(cmd, serverResource.getStoragePoolMgr()); + if (cmd.getTargetStorageType() == VmwareCbtTargetStorageType.RBD_RAW) { + validateRbdTargetPool(cmd, targetPool); + } else if (cmd.getTargetStorageType() == VmwareCbtTargetStorageType.RAW_BLOCK_DEVICE) { + validateBlockDeviceTargetPool(cmd, targetPool); + } + + String passwordFilePath = writePasswordFile(cmd); + try { + List<VmwareCbtDiskSyncResultTO> diskResults = new ArrayList<>(); + for (VmwareCbtSyncPlan.DiskPlan diskPlan : syncPlan.getDiskPlans()) { + diskResults.add(syncDiskChangedBlocks(cmd, diskPlan, targetPool, passwordFilePath, vddkLibDir, vddkThumbprint, + StringUtils.defaultIfBlank(cmd.getVddkTransports(), serverResource.getVddkTransports()))); + } + long durationSeconds = Math.max(1L, (System.currentTimeMillis() - startTime) / 1000L); + long dirtyRate = getDirtyRateBytesPerSecond(syncPlan.getChangedBytes(), durationSeconds); + String msg = String.format("VMware CBT cycle %s for migration %s copied %s changed block range(s), " + + "coalesced into %s copy range(s) across %s disk(s), totaling %s bytes.", + cmd.getCycleNumber(), cmd.getMigrationUuid(), syncPlan.getChangedRangeCount(), + syncPlan.getCopyRangeCount(), syncPlan.getDiskPlans().size(), syncPlan.getChangedBytes()); + logger.info(msg); + return new VmwareCbtMigrationAnswer(cmd, true, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + syncPlan.getChangedBytes(), dirtyRate, durationSeconds, false, diskResults); + } finally { + Files.deleteIfExists(Path.of(passwordFilePath)); + } + } catch (Exception e) { + String msg = String.format("Cannot synchronize VMware CBT cycle %s for migration %s: %s", + cmd.getCycleNumber(), cmd.getMigrationUuid(), + StringUtils.defaultIfBlank(e.getMessage(), e.getClass().getSimpleName())); + logger.error(msg, e); + return new VmwareCbtMigrationAnswer(cmd, false, msg, cmd.getMigrationUuid(), cmd.getCycleNumber(), + syncPlan.getChangedBytes(), 0, 0, false, null); + } + } + + private void validateSyncCommand(VmwareCbtSyncCommand cmd) { + RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); + if (sourceInstance == null) { + throw new IllegalArgumentException("source VMware instance information is missing"); + } + if (StringUtils.isAnyBlank(sourceInstance.getVcenterHost(), sourceInstance.getVcenterUsername(), + sourceInstance.getVcenterPassword(), sourceInstance.getInstanceName())) { + throw new IllegalArgumentException("source vCenter host, username, password and VM name are required"); + } + if (StringUtils.isBlank(sourceInstance.getVmwareMoref())) { + throw new IllegalArgumentException("source VMware VM managed object reference is missing"); + } + if (StringUtils.isBlank(cmd.getSnapshotMor())) { + throw new IllegalArgumentException("VMware snapshot managed object reference is missing"); + } + } + + private VmwareCbtDiskSyncResultTO syncDiskChangedBlocks(VmwareCbtSyncCommand cmd, VmwareCbtSyncPlan.DiskPlan diskPlan, + KVMStoragePool targetPool, String passwordFilePath, String vddkLibDir, + String vddkThumbprint, String vddkTransports) throws Exception { + VmwareCbtDiskTO disk = diskPlan.getDisk(); + validateDisk(cmd, disk); + long startTime = System.currentTimeMillis(); + String scriptPath = writeDiskSyncScript(cmd, diskPlan, targetPool); + try { + String command = buildNbdkitDeltaSyncCommand(cmd, disk, passwordFilePath, vddkLibDir, vddkThumbprint, + vddkTransports, scriptPath); + VmwareCbtCommandResult commandResult = executeLoggedBash(command, getTimeout(cmd), + String.format("(%s) VMware CBT delta sync disk %s", cmd.getMigrationUuid(), disk.getDiskId())); + long durationSeconds = Math.max(1L, (System.currentTimeMillis() - startTime) / 1000L); + if (commandResult.getExitValue() != 0) { + String details = String.format("changed-block copy from VMware VDDK snapshot failed for source disk %s with exit code %s", + disk.getDiskId(), commandResult.getExitValue()); + throw new IllegalStateException(commandResult.appendLastCommandOutput(details)); + } + return new VmwareCbtDiskSyncResultTO(disk.getDiskId(), disk.getTargetPath(), disk.getChangeId(), Review Comment: Confirmed. The agent echoes the input changeId; after a successful sync, the management server is meant to persist the next ID. The missing ID in the VMware query result prevented that. Commit a5578d6f53 supplies the snapshot backing changeId, and a unit test verifies that it is persisted for the next cycle. Live multi-cycle regression on this commit is still pending. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
