This is an automated email from the ASF dual-hosted git repository. dahn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit dbfc7f23a7f77ad0208484fffbfe67edf879ee1e Merge: 046870ef764 0602f46d82b Author: Daan Hoogland <d...@onecht.net> AuthorDate: Fri Oct 11 17:59:46 2024 +0200 Merge branch '4.19' .../main/java/com/cloud/user/AccountService.java | 2 + .../motion/StorageSystemDataMotionStrategy.java | 65 +++-- .../cloudstack/api/command/QuotaBalanceCmd.java | 14 +- .../cloudstack/api/command/QuotaCreditsCmd.java | 6 + .../cloudstack/api/command/QuotaStatementCmd.java | 6 + .../hypervisor/kvm/resource/MigrateKVMAsync.java | 2 + .../contrail/management/MockAccountManager.java | 5 + .../api/command/ListAndSwitchSAMLAccountCmd.java | 7 +- .../apache/cloudstack/saml/SAML2AuthManager.java | 3 + .../cloudstack/saml/SAML2AuthManagerImpl.java | 3 +- .../java/org/apache/cloudstack/saml/SAMLUtils.java | 25 +- .../command/ListAndSwitchSAMLAccountCmdTest.java | 25 +- server/src/main/java/com/cloud/api/ApiServer.java | 26 +- server/src/main/java/com/cloud/api/ApiServlet.java | 54 +++-- .../com/cloud/api/dispatch/ParamProcessWorker.java | 40 ++- .../cloud/network/vpn/Site2SiteVpnManagerImpl.java | 114 +++++---- .../com/cloud/server/ManagementServerImpl.java | 4 +- .../java/com/cloud/user/AccountManagerImpl.java | 14 ++ .../cloud/api/dispatch/ParamProcessWorkerTest.java | 138 +++++++++-- .../com/cloud/user/MockAccountManagerImpl.java | 5 + .../storage/formatinspector/Qcow2HeaderField.java | 51 ++++ .../storage/formatinspector/Qcow2Inspector.java | 267 +++++++++++++++++++++ .../resource/NfsSecondaryStorageResource.java | 14 +- .../storage/template/DownloadManagerImpl.java | 39 ++- test/integration/smoke/test_login.py | 1 + ui/src/api/index.js | 11 +- ui/src/store/modules/user.js | 10 +- ui/src/views/compute/EditVM.vue | 35 ++- ui/src/views/network/VpcTab.vue | 8 +- ui/vue.config.js | 6 +- utils/src/main/java/com/cloud/utils/HttpUtils.java | 42 +++- .../test/java/com/cloud/utils/HttpUtilsTest.java | 86 ++++++- 32 files changed, 947 insertions(+), 181 deletions(-) diff --cc engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 07b53842640,82f9a84cdb5..22a3bfbf93a --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@@ -2024,16 -2025,9 +2026,19 @@@ public class StorageSystemDataMotionStr continue; } + VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId()); + if (srcVolumeInfo.getTemplateId() != null && + Objects.nonNull(vmTemplate) && + !Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) { + logger.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId())); + copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost); + } else { + logger.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a template.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId())); + } + + MigrationOptions.Type migrationType = decideMigrationTypeAndCopyTemplateIfNeeded(destHost, vmInstance, srcVolumeInfo, sourceStoragePool, destStoragePool, destDataStore); + migrateNonSharedInc = migrateNonSharedInc || MigrationOptions.Type.LinkedClone.equals(migrationType); + VolumeVO destVolume = duplicateVolumeOnAnotherStorage(srcVolume, destStoragePool); VolumeInfo destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); diff --cc plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 218e3c2b2f9,ecb6531d0cf..628eca56c2d --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@@ -21,6 -21,9 +21,9 @@@ import java.util.List import javax.inject.Inject; + import com.cloud.user.Account; ++ + import org.apache.cloudstack.api.ACL; -import org.apache.log4j.Logger; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; @@@ -43,14 -48,13 +47,15 @@@ public class QuotaBalanceCmd extends Ba @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End date range for quota query. Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-03.") + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End of the period of the Quota balance." + + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start date range quota query. Use yyyy-MM-dd as the date format, e.g. startDate=2009-06-01.") + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start of the period of the Quota balance. " + + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; + @ACL @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account") private Long accountId; diff --cc plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java index 3e6b093abe1,c2f81cd3356..4851973c2ad --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java @@@ -46,7 -46,9 +46,8 @@@ import org.apache.cloudstack.api.respon import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.saml.SAML2AuthManager; import org.apache.cloudstack.saml.SAMLUtils; -import org.apache.log4j.Logger; + import com.cloud.api.ApiServer; import com.cloud.api.response.ApiResponseSerializer; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; @@@ -59,8 -61,11 +60,10 @@@ import com.cloud.user.dao.UserAccountDa import com.cloud.user.dao.UserDao; import com.cloud.utils.HttpUtils; + import org.apache.commons.lang3.EnumUtils; + @APICommand(name = "listAndSwitchSamlAccount", description = "Lists and switches to other SAML accounts owned by the SAML user", responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListAndSwitchSAMLAccountCmd extends BaseCmd implements APIAuthenticator { - public static final Logger s_logger = Logger.getLogger(ListAndSwitchSAMLAccountCmd.class.getName()); @Inject ApiServerService _apiServer; diff --cc plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java index 7ffe07a8609,bb94c8af4c2..6efe454a792 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java @@@ -102,10 -103,12 +104,12 @@@ import org.w3c.dom.Document import org.w3c.dom.Element; import org.xml.sax.SAXException; + import com.cloud.api.ApiServlet; import com.cloud.utils.HttpUtils; + import com.cloud.utils.exception.CloudRuntimeException; public class SAMLUtils { - public static final Logger s_logger = Logger.getLogger(SAMLUtils.class); + protected static Logger LOGGER = LogManager.getLogger(SAMLUtils.class); static final String charset = "abcdefghijklmnopqrstuvwxyz"; diff --cc server/src/main/java/com/cloud/api/ApiServlet.java index f2b5d3c4797,d9654f03916..e2ff411f8f4 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@@ -47,8 -46,8 +47,10 @@@ import org.apache.cloudstack.api.comman import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.cloudstack.utils.consoleproxy.ConsoleAccessUtils; -import org.apache.log4j.Logger; ++ +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; diff --cc server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java index 314b83acdb5,b11a5282e62..16b9673f198 --- a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java @@@ -334,19 -295,35 +332,35 @@@ public class ParamProcessWorker impleme _accountMgr.checkAccess(caller, null, false, owners); } - if (!entitiesToAccess.isEmpty()) { - // check that caller can access the owner account. - _accountMgr.checkAccess(caller, null, false, owners); - for (Map.Entry<Object,AccessType>entry : entitiesToAccess.entrySet()) { - Object entity = entry.getKey(); - if (entity instanceof ControlledEntity) { - _accountMgr.checkAccess(caller, entry.getValue(), true, (ControlledEntity) entity); - } else if (entity instanceof InfrastructureEntity) { - // FIXME: Move this code in adapter, remove code from - // Account manager - } + checkCallerAccessToEntities(caller, owners, entitiesToAccess); + } + + protected Account[] getEntityOwners(BaseCmd cmd) { + List<Long> entityOwners = cmd.getEntityOwnerIds(); + if (entityOwners != null) { + return entityOwners.stream().map(id -> _accountMgr.getAccount(id)).toArray(Account[]::new); + } + + if (cmd.getEntityOwnerId() == Account.ACCOUNT_ID_SYSTEM && cmd instanceof BaseAsyncCmd && cmd.getApiResourceType() == ApiCommandResourceType.Network) { - s_logger.debug("Skipping access check on the network owner if the owner is ROOT/system."); ++ logger.debug("Skipping access check on the network owner if the owner is ROOT/system."); + } else { + Account owner = _accountMgr.getAccount(cmd.getEntityOwnerId()); + if (owner != null) { + return new Account[]{owner}; } } + return new Account[]{}; + } + + protected void checkCallerAccessToEntities(Account caller, Account[] owners, Map<Object, AccessType> entitiesToAccess) { + if (entitiesToAccess.isEmpty()) { + return; + } + _accountMgr.checkAccess(caller, null, false, owners); + for (Map.Entry<Object, AccessType> entry : entitiesToAccess.entrySet()) { + Object entity = entry.getKey(); + _accountMgr.validateAccountHasAccessToResource(caller, entry.getValue(), entity); + } } @SuppressWarnings({"unchecked", "rawtypes"}) diff --cc server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java index e76c52b9ebf,3efe9a8788f..094f81607fe --- a/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java @@@ -23,10 -23,12 +23,11 @@@ import java.util.Map import javax.inject.Inject; import javax.naming.ConfigurationException; - import org.apache.cloudstack.annotation.AnnotationService; - import org.apache.cloudstack.annotation.dao.AnnotationDao; + import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Logger; import org.springframework.stereotype.Component; + import org.apache.cloudstack.annotation.AnnotationService; + import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.command.user.vpn.CreateVpnConnectionCmd; import org.apache.cloudstack.api.command.user.vpn.CreateVpnCustomerGatewayCmd; import org.apache.cloudstack.api.command.user.vpn.CreateVpnGatewayCmd; diff --cc server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java index da70bc1c1bf,0604405f4a5..a1c97e21d5b --- a/server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java +++ b/server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java @@@ -38,9 -29,27 +29,28 @@@ import org.junit.Test import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; + import org.mockito.InjectMocks; + import org.mockito.Spy; + import org.mockito.junit.MockitoJUnitRunner; - import java.util.HashMap; ++import org.apache.cloudstack.api.ApiArgValidator; + import org.apache.cloudstack.api.BaseCmd; + import org.apache.cloudstack.api.Parameter; + import org.apache.cloudstack.api.ServerApiException; + import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd; + import org.apache.cloudstack.acl.SecurityChecker; + import org.apache.cloudstack.context.CallContext; + + import com.cloud.exception.ConcurrentOperationException; + import com.cloud.exception.InsufficientCapacityException; + import com.cloud.exception.NetworkRuleConflictException; + import com.cloud.exception.ResourceAllocationException; + import com.cloud.exception.ResourceUnavailableException; + import com.cloud.user.Account; + import com.cloud.user.AccountManager; + import com.cloud.user.User; + import com.cloud.vm.VMInstanceVO; @RunWith(MockitoJUnitRunner.class) public class ParamProcessWorkerTest { @@@ -64,12 -91,9 +92,12 @@@ @Parameter(name = "doubleparam1", type = CommandType.DOUBLE) double doubleparam1; + @Parameter(name = "vmHostNameParam", type = CommandType.STRING, validations = {ApiArgValidator.RFCComplianceDomainName}) + String vmHostNameParam; + @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, - ResourceAllocationException, NetworkRuleConflictException { + ResourceAllocationException, NetworkRuleConflictException { // well documented nothing } @@@ -104,44 -126,72 +130,106 @@@ params.put("intparam1", "100"); params.put("boolparam1", "true"); params.put("doubleparam1", "11.89"); + params.put("vmHostNameParam", "test-host-name-123"); final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); + paramProcessWorkerSpy.processParameters(cmd, params); Assert.assertEquals("foo", cmd.strparam1); Assert.assertEquals(100, cmd.intparam1); Assert.assertTrue(Double.compare(cmd.doubleparam1, 11.89) == 0); + Assert.assertEquals("test-host-name-123", cmd.vmHostNameParam); + } + + @Test(expected = ServerApiException.class) + public void processVmHostNameParameter_CannotStartWithDigit() { + final HashMap<String, String> params = new HashMap<String, String>(); + params.put("vmHostNameParam", "123test"); + final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); ++ paramProcessWorkerSpy.processParameters(cmd, params); + } + + @Test(expected = ServerApiException.class) + public void processVmHostNameParameter_CannotStartWithHypen() { + final HashMap<String, String> params = new HashMap<String, String>(); + params.put("vmHostNameParam", "-test"); + final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); ++ paramProcessWorkerSpy.processParameters(cmd, params); + } + + @Test(expected = ServerApiException.class) + public void processVmHostNameParameter_CannotEndWithHypen() { + final HashMap<String, String> params = new HashMap<String, String>(); + params.put("vmHostNameParam", "test-"); + final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); ++ paramProcessWorkerSpy.processParameters(cmd, params); + } + + @Test(expected = ServerApiException.class) + public void processVmHostNameParameter_NotMoreThan63Chars() { + final HashMap<String, String> params = new HashMap<String, String>(); + params.put("vmHostNameParam", "test-f2405112-d5a1-47c1-9f00-976909e3a6d3-1e6f3264-955ee76011a99"); + final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); ++ paramProcessWorkerSpy.processParameters(cmd, params); + Mockito.verify(paramProcessWorkerSpy).doAccessChecks(Mockito.any(), Mockito.any()); + } + + @Test + public void doAccessChecksTestChecksCallerAccessToOwnerWhenCmdExtendsBaseAsyncCreateCmd() { + Mockito.doReturn(owners).when(paramProcessWorkerSpy).getEntityOwners(Mockito.any()); + Mockito.doNothing().when(paramProcessWorkerSpy).checkCallerAccessToEntities(Mockito.any(), Mockito.any(), Mockito.any()); + + paramProcessWorkerSpy.doAccessChecks(new AssociateIPAddrCmd(), entities); + + Mockito.verify(accountManagerMock).checkAccess(callingAccountMock, null, false, owners); + } + + @Test + public void doAccessChecksTestChecksCallerAccessToEntities() { + Mockito.doReturn(owners).when(paramProcessWorkerSpy).getEntityOwners(Mockito.any()); + Mockito.doNothing().when(paramProcessWorkerSpy).checkCallerAccessToEntities(Mockito.any(), Mockito.any(), Mockito.any()); + + paramProcessWorkerSpy.doAccessChecks(new AssociateIPAddrCmd(), entities); + + Mockito.verify(paramProcessWorkerSpy).checkCallerAccessToEntities(callingAccountMock, owners, entities); + } + + @Test + public void getEntityOwnersTestReturnsAccountsWhenCmdHasMultipleEntityOwners() { + Mockito.when(baseCmdMock.getEntityOwnerIds()).thenReturn(List.of(1L, 2L)); + Mockito.doReturn(callingAccountMock).when(accountManagerMock).getAccount(1L); + Mockito.doReturn(ownerAccountMock).when(accountManagerMock).getAccount(2L); + + List<Account> result = List.of(paramProcessWorkerSpy.getEntityOwners(baseCmdMock)); + + Assert.assertEquals(List.of(callingAccountMock, ownerAccountMock), result); + } + + @Test + public void getEntityOwnersTestReturnsAccountWhenCmdHasOneEntityOwner() { + Mockito.when(baseCmdMock.getEntityOwnerId()).thenReturn(1L); + Mockito.when(baseCmdMock.getEntityOwnerIds()).thenReturn(null); + Mockito.doReturn(ownerAccountMock).when(accountManagerMock).getAccount(1L); + + List<Account> result = List.of(paramProcessWorkerSpy.getEntityOwners(baseCmdMock)); + + Assert.assertEquals(List.of(ownerAccountMock), result); + } + + @Test + public void checkCallerAccessToEntitiesTestChecksCallerAccessToOwners() { + entities.put(ownerAccountMock, SecurityChecker.AccessType.UseEntry); + + paramProcessWorkerSpy.checkCallerAccessToEntities(callingAccountMock, owners, entities); + + Mockito.verify(accountManagerMock).checkAccess(callingAccountMock, null, false, owners); + } + + @Test + public void checkCallerAccessToEntitiesTestChecksCallerAccessToResource() { + VMInstanceVO vmInstanceVo = new VMInstanceVO(); + entities.put(vmInstanceVo, SecurityChecker.AccessType.UseEntry); + + paramProcessWorkerSpy.checkCallerAccessToEntities(callingAccountMock, owners, entities); + + Mockito.verify(accountManagerMock).validateAccountHasAccessToResource(callingAccountMock, SecurityChecker.AccessType.UseEntry, vmInstanceVo); } } diff --cc services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index ad73a9bc708,c946614934b..e2eecbfcf22 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@@ -51,10 -49,13 +49,11 @@@ import org.apache.cloudstack.storage.co import org.apache.cloudstack.storage.resource.IpTablesHelper; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; + import org.apache.cloudstack.storage.formatinspector.Qcow2HeaderField; + import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.cloudstack.utils.security.ChecksumValue; import org.apache.cloudstack.utils.security.DigestHelper; - import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; @@@ -89,13 -90,12 +88,16 @@@ import com.cloud.utils.NumbersUtil import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Proxy; -import com.cloud.utils.script.Script; + import com.cloud.utils.StringUtils; +import com.cloud.utils.script.Script; - import com.cloud.utils.storage.QCOW2Utils; ++ +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class DownloadManagerImpl extends ManagerBase implements DownloadManager { + protected static Logger LOGGER = LogManager.getLogger(DownloadManagerImpl.class); private String _name; StorageLayer _storage; public Map<String, Processor> _processors;