This is an automated email from the ASF dual-hosted git repository. dahn pushed a commit to branch 4.20 in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 03dfe4d1f3e165b943346437b421ce72f87ae249 Author: abh1sar <[email protected]> AuthorDate: Mon Mar 2 11:06:13 2026 +0530 secondary storage resource limit for download --- .../storage/VMTemplateStorageResourceAssoc.java | 3 +- .../cloud/agent/api/storage/DownloadAnswer.java | 2 +- .../storage/image/BaseImageStoreDriverImpl.java | 9 ++- .../resourcelimit/ResourceLimitManagerImpl.java | 4 +- .../storage/download/DownloadActiveState.java | 5 ++ .../cloud/storage/download/DownloadErrorState.java | 5 ++ .../storage/download/DownloadInactiveState.java | 6 ++ .../download/DownloadLimitReachedState.java | 54 +++++++++++++ .../cloud/storage/download/DownloadListener.java | 89 ++++++++++++++++++++-- .../com/cloud/storage/download/DownloadState.java | 6 +- .../cloud/template/HypervisorTemplateAdapter.java | 9 ++- .../storage/template/DownloadManagerImpl.java | 2 +- 12 files changed, 173 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java index db702a61f2b..7d5b2d7c57d 100644 --- a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java +++ b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java @@ -23,9 +23,10 @@ import org.apache.cloudstack.api.InternalIdentity; public interface VMTemplateStorageResourceAssoc extends InternalIdentity { public static enum Status { - UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED + UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, LIMIT_REACHED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED } + List<Status> ERROR_DOWNLOAD_STATES = List.of(Status.DOWNLOAD_ERROR, Status.ABANDONED, Status.LIMIT_REACHED, Status.UNKNOWN); List<Status> PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS); String getInstallPath(); diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 0c6373134b1..1c5eb7b9a9a 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -140,7 +140,7 @@ public class DownloadAnswer extends Answer { } public Long getTemplateSize() { - return templateSize; + return templateSize == 0 ? templatePhySicalSize : templateSize; } public void setTemplatePhySicalSize(long templatePhySicalSize) { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index a2e9eff2a08..61b1a84cdc6 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -230,8 +230,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { updateBuilder.setJobId(answer.getJobId()); updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); updateBuilder.setInstallPath(answer.getInstallPath()); - updateBuilder.setSize(answer.getTemplateSize()); - updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + } _templateStoreDao.update(tmpltStoreVO.getId(), updateBuilder); // update size in vm_template table VMTemplateVO tmlptUpdater = _templateDao.createForUpdate(); @@ -241,8 +243,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { AsyncCompletionCallback<CreateCmdResult> caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 09a0dda3aaa..43c3b383258 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -258,7 +258,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim templateSizeSearch = _vmTemplateStoreDao.createSearchBuilder(SumCount.class); templateSizeSearch.select("sum", Func.SUM, templateSizeSearch.entity().getSize()); - templateSizeSearch.and("downloadState", templateSizeSearch.entity().getDownloadState(), Op.EQ); + templateSizeSearch.and("downloadState", templateSizeSearch.entity().getDownloadState(), Op.IN); templateSizeSearch.and("destroyed", templateSizeSearch.entity().getDestroyed(), Op.EQ); SearchBuilder<VMTemplateVO> join1 = _vmTemplateDao.createSearchBuilder(); join1.and("accountId", join1.entity().getAccountId(), Op.EQ); @@ -1410,7 +1410,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim long totalTemplatesSize = 0; SearchCriteria<SumCount> sc = templateSizeSearch.create(); - sc.setParameters("downloadState", Status.DOWNLOADED); + sc.setParameters("downloadState", Status.DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS); sc.setParameters("destroyed", false); sc.setJoinParameters("templates", "accountId", accountId); List<SumCount> templates = _vmTemplateStoreDao.customSearch(sc, null); diff --git a/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java b/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java index 889ffbc0b1c..35b23985dd2 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadActiveState.java @@ -95,6 +95,11 @@ public abstract class DownloadActiveState extends DownloadState { return Status.ABANDONED.toString(); } + @Override + public String handleLimitReached() { + return Status.LIMIT_REACHED.toString(); + } + @Override public String handleDisconnect() { diff --git a/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java b/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java index a0834456a2d..5dddefb0192 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadErrorState.java @@ -60,6 +60,11 @@ public class DownloadErrorState extends DownloadInactiveState { return Status.ABANDONED.toString(); } + @Override + public String handleLimitReached() { + return Status.LIMIT_REACHED.toString(); + } + @Override public String getName() { return Status.DOWNLOAD_ERROR.toString(); diff --git a/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java b/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java index 8fee3d0437c..69d46879ebe 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadInactiveState.java @@ -36,6 +36,12 @@ public abstract class DownloadInactiveState extends DownloadState { return getName(); } + @Override + public String handleLimitReached() { + // ignore and stay put + return getName(); + } + @Override public String handleDisconnect() { //ignore and stay put diff --git a/server/src/main/java/com/cloud/storage/download/DownloadLimitReachedState.java b/server/src/main/java/com/cloud/storage/download/DownloadLimitReachedState.java new file mode 100644 index 00000000000..8ce5668299e --- /dev/null +++ b/server/src/main/java/com/cloud/storage/download/DownloadLimitReachedState.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.download; + +import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; +import org.apache.logging.log4j.Level; + +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; + +public class DownloadLimitReachedState extends DownloadInactiveState { + + public DownloadLimitReachedState(DownloadListener dl) { + super(dl); + } + + @Override + public String getName() { + return Status.LIMIT_REACHED.toString(); + } + + @Override + public String handleEvent(DownloadEvent event, Object eventObj) { + if (logger.isTraceEnabled()) { + getDownloadListener().log("handleEvent, event type=" + event + ", curr state=" + getName(), Level.TRACE); + } + return Status.LIMIT_REACHED.toString(); + } + + @Override + public void onEntry(String prevState, DownloadEvent event, Object evtObj) { + if (!prevState.equalsIgnoreCase(getName())) { + DownloadAnswer answer = new DownloadAnswer("Storage Limit Reached", Status.LIMIT_REACHED); + getDownloadListener().callback(answer); + getDownloadListener().cancelStatusTask(); + getDownloadListener().cancelTimeoutTask(); + getDownloadListener().scheduleImmediateStatusCheck(RequestType.PURGE); + } + } +} diff --git a/server/src/main/java/com/cloud/storage/download/DownloadListener.java b/server/src/main/java/com/cloud/storage/download/DownloadListener.java index 42b0e394db4..058881fdb54 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadListener.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadListener.java @@ -25,6 +25,15 @@ import java.util.Timer; import javax.inject.Inject; +import com.cloud.configuration.Resource; +import com.cloud.resourcelimit.CheckedReservation; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.ResourceLimitService; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -34,10 +43,13 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; +import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.utils.cache.LazyCache; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -107,6 +119,7 @@ public class DownloadListener implements Listener { public static final String DOWNLOAD_ERROR = Status.DOWNLOAD_ERROR.toString(); public static final String DOWNLOAD_IN_PROGRESS = Status.DOWNLOAD_IN_PROGRESS.toString(); public static final String DOWNLOAD_ABANDONED = Status.ABANDONED.toString(); + public static final String DOWNLOAD_LIMIT_REACHED = Status.LIMIT_REACHED.toString(); private EndPoint _ssAgent; @@ -137,6 +150,18 @@ public class DownloadListener implements Listener { private DataStoreManager _storeMgr; @Inject private VolumeService _volumeSrv; + @Inject + private VMTemplateDao _templateDao; + @Inject + private TemplateDataStoreDao _templateDataStoreDao; + @Inject + private VolumeDao _volumeDao; + @Inject + private ResourceLimitService _resourceLimitMgr; + @Inject + private AccountManager _accountMgr; + @Inject + ReservationDao _reservationDao; private LazyCache<Long, List<Hypervisor.HypervisorType>> zoneHypervisorsCache; @@ -160,7 +185,7 @@ public class DownloadListener implements Listener { _downloadMonitor = downloadMonitor; _cmd = cmd; initStateMachine(); - _currState = getState(Status.NOT_DOWNLOADED.toString()); + _currState = getState(NOT_DOWNLOADED); this._timer = timer; _timeoutTask = new TimeoutTask(this); this._timer.schedule(_timeoutTask, 3 * STATUS_POLL_INTERVAL); @@ -184,11 +209,12 @@ public class DownloadListener implements Listener { } private void initStateMachine() { - _stateMap.put(Status.NOT_DOWNLOADED.toString(), new NotDownloadedState(this)); - _stateMap.put(Status.DOWNLOADED.toString(), new DownloadCompleteState(this)); - _stateMap.put(Status.DOWNLOAD_ERROR.toString(), new DownloadErrorState(this)); - _stateMap.put(Status.DOWNLOAD_IN_PROGRESS.toString(), new DownloadInProgressState(this)); - _stateMap.put(Status.ABANDONED.toString(), new DownloadAbandonedState(this)); + _stateMap.put(NOT_DOWNLOADED, new NotDownloadedState(this)); + _stateMap.put(DOWNLOADED, new DownloadCompleteState(this)); + _stateMap.put(DOWNLOAD_ERROR, new DownloadErrorState(this)); + _stateMap.put(DOWNLOAD_IN_PROGRESS, new DownloadInProgressState(this)); + _stateMap.put(DOWNLOAD_ABANDONED, new DownloadAbandonedState(this)); + _stateMap.put(DOWNLOAD_LIMIT_REACHED, new DownloadLimitReachedState(this)); } private DownloadState getState(String stateName) { @@ -239,10 +265,53 @@ public class DownloadListener implements Listener { return false; } + private Long getAccountIdForDataObject() { + if (object == null) { + return null; + } + if (DataObjectType.TEMPLATE.equals(object.getType())) { + VMTemplateVO t = _templateDao.findById(object.getId()); + return t != null ? t.getAccountId() : null; + } else if (DataObjectType.VOLUME.equals(object.getType())) { + VolumeVO v = _volumeDao.findById(object.getId()); + return v != null ? v.getAccountId() : null; + } + return null; + } + + private Long getSizeFromDB() { + Long lastSize = 0L; + if (DataObjectType.TEMPLATE.equals(object.getType())) { + TemplateDataStoreVO t = _templateDataStoreDao.findByStoreTemplate(object.getDataStore().getId(), object.getId()); + lastSize = t.getSize(); + } else if (DataObjectType.VOLUME.equals(object.getType())) { + VolumeVO v = _volumeDao.findById(object.getId()); + lastSize = v.getSize(); + } + return lastSize; + } + + private Boolean checkAndUpdateResourceLimits(DownloadAnswer answer) { + Long lastSize = getSizeFromDB(); + Long currentSize = answer.getTemplateSize(); + + if (currentSize > lastSize) { + Long accountId = getAccountIdForDataObject(); + Account account = _accountMgr.getAccount(accountId); + Long usage = currentSize - lastSize; + try (CheckedReservation secStorageReservation = new CheckedReservation(account, Resource.ResourceType.secondary_storage, usage, _reservationDao, _resourceLimitMgr)) { + _resourceLimitMgr.incrementResourceCount(accountId, Resource.ResourceType.secondary_storage, usage); + } catch (Exception e) { + return false; + } + } + return true; + } + @Override public boolean processAnswers(long agentId, long seq, Answer[] answers) { boolean processed = false; - if (answers != null & answers.length > 0) { + if (answers != null && answers.length > 0) { if (answers[0] instanceof DownloadAnswer) { final DownloadAnswer answer = (DownloadAnswer)answers[0]; if (getJobId() == null) { @@ -250,7 +319,11 @@ public class DownloadListener implements Listener { } else if (!getJobId().equalsIgnoreCase(answer.getJobId())) { return false;//TODO } - transition(DownloadEvent.DOWNLOAD_ANSWER, answer); + if (!checkAndUpdateResourceLimits(answer)) { + transition(DownloadEvent.LIMIT_REACHED, answer); + } else { + transition(DownloadEvent.DOWNLOAD_ANSWER, answer); + } processed = true; } } diff --git a/server/src/main/java/com/cloud/storage/download/DownloadState.java b/server/src/main/java/com/cloud/storage/download/DownloadState.java index 68723b53e35..e58d1f3f39f 100644 --- a/server/src/main/java/com/cloud/storage/download/DownloadState.java +++ b/server/src/main/java/com/cloud/storage/download/DownloadState.java @@ -26,7 +26,7 @@ import com.cloud.agent.api.storage.DownloadAnswer; public abstract class DownloadState { public static enum DownloadEvent { - DOWNLOAD_ANSWER, ABANDON_DOWNLOAD, TIMEOUT_CHECK, DISCONNECT + DOWNLOAD_ANSWER, ABANDON_DOWNLOAD, LIMIT_REACHED, TIMEOUT_CHECK, DISCONNECT }; protected Logger logger = LogManager.getLogger(getClass()); @@ -51,6 +51,8 @@ public abstract class DownloadState { return handleAnswer(answer); case ABANDON_DOWNLOAD: return handleAbort(); + case LIMIT_REACHED: + return handleLimitReached(); case TIMEOUT_CHECK: Date now = new Date(); long update = now.getTime() - dl.getLastUpdated().getTime(); @@ -78,6 +80,8 @@ public abstract class DownloadState { public abstract String handleAbort(); + public abstract String handleLimitReached(); + public abstract String handleDisconnect(); public abstract String handleAnswer(DownloadAnswer answer); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index c096ef0eb1d..7e5811a2ece 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -481,13 +481,12 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { CreateTemplateContext<TemplateApiResult> context) { TemplateApiResult result = callback.getResult(); TemplateInfo template = context.template; + VMTemplateVO tmplt = _tmpltDao.findById(template.getId()); if (result.isSuccess()) { - VMTemplateVO tmplt = _tmpltDao.findById(template.getId()); // need to grant permission for public templates if (tmplt.isPublicTemplate()) { _messageBus.publish(_name, TemplateManager.MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT, PublishScope.LOCAL, tmplt.getId()); } - long accountId = tmplt.getAccountId(); if (template.getSize() != null) { // publish usage event String etype = EventTypes.EVENT_TEMPLATE_CREATE; @@ -517,7 +516,11 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { template.getSize(), VirtualMachineTemplate.class.getName(), template.getUuid()); } } - _resourceLimitMgr.recalculateResourceCount(accountId, tmplt.getDomainId(), ResourceType.secondary_storage.getOrdinal()); + } + if (tmplt != null) { + long accountId = tmplt.getAccountId(); + Account account = _accountDao.findById(accountId); + _resourceLimitMgr.recalculateResourceCount(accountId, account.getDomainId(), ResourceType.secondary_storage.getOrdinal()); } return null; diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 43a9422e8ad..c695b3d4b44 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -996,7 +996,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager break; // TODO } return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), - getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); + getDownloadTemplateSize(jobId), td.getDownloadedBytes(), getDownloadCheckSum(jobId)); } private String getInstallPath(String jobId) {
