This is an automated email from the ASF dual-hosted git repository.
weizhouapache pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/main by this push:
new 5ed4894e97b storage: Add config keys for controlling public/private
template secondary storage replica counts (#12877)
5ed4894e97b is described below
commit 5ed4894e97b03b8f49cc10e2094bde980f7adfa0
Author: Daman Arora <[email protected]>
AuthorDate: Mon Jun 29 02:14:27 2026 -0400
storage: Add config keys for controlling public/private template secondary
storage replica counts (#12877)
Adds two new operator-level configuration keys to control the number of
secondary storage copies made for public and private templates, decoupling
replica count from template visibility.
- secstorage.public.template.copy.max (default: 0 = all stores, preserving
existing behavior)
- secstorage.private.template.copy.max (default: 1, preserving existing
behavior)
---
.../subsystem/api/storage/TemplateService.java | 6 +
.../java/com/cloud/template/TemplateManager.java | 20 ++
.../storage/image/TemplateServiceImpl.java | 217 ++++++++++++++++++++-
.../storage/image/TemplateServiceImplTest.java | 29 ++-
.../cloud/storage/ImageStoreUploadMonitorImpl.java | 6 +
.../cloud/template/HypervisorTemplateAdapter.java | 19 +-
.../com/cloud/template/TemplateAdapterBase.java | 45 ++---
.../com/cloud/template/TemplateManagerImpl.java | 37 +++-
.../template/HypervisorTemplateAdapterTest.java | 80 ++++----
9 files changed, 378 insertions(+), 81 deletions(-)
diff --git
a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
index 269eb4f1c21..2f8d57171bc 100644
---
a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
+++
b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
@@ -67,6 +67,12 @@ public interface TemplateService {
void handleTemplateSync(DataStore store);
+ void enforceSecStorageCopyLimit(long templateId, long zoneId);
+
+ boolean canCopyTemplateToImageStore(long templateId, long zoneId);
+
+ void replicateTemplateUpToCap(long templateId, long zoneId);
+
void downloadBootstrapSysTemplate(DataStore store);
void addSystemVMTemplatesToSecondary(DataStore store);
diff --git
a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java
b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java
index 24d7bf621f6..8c11fe6c93a 100644
---
a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java
+++
b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java
@@ -45,6 +45,8 @@ import com.cloud.vm.VirtualMachineProfile;
public interface TemplateManager {
static final String AllowPublicUserTemplatesCK =
"allow.public.user.templates";
static final String TemplatePreloaderPoolSizeCK =
"template.preloader.pool.size";
+ static final String PublicTemplateSecStorageCopyCK =
"secstorage.public.template.copy.max";
+ static final String PrivateTemplateSecStorageCopyCK =
"secstorage.private.template.copy.max";
static final ConfigKey<Boolean> AllowPublicUserTemplates = new
ConfigKey<Boolean>("Advanced", Boolean.class, AllowPublicUserTemplatesCK,
"true",
"If false, users will not be able to create public Templates.", true,
ConfigKey.Scope.Account);
@@ -64,6 +66,18 @@ public interface TemplateManager {
true,
ConfigKey.Scope.Global);
+ ConfigKey<Integer> PublicTemplateSecStorageCopy = new
ConfigKey<Integer>("Advanced", Integer.class,
+ PublicTemplateSecStorageCopyCK, "0",
+ "Maximum number of secondary storage pools to which a public
template is copied. " +
+ "0 means copy to all secondary storage pools (default behavior).",
+ true, ConfigKey.Scope.Zone);
+
+ ConfigKey<Integer> PrivateTemplateSecStorageCopy = new
ConfigKey<Integer>("Advanced", Integer.class,
+ PrivateTemplateSecStorageCopyCK, "1",
+ "Maximum number of secondary storage pools to which a private
template is copied. " +
+ "Default is 1 to preserve existing behavior.",
+ true, ConfigKey.Scope.Zone);
+
ConfigKey<Integer> VmIsoMaxCount = new ConfigKey<Integer>("Advanced",
Integer.class,
"vm.iso.max.count", "1",
@@ -153,6 +167,12 @@ public interface TemplateManager {
List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);
+ /**
+ * Max number of secondary storage copies for the template in this zone;
{@code 0} means no limit.
+ * SYSTEM/ROUTING/BUILTIN templates are always exempt (returns {@code 0}).
+ */
+ int getSecStorageCopyLimit(VMTemplateVO template, long zoneId);
+
TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId);
diff --git
a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index e29e89cf431..6e32df5d5e3 100644
---
a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++
b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -295,6 +295,171 @@ public class TemplateServiceImpl implements
TemplateService {
}
}
+ private int countActiveSecStorageCopies(long templateId, long zoneId) {
+ List<DataStore> stores = _storeMgr.getImageStoresByScope(new
ZoneScope(zoneId));
+ if (stores == null || stores.isEmpty()) {
+ return 0;
+ }
+ int count = 0;
+ for (DataStore ds : stores) {
+ List<TemplateDataStoreVO> rows =
_vmTemplateStoreDao.listByTemplateStore(templateId, ds.getId());
+ if (rows == null) {
+ continue;
+ }
+ for (TemplateDataStoreVO row : rows) {
+ State st = row.getState();
+ Status ds_state = row.getDownloadState();
+ if (st != State.Failed && st != State.Destroyed
+ && ds_state != Status.ABANDONED && ds_state !=
Status.DOWNLOAD_ERROR) {
+ count++;
+ break;
+ }
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Central gate for the secondary storage copy limit
(secstorage.public/private.template.copy.max).
+ * Every template-landing path (periodic sync, cross-zone copy, register,
upload) should consult this
+ * single method before placing another copy of a template on a secondary
store in a zone, so the limit
+ * is enforced consistently instead of being re-implemented per call site.
+ *
+ * SYSTEM/ROUTING/BUILTIN templates and a limit of 0 mean "unlimited"
(return true). The per-template,
+ * per-zone {@link GlobalLock} serializes concurrent placement decisions
so racing SSVM syncs / copies
+ * cannot collectively exceed the limit.
+ */
+ @Override
+ public boolean canCopyTemplateToImageStore(long templateId, long zoneId) {
+ VMTemplateVO template = _templateDao.findById(templateId);
+ if (template == null) {
+ return false;
+ }
+ int copyLimit = _tmpltMgr.getSecStorageCopyLimit(template, zoneId);
+ if (copyLimit <= 0) {
+ logger.debug("Template [{}] has no secondary storage copy limit in
zone [{}] (limit={}); copy allowed.",
+ template.getUniqueName(), zoneId, copyLimit);
+ return true;
+ }
+ int count = countActiveSecStorageCopies(templateId, zoneId);
+ logger.debug("Template [{}] secstorage copy check in zone [{}]:
count={}, limit={}",
+ template.getUniqueName(), zoneId, count, copyLimit);
+ return count < copyLimit;
+ }
+
+ private boolean hasReachedSecStorageCopyLimit(VMTemplateVO template, long
zoneId) {
+ return !canCopyTemplateToImageStore(template.getId(), zoneId);
+ }
+
+ @Override
+ public void replicateTemplateUpToCap(long templateId, long zoneId) {
+ VMTemplateVO template = _templateDao.findById(templateId);
+ if (template == null) {
+ return;
+ }
+ int copyLimit = _tmpltMgr.getSecStorageCopyLimit(template, zoneId);
+ if (copyLimit <= 0) {
+ return;
+ }
+ int needed = copyLimit - countActiveSecStorageCopies(templateId,
zoneId);
+ if (needed <= 0) {
+ return;
+ }
+ List<DataStore> stores = _storeMgr.getImageStoresByScope(new
ZoneScope(zoneId));
+ if (stores == null || stores.isEmpty()) {
+ return;
+ }
+ int kicked = 0;
+ for (DataStore store : stores) {
+ if (kicked >= needed) {
+ break;
+ }
+ if (hasActiveTemplateCopyOnStore(templateId, store.getId())) {
+ continue;
+ }
+ try {
+
storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(templateId,
store);
+ kicked++;
+ } catch (Exception e) {
+ logger.warn("Failed to proactively replicate template [{}] to
image store [{}] in zone [{}]: {}",
+ template.getUniqueName(), store.getName(), zoneId,
e.getMessage());
+ }
+ }
+ }
+
+ private boolean hasActiveTemplateCopyOnStore(long templateId, long
storeId) {
+ List<TemplateDataStoreVO> rows =
_vmTemplateStoreDao.listByTemplateStore(templateId, storeId);
+ if (rows == null) {
+ return false;
+ }
+ for (TemplateDataStoreVO row : rows) {
+ State st = row.getState();
+ Status ds = row.getDownloadState();
+ if (st != State.Failed && st != State.Destroyed
+ && ds != Status.ABANDONED && ds != Status.DOWNLOAD_ERROR) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void enforceSecStorageCopyLimit(long templateId, long zoneId) {
+ VMTemplateVO template = _templateDao.findById(templateId);
+ if (template == null) {
+ return;
+ }
+ int copyLimit = _tmpltMgr.getSecStorageCopyLimit(template, zoneId);
+ if (copyLimit <= 0) {
+ return;
+ }
+ if (_tmpltMgr.verifyHeuristicRulesForZone(template, zoneId) != null) {
+ return;
+ }
+ GlobalLock lock = GlobalLock.getInternLock("template.copy.limit." +
templateId + "." + zoneId);
+ try {
+ if (!lock.lock(30)) {
+ logger.warn("Could not acquire lock to enforce secondary
storage copy limit for template [{}] in zone [{}].",
+ template.getUniqueName(), zoneId);
+ return;
+ }
+ List<DataStore> stores = _storeMgr.getImageStoresByScope(new
ZoneScope(zoneId));
+ if (stores == null) {
+ return;
+ }
+ List<TemplateDataStoreVO> removable = new ArrayList<>();
+ for (DataStore ds : stores) {
+ TemplateDataStoreVO ref =
_vmTemplateStoreDao.findByStoreTemplate(ds.getId(), templateId);
+ if (ref != null
+ && ref.getState() == State.Ready
+ && ref.getDownloadState() == Status.DOWNLOADED
+ && (ref.getRefCnt() == null || ref.getRefCnt() == 0)) {
+ removable.add(ref);
+ }
+ }
+ int excess = removable.size() - copyLimit;
+ if (excess <= 0) {
+ return;
+ }
+ logger.info("Template [{}] has [{}] removable secondary storage
copies in zone [{}], limit is [{}]; removing [{}] excess copies.",
+ template.getUniqueName(), removable.size(), zoneId,
copyLimit, excess);
+ for (int i = 0; i < excess; i++) {
+ DataStore ds =
_storeMgr.getDataStore(removable.get(i).getDataStoreId(), DataStoreRole.Image);
+ try {
+
deleteTemplateAsync(_templateFactory.getTemplate(templateId, ds));
+ logger.info("Removed excess copy of template [{}] from
image store [{}] to honor the secondary storage copy limit.",
+ template.getUniqueName(), ds.getName());
+ } catch (Exception e) {
+ logger.warn("Failed to remove excess copy of template [{}]
from image store [{}]: {}",
+ template.getUniqueName(), ds, e.getMessage());
+ }
+ }
+ } finally {
+ lock.unlock();
+ lock.releaseRef();
+ }
+ }
+
protected boolean shouldDownloadTemplateToStore(VMTemplateVO template,
DataStore store) {
Long zoneId = store.getScope().getScopeId();
DataStore directedStore =
_tmpltMgr.verifyHeuristicRulesForZone(template, zoneId);
@@ -304,6 +469,12 @@ public class TemplateServiceImpl implements
TemplateService {
return false;
}
+ if (zoneId != null && hasReachedSecStorageCopyLimit(template, zoneId))
{
+ logger.info("Skipping sync of template [{}] to image store [{}]:
zone [{}] has reached the configured copy limit.",
+ template.getUniqueName(), store.getName(), zoneId);
+ return false;
+ }
+
if (template.isPublicTemplate()) {
logger.debug("Download of template [{}] to image store [{}] cannot
be skipped, as it is public.", template.getUniqueName(),
store.getName());
@@ -328,8 +499,9 @@ public class TemplateServiceImpl implements TemplateService
{
return true;
}
- logger.info("Skipping download of template [{}] to image store [{}].",
template.getUniqueName(), store.getName());
- return false;
+ logger.debug("Copying template [{}] to image store [{}] to reach the
configured secondary storage copy limit in zone [{}].",
+ template.getUniqueName(), store.getName(), zoneId);
+ return true;
}
@Override
@@ -531,10 +703,13 @@ public class TemplateServiceImpl implements
TemplateService {
&& tmpltStore.getState() == State.Ready
&& tmpltStore.getInstallPath() ==
null) {
logger.info("Keep fake entry in template
store table for migration of previous NFS to object store");
- } else {
+ } else if (tmpltStore.getDownloadState() ==
VMTemplateStorageResourceAssoc.Status.DOWNLOADED
+ || tmpltStore.getState() ==
State.Ready) {
logger.info("Removing leftover template {}
entry from template store table", tmplt);
- // remove those leftover entries
_vmTemplateStoreDao.remove(tmpltStore.getId());
+ } else {
+ logger.debug("Template {} entry on store
{} is in pre-download state ({}/{}); not treating as leftover.",
+ tmplt, store,
tmpltStore.getState(), tmpltStore.getDownloadState());
}
}
}
@@ -556,7 +731,7 @@ public class TemplateServiceImpl implements TemplateService
{
availHypers.add(HypervisorType.None); // bug 9809:
resume ISO
// download.
for (VMTemplateVO tmplt : toBeDownloaded) {
- // if this is private template, skip sync to a new
image store
+ // skip stores excluded by heuristic rules or
already at the configured copy limit
if (!shouldDownloadTemplateToStore(tmplt, store)) {
continue;
}
@@ -580,6 +755,12 @@ public class TemplateServiceImpl implements
TemplateService {
}
}
+ if (zoneId != null) {
+ for (VMTemplateVO tmplt : allTemplates) {
+ enforceSecStorageCopyLimit(tmplt.getId(), zoneId);
+ }
+ }
+
for (String uniqueName : templateInfos.keySet()) {
TemplateProp tInfo = templateInfos.get(uniqueName);
if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {
@@ -965,6 +1146,15 @@ public class TemplateServiceImpl implements
TemplateService {
return null;
}
+ try {
+ DataStore destStore = template.getDataStore();
+ if (destStore != null && destStore.getScope() != null &&
destStore.getScope().getScopeId() != null) {
+ enforceSecStorageCopyLimit(template.getId(),
destStore.getScope().getScopeId());
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to enforce secstorage copy limit after
template [{}] became Ready: {}", template.getUuid(), e.getMessage());
+ }
+
if (parentCallback != null) {
parentCallback.complete(result);
}
@@ -1406,6 +1596,14 @@ public class TemplateServiceImpl implements
TemplateService {
destTemplate.processEvent(Event.OperationFailed);
} else {
destTemplate.processEvent(Event.OperationSucceeded,
result.getAnswer());
+ try {
+ DataStore destStore = destTemplate.getDataStore();
+ if (destStore != null && destStore.getScope() != null &&
destStore.getScope().getScopeId() != null) {
+ enforceSecStorageCopyLimit(destTemplate.getId(),
destStore.getScope().getScopeId());
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to enforce secstorage copy limit after
copy of template [{}] became Ready: {}", destTemplate.getUuid(),
e.getMessage());
+ }
}
future.complete(res);
} catch (Exception e) {
@@ -1431,6 +1629,15 @@ public class TemplateServiceImpl implements
TemplateService {
destTemplate.processEvent(Event.OperationFailed);
} else {
destTemplate.processEvent(Event.OperationSucceeded,
result.getAnswer());
+ try {
+ DataStore destStore = destTemplate.getDataStore();
+ if (destStore != null && destStore.getScope() != null &&
destStore.getScope().getScopeId() != null) {
+ replicateTemplateUpToCap(destTemplate.getId(),
destStore.getScope().getScopeId());
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to schedule additional copies for
cross-zone copied template [{}]: {}",
+ destTemplate.getUuid(), e.getMessage());
+ }
}
future.complete(res);
} catch (Exception e) {
diff --git
a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java
b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java
index e9eac045869..315fb697894 100644
---
a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java
+++
b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java
@@ -119,6 +119,7 @@ public class TemplateServiceImplTest {
Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L,
sourceStoreMock);
Mockito.doReturn(3L).when(dataStoreMock).getId();
Mockito.doReturn(zoneScopeMock).when(dataStoreMock).getScope();
+ Mockito.lenient().doReturn(tmpltMock).when(templateDao).findById(2L);
}
@Test
@@ -153,11 +154,37 @@ public class TemplateServiceImplTest {
}
@Test
- public void
shouldDownloadTemplateToStoreTestSkipsPrivateExistingTemplate() {
+ public void
shouldDownloadTemplateToStoreTestReplicatesPrivateTemplateUnderCopyLimit() {
+ DataStore storeWithCopy = Mockito.mock(DataStore.class);
+ Mockito.doReturn(10L).when(storeWithCopy).getId();
+ Mockito.when(templateManagerMock.getSecStorageCopyLimit(tmpltMock,
zoneScopeMock.getScopeId())).thenReturn(2);
+
Mockito.doReturn(List.of(storeWithCopy)).when(dataStoreManagerMock).getImageStoresByScope(Mockito.any());
+
Mockito.doReturn(List.of(Mockito.mock(TemplateDataStoreVO.class))).when(templateDataStoreDao).listByTemplateStore(2L,
10L);
Mockito.when(templateDataStoreDao.findByTemplateZone(tmpltMock.getId(),
zoneScopeMock.getScopeId(),
DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class));
+
Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock,
dataStoreMock));
+ }
+
+ @Test
+ public void
shouldDownloadTemplateToStoreTestSkipsPrivateTemplateAtCopyLimit() {
+ DataStore storeWithCopy = Mockito.mock(DataStore.class);
+ Mockito.doReturn(10L).when(storeWithCopy).getId();
+ Mockito.when(templateManagerMock.getSecStorageCopyLimit(tmpltMock,
zoneScopeMock.getScopeId())).thenReturn(1);
+
Mockito.doReturn(List.of(storeWithCopy)).when(dataStoreManagerMock).getImageStoresByScope(Mockito.any());
+
Mockito.doReturn(List.of(Mockito.mock(TemplateDataStoreVO.class))).when(templateDataStoreDao).listByTemplateStore(2L,
10L);
Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock,
dataStoreMock));
}
+ @Test
+ public void canCopyTemplateToImageStoreTestUnlimitedWhenLimitIsZero() {
+ Mockito.when(templateManagerMock.getSecStorageCopyLimit(tmpltMock,
1L)).thenReturn(0);
+ Assert.assertTrue(templateService.canCopyTemplateToImageStore(2L, 1L));
+ }
+
+ // The under-limit / at-limit behavior of canCopyTemplateToImageStore is
exercised through
+ // shouldDownloadTemplateToStore above (Replicates*UnderCopyLimit /
Skips*AtCopyLimit), which run it via
+ // the real call path. Calling the GlobalLock-wrapped method directly on
the Mockito spy is not reliable
+ // in the unit-test JVM, so it is not duplicated here.
+
@Test
public void
tryDownloadingTemplateToImageStoreTestDownloadsTemplateWhenUrlIsNotNull() {
Mockito.doReturn("url").when(tmpltMock).getUrl();
diff --git
a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
index c670b631645..64e066f0c91 100755
--- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
+++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java
@@ -574,6 +574,12 @@ public class ImageStoreUploadMonitorImpl extends
ManagerBase implements ImageSto
if (logger.isDebugEnabled()) {
logger.debug("Template {} uploaded
successfully", tmpTemplate);
}
+ try {
+
templateService.replicateTemplateUpToCap(tmpTemplate.getId(),
vo.getDataCenterId());
+ } catch (Exception e) {
+ logger.warn("Failed to schedule additional
copies for uploaded template [{}] in zone [{}]: {}",
+ tmpTemplate.getUuid(),
vo.getDataCenterId(), e.getMessage());
+ }
break;
case IN_PROGRESS:
if
(!checkAndUpdateTemplateResourceLimit(tmpTemplate, tmpTemplateDataStore,
answer)) {
diff --git
a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
index fdde8f47a67..9417d63c594 100644
--- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
@@ -19,10 +19,10 @@ package com.cloud.template;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
-import java.util.HashSet;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
+import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@@ -264,9 +264,10 @@ public class HypervisorTemplateAdapter extends
TemplateAdapterBase {
if (imageStore == null) {
List<DataStore> imageStores =
getImageStoresThrowsExceptionIfNotFound(zoneId, profile);
- standardImageStoreAllocation(imageStores, template);
+ standardImageStoreAllocation(imageStores, template, zoneId);
} else {
- validateSecondaryStorageAndCreateTemplate(List.of(imageStore),
template, null);
+ int copyLimit = getSecStorageCopyLimit(template, zoneId);
+ validateSecondaryStorageAndCreateTemplate(List.of(imageStore),
template, new HashMap<>(), copyLimit);
}
}
}
@@ -279,17 +280,17 @@ public class HypervisorTemplateAdapter extends
TemplateAdapterBase {
return imageStores;
}
- protected void standardImageStoreAllocation(List<DataStore> imageStores,
VMTemplateVO template) {
- Set<Long> zoneSet = new HashSet<Long>();
+ protected void standardImageStoreAllocation(List<DataStore> imageStores,
VMTemplateVO template, long zoneId) {
+ int copyLimit = getSecStorageCopyLimit(template, zoneId);
Collections.shuffle(imageStores);
- validateSecondaryStorageAndCreateTemplate(imageStores, template,
zoneSet);
+ validateSecondaryStorageAndCreateTemplate(imageStores, template, new
HashMap<>(), copyLimit);
}
- protected void validateSecondaryStorageAndCreateTemplate(List<DataStore>
imageStores, VMTemplateVO template, Set<Long> zoneSet) {
+ protected void validateSecondaryStorageAndCreateTemplate(List<DataStore>
imageStores, VMTemplateVO template, Map<Long, Integer> zoneCopyCount, int
copyLimit) {
for (DataStore imageStore : imageStores) {
Long zoneId = imageStore.getScope().getScopeId();
- if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneSet,
isPrivateTemplate(template))) {
+ if (!isZoneAndImageStoreAvailable(imageStore, zoneId,
zoneCopyCount, copyLimit)) {
continue;
}
diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java
b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java
index 8f508135605..1aa3bedbdf0 100644
--- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java
+++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java
@@ -20,11 +20,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
import javax.inject.Inject;
@@ -169,7 +167,11 @@ public abstract class TemplateAdapterBase extends
AdapterBase implements Templat
return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId,
heuristicType, template);
}
- protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long
zoneId, Set<Long> zoneSet, boolean isTemplatePrivate) {
+ protected int getSecStorageCopyLimit(VMTemplateVO template, long zoneId) {
+ return templateMgr.getSecStorageCopyLimit(template, zoneId);
+ }
+
+ protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long
zoneId, Map<Long, Integer> zoneCopyCount, int copyLimit) {
if (zoneId == null) {
logger.warn(String.format("Zone ID is null, cannot allocate
ISO/template in image store [%s].", imageStore));
return false;
@@ -191,33 +193,30 @@ public abstract class TemplateAdapterBase extends
AdapterBase implements Templat
return false;
}
- if (zoneSet == null) {
- logger.info(String.format("Zone set is null; therefore, the
ISO/template should be allocated in every secondary storage of zone [%s].",
zone));
- return true;
- }
-
- if (isTemplatePrivate && zoneSet.contains(zoneId)) {
- logger.info(String.format("The template is private and it is
already allocated in a secondary storage in zone [%s]; therefore, image store
[%s] will be skipped.",
- zone, imageStore));
+ int currentCount = zoneCopyCount.getOrDefault(zoneId, 0);
+ if (copyLimit > 0 && currentCount >= copyLimit) {
+ logger.info("Copy limit of {} reached for zone [{}]; skipping
image store [{}].", copyLimit, zone, imageStore);
return false;
}
- logger.info(String.format("Private template will be allocated in image
store [%s] in zone [%s].", imageStore, zone));
- zoneSet.add(zoneId);
+ zoneCopyCount.put(zoneId, currentCount + 1);
return true;
}
/**
- * If the template/ISO is marked as private, then it is allocated to a
random secondary storage; otherwise, allocates to every storage pool in every
zone given by the
- * {@link TemplateProfile#getZoneIdList()}.
+ * Allocates the template/ISO to a single image store - the one the file
will be uploaded to. The upload can only
+ * target one secondary store, so additional copies (up to the configured
secstorage.public/private.template.copy.max)
+ * are propagated later by template sync instead of being pre-allocated
here as empty placeholder entries that never
+ * receive the data.
*/
protected void postUploadAllocation(List<DataStore> imageStores,
VMTemplateVO template, List<TemplateOrVolumePostUploadCommand> payloads) {
- Set<Long> zoneSet = new HashSet<>();
+ Map<Long, Integer> zoneCopyCount = new HashMap<>();
Collections.shuffle(imageStores);
for (DataStore imageStore : imageStores) {
Long zoneId_is = imageStore.getScope().getScopeId();
+ int copyLimit = zoneId_is == null ? 0 :
getSecStorageCopyLimit(template, zoneId_is);
- if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneSet,
isPrivateTemplate(template))) {
+ if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is,
zoneCopyCount, copyLimit)) {
continue;
}
@@ -251,15 +250,11 @@ public abstract class TemplateAdapterBase extends
AdapterBase implements Templat
payload.setRequiresHvm(template.requiresHvm());
payload.setDescription(template.getDisplayText());
payloads.add(payload);
- }
- }
- protected boolean isPrivateTemplate(VMTemplateVO template){
- // if public OR featured OR system template
- if (template.isPublicTemplate() || template.isFeatured() ||
template.getTemplateType() == TemplateType.SYSTEM) {
- return false;
- } else {
- return true;
+ // The file can only be uploaded to a single secondary store.
Allocate just this one; additional copies
+ // up to the configured secondary storage copy limit are
propagated afterwards by template sync, so we do
+ // not create empty placeholder template_store_ref rows on the
other stores.
+ break;
}
}
diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
index 6cac485c4e1..d9049af679b 100755
--- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java
@@ -943,6 +943,12 @@ public class TemplateManagerImpl extends ManagerBase
implements TemplateManager,
_tmplStoreDao.removeByTemplateStore(tmpltId,
dstSecStore.getId());
}
+ if (!_tmpltSvr.canCopyTemplateToImageStore(tmpltId, dstZoneId)) {
+ logger.info("Not copying template {} to image store {}: zone
{} has reached the configured secondary storage copy limit.",
+ template, dstSecStore, dstZone);
+ continue;
+ }
+
AsyncCallFuture<TemplateApiResult> future =
_tmpltSvr.copyTemplate(srcTemplate, dstSecStore);
try {
TemplateApiResult result = future.get();
@@ -1914,6 +1920,13 @@ public class TemplateManagerImpl extends ManagerBase
implements TemplateManager,
_launchPermissionDao.removeAllPermissions(id);
_messageBus.publish(_name,
TemplateManager.MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT, PublishScope.LOCAL,
template.getId());
}
+
+ if (isPublic != null || isFeatured != null ||
"reset".equalsIgnoreCase(operation)) {
+ for (VMTemplateZoneVO templateZone :
_tmpltZoneDao.listByTemplateId(template.getId())) {
+ _tmpltSvr.enforceSecStorageCopyLimit(template.getId(),
templateZone.getZoneId());
+ _tmpltSvr.replicateTemplateUpToCap(template.getId(),
templateZone.getZoneId());
+ }
+ }
return true;
}
@@ -1931,10 +1944,10 @@ public class TemplateManagerImpl extends ManagerBase
implements TemplateManager,
Account caller = CallContext.current().getCallingAccount();
boolean kvmSnapshotOnlyInPrimaryStorage = false;
SnapshotInfo snapInfo = null;
+ long zoneId = 0;
try {
TemplateInfo tmplInfo = _tmplFactory.getTemplate(templateId,
DataStoreRole.Image);
- long zoneId = 0;
if (snapshotId != null) {
snapshot = _snapshotDao.findById(snapshotId);
if (command.getZoneId() == null) {
@@ -2074,6 +2087,12 @@ public class TemplateManagerImpl extends ManagerBase
implements TemplateManager,
}
if (privateTemplate != null) {
+ try {
+ _tmpltSvr.replicateTemplateUpToCap(privateTemplate.getId(),
zoneId);
+ } catch (Exception e) {
+ logger.warn("Failed to schedule additional copies for template
[{}] in zone [{}]: {}",
+ privateTemplate.getUniqueName(), zoneId,
e.getMessage());
+ }
return privateTemplate;
} else {
throw new CloudRuntimeException("Failed to create a Template");
@@ -2397,6 +2416,20 @@ public class TemplateManagerImpl extends ManagerBase
implements TemplateManager,
return stores;
}
+ @Override
+ public int getSecStorageCopyLimit(VMTemplateVO template, long zoneId) {
+ if (template == null) {
+ return 0;
+ }
+ TemplateType type = template.getTemplateType();
+ if (type == TemplateType.SYSTEM || type == TemplateType.ROUTING ||
type == TemplateType.BUILTIN) {
+ return 0;
+ }
+ return template.isPublicTemplate()
+ ? PublicTemplateSecStorageCopy.valueIn(zoneId)
+ : PrivateTemplateSecStorageCopy.valueIn(zoneId);
+ }
+
@Override
@ActionEvent(eventType = EventTypes.EVENT_ISO_UPDATE, eventDescription =
"Updating ISO", async = false)
public VMTemplateVO updateTemplate(UpdateIsoCmd cmd) {
@@ -2718,6 +2751,8 @@ public class TemplateManagerImpl extends ManagerBase
implements TemplateManager,
TemplatePreloaderPoolSize,
ValidateUrlIsResolvableBeforeRegisteringTemplate,
TemplateDeleteFromPrimaryStorage,
+ PublicTemplateSecStorageCopy,
+ PrivateTemplateSecStorageCopy,
VmIsoMaxCount};
}
diff --git
a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java
b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java
index e2a97be469f..4cd48e686b0 100644
--- a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java
+++ b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java
@@ -32,10 +32,8 @@ import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ExecutionException;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -339,7 +337,7 @@ public class HypervisorTemplateAdapterTest {
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class),
Mockito.any(TemplateProfile.class));
Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class),
Mockito.anyLong());
-
Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(),
Mockito.any(VMTemplateVO.class));
+
Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(),
Mockito.any(VMTemplateVO.class), Mockito.anyLong());
_adapter.createTemplateWithinZones(templateProfileMock,
vmTemplateVOMock);
@@ -355,11 +353,11 @@ public class HypervisorTemplateAdapterTest {
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class),
Mockito.any(TemplateProfile.class));
Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class),
Mockito.anyLong());
-
Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(),
Mockito.any(VMTemplateVO.class));
+
Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(),
Mockito.any(VMTemplateVO.class), Mockito.anyLong());
_adapter.createTemplateWithinZones(templateProfileMock,
vmTemplateVOMock);
- Mockito.verify(_adapter,
Mockito.times(1)).standardImageStoreAllocation(Mockito.isNull(),
Mockito.any(VMTemplateVO.class));
+ Mockito.verify(_adapter,
Mockito.times(1)).standardImageStoreAllocation(Mockito.isNull(),
Mockito.any(VMTemplateVO.class), Mockito.anyLong());
}
@Test
@@ -371,11 +369,11 @@ public class HypervisorTemplateAdapterTest {
Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
Mockito.doReturn(dataStoreMock).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class),
Mockito.anyLong());
-
Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class),
Mockito.any(VMTemplateVO.class), Mockito.isNull());
+
Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class),
Mockito.any(VMTemplateVO.class), Mockito.any(Map.class), Mockito.anyInt());
_adapter.createTemplateWithinZones(templateProfileMock,
vmTemplateVOMock);
- Mockito.verify(_adapter,
Mockito.times(1)).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class),
Mockito.any(VMTemplateVO.class), Mockito.isNull());
+ Mockito.verify(_adapter,
Mockito.times(1)).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class),
Mockito.any(VMTemplateVO.class), Mockito.any(Map.class), Mockito.anyInt());
}
@Test(expected = CloudRuntimeException.class)
@@ -411,11 +409,8 @@ public class HypervisorTemplateAdapterTest {
@Test
public void
isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
- Long zoneId = null;
- Set<Long> zoneSet = null;
- boolean isTemplatePrivate = false;
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
null, new HashMap<>(), 0);
Mockito.verify(loggerMock, Mockito.times(1)).warn(String.format("Zone
ID is null, cannot allocate ISO/template in image store [%s].", dataStoreMock));
Assert.assertFalse(result);
@@ -425,13 +420,10 @@ public class HypervisorTemplateAdapterTest {
public void isZoneAndImageStoreAvailableTestZoneIsNullShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
- Set<Long> zoneSet = null;
- boolean isTemplatePrivate = false;
- DataCenterVO dataCenterVOMock = null;
-
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
+ Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(null);
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, new HashMap<>(), 0);
Mockito.verify(loggerMock, Mockito.times(1)).warn("Unable to find zone
by id [{}], so skip downloading template to its image store [{}].",
zoneId, dataStoreMock);
@@ -442,14 +434,12 @@ public class HypervisorTemplateAdapterTest {
public void
isZoneAndImageStoreAvailableTestZoneIsDisabledShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
- Set<Long> zoneSet = null;
- boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled);
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, new HashMap<>(), 0);
Mockito.verify(loggerMock, Mockito.times(1)).info("Zone [{}] is
disabled. Skip downloading template to its image store [{}].",
dataCenterVOMock, dataStoreMock);
Assert.assertFalse(result);
@@ -459,15 +449,13 @@ public class HypervisorTemplateAdapterTest {
public void
isZoneAndImageStoreAvailableTestImageStoreDoesNotHaveEnoughCapacityShouldReturnFalse()
{
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
- Set<Long> zoneSet = null;
- boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(false);
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, new HashMap<>(), 0);
Mockito.verify(loggerMock, times(1)).info("Image store doesn't have
enough capacity. Skip downloading template to this image store [{}].",
dataStoreMock);
@@ -475,60 +463,72 @@ public class HypervisorTemplateAdapterTest {
}
@Test
- public void
isZoneAndImageStoreAvailableTestImageStoreHasEnoughCapacityAndZoneSetIsNullShouldReturnTrue()
{
+ public void
isZoneAndImageStoreAvailableTestReplicaLimitZeroShouldCopyToAllStores() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
- Set<Long> zoneSet = null;
- boolean isTemplatePrivate = false;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
+ Map<Long, Integer> zoneCopyCount = new HashMap<>();
+ zoneCopyCount.put(zoneId, 999);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneCopyCount, 0);
- Mockito.verify(loggerMock, times(1)).info(String.format("Zone set is
null; therefore, the ISO/template should be allocated in every secondary
storage " +
- "of zone [%s].", dataCenterVOMock));
Assert.assertTrue(result);
+ Assert.assertEquals(1000, (int) zoneCopyCount.get(zoneId));
}
@Test
- public void
isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsAlreadyAllocatedToTheSameZoneShouldReturnFalse()
{
+ public void
isZoneAndImageStoreAvailableTestReplicaLimitReachedShouldReturnFalse() {
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
- Set<Long> zoneSet = Set.of(1L);
- boolean isTemplatePrivate = true;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
+ Map<Long, Integer> zoneCopyCount = new HashMap<>();
+ zoneCopyCount.put(zoneId, 1);
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneCopyCount, 1);
- Mockito.verify(loggerMock, times(1)).info(String.format("The template
is private and it is already allocated in a secondary storage in zone [%s]; " +
- "therefore, image store [%s] will be skipped.",
dataCenterVOMock, dataStoreMock));
+ Mockito.verify(loggerMock, times(1)).info("Copy limit of {} reached
for zone [{}]; skipping image store [{}].", 1, dataCenterVOMock, dataStoreMock);
Assert.assertFalse(result);
}
@Test
- public void
isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsNotAlreadyAllocatedToTheSameZoneShouldReturnTrue()
{
+ public void
isZoneAndImageStoreAvailableTestReplicaLimitNotYetReachedShouldReturnTrueAndIncrementCount()
{
DataStore dataStoreMock = Mockito.mock(DataStore.class);
Long zoneId = 1L;
- Set<Long> zoneSet = new HashSet<>();
- boolean isTemplatePrivate = true;
DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
+ Map<Long, Integer> zoneCopyCount = new HashMap<>();
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
- boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneSet, isTemplatePrivate);
+ boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock,
zoneId, zoneCopyCount, 2);
- Mockito.verify(loggerMock, times(1)).info(String.format("Private
template will be allocated in image store [%s] in zone [%s].",
- dataStoreMock, dataCenterVOMock));
Assert.assertTrue(result);
+ Assert.assertEquals(1, (int) zoneCopyCount.get(zoneId));
+ }
+
+ @Test
+ public void
isZoneAndImageStoreAvailableTestReplicaLimitOfTwoShouldCopyToExactlyTwoStores()
{
+ Long zoneId = 1L;
+ DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class);
+ Map<Long, Integer> zoneCopyCount = new HashMap<>();
+
+
Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock);
+
Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled);
+
Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true);
+
+
Assert.assertTrue(_adapter.isZoneAndImageStoreAvailable(Mockito.mock(DataStore.class),
zoneId, zoneCopyCount, 2));
+
Assert.assertTrue(_adapter.isZoneAndImageStoreAvailable(Mockito.mock(DataStore.class),
zoneId, zoneCopyCount, 2));
+
Assert.assertFalse(_adapter.isZoneAndImageStoreAvailable(Mockito.mock(DataStore.class),
zoneId, zoneCopyCount, 2));
+ Assert.assertEquals(2, (int) zoneCopyCount.get(zoneId));
}
@Test
