nizhikov commented on code in PR #11939: URL: https://github.com/apache/ignite/pull/11939#discussion_r2013901049
########## modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/SnapshotCompatibilityTest.java: ########## @@ -0,0 +1,459 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.compatibility.persistence; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.binary.BinaryType; +import org.apache.ignite.cdc.TypeMapping; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.compatibility.testframework.junits.IgniteCompatibilityAbstractTest; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.dump.DumpConsumer; +import org.apache.ignite.dump.DumpEntry; +import org.apache.ignite.dump.DumpReader; +import org.apache.ignite.dump.DumpReaderConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.StoredCacheData; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteInClosure; +import org.jetbrains.annotations.NotNull; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * + */ +@RunWith(Parameterized.class) +public class SnapshotCompatibilityTest extends IgniteCompatibilityAbstractTest { + /** */ + @Parameterized.Parameter + public boolean incrementalSnp; + + /** */ + @Parameterized.Parameter(1) + @Nullable public String consistentId; + + /** */ + @Parameterized.Parameter(2) + public int oldNodesCnt; + + /** */ + @Parameterized.Parameter(3) + public boolean cacheDump; + + /** */ + @Parameterized.Parameter(4) + public boolean customSnpPath; + + /** */ + @Parameterized.Parameter(5) + public boolean testCacheGrp; + + /** */ + private static final String OLD_IGNITE_VERSION = "2.16.0"; + + /** */ + private static final String SNAPSHOT_NAME = "test_snapshot"; + + /** */ + private static final String CACHE_DUMP_NAME = "test_cache_dump"; + + /** */ + private static final int BASE_CACHE_SIZE = 10_000; + + /** */ + private static final int ENTRIES_CNT_FOR_INCREMENT = 10_000; + + /** */ + private static final String CUSTOM_SNP_RELATIVE_PATH = "ex_snapshots"; + + /** */ + private static final String CONSISTENT_ID = "db3e5e20-91c1-4b2d-95c9-f7e5f7a0b8d3"; + + /** */ + private CacheGroupInfo cacheGrpInfo; + + /** + * The test is parameterized by whether an incremental snapshot is taken and by consistentId. + * Restore incremental snapshot if consistentId is null is fixed in 2.17.0, see here https://issues.apache.org/jira/browse/IGNITE-23222. + * Also restoring cache dump and any kind of snapshot is pointless. + */ + @Parameters(name = "incrementalSnp={0}, consistentID={1}, oldNodesCnt={2}, cacheDump={3}, customSnpPath={4}, testCacheGrp={5}") + public static Collection<Object[]> data() { + List<Object[]> data = new ArrayList<>(); + + List<Boolean> incrementalSnpValues = Arrays.asList(true, false); + List<String> consistentIdValues = Arrays.asList(CONSISTENT_ID, null); + List<Integer> oldNodesCntValues = Arrays.asList(1, 3); + List<Boolean> createDumpValues = Arrays.asList(true, false); + List<Boolean> customSnpPathValues = Arrays.asList(true, false); + List<Boolean> cachesCntValues = Arrays.asList(true, false); + + for (Boolean incrementalSnp : incrementalSnpValues) + for (String consistentId : consistentIdValues) + for (Integer oldNodesCnt : oldNodesCntValues) + for (Boolean cacheDump : createDumpValues) + for (Boolean customSnpPath : customSnpPathValues) + for (Boolean testCacheGrp : cachesCntValues) + if ((!incrementalSnp || !cacheDump) && (!incrementalSnp || consistentId != null)) + data.add( + new Object[]{incrementalSnp, consistentId, oldNodesCnt, cacheDump, customSnpPath, testCacheGrp} + ); + + return data; + } + + /** */ + @Before + public void setUp() { + final int cachesCnt = testCacheGrp ? 2 : 1; + + List<String> cacheNames = new ArrayList<>(); + + for (int i = 0; i < cachesCnt; ++i) + cacheNames.add("test-cache-" + i); + + cacheGrpInfo = new CacheGroupInfo("test-cache", cacheNames); + } + + /** */ + @Test + public void testSnapshotRestore() throws Exception { + try { + startGrid( + oldNodesCnt, + OLD_IGNITE_VERSION, + new ConfigurationClosure(incrementalSnp, consistentId, customSnpPath, true, cacheGrpInfo), + new PostStartupClosure(incrementalSnp, cacheDump, cacheGrpInfo) + ); + + stopAllGrids(); + + cleanPersistenceDir(true); + + IgniteEx curIgn = startGrid(getCurrentIgniteConfiguration(incrementalSnp, consistentId, customSnpPath)); + + curIgn.cluster().state(ClusterState.ACTIVE); + + if (cacheDump) + checkCacheDump(curIgn); + else if (incrementalSnp) + checkIncrementalSnapshot(curIgn); + else + checkSnapshot(curIgn); + } + finally { + stopAllGrids(); + + cleanPersistenceDir(); + } + } + + /** */ + private void checkSnapshot(IgniteEx curIgn) { + curIgn.snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(cacheGrpInfo.getName())).get(); + + checkCaches(curIgn, cacheGrpInfo, BASE_CACHE_SIZE); + } + + /** */ + private void checkIncrementalSnapshot(IgniteEx curIgn) { + curIgn.snapshot().restoreSnapshot(SNAPSHOT_NAME, Collections.singleton(cacheGrpInfo.getName()), 1).get(); + + checkCaches(curIgn, cacheGrpInfo, BASE_CACHE_SIZE + ENTRIES_CNT_FOR_INCREMENT); + } + + /** */ + private void checkCacheDump(IgniteEx curIgn) throws IgniteCheckedException { + Map<String, Integer> foundCacheSizes = new ConcurrentHashMap<>(); + + Set<String> foundCacheNames = ConcurrentHashMap.newKeySet(); + + DumpConsumer consumer = new DumpConsumer() { + @Override public void start() { + // No-op. + } + + @Override public void onMappings(Iterator<TypeMapping> mappings) { + // No-op. + } + + @Override public void onTypes(Iterator<BinaryType> types) { + // No-op. + } + + @Override public void onCacheConfigs(Iterator<StoredCacheData> caches) { + assertNotNull(cacheGrpInfo); + + caches.forEachRemaining(cache -> { + CacheConfiguration<?, ?> ccfg = cache.config(); + + assertNotNull(ccfg); + + assertEquals(cacheGrpInfo.getName(), ccfg.getGroupName()); + + foundCacheNames.add(ccfg.getName()); + }); + } + + @Override public void onPartition(int grp, int part, Iterator<DumpEntry> data) { + assertNotNull(cacheGrpInfo); + + data.forEachRemaining(de -> { + assertNotNull(de); + + Integer key = (Integer)de.key(); + String val = (String)de.value(); + + for (String cacheName : cacheGrpInfo.getCacheNamesList()) { + if (val.startsWith(cacheName)) { + assertEquals(calcValue(cacheName, key), val); + + foundCacheSizes.put(cacheName, foundCacheSizes.getOrDefault(cacheName, 0) + 1); + + break; + } + } + }); + } + + @Override public void stop() { + // No-op. + } + }; + + new DumpReader(new DumpReaderConfiguration( + CACHE_DUMP_NAME, + customSnpPath ? getCustomSnapshotPath(CUSTOM_SNP_RELATIVE_PATH, false) : null, + curIgn.configuration(), + consumer + ), log).run(); + + cacheGrpInfo.getCacheNamesList().forEach( + cacheName -> assertEquals(BASE_CACHE_SIZE, (int)foundCacheSizes.get(cacheName)) + ); + + assertEquals(cacheGrpInfo.getCacheNamesSet(), foundCacheNames); + } + + /** */ + private @NotNull IgniteConfiguration getCurrentIgniteConfiguration( + boolean incrementalSnp, + String consistentId, + boolean customSnpPath + ) throws Exception { + IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(0)); + + // We configure current Ignite version in the same way as the old one. + new ConfigurationClosure(incrementalSnp, consistentId, customSnpPath, false, cacheGrpInfo).apply(cfg); + + return cfg; + } + + /** */ + private static String getCustomSnapshotPath(String relativePath, boolean forSnapshotTake) throws IgniteCheckedException { Review Comment: getCustomSnapshotPath -> customSnapshotPath get must be omitted for internal code. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@ignite.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
