This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new abe7ed1c44 Sped up SplitMillionIT by not hosting clone table tablets.
(#5294)
abe7ed1c44 is described below
commit abe7ed1c44134a3df8411cf0b400154cc30d9ffc
Author: Dave Marion <[email protected]>
AuthorDate: Tue Feb 4 14:45:43 2025 -0500
Sped up SplitMillionIT by not hosting clone table tablets. (#5294)
The test table is cloned and some of the tablets are hosted. Then
the cloned table is merged, which closes the hosted tablets. This
change sets the clone table tablet availability to unhosted to try
and speed up the merge.
---
.../accumulo/test/functional/SplitMillionIT.java | 35 ++++++++++++++++++----
1 file changed, 29 insertions(+), 6 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
index 2094c15430..670223967f 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitMillionIT.java
@@ -33,11 +33,16 @@ import java.util.stream.IntStream;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase.ConsistencyLevel;
import org.apache.accumulo.core.client.admin.CloneConfiguration;
import org.apache.accumulo.core.client.admin.CompactionConfig;
+import org.apache.accumulo.core.client.admin.TabletAvailability;
+import org.apache.accumulo.core.data.ArrayByteSequence;
+import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
import org.apache.accumulo.core.metadata.AccumuloTable;
@@ -57,10 +62,11 @@ public class SplitMillionIT extends ConfigurableMacBase {
private static final Logger log =
LoggerFactory.getLogger(SplitMillionIT.class);
public static class XFilter extends Filter {
+ private static final ByteSequence X_BS = new ArrayByteSequence("x");
@Override
public boolean accept(Key k, Value v) {
- return !k.getColumnQualifierData().toString().equals("x");
+ return !k.getColumnQualifierData().equals(X_BS);
}
}
@@ -162,7 +168,15 @@ public class SplitMillionIT extends ConfigurableMacBase {
log.info("Time to compact all tablets : {}ms", t2 - t1);
var expected = Map.of("y", "900", "z", "300");
- vefifyData(rows, c, tableName, expected);
+ vefifyData(rows, c, tableName, expected, ConsistencyLevel.IMMEDIATE);
+
+ // We are done with tableName, except for deleting it later. Modify the
tablets
+ // availability to UNHOSTED so that when the clone operation happens
below none
+ // of the tablets for the clone table will be hosted. The subsequent
merge operation
+ // is a metadata-only operation unless the tablet is hosted. If the
tablet is hosted
+ // then the tablet has to be closed making the merge operation take
longer.
+ c.tableOperations().setTabletAvailability(tableName, new Range(),
+ TabletAvailability.UNHOSTED);
// clone the table to test cloning with lots of tablets and also to give
merge its own table
// to work on
@@ -171,7 +185,10 @@ public class SplitMillionIT extends ConfigurableMacBase {
c.tableOperations().clone(tableName, cloneName,
CloneConfiguration.builder().build());
t2 = System.currentTimeMillis();
log.info("Time to clone table : {}ms", t2 - t1);
- vefifyData(rows, c, cloneName, expected);
+ vefifyData(rows, c, cloneName, expected, ConsistencyLevel.EVENTUAL);
+
+ TableId tid =
TableId.of(c.tableOperations().tableIdMap().get(cloneName));
+ assertEquals(0, ManagerAssignmentIT.countTabletsWithLocation(c, tid));
// merge the clone, so that delete table can run later on tablet with
lots and lots of tablets
t1 = System.currentTimeMillis();
@@ -179,23 +196,29 @@ public class SplitMillionIT extends ConfigurableMacBase {
t2 = System.currentTimeMillis();
log.info("Time to merge all tablets : {}ms", t2 - t1);
- vefifyData(rows, c, cloneName, expected);
+ vefifyData(rows, c, cloneName, expected, ConsistencyLevel.EVENTUAL);
t1 = System.currentTimeMillis();
c.tableOperations().delete(tableName);
t2 = System.currentTimeMillis();
- log.info("Time to delete table : {}ms", t2 - t1);
+ log.info("Time to delete original table : {}ms", t2 - t1);
+
+ t1 = System.currentTimeMillis();
+ c.tableOperations().delete(cloneName);
+ t2 = System.currentTimeMillis();
+ log.info("Time to delete clone table : {}ms", t2 - t1);
}
}
private void vefifyData(int[] rows, AccumuloClient c, String tableName,
- Map<String,String> expected) throws Exception {
+ Map<String,String> expected, ConsistencyLevel level) throws Exception {
// use a batch scanner so that many hosting request can be submitted at
the same time
long t1 = System.currentTimeMillis();
try (var scanner = c.createBatchScanner(tableName)) {
var ranges = IntStream.of(rows).mapToObj(row -> String.format("%010d",
row)).map(Range::new)
.collect(Collectors.toList());
scanner.setRanges(ranges);
+ scanner.setConsistencyLevel(level);
Map<String,Map<String,String>> allCoords = new HashMap<>();
scanner.forEach((k, v) -> {
var row = k.getRowData().toString();
