This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new ba9c376ecc Increase majc priority for tablets over file size threshold
(#5026)
ba9c376ecc is described below
commit ba9c376ecc9f1475fceaa228db405f7236d56d64
Author: Dave Marion <[email protected]>
AuthorDate: Fri Nov 8 14:55:50 2024 -0500
Increase majc priority for tablets over file size threshold (#5026)
CompactionManager.mainLoop runs in the TabletServer looking
for tablets that need to be compacted. It ends up calling
CompactionService.submitCompaction at some interval for each
hosted Tablet. CompactionService.getCompactionPlan will make
a CompactionPlan for the tablet and log a warning if no
CompactionPlan is created but the number of files is larger
than TSERV_SCAN_MAX_OPENFILES. When there are no compactions
running for the tablet and no plan is calculated, then
DefaultCompactionPlanner.makePlan takes into account
TABLE_FILE_MAX and TSERV_SCAN_MAX_OPENFILES and will create
a system compaction that considers all of the files and
calculates which ones need to be compacted to get below the
limit. Finally, a priority is calculated by calling
CompactionJobPrioritizer.createPriority. However, given that
this compaction is a SYSTEM compaction it will have a lower
priority than all current USER compactions and it's priority
will still be based on the total number of files. Given that
the TABLE_FILE_MAX is per-table it's possible to have two
tablets from different tables and the tablet that is over the
size threshold has a lower priority than the tablet that is
not over the size threshold. This change modifies
CompactionJobPrioritizer.createPriority to take into account
whether or not the tablet is over the threshold to give it a
higher priority.
Related to #4610
---
.../core/spi/compaction/CompactionPlanner.java | 9 +
.../spi/compaction/DefaultCompactionPlanner.java | 18 +-
.../util/compaction/CompactionJobPrioritizer.java | 144 +++++++++++---
.../compaction/DefaultCompactionPlannerTest.java | 8 +
.../util/compaction/CompactionPrioritizerTest.java | 213 ++++++++++++++++++---
.../tserver/compactions/CompactionService.java | 7 +
6 files changed, 339 insertions(+), 60 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
index 4b933fc744..9f4946b357 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/CompactionPlanner.java
@@ -21,8 +21,10 @@ package org.apache.accumulo.core.spi.compaction;
import java.util.Collection;
import java.util.Map;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
@@ -79,6 +81,13 @@ public interface CompactionPlanner {
*/
public interface PlanningParameters {
+ /**
+ * @return The id of the namespace that the table is assigned to
+ * @throws TableNotFoundException thrown when the namespace for a table
cannot be calculated
+ * @since 2.1.4
+ */
+ NamespaceId getNamespaceId() throws TableNotFoundException;
+
/**
* @return The id of the table that compactions are being planned for.
* @see ServiceEnvironment#getTableName(TableId)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index 73952b7900..6c60d5095c 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
@@ -250,6 +251,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
@Override
public CompactionPlan makePlan(PlanningParameters params) {
+ int maxTabletFiles = 0;
try {
if (params.getCandidates().isEmpty()) {
@@ -322,7 +324,7 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
} else if (params.getKind() == CompactionKind.SYSTEM
&& params.getRunningCompactions().isEmpty()
&& params.getAll().size() == params.getCandidates().size()) {
- int maxTabletFiles = getMaxTabletFiles(
+ maxTabletFiles = getMaxTabletFiles(
params.getServiceEnvironment().getConfiguration(params.getTableId()));
if (params.getAll().size() > maxTabletFiles) {
// The tablet is above its max files, there are no compactions
running, all files are
@@ -339,11 +341,13 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
// determine which executor to use based on the size of the files
var ceid = getExecutor(group);
- return params.createPlanBuilder().addJob(createPriority(params,
group), ceid, group)
- .build();
+ return params.createPlanBuilder()
+ .addJob(createPriority(params, group, maxTabletFiles), ceid,
group).build();
}
} catch (RuntimeException e) {
throw e;
+ } catch (TableNotFoundException e) {
+ throw new RuntimeException("Error getting namespace for table: " +
params.getTableId(), e);
}
}
@@ -415,10 +419,10 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
return found;
}
- private static short createPriority(PlanningParameters params,
- Collection<CompactableFile> group) {
- return CompactionJobPrioritizer.createPriority(params.getKind(),
params.getAll().size(),
- group.size());
+ private static short createPriority(PlanningParameters params,
Collection<CompactableFile> group,
+ int maxTabletFiles) throws TableNotFoundException {
+ return CompactionJobPrioritizer.createPriority(params.getNamespaceId(),
params.getTableId(),
+ params.getKind(), params.getAll().size(), group.size(),
maxTabletFiles);
}
private long getMaxSizeToCompact(CompactionKind kind) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
index 1f87459f11..686e589f5c 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionJobPrioritizer.java
@@ -19,9 +19,22 @@
package org.apache.accumulo.core.util.compaction;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import org.apache.accumulo.core.clientImpl.Namespace;
+import org.apache.accumulo.core.data.NamespaceId;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.commons.lang3.Range;
+
+import com.google.common.base.Preconditions;
public class CompactionJobPrioritizer {
@@ -29,32 +42,113 @@ public class CompactionJobPrioritizer {
Comparator.comparingInt(CompactionJob::getPriority)
.thenComparingInt(job -> job.getFiles().size()).reversed();
- public static short createPriority(CompactionKind kind, int totalFiles, int
compactingFiles) {
-
- int prio = totalFiles + compactingFiles;
-
- switch (kind) {
- case USER:
- case CHOP:
- // user-initiated compactions will have a positive priority
- // based on number of files
- if (prio > Short.MAX_VALUE) {
- return Short.MAX_VALUE;
- }
- return (short) prio;
- case SELECTOR:
- case SYSTEM:
- // system-initiated compactions will have a negative priority
- // starting at -32768 and increasing based on number of files
- // maxing out at -1
- if (prio > Short.MAX_VALUE) {
- return -1;
- } else {
- return (short) (Short.MIN_VALUE + prio);
- }
- default:
- throw new AssertionError("Unknown kind " + kind);
+ private static final Map<Pair<TableId,CompactionKind>,Range<Short>>
SYSTEM_TABLE_RANGES =
+ new HashMap<>();
+ private static final Map<Pair<NamespaceId,CompactionKind>,
+ Range<Short>> ACCUMULO_NAMESPACE_RANGES = new HashMap<>();
+
+ // Create ranges of possible priority values where each range has
+ // 2000 possible values. Priority order is:
+ // root table user initiated
+ // root table system initiated
+ // metadata table user initiated
+ // metadata table system initiated
+ // other tables in accumulo namespace user initiated
+ // other tables in accumulo namespace system initiated
+ // user tables that have more files that configured system initiated
+ // user tables user initiated
+ // user tables system initiated
+ static final Range<Short> ROOT_TABLE_USER = Range.of((short) 30768, (short)
32767);
+ static final Range<Short> ROOT_TABLE_SYSTEM = Range.of((short) 28768,
(short) 30767);
+
+ static final Range<Short> METADATA_TABLE_USER = Range.of((short) 26768,
(short) 28767);
+ static final Range<Short> METADATA_TABLE_SYSTEM = Range.of((short) 24768,
(short) 26767);
+
+ static final Range<Short> SYSTEM_NS_USER = Range.of((short) 22768, (short)
24767);
+ static final Range<Short> SYSTEM_NS_SYSTEM = Range.of((short) 20768, (short)
22767);
+
+ static final Range<Short> TABLE_OVER_SIZE = Range.of((short) 18768, (short)
20767);
+
+ static final Range<Short> USER_TABLE_USER = Range.of((short) 1, (short)
18767);
+ static final Range<Short> USER_TABLE_SYSTEM = Range.of((short) -32768,
(short) 0);
+
+ static {
+ // root table
+ SYSTEM_TABLE_RANGES.put(new Pair<>(RootTable.ID, CompactionKind.USER),
ROOT_TABLE_USER);
+ SYSTEM_TABLE_RANGES.put(new Pair<>(RootTable.ID, CompactionKind.SYSTEM),
ROOT_TABLE_SYSTEM);
+
+ // metadata table
+ SYSTEM_TABLE_RANGES.put(new Pair<>(MetadataTable.ID, CompactionKind.USER),
METADATA_TABLE_USER);
+ SYSTEM_TABLE_RANGES.put(new Pair<>(MetadataTable.ID,
CompactionKind.SYSTEM),
+ METADATA_TABLE_SYSTEM);
+
+ // metadata table
+ ACCUMULO_NAMESPACE_RANGES.put(new Pair<>(Namespace.ACCUMULO.id(),
CompactionKind.USER),
+ SYSTEM_NS_USER);
+ ACCUMULO_NAMESPACE_RANGES.put(new Pair<>(Namespace.ACCUMULO.id(),
CompactionKind.SYSTEM),
+ SYSTEM_NS_SYSTEM);
+ }
+
+ public static short createPriority(final NamespaceId nsId, final TableId
tableId,
+ final CompactionKind kind, final int totalFiles, final int
compactingFiles,
+ final int maxFilesPerTablet) {
+
+ Objects.requireNonNull(nsId, "nsId cannot be null");
+ Objects.requireNonNull(tableId, "tableId cannot be null");
+ Preconditions.checkArgument(totalFiles >= 0, "totalFiles is negative %s",
totalFiles);
+ Preconditions.checkArgument(compactingFiles >= 0, "compactingFiles is
negative %s",
+ compactingFiles);
+
+ final Function<Range<Short>,Short> normalPriorityFunction = new
Function<>() {
+ @Override
+ public Short apply(Range<Short> f) {
+ return (short) Math.min(f.getMaximum(), f.getMinimum() + totalFiles +
compactingFiles);
+ }
+ };
+
+ final Function<Range<Short>,Short> tabletOverSizeFunction = new
Function<>() {
+ @Override
+ public Short apply(Range<Short> f) {
+ return (short) Math.min(f.getMaximum(),
+ f.getMinimum() + compactingFiles + (totalFiles -
maxFilesPerTablet));
+ }
+ };
+
+ // Handle the case of a CHOP compaction. For the purposes of determining
+ // a priority, treat them as a USER compaction.
+ CompactionKind calculationKind = kind;
+ if (kind == CompactionKind.CHOP) {
+ calculationKind = CompactionKind.USER;
+ } else if (kind == CompactionKind.SELECTOR) {
+ calculationKind = CompactionKind.SYSTEM;
}
+
+ Range<Short> range = null;
+ Function<Range<Short>,Short> func = normalPriorityFunction;
+ if (Namespace.ACCUMULO.id() == nsId) {
+ // Handle system tables
+ range = SYSTEM_TABLE_RANGES.get(new Pair<>(tableId, calculationKind));
+ if (range == null) {
+ range = ACCUMULO_NAMESPACE_RANGES.get(new Pair<>(nsId,
calculationKind));
+ }
+ } else {
+ // Handle user tables
+ if (totalFiles > maxFilesPerTablet && calculationKind ==
CompactionKind.SYSTEM) {
+ range = TABLE_OVER_SIZE;
+ func = tabletOverSizeFunction;
+ } else if (calculationKind == CompactionKind.SYSTEM) {
+ range = USER_TABLE_SYSTEM;
+ } else {
+ range = USER_TABLE_USER;
+ }
+ }
+
+ if (range == null) {
+ throw new IllegalStateException(
+ "Error calculating compaction priority for table: " + tableId);
+ }
+ return func.apply(range);
+
}
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
index 8423ee86e6..f3186701ad 100644
---
a/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java
@@ -35,12 +35,15 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+import org.apache.accumulo.core.clientImpl.Namespace;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
import org.apache.accumulo.core.spi.common.ServiceEnvironment.Configuration;
@@ -741,6 +744,11 @@ public class DefaultCompactionPlannerTest {
CompactionKind kind, Configuration conf) {
return new CompactionPlanner.PlanningParameters() {
+ @Override
+ public NamespaceId getNamespaceId() throws TableNotFoundException {
+ return Namespace.ACCUMULO.id();
+ }
+
@Override
public TableId getTableId() {
return TableId.of("42");
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
index 70f18be1cd..1aa62c4f6b 100644
---
a/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/compaction/CompactionPrioritizerTest.java
@@ -18,22 +18,41 @@
*/
package org.apache.accumulo.core.util.compaction;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.METADATA_TABLE_SYSTEM;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.METADATA_TABLE_USER;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.ROOT_TABLE_SYSTEM;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.ROOT_TABLE_USER;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.SYSTEM_NS_SYSTEM;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.SYSTEM_NS_USER;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.TABLE_OVER_SIZE;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.USER_TABLE_SYSTEM;
+import static
org.apache.accumulo.core.util.compaction.CompactionJobPrioritizer.USER_TABLE_USER;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+import org.apache.accumulo.core.clientImpl.Namespace;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.spi.compaction.CompactionJob;
import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.commons.lang3.Range;
import org.junit.jupiter.api.Test;
public class CompactionPrioritizerTest {
+ private static final int TABLET_FILE_MAX = 3001;
+
public CompactionJob createJob(CompactionKind kind, String tablet, int
numFiles, int totalFiles) {
Collection<CompactableFile> files = new ArrayList<>();
@@ -41,43 +60,181 @@ public class CompactionPrioritizerTest {
files.add(CompactableFile
.create(URI.create("hdfs://foonn/accumulo/tables/5/" + tablet + "/"
+ i + ".rf"), 4, 4));
}
- // TODO pass numFiles
return new CompactionJobImpl(
- CompactionJobPrioritizer.createPriority(kind, totalFiles, numFiles),
+ CompactionJobPrioritizer.createPriority(Namespace.DEFAULT.id(),
TableId.of("5"), kind,
+ totalFiles, numFiles, totalFiles * 2),
CompactionExecutorIdImpl.externalId("test"), files, kind,
Optional.of(false));
}
@Test
- public void testPrioritizer() throws Exception {
- assertEquals((short) 0,
CompactionJobPrioritizer.createPriority(CompactionKind.USER, 0, 0));
- assertEquals((short) 10000,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER, 10000,
0));
- assertEquals((short) 32767,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER, 32767,
0));
- assertEquals((short) 32767,
- CompactionJobPrioritizer.createPriority(CompactionKind.USER,
Integer.MAX_VALUE, 0));
-
- assertEquals((short) -32768,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 0, 0));
- assertEquals((short) -22768,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 10000,
0));
- assertEquals((short) -1,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM, 32767,
0));
- assertEquals((short) -1,
- CompactionJobPrioritizer.createPriority(CompactionKind.SYSTEM,
Integer.MAX_VALUE, 0));
+ public void testNonOverlappingRanges() {
+ List<Range<Short>> ranges = new ArrayList<>();
+ ranges.add(ROOT_TABLE_USER);
+ ranges.add(ROOT_TABLE_SYSTEM);
+ ranges.add(METADATA_TABLE_USER);
+ ranges.add(METADATA_TABLE_SYSTEM);
+ ranges.add(SYSTEM_NS_USER);
+ ranges.add(SYSTEM_NS_SYSTEM);
+ ranges.add(TABLE_OVER_SIZE);
+ ranges.add(USER_TABLE_USER);
+ ranges.add(USER_TABLE_SYSTEM);
+
+ for (Range<Short> r1 : ranges) {
+ for (Range<Short> r2 : ranges) {
+ if (r1 == r2) {
+ continue;
+ }
+ assertFalse(r1.isOverlappedBy(r2), r1.toString() + " is overlapped by
" + r2.toString());
+ }
+ }
+
+ Collections.sort(ranges, new Comparator<Range<Short>>() {
+ @Override
+ public int compare(Range<Short> r1, Range<Short> r2) {
+ return Short.compare(r1.getMinimum(), r2.getMinimum());
+ }
+ });
+ assertEquals(Short.MIN_VALUE, ranges.get(0).getMinimum());
+ assertEquals(Short.MAX_VALUE, ranges.get(ranges.size() - 1).getMaximum());
+ // check that the max of the previous range is one less than the
+ // minimum of the current range to make sure there are no holes.
+ short lastMax = Short.MIN_VALUE;
+ for (Range<Short> r : ranges) {
+ if (lastMax != Short.MIN_VALUE) {
+ assertTrue(r.getMinimum() - lastMax == 1);
+ }
+ lastMax = r.getMaximum();
+ }
+ }
+
+ @Test
+ public void testRootTablePriorities() {
+ assertEquals(ROOT_TABLE_USER.getMinimum() + 1,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.USER, 0, 1,
TABLET_FILE_MAX));
+ assertEquals(ROOT_TABLE_USER.getMinimum() + 1010,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.USER, 1000, 10,
TABLET_FILE_MAX));
+ assertEquals(ROOT_TABLE_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.USER, 3000, 100,
TABLET_FILE_MAX));
+
+ assertEquals(ROOT_TABLE_USER.getMinimum() + 2,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.CHOP, 0, 2,
TABLET_FILE_MAX));
+ assertEquals(ROOT_TABLE_USER.getMinimum() + 1020,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.CHOP, 1000, 20,
TABLET_FILE_MAX));
+ assertEquals(ROOT_TABLE_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.CHOP, 3000, 200,
TABLET_FILE_MAX));
+
+ assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 3,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 0, 3,
TABLET_FILE_MAX));
+ assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 1030,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 1000,
30, TABLET_FILE_MAX));
+ assertEquals(ROOT_TABLE_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 3000,
300, TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testMetaTablePriorities() {
+ assertEquals(METADATA_TABLE_USER.getMinimum() + 4,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.USER, 0, 4,
TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_USER.getMinimum() + 1040,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.USER, 1000,
40, TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_USER.getMaximum(),
+ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
+ CompactionKind.USER, 3000, 400, TABLET_FILE_MAX));
+
+ assertEquals(METADATA_TABLE_USER.getMinimum() + 5,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.CHOP, 0, 5,
TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_USER.getMinimum() + 1050,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.CHOP, 1000,
50, TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_USER.getMaximum(),
+ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
+ CompactionKind.CHOP, 3000, 500, TABLET_FILE_MAX));
+
+ assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 6,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.SYSTEM, 0,
6, TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 1060,
+ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
+ CompactionKind.SYSTEM, 1000, 60, TABLET_FILE_MAX));
+ assertEquals(METADATA_TABLE_SYSTEM.getMaximum(),
+ CompactionJobPrioritizer.createPriority(Namespace.ACCUMULO.id(),
MetadataTable.ID,
+ CompactionKind.SYSTEM, 3000, 600, TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testSystemNamespacePriorities() {
+ TableId tid = TableId.of("someOtherSystemTable");
+ assertEquals(SYSTEM_NS_USER.getMinimum() + 7, CompactionJobPrioritizer
+ .createPriority(Namespace.ACCUMULO.id(), tid, CompactionKind.USER, 0,
7, TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_USER.getMinimum() + 1070,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.USER, 1000, 70,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.USER, 3000, 700,
TABLET_FILE_MAX));
+
+ assertEquals(SYSTEM_NS_USER.getMinimum() + 8, CompactionJobPrioritizer
+ .createPriority(Namespace.ACCUMULO.id(), tid, CompactionKind.CHOP, 0,
8, TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_USER.getMinimum() + 1080,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.CHOP, 1000, 80,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_USER.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.CHOP, 3000, 800,
TABLET_FILE_MAX));
+
+ assertEquals(SYSTEM_NS_SYSTEM.getMinimum() + 9,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 0, 9,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_SYSTEM.getMinimum() + 1090,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 1000, 90,
TABLET_FILE_MAX));
+ assertEquals(SYSTEM_NS_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 3000, 900,
TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testUserTablePriorities() {
+ TableId tid = TableId.of("someUserTable");
+ assertEquals(USER_TABLE_USER.getMinimum() + 10, CompactionJobPrioritizer
+ .createPriority(Namespace.DEFAULT.id(), tid, CompactionKind.USER, 0,
10, TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_USER.getMinimum() + 1100,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.USER, 1000, 100,
TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_USER.getMinimum() + 4000,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.USER, 3000, 1000,
TABLET_FILE_MAX));
+
+ assertEquals(USER_TABLE_SYSTEM.getMinimum() + 11,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 0, 11,
TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_SYSTEM.getMinimum() + 1110,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 1000, 110,
TABLET_FILE_MAX));
+ assertEquals(USER_TABLE_SYSTEM.getMinimum() + 4100,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 3000, 1100,
TABLET_FILE_MAX));
+ }
+
+ @Test
+ public void testTableOverSize() {
+ final int tabletFileMax = 30;
+ final TableId tid = TableId.of("someTable");
+ assertEquals(ROOT_TABLE_SYSTEM.getMinimum() + 150,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
+ assertEquals(METADATA_TABLE_SYSTEM.getMinimum() + 150,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.SYSTEM, 100,
50, tabletFileMax));
+ assertEquals(SYSTEM_NS_SYSTEM.getMinimum() + 150,
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
+ assertEquals(TABLE_OVER_SIZE.getMinimum() + 120,
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 100, 50,
tabletFileMax));
+ assertEquals(ROOT_TABLE_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), RootTable.ID, CompactionKind.SYSTEM, 3000,
50, tabletFileMax));
+ assertEquals(METADATA_TABLE_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), MetadataTable.ID, CompactionKind.SYSTEM,
3000, 50, tabletFileMax));
+ assertEquals(SYSTEM_NS_SYSTEM.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.ACCUMULO.id(), tid, CompactionKind.SYSTEM, 3000, 50,
tabletFileMax));
+ assertEquals(TABLE_OVER_SIZE.getMaximum(),
CompactionJobPrioritizer.createPriority(
+ Namespace.DEFAULT.id(), tid, CompactionKind.SYSTEM, 3000, 50,
tabletFileMax));
}
@Test
public void testCompactionJobComparator() {
- var j1 = createJob(CompactionKind.USER, "t-009", 10, 20);
- var j2 = createJob(CompactionKind.USER, "t-010", 11, 25);
- var j3 = createJob(CompactionKind.USER, "t-011", 11, 20);
- var j4 = createJob(CompactionKind.SYSTEM, "t-012", 11, 30);
- var j5 = createJob(CompactionKind.SYSTEM, "t-013", 5, 10);
- var j6 = createJob(CompactionKind.CHOP, "t-014", 5, 40);
- var j7 = createJob(CompactionKind.CHOP, "t-015", 5, 7);
- var j8 = createJob(CompactionKind.SELECTOR, "t-014", 5, 21);
- var j9 = createJob(CompactionKind.SELECTOR, "t-015", 7, 20);
+ var j1 = createJob(CompactionKind.USER, "t-009", 10, 20); // 30
+ var j2 = createJob(CompactionKind.USER, "t-010", 11, 25); // 36
+ var j3 = createJob(CompactionKind.USER, "t-011", 11, 20); // 31
+ var j4 = createJob(CompactionKind.SYSTEM, "t-012", 11, 30); // 40
+ var j5 = createJob(CompactionKind.SYSTEM, "t-013", 5, 10); // 15
+ var j6 = createJob(CompactionKind.CHOP, "t-014", 5, 40); // 45
+ var j7 = createJob(CompactionKind.CHOP, "t-015", 5, 7); // 12
+ var j8 = createJob(CompactionKind.SELECTOR, "t-014", 5, 21); // 26
+ var j9 = createJob(CompactionKind.SELECTOR, "t-015", 7, 20); // 27
var expected = List.of(j6, j2, j3, j1, j7, j4, j9, j8, j5);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
index 593be6b67c..52107f19d0 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
@@ -41,9 +41,11 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer;
import java.util.function.Function;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.NamespaceId;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.spi.common.ServiceEnvironment;
@@ -235,6 +237,11 @@ public class CompactionService {
private final ServiceEnvironment senv = new
ServiceEnvironmentImpl(context);
+ @Override
+ public NamespaceId getNamespaceId() throws TableNotFoundException {
+ return context.getNamespaceId(comp.getTableId());
+ }
+
@Override
public TableId getTableId() {
return comp.getTableId();
