yujun777 commented on code in PR #42401:
URL: https://github.com/apache/doris/pull/42401#discussion_r1823707947
##########
fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java:
##########
@@ -193,19 +209,115 @@ public synchronized void process(String rawSql,
List<AlterClause> alterClauses,
/*
* check if the specified backends can be dropped
* 1. backend does not have any tablet.
- * 2. all tablets in backend have been recycled.
+ * 2. all tablets in backend have been recycled or leaky.
+ *
+ * sampleTablets: sample normal tablets
+ * sampleLeakyTablets: sample leaky tablets
+ *
*/
- private boolean checkTablets(Long beId, List<Long> backendTabletIds) {
+ private boolean checkMigrateTablets(long beId, int sampleLimit, List<Long>
sampleTablets,
+ List<Long> sampleLeakyTablets, AtomicInteger totalTabletNum) {
+ TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
+ List<Long> backendTabletIds =
invertedIndex.getTabletIdsByBackendId(beId);
+ totalTabletNum.set(backendTabletIds.size());
if (backendTabletIds.isEmpty()) {
return true;
}
- if (backendTabletIds.size() <
Config.decommission_tablet_check_threshold
- &&
Env.getCurrentRecycleBin().allTabletsInRecycledStatus(backendTabletIds)) {
- LOG.info("tablet size is {}, all tablets on decommissioned backend
{} have been recycled,"
- + " so this backend will be dropped immediately",
backendTabletIds.size(), beId);
- return true;
+ // if too many tablets, no check for efficiency
+ if (backendTabletIds.size() >
Config.decommission_tablet_check_threshold) {
+ return false;
+ }
+ // dbId -> tableId -> partitionId -> tablet list
+ Map<Long, Map<Long, Map<Long, List<Long>>>> tabletsMap =
Maps.newHashMap();
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(backendTabletIds);
+ for (int i = 0; i < backendTabletIds.size(); i++) {
+ long tabletId = backendTabletIds.get(i);
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ tabletsMap.computeIfAbsent(tabletMeta.getDbId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getTableId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getPartitionId(), k ->
Lists.newArrayList())
+ .add(tabletId);
+ }
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin();
+ long now = System.currentTimeMillis();
+ Map<Long, Long> leakyTablets = Maps.newHashMap();
+ boolean searchedFirstTime = !backendLeakyTablets.containsKey(beId);
+ Map<Long, Long> lastLeakyTablets =
backendLeakyTablets.computeIfAbsent(beId, k -> Maps.newHashMap());
+ backendLeakyTablets.put(beId, leakyTablets);
+ Consumer<List<Long>> addPartitionLeakyTablets = tabletsOfPartition -> {
+ tabletsOfPartition.forEach(tabletId -> {
+ leakyTablets.put(tabletId,
lastLeakyTablets.getOrDefault(tabletId, now));
+ });
+ };
+ Consumer<Map<Long, List<Long>>> addTableLeakyTablets = tabletsOfTable
-> {
+ tabletsOfTable.values().forEach(addPartitionLeakyTablets);
+ };
+ Consumer<Map<Long, Map<Long, List<Long>>>> addDbLeakyTablets =
tabletsOfDb -> {
+ tabletsOfDb.values().forEach(addTableLeakyTablets);
+ };
+ boolean searchedAllTablets = true;
+ OUTER:
+ for (Map.Entry<Long, Map<Long, Map<Long, List<Long>>>> dbEntry :
tabletsMap.entrySet()) {
+ long dbId = dbEntry.getKey();
+ Database db = catalog.getDbNullable(dbId);
+ if (db == null) {
+ if (!recycleBin.isRecycleDatabase(dbId)) {
+ addDbLeakyTablets.accept(dbEntry.getValue());
+ }
+ continue;
+ }
+
+ for (Map.Entry<Long, Map<Long, List<Long>>> tableEntry :
dbEntry.getValue().entrySet()) {
+ long tableId = tableEntry.getKey();
+ Table tbl = db.getTableNullable(tableId);
+ if (tbl == null || !tbl.isManagedTable()) {
+ if (!recycleBin.isRecycleTable(dbId, tableId)) {
+ addTableLeakyTablets.accept(tableEntry.getValue());
+ }
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) tbl;
Review Comment:
system handler 的周期是10s， 10s 内最多只拿10个table的lock锁
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
