yujun777 commented on code in PR #42401:
URL: https://github.com/apache/doris/pull/42401#discussion_r1823812345
##########
fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java:
##########
@@ -193,19 +209,115 @@ public synchronized void process(String rawSql,
List<AlterClause> alterClauses,
/*
* check if the specified backends can be dropped
* 1. backend does not have any tablet.
- * 2. all tablets in backend have been recycled.
+ * 2. all tablets in backend have been recycled or leaky.
+ *
+ * sampleTablets: sample normal tablets
+ * sampleLeakyTablets: sample leaky tablets
+ *
*/
- private boolean checkTablets(Long beId, List<Long> backendTabletIds) {
+ private boolean checkMigrateTablets(long beId, int sampleLimit, List<Long>
sampleTablets,
+ List<Long> sampleLeakyTablets, AtomicInteger totalTabletNum) {
+ TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
+ List<Long> backendTabletIds =
invertedIndex.getTabletIdsByBackendId(beId);
+ totalTabletNum.set(backendTabletIds.size());
if (backendTabletIds.isEmpty()) {
return true;
}
- if (backendTabletIds.size() <
Config.decommission_tablet_check_threshold
- &&
Env.getCurrentRecycleBin().allTabletsInRecycledStatus(backendTabletIds)) {
- LOG.info("tablet size is {}, all tablets on decommissioned backend
{} have been recycled,"
- + " so this backend will be dropped immediately",
backendTabletIds.size(), beId);
- return true;
+ // if too many tablets, no check for efficiency
+ if (backendTabletIds.size() >
Config.decommission_tablet_check_threshold) {
+ return false;
+ }
+ // dbId -> tableId -> partitionId -> tablet list
+ Map<Long, Map<Long, Map<Long, List<Long>>>> tabletsMap =
Maps.newHashMap();
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(backendTabletIds);
+ for (int i = 0; i < backendTabletIds.size(); i++) {
+ long tabletId = backendTabletIds.get(i);
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ tabletsMap.computeIfAbsent(tabletMeta.getDbId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getTableId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getPartitionId(), k ->
Lists.newArrayList())
+ .add(tabletId);
+ }
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin();
+ long now = System.currentTimeMillis();
+ Map<Long, Long> leakyTablets = Maps.newHashMap();
+ boolean searchedFirstTime = !backendLeakyTablets.containsKey(beId);
+ Map<Long, Long> lastLeakyTablets =
backendLeakyTablets.computeIfAbsent(beId, k -> Maps.newHashMap());
+ backendLeakyTablets.put(beId, leakyTablets);
+ Consumer<List<Long>> addPartitionLeakyTablets = tabletsOfPartition -> {
+ tabletsOfPartition.forEach(tabletId -> {
+ leakyTablets.put(tabletId,
lastLeakyTablets.getOrDefault(tabletId, now));
+ });
+ };
+ Consumer<Map<Long, List<Long>>> addTableLeakyTablets = tabletsOfTable
-> {
+ tabletsOfTable.values().forEach(addPartitionLeakyTablets);
+ };
+ Consumer<Map<Long, Map<Long, List<Long>>>> addDbLeakyTablets =
tabletsOfDb -> {
+ tabletsOfDb.values().forEach(addTableLeakyTablets);
+ };
+ boolean searchedAllTablets = true;
+ OUTER:
+ for (Map.Entry<Long, Map<Long, Map<Long, List<Long>>>> dbEntry :
tabletsMap.entrySet()) {
+ long dbId = dbEntry.getKey();
+ Database db = catalog.getDbNullable(dbId);
+ if (db == null) {
+ if (!recycleBin.isRecycleDatabase(dbId)) {
+ addDbLeakyTablets.accept(dbEntry.getValue());
+ }
+ continue;
+ }
+
+ for (Map.Entry<Long, Map<Long, List<Long>>> tableEntry :
dbEntry.getValue().entrySet()) {
+ long tableId = tableEntry.getKey();
+ Table tbl = db.getTableNullable(tableId);
+ if (tbl == null || !tbl.isManagedTable()) {
+ if (!recycleBin.isRecycleTable(dbId, tableId)) {
+ addTableLeakyTablets.accept(tableEntry.getValue());
+ }
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) tbl;
Review Comment:
不会，全量扫没那么严重，都是纯内存操作，很轻。像TabletChecker 就每20s 对所有tablet 作一次全量扫了。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
