This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 8bbdb58903c branch-3.0: [Fix](log) correct tablet diff log in
`TabletInvertedIndex.tabletReport` #49390 (#49434)
8bbdb58903c is described below
commit 8bbdb58903c5f928823d6a3231bac88314f234fe
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Mar 25 17:12:58 2025 +0800
branch-3.0: [Fix](log) correct tablet diff log in
`TabletInvertedIndex.tabletReport` #49390 (#49434)
Cherry-picked from #49390
Co-authored-by: bobhan1 <[email protected]>
---
.../apache/doris/catalog/TabletInvertedIndex.java | 20 +++++++++++++-------
.../java/org/apache/doris/master/ReportHandler.java | 7 ++++---
2 files changed, 17 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index a51d1f55014..0362205dbde 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -137,7 +137,7 @@ public class TabletInvertedIndex {
ListMultimap<TStorageMedium, Long>
tabletMigrationMap,
Map<Long, Long> partitionVersionSyncMap,
Map<Long, SetMultimap<Long,
TPartitionVersionInfo>> transactionsToPublish,
- ListMultimap<Long, Long> transactionsToClear,
+ SetMultimap<Long, Long> transactionsToClear,
ListMultimap<Long, Long> tabletRecoveryMap,
List<TTabletMetaInfo> tabletToUpdate,
List<CooldownConf> cooldownConfToPush,
@@ -303,20 +303,26 @@ public class TabletInvertedIndex {
cooldownTablets.forEach(p -> handleCooldownConf(p.first, p.second,
cooldownConfToPush, cooldownConfToUpdate));
long end = System.currentTimeMillis();
+ long toClearTransactionsNum = transactionsToClear.keySet().size();
+ long toClearTransactionsPartitions =
transactionsToClear.values().size();
+ long toPublishTransactionsNum = transactionsToPublish.values().stream()
+ .mapToLong(m ->
m.keySet().size()).sum();
+ long toPublishTransactionsPartitions =
transactionsToPublish.values().stream()
+ .mapToLong(m ->
m.values().size()).sum();
LOG.info("finished to do tablet diff with backend[{}]. fe tablet num:
{}, backend tablet num: {}. sync: {}."
+ " metaDel: {}. foundInMeta: {}. migration: {}.
backend partition num: {}, backend need "
- + "update: {}. found invalid transactions {}. found
republish "
- + "transactions {}. tabletToUpdate: {}. need recovery:
{}. cost: {} ms",
+ + "update: {}. found invalid transactions
{}(partitions: {}). found republish "
+ + "transactions {}(partitions: {}). tabletToUpdate:
{}. need recovery: {}. cost: {} ms",
backendId, feTabletNum, backendTablets.size(),
tabletSyncMap.size(),
tabletDeleteFromMeta.size(), tabletFoundInMeta.size(),
tabletMigrationMap.size(),
- backendPartitionsVersion.size(),
partitionVersionSyncMap.size(),
- transactionsToClear.size(), transactionsToPublish.size(),
tabletToUpdate.size(),
- tabletRecoveryMap.size(), (end - start));
+ backendPartitionsVersion.size(),
partitionVersionSyncMap.size(), toClearTransactionsNum,
+ toClearTransactionsPartitions, toPublishTransactionsNum,
toPublishTransactionsPartitions,
+ tabletToUpdate.size(), tabletRecoveryMap.size(), (end -
start));
}
private void handleBackendTransactions(long backendId, List<Long>
transactionIds, long tabletId,
TabletMeta tabletMeta, Map<Long, SetMultimap<Long,
TPartitionVersionInfo>> transactionsToPublish,
- ListMultimap<Long, Long> transactionsToClear) {
+ SetMultimap<Long, Long> transactionsToClear) {
GlobalTransactionMgrIface transactionMgr =
Env.getCurrentGlobalTransactionMgr();
long partitionId = tabletMeta.getPartitionId();
for (Long transactionId : transactionIds) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index a6a81d6efe3..4ebd59c38d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -91,6 +91,7 @@ import org.apache.doris.thrift.TTabletMetaInfo;
import org.apache.doris.thrift.TTaskType;
import com.google.common.base.Preconditions;
+import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
@@ -508,7 +509,7 @@ public class ReportHandler extends Daemon {
// dbid -> txn id -> [partition info]
Map<Long, SetMultimap<Long, TPartitionVersionInfo>>
transactionsToPublish = Maps.newHashMap();
- ListMultimap<Long, Long> transactionsToClear =
LinkedListMultimap.create();
+ SetMultimap<Long, Long> transactionsToClear =
LinkedHashMultimap.create();
// db id -> tablet id
ListMultimap<Long, Long> tabletRecoveryMap =
LinkedListMultimap.create();
@@ -1297,11 +1298,11 @@ public class ReportHandler extends Daemon {
AgentTaskExecutor.submit(batchTask);
}
- private static void handleClearTransactions(ListMultimap<Long, Long>
transactionsToClear, long backendId) {
+ private static void handleClearTransactions(SetMultimap<Long, Long>
transactionsToClear, long backendId) {
AgentBatchTask batchTask = new AgentBatchTask();
for (Long transactionId : transactionsToClear.keySet()) {
ClearTransactionTask clearTransactionTask = new
ClearTransactionTask(backendId,
- transactionId, transactionsToClear.get(transactionId));
+ transactionId,
Lists.newArrayList(transactionsToClear.get(transactionId)));
batchTask.addTask(clearTransactionTask);
}
AgentTaskExecutor.submit(batchTask);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
