This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new f95221e34e1 [fix](oom) avoid oom when a lot of tablets fail on load (#36962) f95221e34e1 is described below commit f95221e34e167e5149dda9dfcc7ae5962b1582c0 Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com> AuthorDate: Fri Jun 28 21:07:08 2024 +0800 [fix](oom) avoid oom when a lot of tablets fail on load (#36962) When a lot of tablets fail when loading, then detailed information would cause oom. ## Proposed changes Issue Number: close #xxx <!--Describe your changes.--> --- .../doris/transaction/DatabaseTransactionMgr.java | 59 ++++++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 36b32f01069..7ef043136ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -1028,6 +1028,45 @@ public class DatabaseTransactionMgr { return true; } + private class TabletsPublishResultLogs { + public List<String> quorumSuccLogs = Lists.newArrayList(); + public List<String> timeoutSuccLogs = Lists.newArrayList(); + public List<String> failedLogs = Lists.newArrayList(); + + public void addQuorumSuccLog(String log) { + if (quorumSuccLogs.size() < 16) { + quorumSuccLogs.add(log); + } + } + + public void addTimeoutSuccLog(String log) { + if (timeoutSuccLogs.size() < 16) { + timeoutSuccLogs.add(log); + } + } + + public void addFailedLog(String log) { + if (failedLogs.size() < 16) { + failedLogs.add(log); + } + } + + public void log() { + // log failed logs + for (String log : failedLogs) { + LOG.info(log); + } + // log timeout succ logs + for (String log : timeoutSuccLogs) { + LOG.info(log); + } + // log quorum succ logs + for (String log : quorumSuccLogs) { + LOG.info(log); + } + } + } + private PublishResult finishCheckQuorumReplicas(TransactionState transactionState, List<Pair<OlapTable, Partition>> relatedTblPartitions, Set<Long> errorReplicaIds) { @@ -1042,7 +1081,7 @@ public class DatabaseTransactionMgr { List<Replica> tabletSuccReplicas = Lists.newArrayList(); List<Replica> tabletWriteFailedReplicas = Lists.newArrayList(); List<Replica> tabletVersionFailedReplicas = Lists.newArrayList(); - List<String> logs = Lists.newArrayList(); + TabletsPublishResultLogs logs = new TabletsPublishResultLogs(); Map<Long, PublishVersionTask> publishTasks = transactionState.getPublishVersionTasks(); PublishResult publishResult = PublishResult.QUORUM_SUCC; @@ -1092,9 +1131,9 @@ public class DatabaseTransactionMgr { if (hasFailedReplica) { String writeDetail = getTabletWriteDetail(tabletSuccReplicas, tabletWriteFailedReplicas, tabletVersionFailedReplicas); - logs.add(String.format("publish version quorum succ for transaction %s on tablet %s" - + " with version %s, and has failed replicas, load require replica num %s. " - + "table %s, partition %s, tablet detail: %s", + logs.addQuorumSuccLog(String.format("publish version quorum succ for transaction %s on " + + "tablet %s with version %s, and has failed replicas, load require replica num " + + " %s. table %s, partition %s, tablet detail: %s", transactionState, tablet.getId(), newVersion, loadRequiredReplicaNum, tableId, partitionId, writeDetail)); } @@ -1116,8 +1155,8 @@ public class DatabaseTransactionMgr { // that are being publised exists on a few replicas we should go // ahead, otherwise data may be lost and thre // publish task hangs forever. - logs.add(String.format("publish version timeout succ for transaction %s on tablet %s " - + "with version %s, and has failed replicas, load require replica num %s. " + logs.addTimeoutSuccLog(String.format("publish version timeout succ for transaction %s on " + + " tablet %s with version %s, and has failed replicas, load require replica num %s. " + "table %s, partition %s, tablet detail: %s", transactionState, tablet.getId(), newVersion, loadRequiredReplicaNum, tableId, partitionId, writeDetail)); @@ -1129,8 +1168,8 @@ public class DatabaseTransactionMgr { tablet.getId(), healthReplicaNum, loadRequiredReplicaNum, tableId, partitionId, newVersion); transactionState.setErrorMsg(errMsg); - logs.add(String.format("publish version failed for transaction %s on tablet %s with version" - + " %s, and has failed replicas, load required replica num %s. table %s, " + logs.addFailedLog(String.format("publish version failed for transaction %s on tablet %s " + + "with version %s, and has failed replicas, load required replica num %s. table %s, " + "partition %s, tablet detail: %s", transactionState, tablet.getId(), newVersion, loadRequiredReplicaNum, tableId, partitionId, writeDetail)); @@ -1143,9 +1182,7 @@ public class DatabaseTransactionMgr { || now - transactionState.getLastPublishLogTime() > Config.publish_fail_log_interval_second * 1000L; if (needLog) { transactionState.setLastPublishLogTime(now); - for (String log : logs) { - LOG.info("{}. publish times {}", log, transactionState.getPublishCount()); - } + logs.log(); } return publishResult; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org