This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch mtmv_rewrite_performance
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/mtmv_rewrite_performance by
this push:
new 48b1ebdf92a [enhance](mtmv)When calculating whether partitions are
synchronized, first filter based on the partitions used in the query (#48762)
48b1ebdf92a is described below
commit 48b1ebdf92a245bdb3b9537e39d9e047f7b93727
Author: zhangdong <[email protected]>
AuthorDate: Tue Mar 25 11:08:44 2025 +0800
[enhance](mtmv)When calculating whether partitions are synchronized, first
filter based on the partitions used in the query (#48762)
---
.../org/apache/doris/mtmv/MTMVRelationManager.java | 10 ++++--
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 38 +++++++++++++++++++++-
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 24 +++++++-------
3 files changed, 55 insertions(+), 17 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index 5c48a2a982d..b69a18e774d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -84,13 +84,17 @@ public class MTMVRelationManager implements MTMVHookService
{
boolean forceConsistent, BiPredicate<ConnectContext, MTMV>
predicate) {
Set<MTMV> res = Sets.newLinkedHashSet();
Set<BaseTableInfo> mvInfos = getMTMVInfos(tableInfos);
+ Map<List<String>, Set<String>> queryUsedPartitions =
PartitionCompensator.getQueryUsedPartitions(ctx);
+
for (BaseTableInfo tableInfo : mvInfos) {
try {
MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
if (predicate.test(ctx, mtmv)) {
continue;
}
- if (isMVPartitionValid(mtmv, ctx, forceConsistent,
PartitionCompensator.getQueryUsedPartitions(ctx))) {
+ BaseTableInfo relatedTableInfo =
mtmv.getMvPartitionInfo().getRelatedTableInfo();
+ if (isMVPartitionValid(mtmv, ctx, forceConsistent,
+ relatedTableInfo == null ? null :
queryUsedPartitions.get(relatedTableInfo.toList()))) {
res.add(mtmv);
}
} catch (Exception e) {
@@ -120,10 +124,10 @@ public class MTMVRelationManager implements
MTMVHookService {
@VisibleForTesting
public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean
forceConsistent,
- Map<List<String>, Set<String>> queryUsedRelatedTablePartitionsMap)
{
+ Set<String> relatedPartitions) {
long currentTimeMillis = System.currentTimeMillis();
Collection<Partition> mtmvCanRewritePartitions =
MTMVRewriteUtil.getMTMVCanRewritePartitions(
- mtmv, ctx, currentTimeMillis, forceConsistent,
queryUsedRelatedTablePartitionsMap);
+ mtmv, ctx, currentTimeMillis, forceConsistent,
relatedPartitions);
// MTMVRewriteUtil.getMTMVCanRewritePartitions is time-consuming
behavior, So record for used later
ctx.getStatementContext().getMvCanRewritePartitionsMap().putIfAbsent(
new BaseTableInfo(mtmv), mtmvCanRewritePartitions);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
index e1cb899c23c..f0ed6ccf93a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java
@@ -25,6 +25,7 @@ import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -32,6 +33,7 @@ import org.apache.logging.log4j.Logger;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
public class MTMVRewriteUtil {
@@ -46,7 +48,7 @@ public class MTMVRewriteUtil {
*/
public static Collection<Partition> getMTMVCanRewritePartitions(MTMV mtmv,
ConnectContext ctx,
long currentTimeMills, boolean forceConsistent,
- Map<List<String>, Set<String>> queryUsedRelatedTablePartitionsMap)
{
+ Set<String> relatedPartitions) {
List<Partition> res = Lists.newArrayList();
Collection<Partition> allPartitions = mtmv.getPartitions();
MTMVRelation mtmvRelation = mtmv.getRelation();
@@ -58,6 +60,10 @@ public class MTMVRewriteUtil {
if (mtmvStatus.getState() != MTMVState.NORMAL ||
mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) {
return res;
}
+ if (relatedPartitions != null && relatedPartitions.size() == 0) {
+ return res;
+ }
+ Set<String> mtmvNeedComparePartitions = null;
MTMVRefreshContext refreshContext = null;
// check gracePeriod
long gracePeriodMills = mtmv.getGracePeriod();
@@ -76,6 +82,13 @@ public class MTMVRewriteUtil {
return res;
}
}
+ if (mtmvNeedComparePartitions == null) {
+ mtmvNeedComparePartitions =
getMtmvPartitionsByRelatedPartitions(mtmv, refreshContext,
+ relatedPartitions);
+ }
+ if (!mtmvNeedComparePartitions.contains(partition.getName())) {
+ continue;
+ }
try {
if (MTMVPartitionUtil.isMTMVPartitionSync(refreshContext,
partition.getName(),
mtmvRelation.getBaseTablesOneLevel(),
@@ -89,4 +102,27 @@ public class MTMVRewriteUtil {
}
return res;
}
+
+ private static Set<String> getMtmvPartitionsByRelatedPartitions(MTMV mtmv,
MTMVRefreshContext refreshContext,
+ Set<String> relatedPartitions) {
+ if (relatedPartitions == null) {
+ return mtmv.getPartitionNames();
+ }
+ Set<String> res = Sets.newHashSet();
+ Map<String, String> relatedToMv =
getRelatedToMv(refreshContext.getPartitionMappings());
+ for (String relatedPartition : relatedPartitions) {
+ res.add(relatedToMv.get(relatedPartition));
+ }
+ return res;
+ }
+
+ private static Map<String, String> getRelatedToMv(Map<String, Set<String>>
mvToRelated) {
+ Map<String, String> res = Maps.newHashMap();
+ for (Entry<String, Set<String>> entry : mvToRelated.entrySet()) {
+ for (String relatedPartition : entry.getValue()) {
+ res.put(relatedPartition, entry.getKey());
+ }
+ }
+ return res;
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
index 6470b14f914..6ccdbbc4ab3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java
@@ -22,7 +22,6 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState;
import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
-import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
@@ -136,8 +135,7 @@ public class MTMVRewriteUtilTest {
// currentTimeMills is 3, grace period is 2, and partition
getVisibleVersionTime is 1
// if forceConsistent this should get 0 partitions which mtmv can use.
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
- .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, true,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
true, null);
Assert.assertEquals(0, mtmvCanRewritePartitions.size());
}
@@ -145,7 +143,7 @@ public class MTMVRewriteUtilTest {
public void testGetMTMVCanRewritePartitionsNormal() {
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(1, mtmvCanRewritePartitions.size());
}
@@ -167,7 +165,7 @@ public class MTMVRewriteUtilTest {
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(1, mtmvCanRewritePartitions.size());
}
@@ -189,7 +187,7 @@ public class MTMVRewriteUtilTest {
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(0, mtmvCanRewritePartitions.size());
}
@@ -204,7 +202,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
// getMTMVCanRewritePartitions only check the partition is valid or
not, it doesn't care the
// isEnableMaterializedViewRewriteWhenBaseTableUnawareness
Assert.assertEquals(1, mtmvCanRewritePartitions.size());
@@ -223,7 +221,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(0, mtmvCanRewritePartitions.size());
}
@@ -242,7 +240,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(1, mtmvCanRewritePartitions.size());
}
@@ -261,7 +259,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
// getMTMVCanRewritePartitions only check the partition is valid or
not, it doesn't care the
// isEnableMaterializedViewRewriteWhenBaseTableUnawareness
Assert.assertEquals(1, mtmvCanRewritePartitions.size());
@@ -278,7 +276,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(0, mtmvCanRewritePartitions.size());
}
@@ -293,7 +291,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(1, mtmvCanRewritePartitions.size());
}
@@ -308,7 +306,7 @@ public class MTMVRewriteUtilTest {
};
Collection<Partition> mtmvCanRewritePartitions = MTMVRewriteUtil
.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills,
false,
- PartitionCompensator.getQueryUsedPartitions(ctx));
+ null);
Assert.assertEquals(0, mtmvCanRewritePartitions.size());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
