This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9a5ec43f05f [fix](nereids) Fix data wrong using mv rewrite and ignore
case when getting mv related partition table (#28699)
9a5ec43f05f is described below
commit 9a5ec43f05fada36b2ed6eb2d2fe1ae79f34f6f4
Author: seawinde <[email protected]>
AuthorDate: Wed Dec 20 17:59:46 2023 +0800
[fix](nereids) Fix data wrong using mv rewrite and ignore case when getting
mv related partition table (#28699)
1. Fix data wrong using mv rewrite
2. Ignore case when getting mv related partition table
3. Enable infer expression column name without alias when create mv
---
.../main/java/org/apache/doris/catalog/MTMV.java | 6 +-
.../main/java/org/apache/doris/mtmv/MTMVCache.java | 3 +-
.../doris/nereids/pattern/PatternDescriptor.java | 16 +++
.../mv/AbstractMaterializedViewRule.java | 67 +++++----
.../exploration/mv/MaterializationContext.java | 2 +-
.../mv/MaterializedViewAggregateRule.java | 2 +-
.../mv/MaterializedViewProjectAggregateRule.java | 2 +-
.../mv/MaterializedViewProjectJoinRule.java | 2 +-
.../exploration/mv/MaterializedViewUtils.java | 2 +-
.../nereids/rules/exploration/mv/Predicates.java | 7 +-
.../rules/exploration/mv/PredicatesSplitter.java | 33 ++---
.../nereids/rules/exploration/mv/StructInfo.java | 2 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 6 +-
.../exploration/mv/MaterializedViewUtilsTest.java | 40 +++++-
.../rules/expression/PredicatesSplitterTest.java | 2 +-
.../mv/agg_with_roll_up/aggregate_with_roll_up.out | 14 ++
.../aggregate_without_roll_up.out | 38 ++++-
.../nereids_rules_p0/mv/join/inner/inner_join.out | 14 +-
.../agg_with_roll_up/aggregate_with_roll_up.groovy | 154 ++++++++++++---------
.../aggregate_without_roll_up.groovy | 52 +++----
.../mv/join/inner/inner_join.groovy | 18 ++-
21 files changed, 309 insertions(+), 173 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index f6a3500d51b..30160a8ef8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -35,7 +35,6 @@ import org.apache.doris.mtmv.MTMVRefreshInfo;
import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVStatus;
import org.apache.doris.persist.gson.GsonUtils;
-import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
@@ -199,13 +198,12 @@ public class MTMV extends OlapTable {
return Sets.newHashSet(split);
}
- // this should use the same connectContext with query, to use the same
session variable
- public MTMVCache getOrGenerateCache(ConnectContext parent) throws
AnalysisException {
+ public MTMVCache getOrGenerateCache() throws AnalysisException {
if (cache == null) {
writeMvLock();
try {
if (cache == null) {
- this.cache = MTMVCache.from(this, parent);
+ this.cache = MTMVCache.from(this,
MTMVPlanUtil.createMTMVContext(this));
}
} finally {
writeMvUnlock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
index 07fda55e494..5ae62cdd48e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
@@ -63,7 +63,8 @@ public class MTMVCache {
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
LogicalPlan unboundMvPlan = new
NereidsParser().parseSingle(mtmv.getQuerySql());
- // TODO: connect context set current db when create mv by use database
+ // this will be removed in the future when support join derivation
+
connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES,
ELIMINATE_OUTER_JOIN");
StatementContext mvSqlStatementContext = new
StatementContext(connectContext,
new OriginStatement(mtmv.getQuerySql(), 0));
NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java
index 90c143be1aa..5e2eb5757aa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/PatternDescriptor.java
@@ -93,6 +93,22 @@ public class PatternDescriptor<INPUT_TYPE extends Plan> {
return new PatternMatcher<>(pattern, defaultPromise, matchedAction);
}
+ /**
+ * Apply rule to return multi result, catch exception to make sure no
influence on other rule
+ */
+ public <OUTPUT_TYPE extends Plan> PatternMatcher<INPUT_TYPE, OUTPUT_TYPE>
thenApplyMultiNoThrow(
+ MatchedMultiAction<INPUT_TYPE, OUTPUT_TYPE> matchedMultiAction) {
+ MatchedMultiAction<INPUT_TYPE, OUTPUT_TYPE> adaptMatchedMultiAction =
ctx -> {
+ try {
+ return matchedMultiAction.apply(ctx);
+ } catch (Exception ex) {
+ LOG.warn("nereids apply rule failed, because {}",
ex.getMessage(), ex);
+ return null;
+ }
+ };
+ return new PatternMatcher<>(pattern, defaultPromise,
adaptMatchedMultiAction);
+ }
+
public Pattern<INPUT_TYPE> getPattern() {
return pattern;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 4e532366a33..6dcf1885563 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -84,7 +84,7 @@ public abstract class AbstractMaterializedViewRule implements
ExplorationRuleFac
List<MaterializationContext> materializationContexts =
cascadesContext.getMaterializationContexts();
List<Plan> rewriteResults = new ArrayList<>();
if (materializationContexts.isEmpty()) {
- logger.info(currentClassName + " materializationContexts is empty
so return");
+ logger.debug(currentClassName + " materializationContexts is empty
so return");
return rewriteResults;
}
@@ -92,7 +92,7 @@ public abstract class AbstractMaterializedViewRule implements
ExplorationRuleFac
// TODO Just Check query queryPlan firstly, support multi later.
StructInfo queryStructInfo = queryStructInfos.get(0);
if (!checkPattern(queryStructInfo)) {
- logger.info(currentClassName + " queryStructInfo is not valid so
return");
+ logger.debug(currentClassName + " queryStructInfo is not valid so
return");
return rewriteResults;
}
@@ -101,42 +101,42 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
if (queryPlan.getGroupExpression().isPresent()
&& materializationContext.alreadyRewrite(
queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) {
- logger.info(currentClassName + " this group is already
rewritten so skip");
+ logger.debug(currentClassName + " this group is already
rewritten so skip");
continue;
}
MTMV mtmv = materializationContext.getMTMV();
- MTMVCache mtmvCache = getCacheFromMTMV(mtmv, cascadesContext);
+ MTMVCache mtmvCache = getCacheFromMTMV(mtmv);
if (mtmvCache == null) {
- logger.info(currentClassName + " mv cache is null so return");
+ logger.warn(currentClassName + " mv cache is null so return");
return rewriteResults;
}
List<StructInfo> viewStructInfos =
extractStructInfo(mtmvCache.getLogicalPlan(), cascadesContext);
if (viewStructInfos.size() > 1) {
// view struct info should only have one
- logger.info(currentClassName + " the num of view struct info
is more then one so return");
+ logger.warn(currentClassName + " the num of view struct info
is more then one so return");
return rewriteResults;
}
StructInfo viewStructInfo = viewStructInfos.get(0);
if (!checkPattern(viewStructInfo)) {
- logger.info(currentClassName + " viewStructInfo is not valid
so return");
+ logger.debug(currentClassName + " viewStructInfo is not valid
so return");
continue;
}
MatchMode matchMode =
decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
if (MatchMode.COMPLETE != matchMode) {
- logger.info(currentClassName + " match mode is not complete so
return");
+ logger.debug(currentClassName + " match mode is not complete
so return");
continue;
}
List<RelationMapping> queryToViewTableMappings =
RelationMapping.generate(queryStructInfo.getRelations(),
viewStructInfo.getRelations());
// if any relation in query and view can not map, bail out.
if (queryToViewTableMappings == null) {
- logger.info(currentClassName + " query to view table mapping
null so return");
+ logger.warn(currentClassName + " query to view table mapping
null so return");
return rewriteResults;
}
for (RelationMapping queryToViewTableMapping :
queryToViewTableMappings) {
SlotMapping queryToViewSlotMapping =
SlotMapping.generate(queryToViewTableMapping);
if (queryToViewSlotMapping == null) {
- logger.info(currentClassName + " query to view slot
mapping null so continue");
+ logger.warn(currentClassName + " query to view slot
mapping null so continue");
continue;
}
LogicalCompatibilityContext compatibilityContext =
@@ -145,7 +145,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
List<Expression> pulledUpExpressions =
StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
compatibilityContext);
if (pulledUpExpressions == null) {
- logger.info(currentClassName + " graph logical is not
equals so continue");
+ logger.debug(currentClassName + " graph logical is not
equals so continue");
continue;
}
// set pulled up expression to queryStructInfo predicates and
update related predicates
@@ -156,13 +156,13 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
queryToViewSlotMapping);
// Can not compensate, bail out
if (compensatePredicates.isEmpty()) {
- logger.info(currentClassName + " predicate compensate fail
so continue");
+ logger.debug(currentClassName + " predicate compensate
fail so continue");
continue;
}
- Plan rewritedPlan;
+ Plan rewrittenPlan;
Plan mvScan = materializationContext.getMvScanPlan();
if (compensatePredicates.isAlwaysTrue()) {
- rewritedPlan = mvScan;
+ rewrittenPlan = mvScan;
} else {
// Try to rewrite compensate predicates by using mv scan
List<Expression> rewriteCompensatePredicates =
rewriteExpression(
@@ -172,39 +172,52 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
queryToViewSlotMapping,
true);
if (rewriteCompensatePredicates.isEmpty()) {
- logger.info(currentClassName + " compensate predicate
rewrite by view fail so continue");
+ logger.debug(currentClassName + " compensate predicate
rewrite by view fail so continue");
continue;
}
- rewritedPlan = new
LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
+ rewrittenPlan = new
LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
}
// Rewrite query by view
- rewritedPlan = rewriteQueryByView(matchMode,
+ rewrittenPlan = rewriteQueryByView(matchMode,
queryStructInfo,
viewStructInfo,
queryToViewSlotMapping,
- rewritedPlan,
+ rewrittenPlan,
materializationContext);
- if (rewritedPlan == null) {
- logger.info(currentClassName + " rewrite query by view
fail so continue");
+ if (rewrittenPlan == null) {
+ logger.debug(currentClassName + " rewrite query by view
fail so continue");
continue;
}
if (!checkPartitionIsValid(queryStructInfo,
materializationContext, cascadesContext)) {
- logger.info(currentClassName + " check partition
validation fail so continue");
+ logger.debug(currentClassName + " check partition
validation fail so continue");
+ continue;
+ }
+ if (!checkOutput(queryPlan, rewrittenPlan)) {
+ logger.debug(currentClassName + " check output validation
fail so continue");
continue;
}
// run rbo job on mv rewritten plan
CascadesContext rewrittenPlanContext =
-
CascadesContext.initContext(cascadesContext.getStatementContext(), rewritedPlan,
+
CascadesContext.initContext(cascadesContext.getStatementContext(),
rewrittenPlan,
cascadesContext.getCurrentJobContext().getRequiredProperties());
Rewriter.getWholeTreeRewriter(cascadesContext).execute();
- rewritedPlan = rewrittenPlanContext.getRewritePlan();
- logger.info(currentClassName + "rewrite by materialized view
success");
- rewriteResults.add(rewritedPlan);
+ rewrittenPlan = rewrittenPlanContext.getRewritePlan();
+ logger.debug(currentClassName + "rewrite by materialized view
success");
+ rewriteResults.add(rewrittenPlan);
}
}
return rewriteResults;
}
+ protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan) {
+ if (sourcePlan.getGroupExpression().isPresent() &&
!rewrittenPlan.getLogicalProperties().equals(
+
sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) {
+ logger.error("rewrittenPlan output logical properties is not same
with target group");
+ return false;
+ }
+ return true;
+ }
+
/**
* Partition will be pruned in query then add the record the partitions to
select partitions on
* catalog relation.
@@ -276,10 +289,10 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
&&
relatedTalbeValidSet.containsAll(relatedTableSelectedPartitionToCheck);
}
- private MTMVCache getCacheFromMTMV(MTMV mtmv, CascadesContext
cascadesContext) {
+ private MTMVCache getCacheFromMTMV(MTMV mtmv) {
MTMVCache cache;
try {
- cache =
mtmv.getOrGenerateCache(cascadesContext.getConnectContext());
+ cache = mtmv.getOrGenerateCache();
} catch (AnalysisException analysisException) {
logger.warn(this.getClass().getSimpleName() + " get mtmv cache
analysisException", analysisException);
return null;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index f61de5cc2b5..19b1f5be8bb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -67,7 +67,7 @@ public class MaterializationContext {
MTMVCache mtmvCache = null;
try {
- mtmvCache =
mtmv.getOrGenerateCache(cascadesContext.getConnectContext());
+ mtmvCache = mtmv.getOrGenerateCache();
} catch (AnalysisException e) {
LOG.warn("MaterializationContext init mv cache generate fail", e);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
index 2674bf2bf22..9059499d381 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
@@ -36,7 +36,7 @@ public class MaterializedViewAggregateRule extends
AbstractMaterializedViewAggre
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalAggregate(any()).thenApplyMulti(ctx -> {
+ logicalAggregate(any()).thenApplyMultiNoThrow(ctx -> {
LogicalAggregate<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_ONLY_AGGREGATE));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
index f76ec3d26af..106de2a7f9c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectAggregateRule.java
@@ -35,7 +35,7 @@ public class MaterializedViewProjectAggregateRule extends
AbstractMaterializedVi
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalProject(logicalAggregate(any())).thenApplyMulti(ctx -> {
+
logicalProject(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalAggregate<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
index 283ea04ba43..c9e309c0096 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
@@ -37,7 +37,7 @@ public class MaterializedViewProjectJoinRule extends
AbstractMaterializedViewJoi
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
- logicalProject(logicalJoin(any(), any())).thenApplyMulti(ctx
-> {
+ logicalProject(logicalJoin(any(),
any())).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalJoin<Plan, Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_JOIN));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 48f4cb37fbe..e219bec04ba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -63,7 +63,7 @@ public class MaterializedViewUtils {
Slot columnExpr = null;
// get column slot
for (Slot outputSlot : outputExpressions) {
- if (outputSlot.getName().equals(column)) {
+ if (outputSlot.getName().equalsIgnoreCase(column)) {
columnExpr = outputSlot;
break;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
index 96f89a513cf..94799899b9e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
@@ -23,9 +23,10 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
-import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
/**
@@ -34,7 +35,7 @@ import java.util.stream.Collectors;
public class Predicates {
// Predicates that can be pulled up
- private final List<Expression> pulledUpPredicates = new ArrayList<>();
+ private final Set<Expression> pulledUpPredicates = new HashSet<>();
private Predicates() {
}
@@ -49,7 +50,7 @@ public class Predicates {
return predicates;
}
- public List<? extends Expression> getPulledUpPredicates() {
+ public Set<? extends Expression> getPulledUpPredicates() {
return pulledUpPredicates;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
index 5f9a5e242de..de9115468f2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
@@ -20,17 +20,16 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
-import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
-import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* Split the expression to equal, range and residual predicate.
@@ -39,27 +38,26 @@ import java.util.List;
*/
public class PredicatesSplitter {
- private final List<Expression> equalPredicates = new ArrayList<>();
- private final List<Expression> rangePredicates = new ArrayList<>();
- private final List<Expression> residualPredicates = new ArrayList<>();
+ private final Set<Expression> equalPredicates = new HashSet<>();
+ private final Set<Expression> rangePredicates = new HashSet<>();
+ private final Set<Expression> residualPredicates = new HashSet<>();
private final List<Expression> conjunctExpressions;
- private final PredicateExtract instance = new PredicateExtract();
-
public PredicatesSplitter(Expression target) {
this.conjunctExpressions = ExpressionUtils.extractConjunction(target);
+ PredicateExtract instance = new PredicateExtract();
for (Expression expression : conjunctExpressions) {
- expression.accept(instance, expression);
+ expression.accept(instance, null);
}
}
/**
- * PredicateExtract
+ * extract to equal, range, residual predicate set
*/
- public class PredicateExtract extends DefaultExpressionVisitor<Void,
Expression> {
+ public class PredicateExtract extends DefaultExpressionVisitor<Void, Void>
{
@Override
- public Void visitComparisonPredicate(ComparisonPredicate
comparisonPredicate, Expression sourceExpression) {
+ public Void visitComparisonPredicate(ComparisonPredicate
comparisonPredicate, Void context) {
Expression leftArg = comparisonPredicate.getArgument(0);
Expression rightArg = comparisonPredicate.getArgument(1);
boolean leftArgOnlyContainsColumnRef =
containOnlyColumnRef(leftArg, true);
@@ -81,12 +79,9 @@ public class PredicatesSplitter {
}
@Override
- public Void visitCompoundPredicate(CompoundPredicate
compoundPredicate, Expression context) {
- if (compoundPredicate instanceof Or) {
- residualPredicates.add(compoundPredicate);
- return null;
- }
- return super.visitCompoundPredicate(compoundPredicate, context);
+ public Void visit(Expression expr, Void context) {
+ residualPredicates.add(expr);
+ return null;
}
}
@@ -98,7 +93,7 @@ public class PredicatesSplitter {
}
private static boolean containOnlyColumnRef(Expression expression, boolean
allowCast) {
- if (expression instanceof SlotReference && ((SlotReference)
expression).isColumnFromTable()) {
+ if (expression instanceof SlotReference &&
expression.isColumnFromTable()) {
return true;
}
if (allowCast && expression instanceof Cast) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 1d38e893dba..3c9814cdce3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -162,7 +162,7 @@ public class StructInfo {
private void predicatesDerive() {
// construct equivalenceClass according to equals predicates
List<Expression> shuttledExpression =
ExpressionUtils.shuttleExpressionWithLineage(
- this.predicates.getPulledUpPredicates(),
originalPlan).stream()
+ new
ArrayList<>(this.predicates.getPulledUpPredicates()), originalPlan).stream()
.map(Expression.class::cast)
.collect(Collectors.toList());
SplitPredicate splitPredicate =
Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index c8a4bfeebb1..6d81191c8be 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -43,6 +43,7 @@ import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.analyzer.UnboundResultSink;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
@@ -54,6 +55,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import
org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import
org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
@@ -199,7 +201,9 @@ public class CreateMTMVInfo {
public void analyzeQuery(ConnectContext ctx) {
// create table as select
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
- Plan plan = planner.plan(logicalQuery, PhysicalProperties.ANY,
ExplainLevel.ALL_PLAN);
+ // this is for expression column name infer when not use alias
+ LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery);
+ Plan plan = planner.plan(logicalSink, PhysicalProperties.ANY,
ExplainLevel.ALL_PLAN);
if (plan.anyMatch(node -> node instanceof OneRowRelation)) {
throw new AnalysisException("at least contain one table");
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
index 2e402cd5c7a..898b5843031 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
@@ -113,7 +113,7 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", rewrittenPlan);
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"L_SHIPDATE",
@@ -142,7 +142,7 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
MaterializedViewUtils.getRelatedTableInfo("ship_data_alias", rewrittenPlan);
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
- "L_SHIPDATE",
+ "l_shipdate",
true);
});
}
@@ -169,7 +169,7 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", rewrittenPlan);
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"L_SHIPDATE",
@@ -177,6 +177,36 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
});
}
+ @Test
+ public void getRelatedTableInfoUseRightTest() {
+ PlanChecker.from(connectContext)
+ .checkExplain("SELECT t1.L_SHIPDATE, t2.O_ORDERDATE,
t1.L_QUANTITY, t2.O_ORDERSTATUS, "
+ + "count(distinct case when t1.L_SUPPKEY > 0
then t2.O_ORDERSTATUS else null end) as cnt_1 "
+ + "from "
+ + " (select * from "
+ + " lineitem "
+ + " where L_SHIPDATE in ('2017-01-30')) t1 "
+ + "left join "
+ + " (select * from "
+ + " orders "
+ + " where O_ORDERDATE in ('2017-01-30')) t2 "
+ + "on t1.L_ORDERKEY = t2.O_ORDERKEY "
+ + "group by "
+ + "t1.L_SHIPDATE, "
+ + "t2.O_ORDERDATE, "
+ + "t1.L_QUANTITY, "
+ + "t2.O_ORDERSTATUS;",
+ nereidsPlanner -> {
+ Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
+ Optional<RelatedTableInfo> relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("o_orderdate", rewrittenPlan);
+ checkRelatedTableInfo(relatedTableInfo,
+ "orders",
+ "O_ORDERDATE",
+ true);
+ });
+ }
+
@Test
public void getRelatedTableInfoTestWithoutPartitionTest() {
PlanChecker.from(connectContext)
@@ -212,7 +242,7 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
nereidsPlanner -> {
Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
Optional<RelatedTableInfo> relatedTableInfo =
-
MaterializedViewUtils.getRelatedTableInfo("L_SHIPDATE", rewrittenPlan);
+
MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
checkRelatedTableInfo(relatedTableInfo,
"lineitem",
"L_SHIPDATE",
@@ -298,7 +328,7 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
} catch (Exception exception) {
Assertions.fail();
}
- Assertions.assertEquals(relatedTableInfo.get().getColumn(),
expectColumnName);
+
Assertions.assertEquals(relatedTableInfo.get().getColumn().toLowerCase(),
expectColumnName.toLowerCase());
Assertions.assertTrue(pctPossible);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java
index 7c3055b9cef..8bef8afe7a9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java
@@ -48,7 +48,7 @@ public class PredicatesSplitterTest extends
ExpressionRewriteTestHelper {
"c = d or a = 10");
assetEquals("a = b and c + d = e and a > 7 and 10 > d",
"a = b",
- "a > 7 and 10 > d",
+ "10 > d and a > 7",
"c + d = e");
assetEquals("a = b and c + d = e or a > 7 and 10 > d",
"",
diff --git
a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
index b240c02f7ea..e6facb7b890 100644
---
a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
+++
b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
@@ -121,3 +121,17 @@
3 3 2023-12-11 43.20 43.20 43.20 1
4 3 2023-12-09 11.50 11.50 11.50 1
+-- !query1_1_before --
+1 yy 0 0 11.50 11.50 11.50 1
+
+-- !query1_1_after --
+1 yy 0 0 11.50 11.50 11.50 1
+
+-- !query2_0_before --
+2 mi 0 0 57.40 56.20 1.20 2
+2 mm 0 0 43.20 43.20 43.20 1
+
+-- !query2_0_after --
+2 mi 0 0 57.40 56.20 1.20 2
+2 mm 0 0 43.20 43.20 43.20 1
+
diff --git
a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
index 346814bc083..cb924d69bbb 100644
---
a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
+++
b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
@@ -2,9 +2,20 @@
-- !query1_0_before --
-- !query1_0_after --
-1 yy 0 0 77.50 33.50 9.50 5
-2 mi 0 0 57.40 56.20 1.20 2
-2 mm 0 0 43.20 43.20 43.20 1
+
+-- !query1_1_before --
+2023-12-08 1 yy 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-09 1 yy 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-10 1 yy 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-11 2 mm 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-12 2 mi 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+
+-- !query1_1_after --
+2023-12-08 1 yy 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-09 1 yy 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-10 1 yy 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-11 2 mm 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
+2023-12-12 2 mi 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0
-- !query1_2_before --
1 yy 0 0 0 0 0 0 0 0
0 0 0 0 0 0 0
@@ -67,7 +78,6 @@
2 3 2023-12-12 57.40 56.20 1.20 2 0
2 4 2023-12-10 46.00 33.50 12.50 2 0
3 3 2023-12-11 43.20 43.20 43.20 1 0
-4 3 2023-12-09 11.50 11.50 11.50 1 0
-- !query16_0_before --
2 3 2023-12-08 20.00 10.50 9.50 2 0
@@ -89,6 +99,20 @@
-- !query17_0_after --
3 3 2023-12-11 43.20 43.20 43.20 1 0
+-- !query17_1_before --
+1 1 yy 0 0 0 0 0 0 0
0 0 0 0 0 0
+2 1 yy 0 0 0 0 0 0 0
0 0 0 0 0 0
+3 1 yy 0 0 0 0 0 0 0
0 0 0 0 0 0
+4 2 mm 0 0 0 0 0 0 0
0 0 0 0 0 0
+5 2 mi 0 0 0 0 0 0 0
0 0 0 0 0 0
+
+-- !query17_1_after --
+1 1 yy 0 0 0 0 0 0 0
0 0 0 0 0 0
+2 1 yy 0 0 0 0 0 0 0
0 0 0 0 0 0
+3 1 yy 0 0 0 0 0 0 0
0 0 0 0 0 0
+4 2 mm 0 0 0 0 0 0 0
0 0 0 0 0 0
+5 2 mi 0 0 0 0 0 0 0
0 0 0 0 0 0
+
-- !query18_0_before --
-- !query18_0_after --
@@ -125,3 +149,9 @@
4 2 43.20
6 2 57.40
+-- !query20_0_before --
+0 0 0 0 0 0 0 0 0 0
0 0
+
+-- !query20_0_after --
+0 0 0 0 0 0 0 0 0 0
0 0
+
diff --git a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
index 7ae2e05e523..df3e20dc64c 100644
--- a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
+++ b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
@@ -238,8 +238,6 @@
4
4
4
-6
-6
-- !query5_0_before --
4
@@ -261,6 +259,18 @@
6
6
+-- !query6_0_before --
+2 3 2023-12-08
+2 3 2023-12-08
+
+-- !query6_0_after --
+2 3 2023-12-08
+2 3 2023-12-08
+
+-- !query7_0_before --
+
+-- !query7_0_after --
+
-- !query10_0_before --
-- !query10_0_after --
diff --git
a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index d6d96acbb88..e67086deba0 100644
---
a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -144,7 +144,7 @@ suite("aggregate_with_roll_up") {
}
}
- def check_not_match = { mv_sql, query_sql, mv_name ->
+ def check_rewrite_with_force_analyze = { mv_sql, query_sql, mv_name ->
sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
sql"""
@@ -155,78 +155,37 @@ suite("aggregate_with_roll_up") {
AS ${mv_sql}
"""
+ sql "analyze table ${mv_name} with sync;"
+ sql "analyze table lineitem with sync;"
+ sql "analyze table orders with sync;"
+ sql "analyze table partsupp with sync;"
+
def job_name = getJobName(db, mv_name);
waitingMTMVTaskFinished(job_name)
explain {
sql("${query_sql}")
- notContains "(${mv_name})"
+ contains "(${mv_name})"
}
}
- // single table
- // filter + use roll up dimension
- def mv1_1 = "select o_orderdate, o_shippriority, o_comment, " +
- "sum(o_totalprice) as sum_total, " +
- "max(o_totalprice) as max_total, " +
- "min(o_totalprice) as min_total, " +
- "count(*) as count_all, " +
- "bitmap_union(to_bitmap(case when o_shippriority > 1 and
o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, " +
- "bitmap_union(to_bitmap(case when o_shippriority > 2 and
o_orderkey IN (2) then o_custkey else null end)) as cnt_2 " +
- "from orders " +
- "group by " +
- "o_orderdate, " +
- "o_shippriority, " +
- "o_comment "
- def query1_1 = "select o_shippriority, o_comment, " +
- "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end) as cnt_1, " +
- "count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2)
then o_custkey else null end) as cnt_2, " +
- "sum(o_totalprice), " +
- "max(o_totalprice), " +
- "min(o_totalprice), " +
- "count(*) " +
- "from orders " +
- "where o_orderdate = '2023-12-09' " +
- "group by " +
- "o_shippriority, " +
- "o_comment "
- // rewrite success but cbo not chose, tmp
-// order_qt_query1_1_before "${query1_1}"
-// check_rewrite(mv1_1, query1_1, "mv1_1")
-// order_qt_query1_1_after "${query1_1}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
-
+ def check_not_match = { mv_sql, query_sql, mv_name ->
- // filter + not use roll up dimension
- def mv2_0 = "select o_orderdate, o_shippriority, o_comment, " +
- "sum(o_totalprice) as sum_total, " +
- "max(o_totalprice) as max_total, " +
- "min(o_totalprice) as min_total, " +
- "count(*) as count_all, " +
- "bitmap_union(to_bitmap(case when o_shippriority > 1 and
o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, " +
- "bitmap_union(to_bitmap(case when o_shippriority > 2 and
o_orderkey IN (2) then o_custkey else null end)) as cnt_2 " +
- "from orders " +
- "group by " +
- "o_orderdate, " +
- "o_shippriority, " +
- "o_comment "
- def query2_0 = "select o_shippriority, o_comment, " +
- "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end) as cnt_1, " +
- "count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2)
then o_custkey else null end) as cnt_2, " +
- "sum(o_totalprice), " +
- "max(o_totalprice), " +
- "min(o_totalprice), " +
- "count(*) " +
- "from orders " +
- "where o_shippriority = 2 " +
- "group by " +
- "o_shippriority, " +
- "o_comment "
- // rewrite success but cbo not chose, tmp
-// order_qt_query2_0_before "${query2_0}"
-// check_rewrite(mv2_0, query2_0, "mv2_0")
-// order_qt_query2_0_after "${query2_0}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0"""
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS ${mv_sql}
+ """
+ def job_name = getJobName(db, mv_name);
+ waitingMTMVTaskFinished(job_name)
+ explain {
+ sql("${query_sql}")
+ notContains "(${mv_name})"
+ }
+ }
// multi table
// filter inside + left + use roll up dimension
@@ -710,5 +669,72 @@ suite("aggregate_with_roll_up") {
order_qt_query25_0_after "${query25_0}"
sql """ DROP MATERIALIZED VIEW IF EXISTS mv25_0"""
+ // single table
+ // filter + use roll up dimension
+ def mv1_1 = "select o_orderdate, o_shippriority, o_comment, " +
+ "sum(o_totalprice) as sum_total, " +
+ "max(o_totalprice) as max_total, " +
+ "min(o_totalprice) as min_total, " +
+ "count(*) as count_all, " +
+ "bitmap_union(to_bitmap(case when o_shippriority > 1 and
o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, " +
+ "bitmap_union(to_bitmap(case when o_shippriority > 2 and
o_orderkey IN (2) then o_custkey else null end)) as cnt_2 " +
+ "from orders " +
+ "group by " +
+ "o_orderdate, " +
+ "o_shippriority, " +
+ "o_comment "
+ def query1_1 = "select o_shippriority, o_comment, " +
+ "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end) as cnt_1, " +
+ "count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2)
then o_custkey else null end) as cnt_2, " +
+ "sum(o_totalprice), " +
+ "max(o_totalprice), " +
+ "min(o_totalprice), " +
+ "count(*) " +
+ "from orders " +
+ "where o_orderdate = '2023-12-09' " +
+ "group by " +
+ "o_shippriority, " +
+ "o_comment "
+ order_qt_query1_1_before "${query1_1}"
+ // rewrite success, for cbo chose, should force analyze
+ // because data volume is small and mv plan is almost same to query plan
+ check_rewrite_with_force_analyze(mv1_1, query1_1, "mv1_1")
+ order_qt_query1_1_after "${query1_1}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
+
+
+ // filter + not use roll up dimension
+ def mv2_0 = "select o_orderdate, o_shippriority, o_comment, " +
+ "sum(o_totalprice) as sum_total, " +
+ "max(o_totalprice) as max_total, " +
+ "min(o_totalprice) as min_total, " +
+ "count(*) as count_all, " +
+ "bitmap_union(to_bitmap(case when o_shippriority > 1 and
o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, " +
+ "bitmap_union(to_bitmap(case when o_shippriority > 2 and
o_orderkey IN (2) then o_custkey else null end)) as cnt_2 " +
+ "from orders " +
+ "group by " +
+ "o_orderdate, " +
+ "o_shippriority, " +
+ "o_comment "
+ def query2_0 = "select o_shippriority, o_comment, " +
+ "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end) as cnt_1, " +
+ "count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2)
then o_custkey else null end) as cnt_2, " +
+ "sum(o_totalprice), " +
+ "max(o_totalprice), " +
+ "min(o_totalprice), " +
+ "count(*) " +
+ "from orders " +
+ "where o_shippriority = 2 " +
+ "group by " +
+ "o_shippriority, " +
+ "o_comment "
+
+ order_qt_query2_0_before "${query2_0}"
+ // rewrite success, for cbo chose, should force analyze
+ // because data volume is small and mv plan is almost same to query plan
+ check_rewrite_with_force_analyze(mv2_0, query2_0, "mv2_0")
+ order_qt_query2_0_after "${query2_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0"""
+
// can not rewrite, todo
}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 055f8d01804..b76763752fb 100644
---
a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -24,6 +24,7 @@ suite("aggregate_without_roll_up") {
sql "SET enable_nereids_timeout = false"
// tmp disable to rewrite, will be removed in the future
sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+ sql "SET global enable_auto_analyze = false"
sql """
drop table if exists orders
@@ -170,7 +171,7 @@ suite("aggregate_without_roll_up") {
"max(o_totalprice) as max_total, " +
"min(o_totalprice) as min_total, " +
"count(*) as count_all, " +
- "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end) as cnt_1, " +
+ "count(distinct case when o_shippriority > 1 and o_orderkey IN (1,
3) then o_custkey else null end), " +
"count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2)
then o_custkey else null end) as cnt_2 " +
"from orders " +
"group by " +
@@ -194,7 +195,7 @@ suite("aggregate_without_roll_up") {
sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0"""
- def mv1_1 = "select O_SHIPPRIORITY, O_COMMENT, " +
+ def mv1_1 = "select O_SHIPPRIORITY, O_COMMENT, O_ORDERDATE, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (1,
3) then O_ORDERSTATUS else null end) as filter_cnt_1, " +
"count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2)
then O_ORDERSTATUS else null end) as filter_cnt_2, " +
"count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3,
4) then O_ORDERSTATUS else null end) as filter_cnt_3, " +
@@ -218,9 +219,10 @@ suite("aggregate_without_roll_up") {
"from orders " +
"where O_ORDERDATE < '2023-12-30'" +
"group by " +
+ "O_ORDERDATE, " +
"O_SHIPPRIORITY, " +
"O_COMMENT "
- def query1_1 = "select O_SHIPPRIORITY, O_COMMENT, " +
+ def query1_1 = "select O_ORDERDATE, O_SHIPPRIORITY, O_COMMENT, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (1,
3) then O_ORDERSTATUS else null end) as filter_cnt_1, " +
"count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2)
then O_ORDERSTATUS else null end) as filter_cnt_2, " +
"count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3,
4) then O_ORDERSTATUS else null end) as filter_cnt_3, " +
@@ -239,13 +241,13 @@ suite("aggregate_without_roll_up") {
"from orders " +
"where O_ORDERDATE < '2023-12-30' and O_ORDERDATE > '2023-12-01'" +
"group by " +
+ "O_ORDERDATE, " +
"O_SHIPPRIORITY, " +
"O_COMMENT "
- // should support but not, tmp
-// order_qt_query1_1_before "${query1_1}"
-// check_rewrite(mv1_1, query1_1, "mv1_1")
-// order_qt_query1_1_after "${query1_1}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
+ order_qt_query1_1_before "${query1_1}"
+ check_rewrite(mv1_1, query1_1, "mv1_1")
+ order_qt_query1_1_after "${query1_1}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1"""
def mv1_2 = "select O_SHIPPRIORITY, O_COMMENT, " +
@@ -350,11 +352,11 @@ suite("aggregate_without_roll_up") {
// without group, scalar aggregate
def mv3_0 = "select count(distinct case when O_SHIPPRIORITY > 1 and
O_ORDERKEY IN (1, 3) then O_ORDERSTATUS else null end) as filter_cnt_1, " +
- "count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2)
then O_ORDERSTATUS else null end) as filter_cnt_2, " +
- "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3,
4) then O_ORDERSTATUS else null end) as filter_cnt_3, " +
+ "count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2)
then O_ORDERSTATUS else null end), " +
+ "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3,
4) then O_ORDERSTATUS else null end), " +
"count(distinct case when O_SHIPPRIORITY > 4 and O_ORDERKEY IN (5,
6) then O_ORDERSTATUS else null end) as filter_cnt_4, " +
- "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (2,
3) then O_ORDERSTATUS else null end) as filter_cnt_5, " +
- "count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (7,
9) then O_ORDERSTATUS else null end) as filter_cnt_6, " +
+ "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (2,
3) then O_ORDERSTATUS else null end), " +
+ "count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (7,
9) then O_ORDERSTATUS else null end), " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (8,
10) then O_ORDERSTATUS else null end) as filter_cnt_7, " +
"count(distinct case when O_SHIPPRIORITY > 4 and O_ORDERKEY IN
(11, 13) then O_ORDERSTATUS else null end) as filter_cnt_8, " +
"count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN
(12, 11) then O_ORDERSTATUS else null end) as filter_cnt_9, " +
@@ -559,17 +561,17 @@ suite("aggregate_without_roll_up") {
def mv17_1 = "select L_ORDERKEY, O_SHIPPRIORITY, O_COMMENT, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (1,
3) then O_ORDERSTATUS else null end) as filter_cnt_1, " +
"count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (2)
then O_ORDERSTATUS else null end) as filter_cnt_2, " +
- "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3,
4) then O_ORDERSTATUS else null end) as filter_cnt_3, " +
+ "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (3,
4) then O_ORDERSTATUS else null end), " +
"count(distinct case when O_SHIPPRIORITY > 4 and O_ORDERKEY IN (5,
6) then O_ORDERSTATUS else null end) as filter_cnt_4, " +
"count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN (2,
3) then O_ORDERSTATUS else null end) as filter_cnt_5, " +
"count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN (7,
9) then O_ORDERSTATUS else null end) as filter_cnt_6, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN (8,
10) then O_ORDERSTATUS else null end) as filter_cnt_7, " +
"count(distinct case when O_SHIPPRIORITY > 4 and O_ORDERKEY IN
(11, 13) then O_ORDERSTATUS else null end) as filter_cnt_8, " +
- "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN
(12, 11) then O_ORDERSTATUS else null end) as filter_cnt_9, " +
+ "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN
(12, 11) then O_ORDERSTATUS else null end), " +
"count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN
(14, 15) then O_ORDERSTATUS else null end) as filter_cnt_10, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN
(11, 12) then O_ORDERSTATUS else null end) as filter_cnt_11, " +
"count(distinct case when O_SHIPPRIORITY > 4 and O_ORDERKEY IN (3,
6) then O_ORDERSTATUS else null end) as filter_cnt_12, " +
- "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN
(16, 19) then O_ORDERSTATUS else null end) as filter_cnt_13, " +
+ "count(distinct case when O_SHIPPRIORITY > 3 and O_ORDERKEY IN
(16, 19) then O_ORDERSTATUS else null end), " +
"count(distinct case when O_SHIPPRIORITY > 2 and O_ORDERKEY IN
(20, 3) then O_ORDERSTATUS else null end) as filter_cnt_14, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN
(15, 19) then O_ORDERSTATUS else null end) as filter_cnt_15, " +
"count(distinct case when O_SHIPPRIORITY > 1 and O_ORDERKEY IN
(13, 21) then O_ORDERSTATUS else null end) as filter_cnt_16, " +
@@ -609,11 +611,10 @@ suite("aggregate_without_roll_up") {
"lineitem.L_ORDERKEY, " +
"orders.O_SHIPPRIORITY, " +
"orders.O_COMMENT "
- // rewrite success but cbo not chose, tmp
-// order_qt_query17_1_before "${query17_1}"
-// check_rewrite(mv17_1, query17_1, "mv17_1")
-// order_qt_query17_1_after "${query17_1}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv17_1"""
+ order_qt_query17_1_before "${query17_1}"
+ check_rewrite(mv17_1, query17_1, "mv17_1")
+ order_qt_query17_1_after "${query17_1}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv17_1"""
// filter outside + left + right
def mv18_0 = "select l_shipdate, l_suppkey, " +
@@ -677,7 +678,7 @@ suite("aggregate_without_roll_up") {
// without filter
def mv19_0 = "select o_orderdate, l_partkey, l_suppkey, " +
- "sum(o_totalprice) as sum_total, " +
+ "sum(o_totalprice), " +
"max(o_totalprice) as max_total, " +
"min(o_totalprice) as min_total, " +
"count(*) as count_all " +
@@ -760,9 +761,8 @@ suite("aggregate_without_roll_up") {
"orders " +
"on lineitem.L_ORDERKEY = orders.O_ORDERKEY " +
"where orders.O_ORDERDATE < '2023-12-30' and orders.O_ORDERDATE >
'2023-12-01' "
- // rewrite success but cbo not chose, tmp
-// order_qt_query20_0_before "${query20_0}"
-// check_rewrite(mv20_0, query20_0, "mv20_0")
-// order_qt_query20_0_after "${query20_0}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_0"""
+ order_qt_query20_0_before "${query20_0}"
+ check_rewrite(mv20_0, query20_0, "mv20_0")
+ order_qt_query20_0_after "${query20_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv20_0"""
}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
index e0c0452415d..d8025e0bc60 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
@@ -387,11 +387,10 @@ suite("inner_join") {
"from lineitem t1 " +
"inner join (select * from orders where o_orderdate =
'2023-12-08') t2 " +
"on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate "
- // should passed but not as isGraphLogicalEquals is false
-// order_qt_query6_0_before "${query6_0}"
-// check_rewrite(mv6_0, query6_0, "mv6_0")
-// order_qt_query6_0_after "${query6_0}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_0"""
+ order_qt_query6_0_before "${query6_0}"
+ check_rewrite(mv6_0, query6_0, "mv6_0")
+ order_qt_query6_0_after "${query6_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_0"""
// filter inside + inner + right
@@ -405,11 +404,10 @@ suite("inner_join") {
"inner join (select * from orders where o_orderdate =
'2023-12-08') t2 " +
"on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate " +
"where l_partkey = 3"
- // should passed but not, because isGraphLogicalEquals is false
-// order_qt_query7_0_before "${query7_0}"
-// check_rewrite(mv7_0, query7_0, "mv7_0")
-// order_qt_query7_0_after "${query7_0}"
-// sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0"""
+ order_qt_query7_0_before "${query7_0}"
+ check_rewrite(mv7_0, query7_0, "mv7_0")
+ order_qt_query7_0_after "${query7_0}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0"""
// check not match, because use a filed orders.O_SHIPPRIORITY which not in
mv
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
