This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 38bac76b376c317a536a5d4b604594107dbea6f5
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Tue May 14 10:51:15 2024 +0800
[opt](mtmv) Cache materialization check result for performance (#34301)
Need check materialization sql pattern in different abstract rule when
rewrite by materialized view.
Such as the subClass of AbstractMaterializedViewJoinRule,
MaterializedViewScanRule, AbstractMaterializedViewAggregateRule.
This check result can be cached when has checked, this can avoid
unnecessary repeat check
---
.../java/org/apache/doris/nereids/memo/Memo.java | 46 ++++++++++++++++++++++
.../mv/AbstractMaterializedViewRule.java | 40 ++++++++++++++++---
2 files changed, 81 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index d7d46ecc15e..09a0ca69840 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -30,6 +30,7 @@ import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.properties.RequestPropertyDeriver;
import org.apache.doris.nereids.properties.RequirePropertiesSupplier;
+import
org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.LeafPlan;
@@ -53,6 +54,7 @@ import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -74,6 +76,10 @@ public class Memo {
private static long stateId = 0;
private final ConnectContext connectContext;
private final AtomicLong refreshVersion = new AtomicLong(1);
+ private final Map<Class<? extends AbstractMaterializedViewRule>,
Set<Long>> materializationCheckSuccessMap =
+ new LinkedHashMap<>();
+ private final Map<Class<? extends AbstractMaterializedViewRule>,
Set<Long>> materializationCheckFailMap =
+ new LinkedHashMap<>();
private final IdGenerator<GroupId> groupIdGenerator =
GroupId.createGenerator();
private final Map<GroupId, Group> groups = Maps.newLinkedHashMap();
// we could not use Set, because Set does not have get method.
@@ -127,6 +133,46 @@ public class Memo {
return refreshVersion.get();
}
+ /**
+ * Record materialization check result for performance
+ */
+ public void recordMaterializationCheckResult(Class<? extends
AbstractMaterializedViewRule> target,
+ Long checkedMaterializationId, boolean isSuccess) {
+ if (isSuccess) {
+ Set<Long> checkedSet = materializationCheckSuccessMap.get(target);
+ if (checkedSet == null) {
+ checkedSet = new HashSet<>();
+ materializationCheckSuccessMap.put(target, checkedSet);
+ }
+ checkedSet.add(checkedMaterializationId);
+ } else {
+ Set<Long> checkResultSet = materializationCheckFailMap.get(target);
+ if (checkResultSet == null) {
+ checkResultSet = new HashSet<>();
+ materializationCheckFailMap.put(target, checkResultSet);
+ }
+ checkResultSet.add(checkedMaterializationId);
+ }
+ }
+
+ /**
+ * Get the info for materialization context is checked
+ *
+ * @return if true, check successfully, if false check fail, if null not
checked
+ */
+ public Boolean materializationHasChecked(Class<? extends
AbstractMaterializedViewRule> target,
+ long materializationId) {
+ Set<Long> checkSuccessSet = materializationCheckSuccessMap.get(target);
+ if (checkSuccessSet != null &&
checkSuccessSet.contains(materializationId)) {
+ return true;
+ }
+ Set<Long> checkFailSet = materializationCheckFailMap.get(target);
+ if (checkFailSet != null && checkFailSet.contains(materializationId)) {
+ return false;
+ }
+ return null;
+ }
+
private Plan skipProject(Plan plan, Group targetGroup) {
// Some top project can't be eliminated
if (plan instanceof LogicalProject && ((LogicalProject<?>)
plan).canEliminate()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index dcff9db69db..41157a6c863 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -108,11 +108,7 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
continue;
}
// check mv plan is valid or not
- boolean valid = checkPattern(context.getStructInfo()) &&
context.getStructInfo().isValid();
- if (!valid) {
- context.recordFailReason(context.getStructInfo(),
- "View struct info is invalid", () -> String.format(",
view plan is %s",
-
context.getStructInfo().getOriginalPlan().treeString()));
+ if (!isMaterializationValid(cascadesContext, context)) {
continue;
}
// get query struct infos according to the view strut info, if
valid query struct infos is empty, bail out
@@ -668,6 +664,40 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
&&
context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId());
}
+ // check mv plan is valid or not, this can use cache for performance
+ private boolean isMaterializationValid(CascadesContext cascadesContext,
MaterializationContext context) {
+ long materializationId =
context.getMaterializationQualifier().hashCode();
+ Boolean cachedCheckResult =
cascadesContext.getMemo().materializationHasChecked(this.getClass(),
+ materializationId);
+ if (cachedCheckResult == null) {
+ // need check in real time
+ boolean checkResult = checkPattern(context.getStructInfo());
+ if (!checkResult) {
+ context.recordFailReason(context.getStructInfo(),
+ "View struct info is invalid", () ->
String.format("view plan is %s",
+
context.getStructInfo().getOriginalPlan().treeString()));
+
cascadesContext.getMemo().recordMaterializationCheckResult(this.getClass(),
materializationId,
+ false);
+ return false;
+ } else {
+
cascadesContext.getMemo().recordMaterializationCheckResult(this.getClass(),
+ materializationId, true);
+ }
+ } else if (!cachedCheckResult) {
+ context.recordFailReason(context.getStructInfo(),
+ "View struct info is invalid", () -> String.format("view
plan is %s",
+
context.getStructInfo().getOriginalPlan().treeString()));
+ return false;
+ }
+ if (!context.getStructInfo().isValid()) {
+ context.recordFailReason(context.getStructInfo(),
+ "View struct info is invalid", () -> String.format("view
plan is %s",
+
context.getStructInfo().getOriginalPlan().treeString()));
+ return false;
+ }
+ return true;
+ }
+
/**
* Query and mv match node
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
