This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dbccbe0fbba [feat](Nereids): add struct info map in group (#31800)
dbccbe0fbba is described below
commit dbccbe0fbbaaa6550eee7b3234c15910615c63a5
Author: 谢健 <jianx...@gmail.com>
AuthorDate: Wed Mar 6 16:49:04 2024 +0800
[feat](Nereids): add struct info map in group (#31800)
---
.../org/apache/doris/mtmv/MTMVRelationManager.java | 10 +-
.../java/org/apache/doris/nereids/memo/Group.java | 7 +-
.../apache/doris/nereids/memo/StructInfoMap.java | 123 +++++++++++++++++++++
.../mv/InitMaterializationContextHook.java | 7 +-
.../doris/nereids/memo/StructInfoMapTest.java | 81 ++++++++++++++
.../org/apache/doris/nereids/util/PlanChecker.java | 2 +
.../apache/doris/utframe/TestWithFeService.java | 28 +++++
7 files changed, 252 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index aa7ffd2426d..723deaff740 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -35,6 +35,7 @@ import
org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
import org.apache.doris.persist.AlterMTMV;
import org.apache.doris.qe.ConnectContext;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -71,8 +72,7 @@ public class MTMVRelationManager implements MTMVHookService {
for (BaseTableInfo tableInfo : mvInfos) {
try {
MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo);
- if (!CollectionUtils
-
.isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, ctx,
System.currentTimeMillis()))) {
+ if (isMVPartitionValid(mtmv, ctx)) {
res.add(mtmv);
}
} catch (AnalysisException e) {
@@ -83,6 +83,12 @@ public class MTMVRelationManager implements MTMVHookService {
return res;
}
+ @VisibleForTesting
+ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
+ return !CollectionUtils
+ .isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv,
ctx, System.currentTimeMillis()));
+ }
+
private Set<BaseTableInfo> getMTMVInfos(List<BaseTableInfo> tableInfos) {
Set<BaseTableInfo> mvInfos = Sets.newHashSet();
for (BaseTableInfo tableInfo : tableInfos) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
index 2a0e83c72a4..5a5abd56f95 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
@@ -19,7 +19,6 @@ package org.apache.doris.nereids.memo;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.cost.Cost;
-import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
@@ -78,6 +77,8 @@ public class Group {
private List<StructInfo> structInfos = new ArrayList<>();
+ private StructInfoMap structInfoMap = new StructInfoMap();
+
/**
* Constructor for Group.
*
@@ -418,8 +419,8 @@ public class Group {
return false;
}
- public List<HyperGraph> getHyperGraphs() {
- return new ArrayList<>();
+ public StructInfoMap getstructInfoMap() {
+ return structInfoMap;
}
public boolean isProjectGroup() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
new file mode 100644
index 00000000000..f7e09fd44d9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.memo;
+
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
+
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+/**
+ * Representation for group in cascades optimizer.
+ */
+public class StructInfoMap {
+ private final Map<BitSet, GroupExpression> groupExpressionMap = new
HashMap<>();
+ private final Map<BitSet, StructInfo> infoMap = new HashMap<>();
+
+ /**
+ * get struct info according to table map
+ * @param mvTableMap the original table map
+ * @param foldTableMap the fold table map
+ * @param group the group that the mv matched
+ * @return struct info or null if not found
+ */
+ public @Nullable StructInfo getStructInfo(BitSet mvTableMap, BitSet
foldTableMap, Group group) {
+ if (!infoMap.containsKey(mvTableMap)) {
+ if ((groupExpressionMap.containsKey(foldTableMap) ||
groupExpressionMap.isEmpty())
+ && !groupExpressionMap.containsKey(mvTableMap)) {
+ refresh(group);
+ }
+ if (groupExpressionMap.containsKey(mvTableMap)) {
+ StructInfo structInfo =
constructStructInfo(groupExpressionMap.get(mvTableMap));
+ infoMap.put(mvTableMap, structInfo);
+ }
+ }
+
+ return infoMap.get(mvTableMap);
+ }
+
+ public Set<BitSet> getTableMaps() {
+ return groupExpressionMap.keySet();
+ }
+
+ private StructInfo constructStructInfo(GroupExpression groupExpression) {
+ throw new RuntimeException("has not been implemented for" +
groupExpression);
+ }
+
+ /**
+ * refresh group expression map
+ * @param group the root group
+ */
+ public void refresh(Group group) {
+ List<Set<BitSet>> childrenTableMap = new ArrayList<>();
+ Set<Group> refreshedGroup = new HashSet<>();
+ for (GroupExpression groupExpression : group.getLogicalExpressions()) {
+ if (groupExpression.children().isEmpty()) {
+ groupExpressionMap.put(constructLeaf(groupExpression),
groupExpression);
+ continue;
+ }
+ for (Group child : groupExpression.children()) {
+ if (!refreshedGroup.contains(child)) {
+ StructInfoMap childStructInfoMap =
child.getstructInfoMap();
+ childStructInfoMap.refresh(child);
+ }
+ refreshedGroup.add(child);
+ childrenTableMap.add(child.getstructInfoMap().getTableMaps());
+ }
+ Set<BitSet> bitSets = cartesianProduct(childrenTableMap);
+ for (BitSet bitSet : bitSets) {
+ groupExpressionMap.put(bitSet, groupExpression);
+ }
+ }
+ }
+
+ private BitSet constructLeaf(GroupExpression groupExpression) {
+ Plan plan = groupExpression.getPlan();
+ BitSet tableMap = new BitSet();
+ if (plan instanceof LogicalCatalogRelation) {
+ // TODO: Bitmap is not compatible with long, use tree map instead
+ tableMap.set((int) ((LogicalCatalogRelation)
plan).getTable().getId());
+ }
+ // one row relation / CTE consumer
+ return tableMap;
+ }
+
+ private Set<BitSet> cartesianProduct(List<Set<BitSet>> childrenTableMap) {
+ return Sets.cartesianProduct(childrenTableMap)
+ .stream()
+ .map(bitSetList -> {
+ BitSet bitSet = new BitSet();
+ for (BitSet b : bitSetList) {
+ bitSet.or(b);
+ }
+ return bitSet;
+ })
+ .collect(Collectors.toSet());
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index baf714fe280..8046639d1ba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -32,6 +32,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import
org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -56,7 +57,11 @@ public class InitMaterializationContextHook implements
PlannerHook {
initMaterializationContext(planner.getCascadesContext());
}
- private void initMaterializationContext(CascadesContext cascadesContext) {
+ /**
+ * init materialization context
+ */
+ @VisibleForTesting
+ public void initMaterializationContext(CascadesContext cascadesContext) {
if
(!cascadesContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite())
{
return;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
new file mode 100644
index 00000000000..0699f04b92f
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.memo;
+
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.mtmv.MTMVRelationManager;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.qe.ConnectContext;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.BitSet;
+import java.util.Set;
+
+class StructInfoMapTest extends SqlTestBase {
+ @Test
+ void testTableMap() throws Exception {
+ CascadesContext c1 = createCascadesContext(
+ "select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id "
+ + "inner join T3 on T1.id = T3.id",
+ connectContext
+ );
+ PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .optimize();
+ Group root = c1.getMemo().getRoot();
+ Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
+ Assertions.assertTrue(tableMaps.isEmpty());
+ root.getstructInfoMap().refresh(root);
+ Assertions.assertEquals(1, tableMaps.size());
+ new MockUp<MTMVRelationManager>() {
+ @Mock
+ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
+ return true;
+ }
+ };
+ connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
+ createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id;");
+ c1 = createCascadesContext(
+ "select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id "
+ + "inner join T3 on T1.id = T3.id",
+ connectContext
+ );
+ PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .optimize()
+ .printlnBestPlanTree();
+ root = c1.getMemo().getRoot();
+ root.getstructInfoMap().refresh(root);
+ tableMaps = root.getstructInfoMap().getTableMaps();
+ Assertions.assertEquals(2, tableMaps.size());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index 0c6cc0e577e..9471e18f6a5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -49,6 +49,7 @@ import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
import
org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver;
+import
org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -121,6 +122,7 @@ public class PlanChecker {
public PlanChecker analyze() {
this.cascadesContext.newAnalyzer().analyze();
this.cascadesContext.toMemo();
+
InitMaterializationContextHook.INSTANCE.initMaterializationContext(this.cascadesContext);
return this;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 58b55f0a7b5..7de8a9a830b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -58,6 +58,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.util.SqlParserUtils;
import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.job.base.AbstractJob;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
@@ -65,10 +66,13 @@ import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
+import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand;
import org.apache.doris.nereids.trees.plans.commands.DropConstraintCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.persist.CreateTableInfo;
+import org.apache.doris.persist.EditLog;
import org.apache.doris.planner.Planner;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.DdlExecutor;
@@ -795,6 +799,30 @@ public abstract class TestWithFeService {
Thread.sleep(100);
}
+ protected void createMvByNereids(String sql) throws Exception {
+ new MockUp<EditLog>() {
+ @Mock
+ public void logCreateTable(CreateTableInfo info) {
+ System.out.println("skip log create table...");
+ }
+
+ @Mock
+ public void logCreateJob(AbstractJob job) {
+ System.out.println("skip log create job...");
+ }
+ };
+ NereidsParser nereidsParser = new NereidsParser();
+ LogicalPlan parsed = nereidsParser.parseSingle(sql);
+ StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql);
+ if (parsed instanceof CreateMTMVCommand) {
+ ((CreateMTMVCommand) parsed).run(connectContext, stmtExecutor);
+ }
+ checkAlterJob();
+ // waiting table state to normal
+ Thread.sleep(1000);
+
+ }
+
private void updateReplicaPathHash() {
com.google.common.collect.Table<Long, Long, Replica> replicaMetaTable
= Env.getCurrentInvertedIndex()
.getReplicaMetaTable();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
