This is an automated email from the ASF dual-hosted git repository. lijibing pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new 4c3d4ee73de [improvement](statistics nereids)Nereids support select mv. (#30267) (#30334) 4c3d4ee73de is described below commit 4c3d4ee73defc212d1beb4171ec7ba7201ee3d7e Author: Jibing-Li <64681310+jibing...@users.noreply.github.com> AuthorDate: Thu Jan 25 12:00:41 2024 +0800 [improvement](statistics nereids)Nereids support select mv. (#30267) (#30334) --- .../antlr4/org/apache/doris/nereids/DorisParser.g4 | 6 +- .../doris/nereids/analyzer/UnboundRelation.java | 20 +++-- .../doris/nereids/parser/LogicalPlanBuilder.java | 7 +- .../doris/nereids/rules/analysis/BindRelation.java | 19 ++++- .../rules/implementation/AggregateStrategies.java | 6 +- .../trees/plans/logical/LogicalOlapScan.java | 35 +++++--- regression-test/data/statistics/test_select_mv.out | 47 +++++++++++ .../suites/statistics/test_select_mv.groovy | 97 ++++++++++++++++++++++ 8 files changed, 214 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index fe15f61d6d5..5037f3f43aa 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -260,7 +260,7 @@ identifierSeq ; relationPrimary - : multipartIdentifier specifiedPartition? + : multipartIdentifier materializedViewName? specifiedPartition? tabletList? tableAlias sample? relationHint? lateralView* #tableName | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN @@ -272,6 +272,10 @@ property : key=propertyItem EQ value=propertyItem ; +materializedViewName + : INDEX indexName=identifier + ; + propertyItem : identifier | constant ; tableAlias diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index dd0033e9e5d..c1971aab721 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -51,21 +51,22 @@ public class UnboundRelation extends LogicalRelation implements Unbound { private final boolean isTempPart; private final List<String> hints; private final Optional<TableSample> tableSample; + private final Optional<String> indexName; public UnboundRelation(RelationId id, List<String> nameParts) { this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of(), - ImmutableList.of(), Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart) { this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of(), - ImmutableList.of(), Optional.empty()); + ImmutableList.of(), Optional.empty(), Optional.empty()); } public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart, - List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample) { + List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample, Optional<String> indexName) { this(id, nameParts, Optional.empty(), Optional.empty(), - partNames, isTempPart, tabletIds, hints, tableSample); + partNames, isTempPart, tabletIds, hints, tableSample, indexName); } /** @@ -73,7 +74,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound { */ public UnboundRelation(RelationId id, List<String> nameParts, Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties, List<String> partNames, boolean isTempPart, - List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample) { + List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample, Optional<String> indexName) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); @@ -81,6 +82,7 @@ public class UnboundRelation extends LogicalRelation implements Unbound { this.isTempPart = isTempPart; this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null.")); this.tableSample = tableSample; + this.indexName = indexName; } public List<String> getNameParts() { @@ -101,14 +103,14 @@ public class UnboundRelation extends LogicalRelation implements Unbound { public Plan withGroupExpression(Optional<GroupExpression> groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample); + partNames, isTempPart, tabletIds, hints, tableSample, indexName); } @Override public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties, List<Plan> children) { return new UnboundRelation(relationId, nameParts, groupExpression, logicalProperties, partNames, - isTempPart, tabletIds, hints, tableSample); + isTempPart, tabletIds, hints, tableSample, indexName); } @Override @@ -151,6 +153,10 @@ public class UnboundRelation extends LogicalRelation implements Unbound { return tabletIds; } + public Optional<String> getIndexName() { + return indexName; + } + public List<String> getHints() { return hints; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 767f906972c..df42620d116 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -635,6 +635,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> { } } + Optional<String> indexName = Optional.empty(); + if (ctx.materializedViewName() != null) { + indexName = Optional.ofNullable(ctx.materializedViewName().indexName.getText()); + } + List<Long> tabletIdLists = new ArrayList<>(); if (ctx.tabletList() != null) { ctx.tabletList().tabletIdList.stream().forEach(tabletToken -> { @@ -653,7 +658,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> { LogicalPlan checkedRelation = withCheckPolicy( new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, partitionNames, isTempPart, tabletIdLists, relationHints, - Optional.ofNullable(tableSample))); + Optional.ofNullable(tableSample), indexName)); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { plan = withGenerate(plan, lateralViewContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index c4d4e3ac9b5..9317569d3f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -189,13 +189,26 @@ public class BindRelation extends OneAnalysisRuleFactory { (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample()); } else { - scan = new LogicalOlapScan(unboundRelation.getRelationId(), + Optional<String> indexName = unboundRelation.getIndexName(); + if (indexName.isPresent()) { + OlapTable olapTable = (OlapTable) table; + Long indexId = olapTable.getIndexIdByName(indexName.get()); + if (indexId == null) { + throw new AnalysisException("Table " + olapTable.getName() + + " doesn't have materialized view " + indexName.get()); + } + scan = new LogicalOlapScan(unboundRelation.getRelationId(), + (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, indexId, + unboundRelation.getHints(), unboundRelation.getTableSample()); + } else { + scan = new LogicalOlapScan(unboundRelation.getRelationId(), (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), tabletIds, unboundRelation.getHints(), unboundRelation.getTableSample()); + } } if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign() - && !ConnectContext.get().getSessionVariable() - .skipDeleteSign()) { + && !ConnectContext.get().getSessionVariable().skipDeleteSign() + && !scan.isDirectMvScan()) { // table qualifier is catalog.db.table, we make db.table.column Slot deleteSlot = null; for (Slot slot : scan.getOutput()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index a0eb011ba92..f7815224dc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -528,10 +528,14 @@ public class AggregateStrategies implements ImplementationRuleFactory { return canNotPush; } if (logicalScan instanceof LogicalOlapScan) { - KeysType keysType = ((LogicalOlapScan) logicalScan).getTable().getKeysType(); + LogicalOlapScan logicalOlapScan = (LogicalOlapScan) logicalScan; + KeysType keysType = logicalOlapScan.getTable().getKeysType(); if (functionClasses.contains(Count.class) && keysType != KeysType.DUP_KEYS) { return canNotPush; } + if (functionClasses.contains(Count.class) && logicalOlapScan.isDirectMvScan()) { + return canNotPush; + } } if (aggregateFunctions.stream().anyMatch(fun -> fun.arity() > 1)) { return canNotPush; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 69166318989..f496eaa47fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -108,6 +108,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan private final Optional<TableSample> tableSample; + private final boolean directMvScan; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } @@ -117,7 +119,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan table.getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of(), - Maps.newHashMap(), Optional.empty()); + Maps.newHashMap(), Optional.empty(), false); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds, @@ -125,7 +127,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, -1, false, PreAggStatus.on(), ImmutableList.of(), hints, Maps.newHashMap(), - tableSample); + tableSample, false); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions, @@ -134,7 +136,15 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, -1, false, PreAggStatus.on(), specifiedPartitions, hints, Maps.newHashMap(), - tableSample); + tableSample, false); + } + + public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds, + long selectedIndexId, List<String> hints, Optional<TableSample> tableSample) { + this(id, table, qualifier, Optional.empty(), Optional.empty(), + table.getPartitionIds(), false, tabletIds, + selectedIndexId, true, PreAggStatus.off("For direct index scan."), + ImmutableList.of(), hints, Maps.newHashMap(), tableSample, true); } /** @@ -146,7 +156,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan List<Long> selectedTabletIds, long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List<Long> specifiedPartitions, List<String> hints, Map<Pair<Long, String>, Slot> cacheSlotWithSlotName, - Optional<TableSample> tableSample) { + Optional<TableSample> tableSample, boolean directMvScan) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); Preconditions.checkArgument(selectedPartitionIds != null, @@ -164,6 +174,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName, "mvNameToSlot can not be null"); this.tableSample = tableSample; + this.directMvScan = directMvScan; } public List<Long> getSelectedPartitionIds() { @@ -220,7 +231,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } @Override @@ -229,7 +240,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) { @@ -237,7 +248,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { @@ -245,7 +256,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, preAgg, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, - tableSample); + tableSample, directMvScan); } public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) { @@ -253,7 +264,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { @@ -261,7 +272,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample); + hints, cacheSlotWithSlotName, tableSample, directMvScan); } @Override @@ -359,4 +370,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan public Optional<TableSample> getTableSample() { return tableSample; } + + public boolean isDirectMvScan() { + return directMvScan; + } } diff --git a/regression-test/data/statistics/test_select_mv.out b/regression-test/data/statistics/test_select_mv.out new file mode 100644 index 00000000000..75b7b8ddc86 --- /dev/null +++ b/regression-test/data/statistics/test_select_mv.out @@ -0,0 +1,47 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !dup_sql1 -- +4 + +-- !dup_sql2 -- +1 +2 + +-- !dup_sql3 -- +2 + +-- !dup_sql4 -- +1 2 2 3 + +-- !dup_sql5 -- +2 +4 + +-- !dup_sql6 -- +2 + +-- !dup_sql7 -- +2 4 2 6 + +-- !agg_sql1 -- +2 + +-- !agg_sql2 -- +1 +2 + +-- !agg_sql3 -- +2 + +-- !agg_sql4 -- +1 2 2 3 + +-- !agg_sql5 -- +2 +4 + +-- !agg_sql6 -- +2 + +-- !agg_sql7 -- +2 4 2 6 + diff --git a/regression-test/suites/statistics/test_select_mv.groovy b/regression-test/suites/statistics/test_select_mv.groovy new file mode 100644 index 00000000000..a35adaeb9ff --- /dev/null +++ b/regression-test/suites/statistics/test_select_mv.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_select_mv") { + + def dup_sql1 = """select count(*) from test_dup;""" + def dup_sql2 = """select mv_key2 from test_dup index dup1 order by mv_key2;""" + def dup_sql3 = """select count(mv_key2) from test_dup index dup1;""" + def dup_sql4 = """select min(mv_key2), max(mv_key2), count(mv_key2), sum(mv_key2) from test_dup index dup1;""" + def dup_sql5 = """select `mva_SUM__CAST(value AS BIGINT)` as a from test_dup index dup1 order by a;""" + def dup_sql6 = """select count(`mva_SUM__CAST(value AS BIGINT)`) from test_dup index dup1;""" + def dup_sql7 = """select min(`mva_SUM__CAST(value AS BIGINT)`), max(`mva_SUM__CAST(value AS BIGINT)`), ndv(`mva_SUM__CAST(value AS BIGINT)`), sum(`mva_SUM__CAST(value AS BIGINT)`) from test_dup index dup1;""" + + def agg_sql1 = """select count(*) from test_agg;""" + def agg_sql2 = """select mv_key2 from test_agg index agg1 order by mv_key2;""" + def agg_sql3 = """select count(mv_key2) from test_agg index agg1;""" + def agg_sql4 = """select min(mv_key2), max(mv_key2), count(mv_key2), sum(mv_key2) from test_agg index agg1;""" + def agg_sql5 = """select `mva_SUM__CAST(value AS BIGINT)` as a from test_agg index agg1 order by a;""" + def agg_sql6 = """select count(`mva_SUM__CAST(value AS BIGINT)`) from test_agg index agg1;""" + def agg_sql7 = """select min(`mva_SUM__CAST(value AS BIGINT)`), max(`mva_SUM__CAST(value AS BIGINT)`), ndv(`mva_SUM__CAST(value AS BIGINT)`), sum(`mva_SUM__CAST(value AS BIGINT)`) from test_agg index agg1;""" + + + sql """drop database if exists test_select_mv""" + sql """create database test_select_mv""" + sql """use test_select_mv""" + + sql """CREATE TABLE test_dup ( + key1 int NOT NULL, + key2 int NOT NULL, + value int NOT NULL + )ENGINE=OLAP + DUPLICATE KEY(`key1`, `key2`) + DISTRIBUTED BY HASH(`key1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + create materialized view dup1 as select key2, sum(value) from test_dup group by key2; + """ + + sql """CREATE TABLE test_agg ( + key1 int NOT NULL, + key2 int NOT NULL, + value int SUM NOT NULL + )ENGINE=OLAP + AGGREGATE KEY(`key1`, `key2`) + DISTRIBUTED BY HASH(`key1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + create materialized view agg1 as select key2, sum(value) from test_agg group by key2; + """ + Thread.sleep(1000) + + sql """insert into test_dup values (1, 1, 1), (2, 2, 2)""" + sql """insert into test_dup values (1, 1, 1), (2, 2, 2)""" + sql """insert into test_agg values (1, 1, 1), (2, 2, 2)""" + sql """insert into test_agg values (1, 1, 1), (2, 2, 2)""" + + qt_dup_sql1 dup_sql1 + qt_dup_sql2 dup_sql2 + qt_dup_sql3 dup_sql3 + qt_dup_sql4 dup_sql4 + qt_dup_sql5 dup_sql5 + qt_dup_sql6 dup_sql6 + qt_dup_sql7 dup_sql7 + + qt_agg_sql1 agg_sql1 + qt_agg_sql2 agg_sql2 + qt_agg_sql3 agg_sql3 + qt_agg_sql4 agg_sql4 + qt_agg_sql5 agg_sql5 + qt_agg_sql6 agg_sql6 + qt_agg_sql7 agg_sql7 + + sql """drop database if exists test_select_mv""" +} + --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org