This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3543f85ae5 [feature](nereids) merge push down and remove redundant
operator rules into one batch (#12569)
3543f85ae5 is described below
commit 3543f85ae57231c59acdebd3ea9fdc7e011e4dba
Author: Kikyou1997 <33112463+kikyou1...@users.noreply.github.com>
AuthorDate: Wed Sep 14 14:37:36 2022 +0800
[feature](nereids) merge push down and remove redundant operator rules into
one batch (#12569)
1. For some related rules, we need to execute them together to get the
expected plan.
2. Add session variables to avoid fallback to stale planner when running
regression tests of nereids for piggyback.
---
.../org/apache/doris/nereids/jobs/batch/RewriteJob.java | 14 +++++++-------
.../main/java/org/apache/doris/qe/ConnectProcessor.java | 11 ++++++++++-
.../src/main/java/org/apache/doris/qe/SessionVariable.java | 10 ++++++++++
.../suites/nereids_syntax_p0/agg_with_const.groovy | 5 +++++
.../suites/nereids_syntax_p0/empty_relation.groovy | 1 +
regression-test/suites/nereids_syntax_p0/explain.groovy | 2 ++
regression-test/suites/nereids_syntax_p0/function.groovy | 1 +
regression-test/suites/nereids_syntax_p0/having.groovy | 1 +
.../suites/nereids_syntax_p0/inpredicate.groovy | 2 ++
regression-test/suites/nereids_syntax_p0/join.groovy | 2 ++
.../suites/nereids_syntax_p0/one_row_relation.groovy | 1 +
regression-test/suites/nereids_syntax_p0/rollup.groovy | 2 ++
.../suites/nereids_syntax_p0/sub_query_alias.groovy | 2 ++
.../suites/nereids_syntax_p0/sub_query_correlated.groovy | 2 ++
regression-test/suites/nereids_syntax_p0/view.groovy | 2 ++
15 files changed, 50 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
index 956ba0c961..2aea7994fa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/RewriteJob.java
@@ -62,17 +62,17 @@ public class RewriteJob extends BatchRulesJob {
.addAll(new ConvertApplyToJoinJob(cascadesContext).rulesJob)
.add(topDownBatch(ImmutableList.of(new
ExpressionNormalization())))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
- .add(topDownBatch(ImmutableList.of(new ReorderJoin())))
+ .add(topDownBatch(ImmutableList.of(new ReorderJoin())))
.add(topDownBatch(ImmutableList.of(new
FindHashConditionForJoin())))
- .add(topDownBatch(ImmutableList.of(new
PushPredicateThroughJoin())))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new ColumnPruning())))
+ .add(topDownBatch(ImmutableList.of(new
PushPredicateThroughJoin(),
+ new PushdownProjectThroughLimit(),
+ new PushdownFilterThroughProject(),
+ new MergeConsecutiveProjects(),
+ new MergeConsecutiveFilters(),
+ new MergeConsecutiveLimits())))
.add(topDownBatch(ImmutableList.of(new
AggregateDisassemble())))
- .add(topDownBatch(ImmutableList.of(new
PushdownProjectThroughLimit())))
- .add(topDownBatch(ImmutableList.of(new
PushdownFilterThroughProject())))
- .add(bottomUpBatch(ImmutableList.of(new
MergeConsecutiveProjects())))
- .add(topDownBatch(ImmutableList.of(new
MergeConsecutiveFilters())))
- .add(bottomUpBatch(ImmutableList.of(new
MergeConsecutiveLimits())))
.add(topDownBatch(ImmutableList.of(new EliminateLimit())))
.add(topDownBatch(ImmutableList.of(new EliminateFilter())))
.add(topDownBatch(ImmutableList.of(new
PruneOlapScanPartition())))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 0be0d61646..7920ae6dae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.QueryStmt;
+import org.apache.doris.analysis.SelectStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@@ -234,11 +235,14 @@ public class ConnectProcessor {
boolean alreadyAddedToAuditInfoList = false;
try {
List<StatementBase> stmts = null;
- if (ctx.getSessionVariable().isEnableNereidsPlanner()) {
+ Exception nereidsParseException = null;
+ // ctx could be null in some unit tests
+ if (ctx != null &&
ctx.getSessionVariable().isEnableNereidsPlanner()) {
NereidsParser nereidsParser = new NereidsParser();
try {
stmts = nereidsParser.parseSQL(originStmt);
} catch (Exception e) {
+ nereidsParseException = e;
// TODO: We should catch all exception here until we
support all query syntax.
LOG.warn(" Fallback to stale planner."
+ " Nereids cannot process this statement:
\"{}\".", originStmt, e);
@@ -255,6 +259,11 @@ public class ConnectProcessor {
ctx.resetReturnRows();
}
parsedStmt = stmts.get(i);
+ if (parsedStmt instanceof SelectStmt) {
+ if
(!ctx.getSessionVariable().enableFallbackToOriginalPlanner) {
+ throw new Exception(String.format("SQL: {}",
parsedStmt.toSql()), nereidsParseException);
+ }
+ }
parsedStmt.setOrigStmt(new OriginStatement(originStmt, i));
parsedStmt.setUserInfo(ctx.getCurrentUserIdentity());
executor = new StmtExecutor(ctx, parsedStmt);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a12ee9059c..e27f5ff2f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -195,6 +195,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_NEREIDS_PLANNER =
"enable_nereids_planner";
+ public static final String ENABLE_FALLBACK_TO_ORIGINAL_PLANNER =
"enable_fallback_to_original_planner";
+
public static final String ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN =
"enable_nereids_reorder_to_eliminate_cross_join";
@@ -534,6 +536,7 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE)
public boolean enableLocalExchange = false;
+
/**
* For debugg purpose, dont' merge unique key and agg key when reading
data.
*/
@@ -546,6 +549,13 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE)
public boolean skipDeletePredicate = false;
+ // This variable is used to avoid FE fallback to the original parser. When
we execute SQL in regression tests
+ // for nereids, fallback will cause the Doris return the correct result
although the syntax is unsupported
+ // in nereids for some mistaken modification. You should set it on the
+ @VariableMgr.VarAttr(name = ENABLE_FALLBACK_TO_ORIGINAL_PLANNER)
+ public boolean enableFallbackToOriginalPlanner = true;
+
+
public String getBlockEncryptionMode() {
return blockEncryptionMode;
}
diff --git a/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy
b/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy
index ed2e5b60f1..fb4fd81fe7 100644
--- a/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy
+++ b/regression-test/suites/nereids_syntax_p0/agg_with_const.groovy
@@ -17,6 +17,9 @@
suite("agg_with_const") {
+ sql "SET enable_vectorized_engine=true"
+ sql "SET enable_nereids_planner=true"
+
sql """
DROP TABLE IF EXISTS t1
"""
@@ -33,6 +36,8 @@ suite("agg_with_const") {
insert into t1 values(1994, 1994, 1995)
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
qt_select """
select count(2) + 1, sum(2) + sum(col1) from t1
"""
diff --git a/regression-test/suites/nereids_syntax_p0/empty_relation.groovy
b/regression-test/suites/nereids_syntax_p0/empty_relation.groovy
index 05e1af14ee..4e1babd166 100644
--- a/regression-test/suites/nereids_syntax_p0/empty_relation.groovy
+++ b/regression-test/suites/nereids_syntax_p0/empty_relation.groovy
@@ -19,6 +19,7 @@ suite("empty_relation") {
// enable nereids and vectorized engine
sql "SET enable_vectorized_engine=true"
sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
test {
sql "select *, substring(s_name, 1, 2) from supplier limit 0"
diff --git a/regression-test/suites/nereids_syntax_p0/explain.groovy
b/regression-test/suites/nereids_syntax_p0/explain.groovy
index 865e08d553..52e13de757 100644
--- a/regression-test/suites/nereids_syntax_p0/explain.groovy
+++ b/regression-test/suites/nereids_syntax_p0/explain.groovy
@@ -24,6 +24,8 @@ suite("explain") {
SET enable_nereids_planner=true
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
explain {
sql("select count(2) + 1, sum(2) + sum(lo_suppkey) from lineorder")
contains "projections: lo_suppkey"
diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy
b/regression-test/suites/nereids_syntax_p0/function.groovy
index f0b44ae2e6..c4099a0798 100644
--- a/regression-test/suites/nereids_syntax_p0/function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/function.groovy
@@ -23,6 +23,7 @@ suite("function") {
sql """
SET enable_nereids_planner=true
"""
+ sql "SET enable_fallback_to_original_planner=false"
order_qt_max """
SELECT max(lo_discount), max(lo_extendedprice) AS max_extendedprice
FROM lineorder;
diff --git a/regression-test/suites/nereids_syntax_p0/having.groovy
b/regression-test/suites/nereids_syntax_p0/having.groovy
index 4f6abf2e61..d7d61f5687 100644
--- a/regression-test/suites/nereids_syntax_p0/having.groovy
+++ b/regression-test/suites/nereids_syntax_p0/having.groovy
@@ -43,6 +43,7 @@ suite("test_nereids_having") {
(3, 3, 9)
"""
+ sql "SET enable_fallback_to_original_planner=false"
order_qt_select "SELECT a1 as value FROM t1 GROUP BY a1 HAVING a1 > 0";
order_qt_select "SELECT a1 as value FROM t1 GROUP BY a1 HAVING value > 0";
diff --git a/regression-test/suites/nereids_syntax_p0/inpredicate.groovy
b/regression-test/suites/nereids_syntax_p0/inpredicate.groovy
index be6f405396..de73bb106b 100644
--- a/regression-test/suites/nereids_syntax_p0/inpredicate.groovy
+++ b/regression-test/suites/nereids_syntax_p0/inpredicate.groovy
@@ -24,6 +24,8 @@ suite("inpredicate") {
SET enable_nereids_planner=true
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
order_qt_in_predicate_1 """
SELECT * FROM supplier WHERE s_suppkey in (1, 2, 3);
"""
diff --git a/regression-test/suites/nereids_syntax_p0/join.groovy
b/regression-test/suites/nereids_syntax_p0/join.groovy
index 5faab7d912..700fe1a49a 100644
--- a/regression-test/suites/nereids_syntax_p0/join.groovy
+++ b/regression-test/suites/nereids_syntax_p0/join.groovy
@@ -24,6 +24,8 @@ suite("join") {
SET enable_nereids_planner=true
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
order_qt_inner_join_1 """
SELECT * FROM lineorder JOIN supplier ON lineorder.lo_suppkey =
supplier.s_suppkey
"""
diff --git a/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
b/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
index 9a73edf1bb..ace60c735d 100644
--- a/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
+++ b/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
@@ -19,6 +19,7 @@ suite("one_row_relation") {
// enable nereids and vectorized engine
sql "SET enable_vectorized_engine=true"
sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
test {
sql "select 100, 'abc', substring('abc', 1, 2),
substring(substring('abcdefg', 4, 3), 1, 2)"
diff --git a/regression-test/suites/nereids_syntax_p0/rollup.groovy
b/regression-test/suites/nereids_syntax_p0/rollup.groovy
index aabb980138..b7830a4484 100644
--- a/regression-test/suites/nereids_syntax_p0/rollup.groovy
+++ b/regression-test/suites/nereids_syntax_p0/rollup.groovy
@@ -67,6 +67,8 @@ suite("rollup") {
sql "set enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
explain {
sql("select k2, sum(v1) from rollup_t1 group by k2")
contains("rollup_t1(r1)")
diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy
b/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy
index 8a50987089..e4cdcec5b5 100644
--- a/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy
+++ b/regression-test/suites/nereids_syntax_p0/sub_query_alias.groovy
@@ -24,6 +24,8 @@ suite("sub_query_alias") {
SET enable_nereids_planner=true
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
qt_select_1 """
select t.c_custkey, t.lo_custkey
from (
diff --git
a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
index abc28073a2..41a3bd7237 100644
--- a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
+++ b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
@@ -89,6 +89,8 @@ suite ("sub_query_correlated") {
insert into subquery4 values (5,4), (5,2), (8,3), (5,4), (6,7), (8,9)
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
//------------------Correlated-----------------
qt_scalar_less_than_corr """
select * from subquery1 where subquery1.k1 < (select sum(subquery3.k3)
from subquery3 where subquery3.v2 = subquery1.k2) order by k1
diff --git a/regression-test/suites/nereids_syntax_p0/view.groovy
b/regression-test/suites/nereids_syntax_p0/view.groovy
index 29ca5a34f3..15956a24cc 100644
--- a/regression-test/suites/nereids_syntax_p0/view.groovy
+++ b/regression-test/suites/nereids_syntax_p0/view.groovy
@@ -46,6 +46,8 @@ suite("view") {
on v1.c_custkey = t.lo_custkey;
"""
+ sql "SET enable_fallback_to_original_planner=false"
+
qt_select_1 """
select *
from v1
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
