This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 35ebef62362 [feature](mtmv) Support to use nondeterministic function
when create async mv (#36111)
35ebef62362 is described below
commit 35ebef62362334fce5d2b901b9f47fd3517abee2
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Fri Jun 21 10:31:58 2024 +0800
[feature](mtmv) Support to use nondeterministic function when create async
mv (#36111)
Support to use current_date() when create async materialized view by
adding
'enable_nondeterministic_function' = 'true' in properties when create
materialized view. `enable_nondeterministic_function` is default false.
Here is a example, it will success
> CREATE MATERIALIZED VIEW mv_name
> BUILD DEFERRED REFRESH AUTO ON MANUAL
> DISTRIBUTED BY RANDOM BUCKETS 2
> PROPERTIES (
> 'replication_num' = '1',
> 'enable_nondeterministic_function' = 'true'
> )
> AS
> SELECT *, unix_timestamp(k3, '%Y-%m-%d %H:%i-%s') from ${tableName}
where current_date() > k3;
Note:
unix_timestamp is nondeterministic when has no params. it is
deterministic when has params which means format column k3 date
another example, it will success
> CREATE MATERIALIZED VIEW mv_name
> BUILD DEFERRED REFRESH AUTO ON MANUAL
> DISTRIBUTED BY RANDOM BUCKETS 2
> PROPERTIES (
> 'replication_num' = '1',
> 'enable_nondeterministic_function' = 'true'
> )
> AS
> SELECT *, unix_timestamp() from ${tableName} where current_date() >
k3;
though unix_timestamp() is nondeterministic, we add
'enable_date_nondeterministic_function' = 'true' in properties
---
.../apache/doris/common/util/PropertyAnalyzer.java | 3 +
.../org/apache/doris/mtmv/MTMVPropertyUtil.java | 7 +-
.../exploration/mv/MaterializedViewUtils.java | 13 ++
.../expressions/functions/ExpressionTrait.java | 22 ++++
.../expressions/functions/Nondeterministic.java | 11 +-
.../functions/scalar/UnixTimestamp.java | 9 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 25 ++--
.../visitor/NondeterministicFunctionCollector.java | 21 ++--
.../doris/nereids/trees/plans/PlanVisitorTest.java | 99 +++++++++++----
...enable_date_non_deterministic_function_mtmv.out | 11 ++
...ble_date_non_deterministic_function_mtmv.groovy | 136 +++++++++++++++++++++
11 files changed, 307 insertions(+), 50 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 69869188c77..6f087d14f4c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -178,6 +178,9 @@ public class PropertyAnalyzer {
public static final String
PROPERTIES_ENABLE_DUPLICATE_WITHOUT_KEYS_BY_DEFAULT =
"enable_duplicate_without_keys_by_default";
public static final String PROPERTIES_GRACE_PERIOD = "grace_period";
+
+ public static final String PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION =
+ "enable_nondeterministic_function";
public static final String PROPERTIES_EXCLUDED_TRIGGER_TABLES =
"excluded_trigger_tables";
public static final String PROPERTIES_REFRESH_PARTITION_NUM =
"refresh_partition_num";
public static final String PROPERTIES_WORKLOAD_GROUP = "workload_group";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
index a9df9b87d72..12287183886 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
@@ -30,14 +30,15 @@ import java.util.Optional;
import java.util.Set;
public class MTMVPropertyUtil {
- public static final Set<String> mvPropertyKeys = Sets.newHashSet(
+ public static final Set<String> MV_PROPERTY_KEYS = Sets.newHashSet(
PropertyAnalyzer.PROPERTIES_GRACE_PERIOD,
PropertyAnalyzer.PROPERTIES_EXCLUDED_TRIGGER_TABLES,
PropertyAnalyzer.PROPERTIES_REFRESH_PARTITION_NUM,
PropertyAnalyzer.PROPERTIES_WORKLOAD_GROUP,
PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT,
PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT,
- PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT
+ PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT,
+ PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION
);
public static void analyzeProperty(String key, String value) {
@@ -63,6 +64,8 @@ public class MTMVPropertyUtil {
case PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT:
analyzePartitionSyncLimit(value);
break;
+ case PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION:
+ break;
default:
throw new AnalysisException("illegal key:" + key);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 7090b745ef8..49e6e7ffc4e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -54,6 +54,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
+import
org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.HashMultimap;
@@ -63,6 +64,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
+import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashSet;
@@ -255,6 +257,17 @@ public class MaterializedViewUtils {
rewrittenPlan);
}
+ /**
+ * Extract nondeterministic function form plan, if the function is in
whiteExpressionSet,
+ * the function would be considered as deterministic function and will not
return
+ * in the result expression result
+ */
+ public static List<Expression> extractNondeterministicFunction(Plan plan) {
+ List<Expression> nondeterministicFunctions = new ArrayList<>();
+ plan.accept(NondeterministicFunctionCollector.INSTANCE,
nondeterministicFunctions);
+ return nondeterministicFunctions;
+ }
+
private static final class TableQueryOperatorChecker extends
DefaultPlanVisitor<Boolean, Void> {
public static final TableQueryOperatorChecker INSTANCE = new
TableQueryOperatorChecker();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
index 2d76c78f4cc..2e69a5ecd16 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java
@@ -26,6 +26,7 @@ import org.apache.doris.nereids.types.DataType;
import com.google.common.collect.ImmutableList;
import java.util.List;
+import java.util.Optional;
/**
* ExpressionTrait.
@@ -76,4 +77,25 @@ public interface ExpressionTrait extends
TreeNode<Expression> {
default boolean foldable() {
return true;
}
+
+ /**
+ * Identify the expression is deterministic or not
+ */
+ default boolean isDeterministic() {
+ boolean isDeterministic = true;
+ List<Expression> children = this.children();
+ if (children.isEmpty()) {
+ return isDeterministic;
+ }
+ for (Expression child : children) {
+ Optional<ExpressionTrait> nonDeterministic =
+ child.collectFirst(expressionTreeNode ->
expressionTreeNode instanceof ExpressionTrait
+ && !((ExpressionTrait)
expressionTreeNode).isDeterministic());
+ if (nonDeterministic.isPresent()) {
+ isDeterministic = false;
+ break;
+ }
+ }
+ return isDeterministic;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Nondeterministic.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Nondeterministic.java
index 56aa5ebb3b9..2d4e2df2fd6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Nondeterministic.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Nondeterministic.java
@@ -19,9 +19,16 @@ package org.apache.doris.nereids.trees.expressions.functions;
/**
* Nondeterministic functions.
- *
+ * <p>
* e.g. 'rand()', 'random()'.
- *
*/
public interface Nondeterministic extends ExpressionTrait {
+
+ /**
+ * Identify the function is deterministic or not, such as UnixTimestamp,
when it's children is not empty
+ * it's deterministic
+ */
+ default boolean isDeterministic() {
+ return false;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UnixTimestamp.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UnixTimestamp.java
index 143bc63aade..633e1e7d4f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UnixTimestamp.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/UnixTimestamp.java
@@ -20,7 +20,6 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateTimeType;
@@ -40,8 +39,7 @@ import java.util.List;
/**
* ScalarFunction 'unix_timestamp'. This class is generated by
GenerateFunction.
*/
-public class UnixTimestamp extends ScalarFunction
- implements ExplicitlyCastableSignature, Nondeterministic {
+public class UnixTimestamp extends ScalarFunction implements
ExplicitlyCastableSignature {
// we got changes when computeSignature
private static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
@@ -142,4 +140,9 @@ public class UnixTimestamp extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitUnixTimestamp(this, context);
}
+
+ @Override
+ public boolean isDeterministic() {
+ return !this.children.isEmpty();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index c4de4dca35d..ab6637964f8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -54,7 +54,6 @@ import org.apache.doris.nereids.analyzer.UnboundResultSink;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
-import org.apache.doris.nereids.trees.TreeNode;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -63,7 +62,6 @@ import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
-import
org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
import org.apache.doris.nereids.types.AggStateType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
@@ -77,7 +75,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -157,7 +154,7 @@ public class CreateMTMVInfo {
throw new AnalysisException(message);
}
analyzeProperties();
- analyzeQuery(ctx);
+ analyzeQuery(ctx, this.mvProperties);
// analyze column
final boolean finalEnableMergeOnWrite = false;
Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
@@ -189,7 +186,7 @@ public class CreateMTMVInfo {
if
(DynamicPartitionUtil.checkDynamicPartitionPropertiesExist(properties)) {
throw new AnalysisException("Not support dynamic partition
properties on async materialized view");
}
- for (String key : MTMVPropertyUtil.mvPropertyKeys) {
+ for (String key : MTMVPropertyUtil.MV_PROPERTY_KEYS) {
if (properties.containsKey(key)) {
MTMVPropertyUtil.analyzeProperty(key, properties.get(key));
mvProperties.put(key, properties.get(key));
@@ -201,7 +198,7 @@ public class CreateMTMVInfo {
/**
* analyzeQuery
*/
- public void analyzeQuery(ConnectContext ctx) {
+ public void analyzeQuery(ConnectContext ctx, Map<String, String>
mvProperties) {
// create table as select
StatementContext statementContext = ctx.getStatementContext();
NereidsPlanner planner = new NereidsPlanner(statementContext);
@@ -214,7 +211,7 @@ public class CreateMTMVInfo {
// can not contain VIEW or MTMV
analyzeBaseTables(planner.getAnalyzedPlan());
// can not contain Random function
- analyzeExpressions(planner.getAnalyzedPlan());
+ analyzeExpressions(planner.getAnalyzedPlan(), mvProperties);
// can not contain partition or tablets
boolean containTableQueryOperator =
MaterializedViewUtils.containTableQueryOperator(planner.getAnalyzedPlan());
if (containTableQueryOperator) {
@@ -340,11 +337,17 @@ public class CreateMTMVInfo {
}
}
- private void analyzeExpressions(Plan plan) {
- List<TreeNode<Expression>> functionCollectResult = new ArrayList<>();
- plan.accept(NondeterministicFunctionCollector.INSTANCE,
functionCollectResult);
+ private void analyzeExpressions(Plan plan, Map<String, String>
mvProperties) {
+ boolean enableNondeterministicFunction = Boolean.parseBoolean(
+
mvProperties.get(PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION));
+ if (enableNondeterministicFunction) {
+ return;
+ }
+ List<Expression> functionCollectResult =
MaterializedViewUtils.extractNondeterministicFunction(plan);
if (!CollectionUtils.isEmpty(functionCollectResult)) {
- throw new AnalysisException("can not contain invalid expression");
+ throw new AnalysisException(String.format(
+ "can not contain invalid expression, the expression is %s",
+
functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(","))));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java
index b17be42d383..5b260144575 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java
@@ -17,31 +17,34 @@
package org.apache.doris.nereids.trees.plans.visitor;
-import org.apache.doris.nereids.trees.TreeNode;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.trees.expressions.functions.FunctionTrait;
import org.apache.doris.nereids.trees.plans.Plan;
import java.util.List;
+import java.util.Set;
/**
* Collect the nondeterministic expr in plan, these expressions will be put
into context
*/
public class NondeterministicFunctionCollector
- extends DefaultPlanVisitor<Void, List<TreeNode<Expression>>> {
+ extends DefaultPlanVisitor<Void, List<Expression>> {
- public static final NondeterministicFunctionCollector INSTANCE
- = new NondeterministicFunctionCollector();
+ public static final NondeterministicFunctionCollector INSTANCE = new
NondeterministicFunctionCollector();
@Override
- public Void visit(Plan plan, List<TreeNode<Expression>>
collectedExpressions) {
+ public Void visit(Plan plan, List<Expression> collectedExpressions) {
List<? extends Expression> expressions = plan.getExpressions();
if (expressions.isEmpty()) {
return super.visit(plan, collectedExpressions);
}
- expressions.forEach(expression -> {
-
collectedExpressions.addAll(expression.collect(Nondeterministic.class::isInstance));
- });
+ for (Expression expression : expressions) {
+ Set<Expression> nondeterministicFunctions =
+ expression.collect(expr -> !((ExpressionTrait)
expr).isDeterministic()
+ && expr instanceof FunctionTrait);
+ collectedExpressions.addAll(nondeterministicFunctions);
+ }
return super.visit(plan, collectedExpressions);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
index 2b3a66b08d7..1afbf6dbbf5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java
@@ -19,14 +19,15 @@ package org.apache.doris.nereids.trees.plans;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
-import org.apache.doris.nereids.trees.TreeNode;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentDate;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentTime;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Now;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Random;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.UnixTimestamp;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Uuid;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
-import
org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import
org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import org.apache.doris.nereids.util.PlanChecker;
@@ -39,7 +40,6 @@ import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashSet;
import java.util.List;
@@ -61,10 +61,11 @@ public class PlanVisitorTest extends TestWithFeService {
+ " `c1` varchar(20) NULL,\n"
+ " `c2` bigint(20) NULL,\n"
+ " `c3` int(20) not NULL,\n"
+ + " `c4` DATE not NULL,\n"
+ " `k4` bitmap BITMAP_UNION ,\n"
+ " `k5` bitmap BITMAP_UNION \n"
+ ") ENGINE=OLAP\n"
- + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`, `c4`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ "PROPERTIES (\n"
@@ -75,10 +76,11 @@ public class PlanVisitorTest extends TestWithFeService {
+ " `c1` bigint(20) NULL,\n"
+ " `c2` bigint(20) NULL,\n"
+ " `c3` bigint(20) not NULL,\n"
+ + " `c4` DATE not NULL,\n"
+ " `k4` bitmap BITMAP_UNION ,\n"
+ " `k5` bitmap BITMAP_UNION \n"
+ ") ENGINE=OLAP\n"
- + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`, `c4`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ "PROPERTIES (\n"
@@ -89,10 +91,11 @@ public class PlanVisitorTest extends TestWithFeService {
+ " `c1` bigint(20) NULL,\n"
+ " `c2` bigint(20) NULL,\n"
+ " `c3` bigint(20) not NULL,\n"
+ + " `c4` DATE not NULL,\n"
+ " `k4` bitmap BITMAP_UNION ,\n"
+ " `k5` bitmap BITMAP_UNION \n"
+ ") ENGINE=OLAP\n"
- + "AGGREGATE KEY(`c1`, `c2`, `c3`)\n"
+ + "AGGREGATE KEY(`c1`, `c2`, `c3`, `c4`)\n"
+ "COMMENT 'OLAP'\n"
+ "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n"
+ "PROPERTIES (\n"
@@ -120,11 +123,11 @@ public class PlanVisitorTest extends TestWithFeService {
+ "WHERE table1.c1 IN (SELECT c1 FROM table2)
OR table1.c1 < 10",
nereidsPlanner -> {
PhysicalPlan physicalPlan =
nereidsPlanner.getPhysicalPlan();
- List<TreeNode<Expression>> collectResult = new
ArrayList<>();
// Check nondeterministic collect
-
physicalPlan.accept(NondeterministicFunctionCollector.INSTANCE, collectResult);
- Assertions.assertEquals(1, collectResult.size());
- Assertions.assertTrue(collectResult.get(0)
instanceof Random);
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(physicalPlan);
+ Assertions.assertEquals(1,
nondeterministicFunctionSet.size());
+
Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random);
// Check get tables
TableCollectorContext collectorContext = new
TableCollector.TableCollectorContext(
Sets.newHashSet(TableType.OLAP), true);
@@ -148,12 +151,12 @@ public class PlanVisitorTest extends TestWithFeService {
+ "WHERE view1.c1 IN (SELECT c1 FROM table2)
OR view1.c1 < 10",
nereidsPlanner -> {
PhysicalPlan physicalPlan =
nereidsPlanner.getPhysicalPlan();
- List<TreeNode<Expression>> collectResult = new
ArrayList<>();
// Check nondeterministic collect
-
physicalPlan.accept(NondeterministicFunctionCollector.INSTANCE, collectResult);
- Assertions.assertEquals(2, collectResult.size());
- Assertions.assertTrue(collectResult.get(0)
instanceof Uuid);
- Assertions.assertTrue(collectResult.get(1)
instanceof Random);
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(physicalPlan);
+ Assertions.assertEquals(2,
nondeterministicFunctionSet.size());
+
Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid);
+
Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof Random);
// Check get tables
TableCollectorContext collectorContext = new
TableCollector.TableCollectorContext(
Sets.newHashSet(TableType.OLAP), true);
@@ -186,9 +189,11 @@ public class PlanVisitorTest extends TestWithFeService {
+ "WHERE mv1.c1 IN (SELECT c1 FROM table2) OR
mv1.c1 < 10",
nereidsPlanner -> {
PhysicalPlan physicalPlan =
nereidsPlanner.getPhysicalPlan();
- List<TreeNode<Expression>> collectResult = new
ArrayList<>();
// Check nondeterministic collect
-
physicalPlan.accept(NondeterministicFunctionCollector.INSTANCE, collectResult);
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(physicalPlan);
+ Assertions.assertEquals(1,
nondeterministicFunctionSet.size());
+
Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid);
// Check get tables
TableCollectorContext collectorContext = new
TableCollector.TableCollectorContext(
Sets.newHashSet(TableType.OLAP), true);
@@ -247,14 +252,62 @@ public class PlanVisitorTest extends TestWithFeService {
PlanChecker.from(connectContext)
.checkExplain("SELECT *, now() FROM table1 "
+ "LEFT SEMI JOIN table2 ON table1.c1 =
table2.c1 "
- + "WHERE table1.c1 IN (SELECT c1 FROM table2)
OR CURDATE() < '2023-01-01'",
+ + "WHERE table1.c1 IN (SELECT c1 FROM table2)
OR current_date() < '2023-01-01' and current_time() = '2023-01-10'",
nereidsPlanner -> {
- List<TreeNode<Expression>> collectResult = new
ArrayList<>();
// Check nondeterministic collect
-
nereidsPlanner.getAnalyzedPlan().accept(NondeterministicFunctionCollector.INSTANCE,
collectResult);
- Assertions.assertEquals(2, collectResult.size());
- Assertions.assertTrue(collectResult.get(0)
instanceof Now);
- Assertions.assertTrue(collectResult.get(1)
instanceof CurrentDate);
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(
+ nereidsPlanner.getAnalyzedPlan());
+ Assertions.assertEquals(3,
nondeterministicFunctionSet.size());
+
Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Now);
+
Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof
CurrentDate);
+
Assertions.assertTrue(nondeterministicFunctionSet.get(2) instanceof
CurrentTime);
+ });
+ }
+
+ @Test
+ public void testCurrentDateFunction() {
+ PlanChecker.from(connectContext)
+ .checkExplain("SELECT * FROM table1 "
+ + "LEFT SEMI JOIN table2 ON table1.c1 =
table2.c1 "
+ + "WHERE table1.c1 IN (SELECT c1 FROM table2)
OR current_date() < '2023-01-01'",
+ nereidsPlanner -> {
+ // Check nondeterministic collect
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(
+ nereidsPlanner.getAnalyzedPlan());
+ Assertions.assertEquals(1,
nondeterministicFunctionSet.size());
+
Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof
CurrentDate);
+ });
+ }
+
+ @Test
+ public void testUnixTimestampWithArgsFunction() {
+ PlanChecker.from(connectContext)
+ .checkExplain("SELECT * FROM table1 "
+ + "LEFT SEMI JOIN table2 ON table1.c1 =
table2.c1 "
+ + "WHERE table1.c1 IN (SELECT c1 FROM table2)
OR unix_timestamp(table1.c4, '%Y-%m-%d %H:%i-%s') < '2023-01-01' and
unix_timestamp(table1.c4) = '2023-01-10'",
+ nereidsPlanner -> {
+ // Check nondeterministic collect
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(
+ nereidsPlanner.getAnalyzedPlan());
+ Assertions.assertEquals(0,
nondeterministicFunctionSet.size());
+ });
+ }
+
+ @Test
+ public void testUnixTimestampWithoutArgsFunction() {
+ PlanChecker.from(connectContext)
+ .checkExplain("SELECT unix_timestamp(), * FROM table1 "
+ + "LEFT SEMI JOIN table2 ON table1.c1 =
table2.c1 ",
+ nereidsPlanner -> {
+ // Check nondeterministic collect
+ List<Expression> nondeterministicFunctionSet =
+
MaterializedViewUtils.extractNondeterministicFunction(
+ nereidsPlanner.getAnalyzedPlan());
+ Assertions.assertEquals(1,
nondeterministicFunctionSet.size());
+
Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof
UnixTimestamp);
});
}
}
diff --git
a/regression-test/data/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.out
b/regression-test/data/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.out
new file mode 100644
index 00000000000..e991951431b
--- /dev/null
+++
b/regression-test/data/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !with_current_date --
+1 1 2024-05-01 1714492800.000000
+2 2 2024-05-02 1714579200.000000
+3 3 2024-05-03 1714665600.000000
+
+-- !with_unix_timestamp_format --
+1 1 2024-05-01 1714492800.000000
+2 2 2024-05-02 1714579200.000000
+3 3 2024-05-03 1714665600.000000
+
diff --git
a/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
new file mode 100644
index 00000000000..c085779e707
--- /dev/null
+++
b/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert;
+
+suite("test_enable_date_non_deterministic_function_mtmv","mtmv") {
+ String suiteName = "test_enable_date_non_deterministic_function_mtmv"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String db = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k1 TINYINT,
+ k2 INT not null,
+ k3 DATE NOT NULL
+ )
+ COMMENT "my first table"
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ insert into ${tableName} values(1,1, '2024-05-01'),(2,2,
'2024-05-02'),(3,3, '2024-05-03');
+ """
+
+ // when not enable date nondeterministic function, create mv should fail
+ // because contains uuid, unix_timestamp, current_date
+ sql """drop materialized view if exists ${mvName};"""
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT uuid(), unix_timestamp() FROM ${tableName} where
current_date() > k3;
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ logger.info(e.getMessage())
+ assertTrue(e.getMessage().contains("can not contain invalid
expression"));
+ }
+ sql """drop materialized view if exists ${mvName};"""
+
+
+ // when not enable date nondeterministic function, create mv should fail
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT * FROM ${tableName} where current_date() > k3;
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ logger.info(e.getMessage())
+ assertTrue(e.getMessage().contains("can not contain invalid
expression"));
+ }
+ sql """drop materialized view if exists ${mvName};"""
+
+ // when enable date nondeterministic function, create mv with current_date
should success
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'enable_nondeterministic_function' = 'true'
+ )
+ AS
+ SELECT *, unix_timestamp(k3, '%Y-%m-%d %H:%i-%s') from ${tableName}
where current_date() > k3;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ order_qt_with_current_date """select * from ${mvName}"""
+ sql """drop materialized view if exists ${mvName};"""
+
+
+ sql """drop materialized view if exists ${mvName};"""
+
+ // when disable date nondeterministic function, create mv with param
unix_timestamp should success
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT *, unix_timestamp(k3, '%Y-%m-%d %H:%i-%s') from ${tableName};
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ order_qt_with_unix_timestamp_format """select * from ${mvName}"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ // when enable date nondeterministic function, create mv with orther
fuction except current_date() should fail
+ try {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT * FROM ${tableName} where now() > k3 and current_time()
> k3;
+ """
+ Assert.fail();
+ } catch (Exception e) {
+ logger.info(e.getMessage())
+ assertTrue(e.getMessage().contains("can not contain invalid
expression"));
+ }
+
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
