This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 8642480d709 [fix](Nereids) forbid sql cache when use nondeterministic
function (#39736)
8642480d709 is described below
commit 8642480d709f4ade28c5921fd1dbebfa865a8673
Author: 924060929 <[email protected]>
AuthorDate: Thu Aug 22 13:01:58 2024 +0800
[fix](Nereids) forbid sql cache when use nondeterministic function (#39736)
The sql cache maybe return outdated cache because we cannot detect the the
result of non-deterministric function changed, so I forbid sql cache when use
nondeterministic function.
This problem not exists in the 2.1.3 because I refactor it
---
.../java/org/apache/doris/nereids/StatementContext.java | 2 ++
.../apache/doris/nereids/rules/analysis/SlotBinder.java | 5 ++++-
.../nereids/rules/expression/rules/FunctionBinder.java | 7 +++++++
.../trees/expressions/functions/scalar/ConnectionId.java | 3 ++-
.../trees/expressions/functions/scalar/CurrentUser.java | 3 ++-
.../trees/expressions/functions/scalar/Database.java | 3 ++-
.../nereids/trees/expressions/functions/scalar/User.java | 3 ++-
.../expressions/functions/table/TableValuedFunction.java | 4 +++-
.../main/java/org/apache/doris/qe/cache/CacheAnalyzer.java | 4 ++++
regression-test/suites/query_p0/cache/sql_cache.groovy | 14 ++++++++++++++
10 files changed, 42 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 9ea5ed09759..357f0286b8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -55,6 +55,8 @@ import javax.annotation.concurrent.GuardedBy;
*/
public class StatementContext {
+ public boolean hasUnsupportedSqlCacheExpression;
+
private ConnectContext connectContext;
private final Stopwatch stopwatch = Stopwatch.createUnstarted();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
index e1cbb7760c4..e958570c47f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java
@@ -90,7 +90,7 @@ public class SlotBinder extends SubExprAnalyzer {
}
public Expression bind(Expression expression) {
- return expression.accept(this, null);
+ return expression.accept(this, getCascadesContext());
}
@Override
@@ -119,6 +119,9 @@ public class SlotBinder extends SubExprAnalyzer {
throw new AnalysisException(e.getMessage());
}
}
+ if (context != null && context.getStatementContext() != null) {
+ context.getStatementContext().hasUnsupportedSqlCacheExpression =
true;
+ }
return new Variable(unboundVariable.getName(),
unboundVariable.getType(), literal);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
index 4a8c62ed13e..fab5377da24 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FunctionBinder.java
@@ -44,6 +44,7 @@ import
org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nvl;
import
org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder;
@@ -114,6 +115,9 @@ public class FunctionBinder extends
AbstractExpressionRewriteRule {
FunctionBuilder builder = functionRegistry.findFunctionBuilder(
unboundFunction.getDbName(), functionName, arguments);
if (builder instanceof AliasUdfBuilder) {
+ if (context != null) {
+
context.cascadesContext.getStatementContext().hasUnsupportedSqlCacheExpression
= true;
+ }
// we do type coercion in build function in alias function, so
it's ok to return directly.
return builder.build(functionName, arguments);
} else {
@@ -131,6 +135,9 @@ public class FunctionBinder extends
AbstractExpressionRewriteRule {
// so wrap COUNT with Nvl to ensure it's result is 0 instead
of null to get the correct result
boundFunction = new Nvl(boundFunction, new BigIntLiteral(0));
}
+ if (context != null && boundFunction instanceof Nondeterministic) {
+
context.cascadesContext.getStatementContext().hasUnsupportedSqlCacheExpression
= true;
+ }
return boundFunction;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
index 7f16fa1a4ac..6f17e247c44 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
@@ -32,7 +33,7 @@ import java.util.List;
* ScalarFunction 'ConnectionId'.
*/
public class ConnectionId extends ScalarFunction
- implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, Nondeterministic {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).args()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CurrentUser.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CurrentUser.java
index f36e4548ebc..5f00e374716 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CurrentUser.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CurrentUser.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.StringType;
@@ -32,7 +33,7 @@ import java.util.List;
* ScalarFunction 'CurrentUser'.
*/
public class CurrentUser extends ScalarFunction
- implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, Nondeterministic {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(StringType.INSTANCE).args()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Database.java
index acb31f5ae4d..c213fc6bc5c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Database.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Database.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.VarcharType;
@@ -32,7 +33,7 @@ import java.util.List;
* ScalarFunction 'database'.
*/
public class Database extends ScalarFunction
- implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, Nondeterministic {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/User.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/User.java
index efbb24340f7..47c6bd4423f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/User.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/User.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.VarcharType;
@@ -32,7 +33,7 @@ import java.util.List;
* ScalarFunction 'User'.
*/
public class User extends ScalarFunction
- implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
+ implements LeafExpression, ExplicitlyCastableSignature,
AlwaysNotNullable, Nondeterministic {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
index e3e25481691..26602435651 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/TableValuedFunction.java
@@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.Properties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Nondeterministic;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
@@ -44,7 +45,8 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
/** TableValuedFunction */
-public abstract class TableValuedFunction extends BoundFunction implements
UnaryExpression, CustomSignature {
+public abstract class TableValuedFunction extends BoundFunction
+ implements UnaryExpression, CustomSignature, Nondeterministic {
protected final Supplier<TableValuedFunctionIf> catalogFunctionCache =
Suppliers.memoize(this::toCatalogFunction);
protected final Supplier<FunctionGenTable> tableCache =
Suppliers.memoize(() -> {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
index 1527cdbeac6..389c5406fe2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java
@@ -344,6 +344,10 @@ public class CacheAnalyzer {
}
return CacheMode.NoNeed;
}
+ if (context != null && context.getStatementContext() != null
+ &&
context.getStatementContext().hasUnsupportedSqlCacheExpression) {
+ return CacheMode.None;
+ }
if (!(parsedStmt instanceof LogicalPlanAdapter) || scanNodes.size() ==
0) {
if (LOG.isDebugEnabled()) {
LOG.debug("not a select stmt or no scan node. queryid {}",
DebugUtil.printId(queryId));
diff --git a/regression-test/suites/query_p0/cache/sql_cache.groovy
b/regression-test/suites/query_p0/cache/sql_cache.groovy
index cccfca4d965..eec817449e3 100644
--- a/regression-test/suites/query_p0/cache/sql_cache.groovy
+++ b/regression-test/suites/query_p0/cache/sql_cache.groovy
@@ -211,5 +211,19 @@ suite("sql_cache") {
k1;
"""
+ def query_nondeterministics = {
+ sql "drop view if exists view_a"
+ sql "CREATE VIEW view_a AS SELECT now() from $tableName"
+
+ sql "set enable_sql_cache=true"
+ sql "admin set frontend config
('cache_last_version_interval_second'='0')"
+ def result1 = sql "select * from view_a"
+
+ sleep(3000)
+
+ def result2 = sql "select * from view_a"
+ assertNotEquals(result1, result2)
+ }()
+
sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' =
'900')"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
