This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 20cc5e9edb8 branch-3.0: [bugfix](fold) Skip constant folding in
from_base64 function to fix binary data handling #50453 (#51412)
20cc5e9edb8 is described below
commit 20cc5e9edb820a4235bef26ca01ebb9302ea4a91
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jun 11 10:58:43 2025 +0800
branch-3.0: [bugfix](fold) Skip constant folding in from_base64 function to
fix binary data handling #50453 (#51412)
Cherry-picked from #50453
Co-authored-by: lw112 <[email protected]>
---
.../rules/expression/rules/FoldConstantRuleOnBE.java | 18 ++++++++----------
.../doris/nereids/trees/expressions/LiteralTest.java | 8 ++------
2 files changed, 10 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
index d4bfd85cc0e..0ed2116d1c5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Match;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import
org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.FromBase64;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep;
@@ -230,6 +231,11 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
return true;
}
+ // Skip from_base64 function to avoid incorrect binary data processing
during constant folding
+ if (expr instanceof FromBase64) {
+ return true;
+ }
+
// TableGeneratingFunction need pass PlanTranslatorContext value
if (expr instanceof TableGeneratingFunction) {
return true;
@@ -488,16 +494,8 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
} else if (type.isStringLikeType()) {
int num = resultContent.getStringValueCount();
for (int i = 0; i < num; ++i) {
- // get the raw byte data to avoid character encoding
conversion problems
- ByteString bytesValues = resultContent.getBytesValue(i);
- // use UTF-8 encoding to ensure proper handling of binary data
- String stringValue = bytesValues.toStringUtf8();
- // handle special NULL value cases
- if ("\\N".equalsIgnoreCase(stringValue) &&
resultContent.hasHasNull()) {
- res.add(new NullLiteral(type));
- } else {
- res.add(new StringLiteral(stringValue));
- }
+ Literal literal = new
StringLiteral(resultContent.getStringValue(i));
+ res.add(literal);
}
} else if (type.isArrayType()) {
ArrayType arrayType = (ArrayType) type;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
index 9c7e2e5b151..fcb64ff0bfa 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
@@ -233,9 +233,7 @@ class LiteralTest {
PValues.Builder resultContentBuilder = PValues.newBuilder();
for (int i = 0; i < elementsArray.length; i = i + 2) {
childBuilder1.addInt32Value(elementsArray[i]);
- String strValue = "str" + (i + 1);
- childBuilder2.addStringValue(strValue);
-
childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
+ childBuilder2.addStringValue("str" + (i + 1));
}
childBuilder1.setType(childTypeBuilder1.build());
childBuilder2.setType(childTypeBuilder2.build());
@@ -282,9 +280,7 @@ class LiteralTest {
PValues.Builder resultContentBuilder = PValues.newBuilder();
for (int i = 0; i < elementsArray.length; i = i + 2) {
childBuilder1.addInt32Value(elementsArray[i]);
- String strValue = "str" + (i + 1);
- childBuilder2.addStringValue(strValue);
-
childBuilder2.addBytesValue(com.google.protobuf.ByteString.copyFromUtf8(strValue));
+ childBuilder2.addStringValue("str" + (i + 1));
}
childBuilder1.setType(childTypeBuilder1.build());
childBuilder2.setType(childTypeBuilder2.build());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
