This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 07b1c842507 branch-4.2: [fix](function) Preserve invalid URL decoded
bytes (#67841) (#69107)
07b1c842507 is described below
commit 07b1c84250758e839e6835cdaff3168884bdedf1
Author: morrySnow <[email protected]>
AuthorDate: Thu Oct 15 19:07:35 2026 +0800
branch-4.2: [fix](function) Preserve invalid URL decoded bytes (#67841)
(#69107)
### What problem does this PR solve?
Issue Number: N/A
Related PR: https://github.com/apache/doris/pull/67841
Problem Summary: FE URL_DECODE constant folding silently replaced
invalid UTF-8 decoded bytes, unlike BE byte-preserving execution.
Validate the decoded byte stream and defer malformed UTF-8 to BE while
retaining valid UTF-8 folding.
### Release note
Preserve invalid UTF-8 bytes in URL_DECODE results consistently with BE
execution.
### Backport validation
- Test: Unit Test: StringArithmeticTest; FE Checkstyle (0 violations).
- Combined eight-backport validation: 283 tests across 13 classes, 0
failures/errors after correcting identity-cast assertions. This
standalone backport has not had a separate unit-test run.
- Targeted regression: url_decode_invalid_utf8 passed through
run-regression-test.sh on the combined eight-backport ASAN deployment.
This standalone branch and the full regression suite were not tested
separately.
- Full build.sh --fe and ASAN build.sh --be succeeded on the combined
eight-backport tree; deployment smoke validation passed.
- Behavior changed: Yes (invalid UTF-8 decoded bytes are preserved)
- Does this need documentation: No
(cherry picked from commit 366fd37b283e55916bbe1a951e703a78a2f7fa3d)
### Scope
This is an independent backport of #67841 onto branch-4.2.
The branch contains one commit and does not include the other seven
picks.
The 283-test result above was obtained from the combined backport tree;
this standalone branch has not had a separate unit-test run.
### Check List (For Author)
- Test:
- [x] Unit Test: 283 targeted tests passed on the combined backport
tree.
- [x] Manual test: standalone patch matches the validated pick; FE
Checkstyle passed on the combined tree.
- [x] Regression test: url_decode_invalid_utf8 passed on the combined
eight-backport ASAN deployment; this standalone branch was not tested
separately. The full regression suite was not run.
- Behavior changed: see the backport validation details above.
- Does this need documentation: No.
---
.../functions/executable/StringArithmetic.java | 51 ++++++++++++++++++++++
.../functions/executable/StringArithmeticTest.java | 25 +++++++++++
.../scalar_function/url_decode_invalid_utf8.out | 9 ++++
.../scalar_function/url_decode_invalid_utf8.groovy | 39 +++++++++++++++++
4 files changed, 124 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
index 1f86b5e19cb..db0742117fa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
@@ -45,10 +45,14 @@ import org.apache.doris.nereids.types.ArrayType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import java.io.ByteArrayOutputStream;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.URLDecoder;
import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -1070,6 +1074,11 @@ public class StringArithmetic {
*/
@ExecFunction(name = "url_decode")
public static Expression urlDecode(StringLikeLiteral first) {
+ if (!isValidUtf8AfterUrlDecode(first.getValue())) {
+ // String literals cannot preserve an invalid UTF-8 byte sequence.
Let BE evaluate it
+ // instead of folding the replacement characters produced by
java.net.URLDecoder.
+ throw new IllegalArgumentException("URL-decoded value is not valid
UTF-8");
+ }
try {
return castStringLikeLiteral(first,
URLDecoder.decode(first.getValue(), StandardCharsets.UTF_8.name()));
} catch (UnsupportedEncodingException e) {
@@ -1077,6 +1086,48 @@ public class StringArithmetic {
}
}
+ private static boolean isValidUtf8AfterUrlDecode(String value) {
+ ByteArrayOutputStream decodedBytes = new
ByteArrayOutputStream(value.length());
+ int index = 0;
+ while (index < value.length()) {
+ char current = value.charAt(index);
+ if (current == '%') {
+ if (index + 2 >= value.length()) {
+ // Preserve URLDecoder's existing malformed-escape
handling.
+ return true;
+ }
+ int high = Character.digit(value.charAt(index + 1), 16);
+ int low = Character.digit(value.charAt(index + 2), 16);
+ if (high < 0 || low < 0) {
+ // Preserve URLDecoder's existing malformed-escape
handling.
+ return true;
+ }
+ decodedBytes.write((high << 4) + low);
+ index += 3;
+ } else if (current == '+') {
+ decodedBytes.write(' ');
+ index++;
+ } else {
+ int start = index;
+ while (index < value.length() && value.charAt(index) != '%' &&
value.charAt(index) != '+') {
+ index++;
+ }
+ byte[] originalBytes = value.substring(start,
index).getBytes(StandardCharsets.UTF_8);
+ decodedBytes.write(originalBytes, 0, originalBytes.length);
+ }
+ }
+
+ try {
+ StandardCharsets.UTF_8.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT)
+ .decode(ByteBuffer.wrap(decodedBytes.toByteArray()));
+ return true;
+ } catch (CharacterCodingException e) {
+ return false;
+ }
+ }
+
/**
* Executable arithmetic functions urlencode
*/
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
index 985b0029b82..5381bc8e062 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmeticTest.java
@@ -17,9 +17,14 @@
package org.apache.doris.nereids.trees.expressions.functions.executable;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.ExpressionEvaluator;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.UrlDecode;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral;
import org.junit.jupiter.api.Assertions;
@@ -74,4 +79,24 @@ class StringArithmeticTest {
Assertions.assertEquals(2, result.getValue());
}
+
+ @Test
+ void testUrlDecodeDoesNotFoldInvalidUtf8() {
+ String[] invalidUtf8Values = {"%80", "%C0%AF", "%E0%80%80",
"%ED%A0%80", "%FF"};
+ for (String value : invalidUtf8Values) {
+ UrlDecode urlDecode = new UrlDecode(new StringLiteral(value));
+ Assertions.assertSame(urlDecode,
ExpressionEvaluator.INSTANCE.eval(urlDecode), value);
+ }
+ }
+
+ @Test
+ void testUrlDecodeStillFoldsValidUtf8() {
+ assertUrlDecodeValue("%E4%B8%AD+text", "中 text");
+ assertUrlDecodeValue("%EF%BF%BD", "�");
+ }
+
+ private void assertUrlDecodeValue(String encoded, String expected) {
+ Expression result = ExpressionEvaluator.INSTANCE.eval(new
UrlDecode(new StringLiteral(encoded)));
+ Assertions.assertEquals(expected, ((StringLikeLiteral)
result).getValue());
+ }
}
diff --git
a/regression-test/data/nereids_function_p0/scalar_function/url_decode_invalid_utf8.out
b/regression-test/data/nereids_function_p0/scalar_function/url_decode_invalid_utf8.out
new file mode 100644
index 00000000000..3a1c28a3f41
--- /dev/null
+++
b/regression-test/data/nereids_function_p0/scalar_function/url_decode_invalid_utf8.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !invalid_utf8_constant --
+80 C0AF E08080 EDA080 FF
+
+-- !invalid_utf8_runtime --
+80 C0AF E08080 EDA080 FF
+
+-- !valid_utf8_constant --
+中 text EFBFBD
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/url_decode_invalid_utf8.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/url_decode_invalid_utf8.groovy
new file mode 100644
index 00000000000..6c9de7d9b03
--- /dev/null
+++
b/regression-test/suites/nereids_function_p0/scalar_function/url_decode_invalid_utf8.groovy
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("url_decode_invalid_utf8") {
+ qt_invalid_utf8_constant """
+ SELECT HEX(URL_DECODE('%80')),
+ HEX(URL_DECODE('%C0%AF')),
+ HEX(URL_DECODE('%E0%80%80')),
+ HEX(URL_DECODE('%ED%A0%80')),
+ HEX(URL_DECODE('%FF'))
+ """
+
+ qt_invalid_utf8_runtime """
+ SELECT HEX(URL_DECODE(IF(number = 0, '%80', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%C0%AF', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%E0%80%80', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%ED%A0%80', ''))),
+ HEX(URL_DECODE(IF(number = 0, '%FF', '')))
+ FROM numbers("number" = "1")
+ """
+
+ qt_valid_utf8_constant """
+ SELECT URL_DECODE('%E4%B8%AD+text'), HEX(URL_DECODE('%EF%BF%BD'))
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
