This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 98e5b46adf691d942c12484161cfebf6f69c4561
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 8 09:33:20 2026 +0800
branch-4.1: [fix](jsonb) Stop truncating JSON text at a U+0000 inside a
string #68244 (#68735)
Cherry-picked from #68244
Co-authored-by: Jerry Hu <[email protected]>
---
be/src/util/jsonb_utils.h | 9 +++-
be/test/util/jsonb_parser_simd_test.cpp | 37 +++++++++++++
.../table_function/test_json_text_embedded_nul.out | 19 +++++++
.../test_json_text_embedded_nul.groovy | 62 ++++++++++++++++++++++
4 files changed, 125 insertions(+), 2 deletions(-)
diff --git a/be/src/util/jsonb_utils.h b/be/src/util/jsonb_utils.h
index abf05b4d311..05b2ab5ad08 100644
--- a/be/src/util/jsonb_utils.h
+++ b/be/src/util/jsonb_utils.h
@@ -107,8 +107,11 @@ private:
break;
}
case JsonbType::T_String: {
+ // getBlobLen() is the stored payload length. length() would drop a
+ // trailing NUL that belongs to the value, so it cannot be used
here;
+ // the writer stores strings by their exact length and never pads.
string_to_json(val->unpack<JsonbStringVal>()->getBlob(),
- val->unpack<JsonbStringVal>()->length());
+ val->unpack<JsonbStringVal>()->getBlobLen());
break;
}
case JsonbType::T_Binary: {
@@ -162,7 +165,9 @@ private:
return;
}
char char_buffer[16];
- for (const char* ptr = str; ptr != str + len && *ptr; ++ptr) {
+ // A JSON string may legally contain U+0000, so the loop must be
bounded by
+ // the length only; the NUL itself is escaped as \u0000 by the default
branch.
+ for (const char* ptr = str; ptr != str + len; ++ptr) {
if ((unsigned char)*ptr > 31 && *ptr != '\"' && *ptr != '\\') {
os_.put(*ptr);
} else {
diff --git a/be/test/util/jsonb_parser_simd_test.cpp
b/be/test/util/jsonb_parser_simd_test.cpp
index 53a666955ba..8f5b418a193 100644
--- a/be/test/util/jsonb_parser_simd_test.cpp
+++ b/be/test/util/jsonb_parser_simd_test.cpp
@@ -400,4 +400,41 @@ TEST_F(JsonbParserTest, ParseJsonWithToLongKey) {
EXPECT_FALSE(st.ok());
std::cout << st.msg() << std::endl;
}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInString) {
+ std::string_view json_with_nul = R"({"key":"a\u0000b"})";
+ std::string_view expected_json_with_nul = R"({"key":"a\u0000b"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInNestedArray) {
+ std::string_view json_with_nul = R"({"key":["a\u0000b","c"]})";
+ std::string_view expected_json_with_nul = R"({"key":["a\u0000b","c"]})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithTrailingNulInString) {
+ std::string_view json_with_nul = R"({"key":"a\u0000"})";
+ std::string_view expected_json_with_nul = R"({"key":"a\u0000"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithOnlyNulInString) {
+ std::string_view json_with_nul = R"({"key":"\u0000"})";
+ std::string_view expected_json_with_nul = R"({"key":"\u0000"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithTrailingNulInNestedArray) {
+ std::string_view json_with_nul = R"({"key":["a\u0000","\u0000"]})";
+ std::string_view expected_json_with_nul =
R"({"key":["a\u0000","\u0000"]})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInKey) {
+ std::string_view json_with_nul = R"({"a\u0000b":1})";
+ std::string_view expected_json_with_nul = R"({"a\u0000b":1})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
} // namespace doris
\ No newline at end of file
diff --git
a/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
b/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
new file mode 100644
index 00000000000..a3c8fb2de52
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !json_each_text_nested_array --
+x 12 ["a\\u0000b"]
+
+-- !json_each_text_nested_object --
+x 16 {"y":"a\\u0000b"}
+
+-- !json_each_text_trailing_nul --
+x 20 ["a\\u0000","\\u0000"]
+
+-- !explode_json_array_string_direct --
+3 610062
+
+-- !explode_json_array_string_nested --
+12 ["a\\u0000b"]
+
+-- !explode_json_array_string_trailing_nul --
+11 ["a\\u0000"]
+
diff --git
a/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
b/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
new file mode 100644
index 00000000000..7e1e3beeea7
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_text_embedded_nul") {
+ // unhex('5C') is a backslash, so every JSON literal below really contains
+ // \u0000, the legal escape for U+0000 inside a JSON string. Rendering such
+ // a nested value back to text must not stop at the NUL.
+
+ qt_json_each_text_nested_array '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'),
'u0000b"]}')) t AS k, v
+ '''
+
+ qt_json_each_text_nested_object '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":{"y":"a', unhex('5C'),
'u0000b"}}')) t AS k, v
+ '''
+
+ // A NUL at the very end of a string used to be trimmed by the
stored-length
+ // helper, so keep a dedicated case for it.
+ qt_json_each_text_trailing_nul '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'),
'u0000","', unhex('5C'), 'u0000"]}')) t AS k, v
+ '''
+
+ // A direct string element is copied by length, so it keeps the raw NUL
byte;
+ // compare it through hex() to keep the expected output printable.
+ qt_explode_json_array_string_direct '''
+ SELECT length(c), hex(c)
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('["a', unhex('5C'),
'u0000b"]')) t AS c
+ '''
+
+ qt_explode_json_array_string_nested '''
+ SELECT length(c), c
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'),
'u0000b"]]')) t AS c
+ '''
+
+ qt_explode_json_array_string_trailing_nul '''
+ SELECT length(c), c
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'),
'u0000"]]')) t AS c
+ '''
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
