This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 256e7866527 [fix](cloud) Preserve VARIANT escape policy presence in
protobuf conversions (#70622)
256e7866527 is described below
commit 256e7866527408297f59db27b631c0b0e31bca80
Author: lihangyu <[email protected]>
AuthorDate: Tue Nov 17 15:07:14 2026 +0800
[fix](cloud) Preserve VARIANT escape policy presence in protobuf
conversions (#70622)
### Background
Doris converts tablet schemas between `TabletSchemaPB` and
`TabletSchemaCloudPB` when exchanging cloud metadata. The optional
`variant_escape_dots_in_keys` field defaults to `false`, while protobuf
separately tracks whether it was explicitly set.
### What problem does this PR solve?
Issue Number: None
Related PR: #69995 (introduced the field)
PR Type: Bugfix
Problem Summary:
Converting a schema without `variant_escape_dots_in_keys` currently
turns the absent field into an explicitly set `false`. Check
`has_variant_escape_dots_in_keys()` before setting it in both conversion
directions and both copy/move implementations, as requested in Gavin's
review of #69995. Explicit `false` and `true` values remain present and
unchanged; the default value remains `false`.
### Details
- **Frequency:** Every conversion of a source schema that omits the
field.
- **Root cause:** Unconditional protobuf setters materialize the default
value and set the presence bit.
- **Why existing tests missed it:** The policy round-trip test checked
values only, so absence and explicit `false` were indistinguishable. The
all-fields tests explicitly populate every field.
- **Fix:** Guard the four setters with the matching protobuf presence
check. Add absent-field coverage and strengthen explicit-value round
trips with presence assertions.
- **Other paths and branches:** Both cloud/local directions and
const-reference/rvalue overloads are covered; the value-returning
wrappers delegate to them. This PR targets master.
- **Behavior change:** An omitted optional field remains omitted after
conversion; explicit values and effective defaults are unchanged.
### Testing
- On dev-x, the new `PBConvertTest.VariantEscapeTablePolicyAbsent` fails
before the fix: presence becomes `true` in all four conversion paths for
absent input fields.
- After the fix, ASAN `./run-be-ut.sh --run
--filter="PbConvert.*:PBConvertTest.*:TabletSchemaTest.*"` passes all 42
tests across three suites. This host's UT invocation uses
`GLIBC_COMPATIBILITY=OFF` to avoid duplicate compatibility symbols with
its LDB toolchain; this is a test-only setting.
- clang-format 16, the format check, and changed-line clang-tidy all
pass.
### Release note
None
### Check List (For Author)
- Behavior changed: Yes (see Details)
- Does this need documentation: No
- Pre-submission status: Ready
### AI Disclosure
- Tool: OpenAI Codex (GPT-6)
- Generated: Code, tests, and this description.
- Human review: Pending author review.
---
be/src/cloud/pb_convert.cpp | 20 ++++++++++++++++----
be/test/storage/pb_convert_test.cpp | 31 ++++++++++++++++++++++++++-----
2 files changed, 42 insertions(+), 9 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 18809d830c7..0ec08bfa036 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -464,7 +464,6 @@ TabletSchemaCloudPB
doris_tablet_schema_to_cloud(TabletSchemaPB&& in) {
void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const
TabletSchemaPB& in) {
// ATTN: please keep the set order aligned with the definition of proto.
out->set_keys_type(in.keys_type());
- out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
out->mutable_column()->CopyFrom(in.column());
out->set_num_short_key_columns(in.num_short_key_columns());
out->set_num_rows_per_row_block(in.num_rows_per_row_block());
@@ -515,6 +514,10 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB*
out, const TabletSchemaPB
out->set_ttl_col_idx(in.ttl_col_idx());
out->set_row_ttl_duration_us(in.row_ttl_duration_us());
out->set_row_ttl_time_zone_offset_seconds(in.row_ttl_time_zone_offset_seconds());
+ // Older schemas omit this field; do not turn absence into an explicit
false.
+ if (in.has_variant_escape_dots_in_keys()) {
+ out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
+ }
if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
@@ -523,7 +526,6 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out,
const TabletSchemaPB
void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&&
in) {
out->set_keys_type(in.keys_type());
- out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
out->mutable_column()->Swap(in.mutable_column());
out->set_num_short_key_columns(in.num_short_key_columns());
out->set_num_rows_per_row_block(in.num_rows_per_row_block());
@@ -574,6 +576,10 @@ void doris_tablet_schema_to_cloud(TabletSchemaCloudPB*
out, TabletSchemaPB&& in)
out->set_ttl_col_idx(in.ttl_col_idx());
out->set_row_ttl_duration_us(in.row_ttl_duration_us());
out->set_row_ttl_time_zone_offset_seconds(in.row_ttl_time_zone_offset_seconds());
+ // Older schemas omit this field; do not turn absence into an explicit
false.
+ if (in.has_variant_escape_dots_in_keys()) {
+ out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
+ }
if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
@@ -595,7 +601,6 @@ TabletSchemaPB
cloud_tablet_schema_to_doris(TabletSchemaCloudPB&& in) {
void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const
TabletSchemaCloudPB& in) {
// ATTN: please keep the set order aligned with the definition of proto.
out->set_keys_type(in.keys_type());
- out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
out->mutable_column()->CopyFrom(in.column());
out->set_num_short_key_columns(in.num_short_key_columns());
out->set_num_rows_per_row_block(in.num_rows_per_row_block());
@@ -646,6 +651,10 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
const TabletSchemaCloudPB
out->set_ttl_col_idx(in.ttl_col_idx());
out->set_row_ttl_duration_us(in.row_ttl_duration_us());
out->set_row_ttl_time_zone_offset_seconds(in.row_ttl_time_zone_offset_seconds());
+ // Older schemas omit this field; do not turn absence into an explicit
false.
+ if (in.has_variant_escape_dots_in_keys()) {
+ out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
+ }
if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
@@ -655,7 +664,6 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
const TabletSchemaCloudPB
void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&&
in) {
// ATTN: please keep the set order aligned with the definition of proto.
out->set_keys_type(in.keys_type());
- out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
out->mutable_column()->Swap(in.mutable_column());
out->set_num_short_key_columns(in.num_short_key_columns());
out->set_num_rows_per_row_block(in.num_rows_per_row_block());
@@ -706,6 +714,10 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
TabletSchemaCloudPB&& in)
out->set_ttl_col_idx(in.ttl_col_idx());
out->set_row_ttl_duration_us(in.row_ttl_duration_us());
out->set_row_ttl_time_zone_offset_seconds(in.row_ttl_time_zone_offset_seconds());
+ // Older schemas omit this field; do not turn absence into an explicit
false.
+ if (in.has_variant_escape_dots_in_keys()) {
+ out->set_variant_escape_dots_in_keys(in.variant_escape_dots_in_keys());
+ }
if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
diff --git a/be/test/storage/pb_convert_test.cpp
b/be/test/storage/pb_convert_test.cpp
index ba7b2ac3b31..64cdd35558f 100644
--- a/be/test/storage/pb_convert_test.cpp
+++ b/be/test/storage/pb_convert_test.cpp
@@ -503,16 +503,37 @@ TEST(PBConvertTest, VariantEscapeTablePolicyRoundTrip) {
TabletSchemaPB schema;
schema.set_variant_escape_dots_in_keys(escaped);
const auto cloud = doris_tablet_schema_to_cloud(schema);
+ EXPECT_TRUE(cloud.has_variant_escape_dots_in_keys());
EXPECT_EQ(cloud.variant_escape_dots_in_keys(), escaped);
-
EXPECT_EQ(cloud_tablet_schema_to_doris(cloud).variant_escape_dots_in_keys(),
escaped);
+ const auto restored_schema = cloud_tablet_schema_to_doris(cloud);
+ EXPECT_TRUE(restored_schema.has_variant_escape_dots_in_keys());
+ EXPECT_EQ(restored_schema.variant_escape_dots_in_keys(), escaped);
auto moving_schema = schema;
auto moving_cloud =
doris_tablet_schema_to_cloud(std::move(moving_schema));
+ EXPECT_TRUE(moving_cloud.has_variant_escape_dots_in_keys());
EXPECT_EQ(moving_cloud.variant_escape_dots_in_keys(), escaped);
- EXPECT_EQ(
-
cloud_tablet_schema_to_doris(std::move(moving_cloud)).variant_escape_dots_in_keys(),
- escaped);
+ const auto moved_schema =
cloud_tablet_schema_to_doris(std::move(moving_cloud));
+ EXPECT_TRUE(moved_schema.has_variant_escape_dots_in_keys());
+ EXPECT_EQ(moved_schema.variant_escape_dots_in_keys(), escaped);
}
-
EXPECT_FALSE(cloud_tablet_schema_to_doris(TabletSchemaCloudPB()).variant_escape_dots_in_keys());
+}
+
+TEST(PBConvertTest, VariantEscapeTablePolicyAbsent) {
+ const TabletSchemaPB schema;
+ const auto cloud = doris_tablet_schema_to_cloud(schema);
+ EXPECT_FALSE(cloud.has_variant_escape_dots_in_keys());
+ EXPECT_FALSE(cloud.variant_escape_dots_in_keys());
+ const TabletSchemaCloudPB cloud_schema;
+ const auto restored_schema = cloud_tablet_schema_to_doris(cloud_schema);
+ EXPECT_FALSE(restored_schema.has_variant_escape_dots_in_keys());
+ EXPECT_FALSE(restored_schema.variant_escape_dots_in_keys());
+
+ const auto moving_cloud = doris_tablet_schema_to_cloud(TabletSchemaPB());
+ EXPECT_FALSE(moving_cloud.has_variant_escape_dots_in_keys());
+ EXPECT_FALSE(moving_cloud.variant_escape_dots_in_keys());
+ const auto moving_schema =
cloud_tablet_schema_to_doris(TabletSchemaCloudPB());
+ EXPECT_FALSE(moving_schema.has_variant_escape_dots_in_keys());
+ EXPECT_FALSE(moving_schema.variant_escape_dots_in_keys());
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]