This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 25dd0542eec [fix](be) Preserve valid types for date defaults (#69120)
25dd0542eec is described below
commit 25dd0542eecd6470150f9b3fc66189ea301dff2b
Author: Chenyang Sun <[email protected]>
AuthorDate: Fri Oct 16 15:49:52 2026 +0800
[fix](be) Preserve valid types for date defaults (#69120)
Related PR: #56079
Problem Summary: PR #56079 introduced VecDateTimeValue::DEFAULT_VALUE
with the
compute-layer TYPE_DATETIME enum where the storage-layer TimeType was
required. The value is stored in a three-bit field, so it becomes an
invalid
type and can serialize to an empty string. The issue was later exposed
by the
write path in #69070. This PR uses valid time types and defines separate
DATE_DEFAULT_VALUE and DATETIME_DEFAULT_VALUE constants.
---
be/src/core/column/column_vector.h | 8 +-
be/src/core/value/vdatetime_value.cpp | 4 +-
be/src/core/value/vdatetime_value.h | 19 ++--
be/test/core/column/common_column_test.h | 114 +++++++++++----------
.../data_type/data_type_insert_default_test.cpp | 12 ++-
be/test/core/value/vdatetime_value_test.cpp | 14 +++
.../runtime_filter_partition_pruner_test.cpp | 9 +-
.../runtime_filter/runtime_filter_wrapper_test.cpp | 16 +--
8 files changed, 119 insertions(+), 77 deletions(-)
diff --git a/be/src/core/column/column_vector.h
b/be/src/core/column/column_vector.h
index d77fc3afa12..9591c13f11e 100644
--- a/be/src/core/column/column_vector.h
+++ b/be/src/core/column/column_vector.h
@@ -413,7 +413,13 @@ public:
// when run function which need_replace_null_data_to_default, use the
value far from 0 to avoid
// raise errors for null cell.
static value_type default_value() {
- if constexpr (is_date_type(T) || T == PrimitiveType::TYPE_TIMESTAMPTZ)
{
+ if constexpr (T == PrimitiveType::TYPE_DATE) {
+ return PrimitiveTypeTraits<T>::CppType::DATE_DEFAULT_VALUE;
+ } else if constexpr (T == PrimitiveType::TYPE_DATETIME) {
+ return PrimitiveTypeTraits<T>::CppType::DATETIME_DEFAULT_VALUE;
+ } else if constexpr (T == PrimitiveType::TYPE_DATEV2 ||
+ T == PrimitiveType::TYPE_DATETIMEV2 ||
+ T == PrimitiveType::TYPE_TIMESTAMPTZ) {
return PrimitiveTypeTraits<T>::CppType::DEFAULT_VALUE;
} else {
return value_type();
diff --git a/be/src/core/value/vdatetime_value.cpp
b/be/src/core/value/vdatetime_value.cpp
index f02378c8ad5..77a138558e4 100644
--- a/be/src/core/value/vdatetime_value.cpp
+++ b/be/src/core/value/vdatetime_value.cpp
@@ -180,12 +180,12 @@ bool VecDateTimeValue::from_date_int64(int64_t value) {
return check_range_and_set_time(year, month, day, hour, minute, second,
_type);
}
-void VecDateTimeValue::set_zero(int type) {
+void VecDateTimeValue::set_zero(TimeType type) {
memset(this, 0, sizeof(*this));
_type = type;
}
-void VecDateTimeValue::set_type(int type) {
+void VecDateTimeValue::set_type(TimeType type) {
_type = type;
if (type == TIME_DATE) {
_hour = 0;
diff --git a/be/src/core/value/vdatetime_value.h
b/be/src/core/value/vdatetime_value.h
index 8eacd5dc57d..42eb268e716 100644
--- a/be/src/core/value/vdatetime_value.h
+++ b/be/src/core/value/vdatetime_value.h
@@ -292,7 +292,8 @@ public:
_year(0) {} // before int128 16 bytes ---> after int64 8 bytes
const static VecDateTimeValue FIRST_DAY;
- const static VecDateTimeValue DEFAULT_VALUE;
+ const static VecDateTimeValue DATE_DEFAULT_VALUE;
+ const static VecDateTimeValue DATETIME_DEFAULT_VALUE;
// The data format of DATE/DATETIME is different in storage layer and
execute layer.
// So we should use different creator to get data from value.
@@ -728,7 +729,7 @@ public:
return (daynr() - rhs.daynr()) * SECOND_PER_HOUR * HOUR_PER_DAY +
time_part_diff(rhs);
}
- void set_type(int type);
+ void set_type(TimeType type);
int type() const { return _type; }
@@ -767,7 +768,7 @@ public:
bool get_date_from_daynr(uint64_t);
// reset 0
- void reset_zero_by_type(int type) { set_zero(type); }
+ void reset_zero_by_type(TimeType type) { set_zero(type); }
private:
// Used to make sure sizeof VecDateTimeValue
@@ -794,7 +795,7 @@ private:
bool disable_lut = false);
// Helper to set max, min, zero
- void set_zero(int type);
+ void set_zero(TimeType type);
void set_max_time(bool neg);
bool from_date_format_str(const char* format, int format_len, const char*
value,
@@ -810,7 +811,7 @@ private:
uint8_t _month;
uint16_t _year;
- VecDateTimeValue(uint8_t neg, uint8_t type, uint8_t hour, uint8_t minute,
uint8_t second,
+ VecDateTimeValue(uint8_t neg, TimeType type, uint8_t hour, uint8_t minute,
uint8_t second,
uint16_t year, uint8_t month, uint8_t day)
: _neg(neg),
_type(type),
@@ -826,9 +827,11 @@
static_assert(std::is_trivially_destructible_v<VecDateTimeValue>,
"VecDateTimeValue must be trivial destructible");
static_assert(std::is_trivially_copyable_v<VecDateTimeValue>,
"VecDateTimeValue must be trivial copyable");
-inline const VecDateTimeValue VecDateTimeValue::FIRST_DAY(false,
TYPE_DATETIME, 0, 0, 0, 1, 1, 1);
-inline const VecDateTimeValue VecDateTimeValue::DEFAULT_VALUE(false,
TYPE_DATETIME, 0, 0, 0, 1970,
- 1, 1);
+inline const VecDateTimeValue VecDateTimeValue::FIRST_DAY(false,
TIME_DATETIME, 0, 0, 0, 1, 1, 1);
+inline const VecDateTimeValue VecDateTimeValue::DATE_DEFAULT_VALUE(false,
TIME_DATE, 0, 0, 0, 1970,
+ 1, 1);
+inline const VecDateTimeValue VecDateTimeValue::DATETIME_DEFAULT_VALUE(false,
TIME_DATETIME, 0, 0,
+ 0,
1970, 1, 1);
template <typename T>
class DateV2Value {
diff --git a/be/test/core/column/common_column_test.h
b/be/test/core/column/common_column_test.h
index c63fbf310fb..8a97ae0f10d 100644
--- a/be/test/core/column/common_column_test.h
+++ b/be/test/core/column/common_column_test.h
@@ -2949,8 +2949,13 @@ auto assert_column_vector_insert_default_callback =
[](auto x,
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE.to_date_int_val()));
} else if constexpr (PType == PrimitiveType::TYPE_DATE ||
PType == PrimitiveType::TYPE_DATETIME) {
- EXPECT_EQ(col_vec_target->get_element(i),
- T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE));
+ if constexpr (PType == PrimitiveType::TYPE_DATE) {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATE_DEFAULT_VALUE));
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATETIME_DEFAULT_VALUE));
+ }
} else if constexpr (PType == PrimitiveType::TYPE_DECIMALV2) {
EXPECT_EQ(col_vec_target->get_element(i), DecimalV2Value());
} else {
@@ -2962,59 +2967,64 @@ auto assert_column_vector_insert_default_callback =
[](auto x,
};
template <PrimitiveType PType>
-auto assert_column_vector_insert_many_defaults_callback =
- [](auto x, const MutableColumnPtr& source_column) {
- using T = decltype(x);
- using ColumnVecType = std::conditional_t<
- std::is_same_v<T, ColumnString>, ColumnString,
- std::conditional_t<std::is_same_v<T, ColumnString64>,
ColumnString64,
- std::conditional_t<IsDecimalNumber<T>,
ColumnDecimal<PType>,
-
ColumnVector<PType>>>>;
- std::vector<size_t> insert_vals_count = {0, 10, 1000};
- auto* col_vec_src =
assert_cast<ColumnVecType*>(source_column.get());
- auto src_size = source_column->size();
+auto assert_column_vector_insert_many_defaults_callback = [](auto x, const
MutableColumnPtr&
+
source_column) {
+ using T = decltype(x);
+ using ColumnVecType = std::conditional_t<
+ std::is_same_v<T, ColumnString>, ColumnString,
+ std::conditional_t<std::is_same_v<T, ColumnString64>,
ColumnString64,
+ std::conditional_t<IsDecimalNumber<T>,
ColumnDecimal<PType>,
+ ColumnVector<PType>>>>;
+ std::vector<size_t> insert_vals_count = {0, 10, 1000};
+ auto* col_vec_src = assert_cast<ColumnVecType*>(source_column.get());
+ auto src_size = source_column->size();
- auto test_func = [&](size_t clone_count) {
- for (auto n : insert_vals_count) {
- size_t actual_clone_count = std::min(clone_count,
src_size);
- auto target_column =
source_column->clone_resized(actual_clone_count);
- auto* col_vec_target =
assert_cast<ColumnVecType*>(target_column.get());
- col_vec_target->insert_many_defaults(n);
- auto target_size = col_vec_target->size();
- EXPECT_EQ(target_size, actual_clone_count + n);
- size_t i = 0;
- for (; i < actual_clone_count; ++i) {
- if constexpr (std::is_same_v<T, ColumnString> ||
- std::is_same_v<T, ColumnString64>) {
- EXPECT_EQ(col_vec_target->get_data_at(i),
col_vec_src->get_data_at(i));
- } else {
- EXPECT_EQ(col_vec_target->get_element(i),
col_vec_src->get_element(i));
- }
- }
- for (; i < target_size; ++i) {
- if constexpr (std::is_same_v<T, ColumnString> ||
- std::is_same_v<T, ColumnString64>) {
-
EXPECT_EQ(col_vec_target->get_data_at(i).to_string(), "");
- } else if constexpr (PType ==
PrimitiveType::TYPE_DATEV2 ||
- PType ==
PrimitiveType::TYPE_DATETIMEV2) {
- EXPECT_EQ(col_vec_target->get_element(i),
-
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE
- .to_date_int_val()));
- } else if constexpr (PType == PrimitiveType::TYPE_DATE
||
- PType ==
PrimitiveType::TYPE_DATETIME) {
- EXPECT_EQ(col_vec_target->get_element(i),
-
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE));
- } else if constexpr (PType ==
PrimitiveType::TYPE_DECIMALV2) {
- EXPECT_EQ(col_vec_target->get_element(i),
DecimalV2Value());
- } else {
- EXPECT_EQ(col_vec_target->get_element(i), T {});
- }
+ auto test_func = [&](size_t clone_count) {
+ for (auto n : insert_vals_count) {
+ size_t actual_clone_count = std::min(clone_count, src_size);
+ auto target_column =
source_column->clone_resized(actual_clone_count);
+ auto* col_vec_target =
assert_cast<ColumnVecType*>(target_column.get());
+ col_vec_target->insert_many_defaults(n);
+ auto target_size = col_vec_target->size();
+ EXPECT_EQ(target_size, actual_clone_count + n);
+ size_t i = 0;
+ for (; i < actual_clone_count; ++i) {
+ if constexpr (std::is_same_v<T, ColumnString> ||
+ std::is_same_v<T, ColumnString64>) {
+ EXPECT_EQ(col_vec_target->get_data_at(i),
col_vec_src->get_data_at(i));
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i),
col_vec_src->get_element(i));
+ }
+ }
+ for (; i < target_size; ++i) {
+ if constexpr (std::is_same_v<T, ColumnString> ||
+ std::is_same_v<T, ColumnString64>) {
+ EXPECT_EQ(col_vec_target->get_data_at(i).to_string(), "");
+ } else if constexpr (PType == PrimitiveType::TYPE_DATEV2 ||
+ PType == PrimitiveType::TYPE_DATETIMEV2) {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DEFAULT_VALUE
+ .to_date_int_val()));
+ } else if constexpr (PType == PrimitiveType::TYPE_DATE ||
+ PType == PrimitiveType::TYPE_DATETIME) {
+ if constexpr (PType == PrimitiveType::TYPE_DATE) {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATE_DEFAULT_VALUE));
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i),
+
T(PrimitiveTypeTraits<PType>::CppType::DATETIME_DEFAULT_VALUE));
}
+ } else if constexpr (PType == PrimitiveType::TYPE_DECIMALV2) {
+ EXPECT_EQ(col_vec_target->get_element(i),
DecimalV2Value());
+ } else {
+ EXPECT_EQ(col_vec_target->get_element(i), T {});
}
- };
- test_func(0);
- test_func(10);
- };
+ }
+ }
+ };
+ test_func(0);
+ test_func(10);
+};
template <PrimitiveType PType>
auto assert_column_vector_get_bool_callback = [](auto x, const
MutableColumnPtr& source_column) {
diff --git a/be/test/core/data_type/data_type_insert_default_test.cpp
b/be/test/core/data_type/data_type_insert_default_test.cpp
index 611db65afbd..d1f1ccdff1e 100644
--- a/be/test/core/data_type/data_type_insert_default_test.cpp
+++ b/be/test/core/data_type/data_type_insert_default_test.cpp
@@ -167,14 +167,22 @@ TEST(DataTypeInsertDefaultTest, AllTypeFamilies) {
SCOPED_TRACE("date");
auto actual = get_default_field(make_simple_type(TYPE_DATE));
ASSERT_EQ(actual.get_type(), TYPE_DATE);
- EXPECT_EQ(actual.get<TYPE_DATE>(), VecDateTimeValue::DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATE>(),
VecDateTimeValue::DATE_DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATE>().type(), TIME_DATE);
+ char buffer[64];
+ actual.get<TYPE_DATE>().to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01");
}
{
SCOPED_TRACE("datetime");
auto actual = get_default_field(make_simple_type(TYPE_DATETIME));
ASSERT_EQ(actual.get_type(), TYPE_DATETIME);
- EXPECT_EQ(actual.get<TYPE_DATETIME>(),
VecDateTimeValue::DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATETIME>(),
VecDateTimeValue::DATETIME_DEFAULT_VALUE);
+ EXPECT_EQ(actual.get<TYPE_DATETIME>().type(), TIME_DATETIME);
+ char buffer[64];
+ actual.get<TYPE_DATETIME>().to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01 00:00:00");
}
{
diff --git a/be/test/core/value/vdatetime_value_test.cpp
b/be/test/core/value/vdatetime_value_test.cpp
index 6b3cd7488ae..bc1faa9d611 100644
--- a/be/test/core/value/vdatetime_value_test.cpp
+++ b/be/test/core/value/vdatetime_value_test.cpp
@@ -30,6 +30,20 @@
namespace doris {
+TEST(VDateTimeValueTest, LegacyDefaultsHaveValidTimeTypes) {
+ EXPECT_EQ(VecDateTimeValue::FIRST_DAY.type(), TIME_DATETIME);
+ EXPECT_EQ(VecDateTimeValue::DATE_DEFAULT_VALUE.type(), TIME_DATE);
+ EXPECT_EQ(VecDateTimeValue::DATETIME_DEFAULT_VALUE.type(), TIME_DATETIME);
+
+ char buffer[64];
+ VecDateTimeValue::FIRST_DAY.to_string(buffer);
+ EXPECT_STREQ(buffer, "0001-01-01 00:00:00");
+ VecDateTimeValue::DATE_DEFAULT_VALUE.to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01");
+ VecDateTimeValue::DATETIME_DEFAULT_VALUE.to_string(buffer);
+ EXPECT_STREQ(buffer, "1970-01-01 00:00:00");
+}
+
TEST(VDateTimeValueTest, date_v2_to_uint32_test) {
uint16_t year = 2022;
uint8_t month = 5;
diff --git
a/be/test/exec/runtime_filter/runtime_filter_partition_pruner_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_partition_pruner_test.cpp
index 903b86b6859..fb91cec4c2e 100644
--- a/be/test/exec/runtime_filter/runtime_filter_partition_pruner_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_partition_pruner_test.cpp
@@ -588,10 +588,11 @@ TEST_F(RuntimeFilterPartitionPrunerTest,
ParseAndPrunePrimitiveTypeMatrix) {
assert_parse_and_prune_type<TYPE_CHAR>(String("aa"), String("bb"));
assert_parse_and_prune_type<TYPE_VARCHAR>(String("alpha"),
String("bravo"));
assert_parse_and_prune_type<TYPE_STRING>(String("left"), String("right"));
- assert_parse_and_prune_type<TYPE_DATE>(VecDateTimeValue(0, 2, 0, 0, 0,
2024, 1, 1),
- VecDateTimeValue(0, 2, 0, 0, 0,
2024, 2, 1));
- assert_parse_and_prune_type<TYPE_DATETIME>(VecDateTimeValue(0, 3, 0, 0, 0,
2024, 1, 1),
- VecDateTimeValue(0, 3, 0, 0, 0,
2024, 2, 1));
+ assert_parse_and_prune_type<TYPE_DATE>(VecDateTimeValue(0, TIME_DATE, 0,
0, 0, 2024, 1, 1),
+ VecDateTimeValue(0, TIME_DATE, 0,
0, 0, 2024, 2, 1));
+ assert_parse_and_prune_type<TYPE_DATETIME>(
+ VecDateTimeValue(0, TIME_DATETIME, 0, 0, 0, 2024, 1, 1),
+ VecDateTimeValue(0, TIME_DATETIME, 0, 0, 0, 2024, 2, 1));
assert_parse_and_prune_type<TYPE_DATEV2>(date_v2(2024, 1, 1),
date_v2(2024, 2, 1));
assert_parse_and_prune_type<TYPE_DATETIMEV2>(datetime_v2(2024, 1, 1),
datetime_v2(2024, 2, 1),
0, 6);
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index b4c1978826b..f31fcc68d67 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -234,10 +234,10 @@ TEST_F(RuntimeFilterWrapperTest, TestInAssign) {
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATETIMEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_TIMESTAMP_NS);
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, 3, 0, 0, 0,
2020, 1, 1),
- VecDateTimeValue(0, 3, 0, 0, 0, 2020, 1, 2));
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, 2, 0, 0, 0, 2020,
1, 1),
- VecDateTimeValue(0, 2, 0, 0, 0, 2020, 1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, TIME_DATETIME,
0, 0, 0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATETIME, 0, 0, 0, 2020,
1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATE, 0, 0, 0, 2020, 1,
2));
APPLY_FOR_PRIMITIVE_TYPE(TYPE_DECIMALV2, DecimalV2Value(1, 1),
DecimalV2Value(1, 2));
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL32);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL64);
@@ -343,10 +343,10 @@ TEST_F(RuntimeFilterWrapperTest, TestMinMaxAssign) {
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DATETIMEV2);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_TIMESTAMP_NS);
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, 3, 0, 0, 0,
2020, 1, 1),
- VecDateTimeValue(0, 3, 0, 0, 0, 2020, 1, 2));
- APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, 2, 0, 0, 0, 2020,
1, 1),
- VecDateTimeValue(0, 2, 0, 0, 0, 2020, 1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATETIME, VecDateTimeValue(0, TIME_DATETIME,
0, 0, 0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATETIME, 0, 0, 0, 2020,
1, 2));
+ APPLY_FOR_PRIMITIVE_TYPE(TYPE_DATE, VecDateTimeValue(0, TIME_DATE, 0, 0,
0, 2020, 1, 1),
+ VecDateTimeValue(0, TIME_DATE, 0, 0, 0, 2020, 1,
2));
APPLY_FOR_PRIMITIVE_TYPE(TYPE_DECIMALV2, DecimalV2Value(1, 1),
DecimalV2Value(1, 2));
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL32);
APPLY_FOR_PRIMITIVE_BASE_TYPE(TYPE_DECIMAL64);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
