This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 470d4e97cdc [refine](column) enforce nullable nested types for array
(#63088) (#64698)
470d4e97cdc is described below
commit 470d4e97cdcebafcd5a91bfce795748424bf161f
Author: Mryange <[email protected]>
AuthorDate: Tue Jun 23 10:16:32 2026 +0800
[refine](column) enforce nullable nested types for array (#63088) (#64698)
This PR makes the nested types inside Array explicitly nullable in BE
type implementations, instead of relying on implicit caller-side
conventions.
DataTypeArray now always stores nullable nested element type
DataTypeArraySerDe updated to follow the same invariant
https://github.com/apache/doris/pull/63088
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/core/data_type/data_type_array.cpp | 17 +++--
be/src/core/data_type/data_type_array.h | 11 ++--
be/src/core/data_type/primitive_type.h | 2 +
.../core/data_type_serde/data_type_array_serde.cpp | 4 +-
.../core/data_type_serde/data_type_array_serde.h | 10 ++-
.../data_type_serde/data_type_nullable_serde.h | 3 +
be/src/exprs/function/cast/cast_to_array.h | 2 +-
be/test/core/block/block_test.cpp | 6 +-
.../data_type_serde_get_name_test.cpp | 4 +-
.../data_type_serde_string_test.cpp | 8 ++-
be/test/core/jsonb/serialize_test.cpp | 6 +-
.../data/vec/native/all_types_single_row.native | Bin 1135 -> 1140 bytes
be/test/exec/common/schema_util_test.cpp | 4 +-
.../exec/operator/table_function_operator_test.cpp | 55 ++++++++--------
be/test/exprs/aggregate/agg_array_agg_test.cpp | 70 ---------------------
be/test/exprs/aggregate/agg_replace_test.cpp | 6 +-
be/test/format/orc/orc_reader_fill_data_test.cpp | 48 +++++++-------
.../java/org/apache/doris/catalog/ArrayType.java | 57 +++++++++++------
.../doris/nereids/parser/LogicalPlanBuilder.java | 2 +-
.../expression/rules/FoldConstantRuleOnBE.java | 4 +-
.../nereids/rules/rewrite/NestedColumnPruning.java | 5 +-
.../trees/expressions/ArrayItemReference.java | 3 +-
.../functions/AggCombinerFunctionBuilder.java | 3 +-
.../functions/ComputeSignatureHelper.java | 2 +-
.../functions/combinator/ForEachCombinator.java | 2 +-
.../expressions/functions/scalar/ArrayMap.java | 2 +-
.../expressions/functions/scalar/ArraySort.java | 4 +-
.../org/apache/doris/nereids/types/ArrayType.java | 28 ++++-----
.../org/apache/doris/nereids/types/DataType.java | 4 +-
.../doris/nereids/util/TypeCoercionUtils.java | 3 +-
.../java/org/apache/doris/catalog/TypeTest.java | 6 +-
.../nereids/trees/expressions/LiteralTest.java | 12 ++--
32 files changed, 187 insertions(+), 206 deletions(-)
diff --git a/be/src/core/data_type/data_type_array.cpp
b/be/src/core/data_type/data_type_array.cpp
index f1841f0e2dc..5a9d2064fa9 100644
--- a/be/src/core/data_type/data_type_array.cpp
+++ b/be/src/core/data_type/data_type_array.cpp
@@ -48,7 +48,13 @@ namespace ErrorCodes {
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
-DataTypeArray::DataTypeArray(const DataTypePtr& nested_) : nested {nested_} {}
+DataTypeArray::DataTypeArray(const DataTypePtr& nested_) {
+ DataTypePtr nullable_nested = make_nullable(nested_);
+ auto nested_type = std::dynamic_pointer_cast<const
DataTypeNullable>(nullable_nested);
+ DORIS_CHECK(nested_type != nullptr);
+ nested = std::move(nested_type);
+ nested_as_base = nested;
+}
MutableColumnPtr DataTypeArray::create_column() const {
return ColumnArray::create(nested->create_column(),
ColumnArray::ColumnOffsets::create());
@@ -66,9 +72,10 @@ bool DataTypeArray::equals(const IDataType& rhs) const {
// here we should remove nullable, otherwise here always be 1
size_t DataTypeArray::get_number_of_dimensions() const {
- const DataTypeArray* nested_array =
- typeid_cast<const DataTypeArray*>(remove_nullable(nested).get());
- if (!nested_array) return 1;
+ auto* nested_array = typeid_cast<const
DataTypeArray*>(remove_nullable(nested).get());
+ if (!nested_array) {
+ return 1;
+ }
return 1 +
nested_array
->get_number_of_dimensions(); /// Every modern C++ compiler
optimizes tail recursion.
@@ -166,7 +173,7 @@ const char* DataTypeArray::deserialize(const char* buf,
MutableColumnPtr* column
void DataTypeArray::to_pb_column_meta(PColumnMeta* col_meta) const {
IDataType::to_pb_column_meta(col_meta);
- auto children = col_meta->add_children();
+ auto* children = col_meta->add_children();
get_nested_type()->to_pb_column_meta(children);
}
diff --git a/be/src/core/data_type/data_type_array.h
b/be/src/core/data_type/data_type_array.h
index f72e2c18537..26479859121 100644
--- a/be/src/core/data_type/data_type_array.h
+++ b/be/src/core/data_type/data_type_array.h
@@ -29,6 +29,7 @@
#include "common/status.h"
#include "core/data_type/data_type.h"
+#include "core/data_type/data_type_nullable.h"
#include "core/data_type/define_primitive_type.h"
#include "core/data_type_serde/data_type_array_serde.h"
#include "core/data_type_serde/data_type_serde.h"
@@ -47,7 +48,8 @@ namespace doris {
class DataTypeArray final : public IDataType {
private:
/// The type of array elements.
- DataTypePtr nested;
+ DataTypeNullablePtr nested;
+ DataTypePtr nested_as_base;
public:
static constexpr PrimitiveType PType = TYPE_ARRAY;
@@ -74,7 +76,8 @@ public:
bool equals(const IDataType& rhs) const override;
- const DataTypePtr& get_nested_type() const { return nested; }
+ const DataTypePtr& get_nested_type() const { return nested_as_base; }
+ const DataTypeNullablePtr& get_nullable_nested_type() const { return
nested; }
/// 1 for plain array, 2 for array of arrays and so on.
size_t get_number_of_dimensions() const;
@@ -94,7 +97,7 @@ public:
void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType*
scalar_type) const override {
node->set_type(TTypeNodeType::ARRAY);
node->set_contains_null(nested->is_nullable());
- nested->to_protobuf(ptype);
+ get_nested_type()->to_protobuf(ptype);
}
#ifdef BE_TEST
@@ -102,7 +105,7 @@ public:
node.type = TTypeNodeType::ARRAY;
node.__isset.contains_nulls = true;
node.contains_nulls.push_back(nested->is_nullable());
- nested->to_thrift(thrift_type);
+ get_nested_type()->to_thrift(thrift_type);
}
#endif
};
diff --git a/be/src/core/data_type/primitive_type.h
b/be/src/core/data_type/primitive_type.h
index 3ecc6590fab..86b81b5a28e 100644
--- a/be/src/core/data_type/primitive_type.h
+++ b/be/src/core/data_type/primitive_type.h
@@ -93,10 +93,12 @@ class DataTypeHLL;
class DataTypeJsonb;
class DataTypeArray;
class DataTypeMap;
+class DataTypeNullable;
class DataTypeVariant;
class DataTypeStruct;
class DataTypeBitMap;
class DataTypeQuantileState;
+using DataTypeNullablePtr = std::shared_ptr<const DataTypeNullable>;
template <PrimitiveType T>
class ColumnVector;
using ColumnUInt8 = ColumnVector<TYPE_BOOLEAN>;
diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp
b/be/src/core/data_type_serde/data_type_array_serde.cpp
index 9a78fc1f6d3..c7b139083df 100644
--- a/be/src/core/data_type_serde/data_type_array_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_array_serde.cpp
@@ -86,7 +86,7 @@ Status
DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice
auto& array_column = assert_cast<ColumnArray&>(column);
auto& offsets = array_column.get_offsets();
IColumn& nested_column = array_column.get_data();
- DCHECK(nested_column.is_nullable());
+ DORIS_CHECK(nested_column.is_nullable());
if (slice[0] != '[') {
return Status::InvalidArgument("Array does not start with '['
character, found '{}'",
slice[0]);
@@ -164,7 +164,7 @@ Status
DataTypeArraySerDe::deserialize_one_cell_from_hive_text(
auto& array_column = assert_cast<ColumnArray&>(column);
auto& offsets = array_column.get_offsets();
IColumn& nested_column = array_column.get_data();
- DCHECK(nested_column.is_nullable());
+ DORIS_CHECK(nested_column.is_nullable());
char collection_delimiter =
options.get_collection_delimiter(hive_text_complex_type_delimiter_level);
diff --git a/be/src/core/data_type_serde/data_type_array_serde.h
b/be/src/core/data_type_serde/data_type_array_serde.h
index a7a37f324d2..ceb97b7d57f 100644
--- a/be/src/core/data_type_serde/data_type_array_serde.h
+++ b/be/src/core/data_type_serde/data_type_array_serde.h
@@ -25,6 +25,7 @@
#include <utility>
#include "common/status.h"
+#include "core/data_type_serde/data_type_nullable_serde.h"
#include "core/data_type_serde/data_type_serde.h"
namespace doris {
@@ -38,7 +39,12 @@ class IDataType;
class DataTypeArraySerDe : public DataTypeSerDe {
public:
DataTypeArraySerDe(DataTypeSerDeSPtr _nested_serde, int nesting_level = 1)
- : DataTypeSerDe(nesting_level),
nested_serde(std::move(_nested_serde)) {}
+ : DataTypeSerDe(nesting_level) {
+ auto nullable_serde =
+
std::dynamic_pointer_cast<DataTypeNullableSerDe>(std::move(_nested_serde));
+ DORIS_CHECK(nullable_serde != nullptr);
+ nested_serde = std::move(nullable_serde);
+ }
std::string get_name() const override { return "Array(" +
nested_serde->get_name() + ")"; }
@@ -132,6 +138,6 @@ private:
template <bool is_strict_mode>
Status _from_string(StringRef& str, IColumn& column, const FormatOptions&
options) const;
- DataTypeSerDeSPtr nested_serde;
+ DataTypeNullableSerDeSPtr nested_serde;
};
} // namespace doris
diff --git a/be/src/core/data_type_serde/data_type_nullable_serde.h
b/be/src/core/data_type_serde/data_type_nullable_serde.h
index 4363e4a573b..31ad0e0c041 100644
--- a/be/src/core/data_type_serde/data_type_nullable_serde.h
+++ b/be/src/core/data_type_serde/data_type_nullable_serde.h
@@ -130,5 +130,8 @@ public:
private:
DataTypeSerDeSPtr nested_serde;
};
+
+using DataTypeNullableSerDeSPtr = std::shared_ptr<DataTypeNullableSerDe>;
+
#include "common/compile_check_end.h"
} // namespace doris
diff --git a/be/src/exprs/function/cast/cast_to_array.h
b/be/src/exprs/function/cast/cast_to_array.h
index add20080e66..cf3413ba94a 100644
--- a/be/src/exprs/function/cast/cast_to_array.h
+++ b/be/src/exprs/function/cast/cast_to_array.h
@@ -52,7 +52,7 @@ WrapperType create_array_wrapper(FunctionContext* context,
const DataTypePtr& fr
"CAST AS Array can only be performed between same-dimensional
array types");
}
- const DataTypePtr& to_nested_type = to_type.get_nested_type();
+ DataTypePtr to_nested_type = to_type.get_nested_type();
/// Prepare nested type conversion
const auto nested_function =
diff --git a/be/test/core/block/block_test.cpp
b/be/test/core/block/block_test.cpp
index 40a6d2ee99e..89f019ff06f 100644
--- a/be/test/core/block/block_test.cpp
+++ b/be/test/core/block/block_test.cpp
@@ -99,7 +99,8 @@ static void fill_block_with_array_int(Block& block) {
data_column->insert_data((const char*)(&v), 0);
}
- auto column_array_ptr = ColumnArray::create(std::move(data_column),
std::move(off_column));
+ auto column_array_ptr =
+ ColumnArray::create(make_nullable(std::move(data_column)),
std::move(off_column));
DataTypePtr nested_type(std::make_shared<DataTypeInt32>());
DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
ColumnWithTypeAndName test_array_int(std::move(column_array_ptr),
array_type, "test_array_int");
@@ -119,7 +120,8 @@ static void fill_block_with_array_string(Block& block) {
data_column->insert_data(v.data(), v.size());
}
- auto column_array_ptr = ColumnArray::create(std::move(data_column),
std::move(off_column));
+ auto column_array_ptr =
+ ColumnArray::create(make_nullable(std::move(data_column)),
std::move(off_column));
DataTypePtr nested_type(std::make_shared<DataTypeString>());
DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
ColumnWithTypeAndName test_array_string(std::move(column_array_ptr),
array_type,
diff --git a/be/test/core/data_type_serde/data_type_serde_get_name_test.cpp
b/be/test/core/data_type_serde/data_type_serde_get_name_test.cpp
index 11212b2bd09..bbeaf81ce87 100644
--- a/be/test/core/data_type_serde/data_type_serde_get_name_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_get_name_test.cpp
@@ -45,7 +45,7 @@ TEST(DataTypeSerDeGetNameTest, test) {
{
auto type =
std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt64>());
auto serde = type->get_serde();
- EXPECT_EQ(serde->get_name(), "Array(BIGINT)");
+ EXPECT_EQ(serde->get_name(), "Array(Nullable(BIGINT))");
}
{
@@ -114,7 +114,7 @@ TEST(DataTypeSerDeGetNameTest, test) {
auto serde = type->get_serde();
EXPECT_EQ(
serde->get_name(),
- R"(Struct(field1:String, field2:BIGINT, field3:DOUBLE,
field4:Array(INT), field5:Map(String, BIGINT), field6:Nullable(String),
field7:Nullable(BIGINT)))");
+ R"(Struct(field1:String, field2:BIGINT, field3:DOUBLE,
field4:Array(Nullable(INT)), field5:Map(String, BIGINT),
field6:Nullable(String), field7:Nullable(BIGINT)))");
}
}
diff --git a/be/test/core/data_type_serde/data_type_serde_string_test.cpp
b/be/test/core/data_type_serde/data_type_serde_string_test.cpp
index c7bb93a77d1..3708145e391 100644
--- a/be/test/core/data_type_serde/data_type_serde_string_test.cpp
+++ b/be/test/core/data_type_serde/data_type_serde_string_test.cpp
@@ -32,6 +32,7 @@
#include "core/assert_cast.h"
#include "core/column/column.h"
#include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
#include "core/data_type/common_data_type_serder_test.h"
#include "core/data_type/common_data_type_test.h"
#include "core/data_type/data_type.h"
@@ -308,9 +309,12 @@ TEST_F(DataTypeStringSerDeTest,
ArrowMemNotAlignedNestedArr) {
EXPECT_EQ(values_address % 4, 1);
// 5.Test read_column_from_arrow
- auto ser_col = ColumnArray::create(ColumnString::create(),
ColumnOffset64::create());
+ auto ser_col = ColumnArray::create(
+ ColumnNullable::create(ColumnString::create(),
ColumnUInt8::create()),
+ ColumnOffset64::create());
cctz::time_zone tz;
- auto serde_list = std::make_shared<DataTypeArraySerDe>(serde_str);
+ auto serde_nullable_str =
std::make_shared<DataTypeNullableSerDe>(serde_str);
+ auto serde_list = std::make_shared<DataTypeArraySerDe>(serde_nullable_str);
auto st = serde_list->read_column_from_arrow(*ser_col, arr.get(), 0, 1,
tz);
EXPECT_TRUE(st.ok());
}
diff --git a/be/test/core/jsonb/serialize_test.cpp
b/be/test/core/jsonb/serialize_test.cpp
index d576523548e..2419383b0ed 100644
--- a/be/test/core/jsonb/serialize_test.cpp
+++ b/be/test/core/jsonb/serialize_test.cpp
@@ -591,7 +591,8 @@ static void fill_block_with_array_int(Block& block) {
data_column->insert_data((const char*)(&v), 0);
}
- auto column_array_ptr = ColumnArray::create(std::move(data_column),
std::move(off_column));
+ auto column_array_ptr =
+ ColumnArray::create(make_nullable(std::move(data_column)),
std::move(off_column));
DataTypePtr nested_type(std::make_shared<DataTypeInt32>());
DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
ColumnWithTypeAndName test_array_int(std::move(column_array_ptr),
array_type, "test_array_int");
@@ -611,7 +612,8 @@ static void fill_block_with_array_string(Block& block) {
data_column->insert_data(v.data(), v.size());
}
- auto column_array_ptr = ColumnArray::create(std::move(data_column),
std::move(off_column));
+ auto column_array_ptr =
+ ColumnArray::create(make_nullable(std::move(data_column)),
std::move(off_column));
DataTypePtr nested_type(std::make_shared<DataTypeString>());
DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
ColumnWithTypeAndName test_array_string(std::move(column_array_ptr),
array_type,
diff --git a/be/test/data/vec/native/all_types_single_row.native
b/be/test/data/vec/native/all_types_single_row.native
index 742d1b7b30c..fe56a0a778b 100644
Binary files a/be/test/data/vec/native/all_types_single_row.native and
b/be/test/data/vec/native/all_types_single_row.native differ
diff --git a/be/test/exec/common/schema_util_test.cpp
b/be/test/exec/common/schema_util_test.cpp
index 2350bed1b76..c663cbc1a23 100644
--- a/be/test/exec/common/schema_util_test.cpp
+++ b/be/test/exec/common/schema_util_test.cpp
@@ -889,8 +889,8 @@ TEST_F(SchemaUtilTest, TestCastColumnWithExecuteFailure) {
auto simple_type = std::make_shared<DataTypeJsonb>();
// Insert some test dataset
- auto nested_array =
- ColumnArray::create(ColumnIPv4::create(),
ColumnArray::ColumnOffsets::create());
+ auto nested_array =
ColumnArray::create(make_nullable(ColumnIPv4::create()),
+
ColumnArray::ColumnOffsets::create());
nested_array->insert(Field::create_field<PrimitiveType::TYPE_ARRAY>(Array(IPv4(1))));
nested_array->insert(Field::create_field<PrimitiveType::TYPE_ARRAY>(Array(IPv4(2))));
diff --git a/be/test/exec/operator/table_function_operator_test.cpp
b/be/test/exec/operator/table_function_operator_test.cpp
index bd7b108b18a..24079a18e75 100644
--- a/be/test/exec/operator/table_function_operator_test.cpp
+++ b/be/test/exec/operator/table_function_operator_test.cpp
@@ -42,6 +42,11 @@
namespace doris {
+static MutableColumnPtr create_nullable_nested_array_column(MutableColumnPtr
nested,
+ MutableColumnPtr
offsets) {
+ return ColumnArray::create(make_nullable(std::move(nested)),
std::move(offsets));
+}
+
class MockTableFunctionChildOperator : public OperatorXBase {
public:
Status get_block_after_projects(RuntimeState* state, Block* block, bool*
eos) override {
@@ -372,7 +377,7 @@ TEST_F(TableFunctionOperatorTest, block_fast_path_explode) {
offsets->insert_value(3);
offsets->insert_value(3);
offsets->insert_value(4);
- auto arr_col = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_col = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
push_child_block(
Block({ColumnWithTypeAndName(id_col, int_type, "id"),
@@ -398,8 +403,8 @@ TEST_F(TableFunctionOperatorTest, block_fast_path_explode) {
expected_offsets->insert_value(3);
expected_offsets->insert_value(5);
expected_offsets->insert_value(6);
- auto expected_arr =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_out = ColumnHelper::create_column<DataTypeInt32>({1, 2,
3, 4});
@@ -431,7 +436,7 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_batch_truncate) {
offsets->insert_value(3);
offsets->insert_value(3);
offsets->insert_value(4);
- auto arr_col = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_col = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
push_child_block(
Block({ColumnWithTypeAndName(id_col, int_type, "id"),
@@ -451,8 +456,8 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_batch_truncate) {
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(1);
expected_offsets->insert_value(3);
- auto expected_arr =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_out = ColumnHelper::create_column<DataTypeInt32>({1, 2});
auto int_type = std::make_shared<DataTypeInt32>();
@@ -475,8 +480,8 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_batch_truncate) {
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(2);
expected_offsets->insert_value(3);
- auto expected_arr =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_out = ColumnHelper::create_column<DataTypeInt32>({3, 4});
auto int_type = std::make_shared<DataTypeInt32>();
@@ -503,7 +508,7 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_array_skip) {
offsets->insert_value(1);
offsets->insert_value(1);
offsets->insert_value(2);
- auto arr_col = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_col = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
push_child_block(
Block({ColumnWithTypeAndName(id_col, int_type, "id"),
@@ -523,8 +528,8 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_array_skip) {
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(1);
expected_offsets->insert_value(2);
- auto expected_arr =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_out = ColumnHelper::create_column<DataTypeInt32>({1, 2});
@@ -552,7 +557,7 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_array_null_ro
offsets->insert_value(1);
offsets->insert_value(1);
offsets->insert_value(2);
- auto arr_data = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_data = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
auto null_map = ColumnUInt8::create();
null_map->insert_value(0);
null_map->insert_value(1);
@@ -576,8 +581,8 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_array_null_ro
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(1);
expected_offsets->insert_value(2);
- auto expected_arr_data =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr_data =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_arr_null = ColumnUInt8::create();
expected_arr_null->insert_value(0);
expected_arr_null->insert_value(0);
@@ -611,7 +616,7 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_array_misalig
offsets->insert_value(1);
offsets->insert_value(2);
offsets->insert_value(3);
- auto arr_data = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_data = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
auto null_map = ColumnUInt8::create();
null_map->insert_value(0);
null_map->insert_value(1);
@@ -635,8 +640,8 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_array_misalig
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(1);
expected_offsets->insert_value(2);
- auto expected_arr_data =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr_data =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_arr_null = ColumnUInt8::create();
expected_arr_null->insert_value(0);
expected_arr_null->insert_value(0);
@@ -674,7 +679,7 @@ TEST_F(TableFunctionOperatorTest,
offsets->insert_value(3);
offsets->insert_value(4);
offsets->insert_value(5);
- auto arr_data = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_data = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
auto null_map = ColumnUInt8::create();
null_map->insert_value(0);
null_map->insert_value(1);
@@ -702,8 +707,8 @@ TEST_F(TableFunctionOperatorTest,
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(3);
expected_offsets->insert_value(6);
- auto expected_arr_data =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr_data =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_arr_null = ColumnUInt8::create();
expected_arr_null->insert_value(0);
expected_arr_null->insert_value(0);
@@ -730,8 +735,8 @@ TEST_F(TableFunctionOperatorTest,
auto expected_offsets = ColumnArray::ColumnOffsets::create();
expected_offsets->insert_value(3);
expected_offsets->insert_value(4);
- auto expected_arr_data =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr_data =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_arr_null = ColumnUInt8::create();
expected_arr_null->insert_value(0);
expected_arr_null->insert_value(0);
@@ -768,7 +773,7 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_elements) {
auto offsets = ColumnArray::ColumnOffsets::create();
offsets->insert_value(3);
- auto arr_col = ColumnArray::create(std::move(nested),
std::move(offsets));
+ auto arr_col = create_nullable_nested_array_column(std::move(nested),
std::move(offsets));
push_child_block(
Block({ColumnWithTypeAndName(id_col, int_type, "id"),
@@ -808,8 +813,8 @@ TEST_F(TableFunctionOperatorTest,
block_fast_path_explode_nullable_elements) {
expected_offsets->insert_value(3);
expected_offsets->insert_value(6);
expected_offsets->insert_value(9);
- auto expected_arr =
- ColumnArray::create(std::move(expected_nested),
std::move(expected_offsets));
+ auto expected_arr =
create_nullable_nested_array_column(std::move(expected_nested),
+
std::move(expected_offsets));
auto expected_out_data = ColumnInt32::create();
expected_out_data->insert_value(1);
diff --git a/be/test/exprs/aggregate/agg_array_agg_test.cpp
b/be/test/exprs/aggregate/agg_array_agg_test.cpp
index 101328496df..97b6d99456b 100644
--- a/be/test/exprs/aggregate/agg_array_agg_test.cpp
+++ b/be/test/exprs/aggregate/agg_array_agg_test.cpp
@@ -122,74 +122,4 @@ TEST_F(AggregateFunctionArrayAggTest,
test_array_agg_astr_nullable) {
ColumnWithTypeAndName(std::move(array_column), array_data_type,
"column"));
}
-TEST_F(AggregateFunctionArrayAggTest, test_array_agg_astr_foreach) {
- auto data_type = make_nullable(std::make_shared<DataTypeString>());
- auto array_data_type = std::make_shared<DataTypeArray>(data_type);
- create_agg("array_agg_foreach", false, {array_data_type}, array_data_type);
-
- auto off_column = ColumnOffset64::create();
- auto data_column = data_type->create_column();
- std::vector<ColumnArray::Offset64> offs = {0, 4};
- std::vector<int64_t> vals = {1, 2, 3};
- for (size_t i = 1; i < offs.size(); ++i) {
- off_column->insert_data((const char*)(&offs[i]), 0);
- }
- data_column->insert_default();
- for (auto& v : vals) {
- data_column->insert_data((const char*)(&v), sizeof(v));
- }
- auto array_column = ColumnArray::create(data_column->clone(),
off_column->clone());
-
- auto off_column2 = ColumnOffset64::create();
- std::vector<ColumnArray::Offset64> offs2 = {0, 1, 2, 3, 4};
- for (size_t i = 1; i < offs2.size(); ++i) {
- off_column2->insert_data((const char*)(&offs2[i]), 0);
- }
-
- auto array_array_data_type =
std::make_shared<DataTypeArray>(array_data_type);
- auto array_array_off_column = ColumnOffset64::create();
- array_array_off_column->insert_value(4);
- auto array_array_column =
- ColumnArray::create(ColumnArray::create(data_column->clone(),
off_column2->clone()),
- array_array_off_column->clone());
-
- execute(Block({ColumnWithTypeAndName(array_column->clone(),
array_data_type, "")}),
- ColumnWithTypeAndName(std::move(array_array_column),
array_array_data_type, "column"));
-}
-
-TEST_F(AggregateFunctionArrayAggTest, test_array_agg_aint64_foreach) {
- auto data_type = make_nullable(std::make_shared<DataTypeInt64>());
- auto array_data_type = std::make_shared<DataTypeArray>(data_type);
- create_agg("array_agg_foreach", false, {array_data_type}, array_data_type);
-
- auto off_column = ColumnOffset64::create();
- auto data_column = data_type->create_column();
- std::vector<ColumnArray::Offset64> offs = {0, 4};
- std::vector<int64_t> vals = {1, 2, 3};
- for (size_t i = 1; i < offs.size(); ++i) {
- off_column->insert_data((const char*)(&offs[i]), 0);
- }
- data_column->insert_default();
- for (auto& v : vals) {
- data_column->insert_data((const char*)(&v), sizeof(v));
- }
- auto array_column = ColumnArray::create(data_column->clone(),
off_column->clone());
-
- auto off_column2 = ColumnOffset64::create();
- std::vector<ColumnArray::Offset64> offs2 = {0, 1, 2, 3, 4};
- for (size_t i = 1; i < offs2.size(); ++i) {
- off_column2->insert_data((const char*)(&offs2[i]), 0);
- }
-
- auto array_array_data_type =
std::make_shared<DataTypeArray>(array_data_type);
- auto array_array_off_column = ColumnOffset64::create();
- array_array_off_column->insert_value(4);
- auto array_array_column =
- ColumnArray::create(ColumnArray::create(data_column->clone(),
off_column2->clone()),
- array_array_off_column->clone());
-
- execute(Block({ColumnWithTypeAndName(array_column->clone(),
array_data_type, "")}),
- ColumnWithTypeAndName(std::move(array_array_column),
array_array_data_type, "column"));
-}
-
} // namespace doris
diff --git a/be/test/exprs/aggregate/agg_replace_test.cpp
b/be/test/exprs/aggregate/agg_replace_test.cpp
index a0e90ede1cf..bb00051cbee 100644
--- a/be/test/exprs/aggregate/agg_replace_test.cpp
+++ b/be/test/exprs/aggregate/agg_replace_test.cpp
@@ -123,7 +123,11 @@ public:
auto* data_col = array_col->get_data_ptr().get();
EXPECT_EQ(data_col->size(), expect_num);
for (size_t i = 0; i < expect_num; ++i) {
- check_column_basic<DataType, nullable>(data_col, i, i);
+ if (data_col->is_nullable()) {
+ check_column_basic<DataType, true>(data_col, i, i);
+ } else {
+ check_column_basic<DataType, false>(data_col, i, i);
+ }
}
}
diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp
b/be/test/format/orc/orc_reader_fill_data_test.cpp
index 12c1dd209c5..eab2b97e38a 100644
--- a/be/test/format/orc/orc_reader_fill_data_test.cpp
+++ b/be/test/format/orc/orc_reader_fill_data_test.cpp
@@ -187,30 +187,30 @@ TEST_F(OrcReaderFillDataTest, ComplexTypeConversionTest) {
std::cout << block.dump_data() << "\n";
ASSERT_EQ(block.dump_data(),
- "+---------------------------+\n"
- "|cc(Struct(col1:Array(INT)))|\n"
- "+---------------------------+\n"
- "| {\"col1\":[0]}|\n"
- "| {\"col1\":[1, 1]}|\n"
- "| {\"col1\":[2, 2, 2]}|\n"
- "| {\"col1\":[3, 3, 3, 3]}|\n"
- "| {\"col1\":[4, 4, 4, 4, 4]}|\n"
- "| {\"col1\":[5]}|\n"
- "| {\"col1\":[6, 6]}|\n"
- "| {\"col1\":[7, 7, 7]}|\n"
- "| {\"col1\":[8, 8, 8, 8]}|\n"
- "| {\"col1\":[9, 9, 9, 9, 9]}|\n"
- "| {\"col1\":[10]}|\n"
- "| {\"col1\":[11, 11]}|\n"
- "| {\"col1\":[12, 12, 12]}|\n"
- "| {\"col1\":[13, 13, 13, 13]}|\n"
- "|{\"col1\":[14, 14, 14, 14,...|\n"
- "| {\"col1\":[15]}|\n"
- "| {\"col1\":[16, 16]}|\n"
- "| {\"col1\":[17, 17, 17]}|\n"
- "| {\"col1\":[18, 18, 18, 18]}|\n"
- "|{\"col1\":[19, 19, 19, 19,...|\n"
- "+---------------------------+\n");
+ "+-------------------------------------+\n"
+ "|cc(Struct(col1:Array(Nullable(INT))))|\n"
+ "+-------------------------------------+\n"
+ "| {\"col1\":[0]}|\n"
+ "| {\"col1\":[1, 1]}|\n"
+ "| {\"col1\":[2, 2, 2]}|\n"
+ "| {\"col1\":[3, 3, 3, 3]}|\n"
+ "| {\"col1\":[4, 4, 4, 4, 4]}|\n"
+ "| {\"col1\":[5]}|\n"
+ "| {\"col1\":[6, 6]}|\n"
+ "| {\"col1\":[7, 7, 7]}|\n"
+ "| {\"col1\":[8, 8, 8, 8]}|\n"
+ "| {\"col1\":[9, 9, 9, 9, 9]}|\n"
+ "| {\"col1\":[10]}|\n"
+ "| {\"col1\":[11, 11]}|\n"
+ "| {\"col1\":[12, 12, 12]}|\n"
+ "| {\"col1\":[13, 13, 13, 13]}|\n"
+ "| {\"col1\":[14, 14, 14, 14, 14]}|\n"
+ "| {\"col1\":[15]}|\n"
+ "| {\"col1\":[16, 16]}|\n"
+ "| {\"col1\":[17, 17, 17]}|\n"
+ "| {\"col1\":[18, 18, 18, 18]}|\n"
+ "| {\"col1\":[19, 19, 19, 19, 19]}|\n"
+ "+-------------------------------------+\n");
}
{
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
index fb53269d630..3c66f6dd24a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -31,6 +31,11 @@ import java.util.Objects;
/**
* Describes an ARRAY type.
+ *
+ * <p>Array elements are always nullable in Doris. The {@code containsNull}
field is retained
+ * only for backward compatibility with serialized metadata (Gson). It is
always treated as
+ * {@code true} at runtime. The two-argument constructor is kept but
deprecated; prefer the
+ * single-argument constructor {@link #ArrayType(Type)} which defaults to
nullable elements.</p>
*/
public class ArrayType extends Type {
@@ -39,6 +44,8 @@ public class ArrayType extends Type {
@SerializedName(value = "itemType")
private final Type itemType;
+ // Retained for Gson deserialization compatibility with older metadata.
+ // Always treated as true at runtime — array elements are always nullable.
@SerializedName(value = "containsNull")
private final boolean containsNull;
@@ -48,20 +55,31 @@ public class ArrayType extends Type {
}
public ArrayType(Type itemType) {
- this(itemType, true);
+ this.itemType = itemType;
+ this.containsNull = true;
}
+ /**
+ * @deprecated Array elements are always nullable. Use {@link
#ArrayType(Type)} instead.
+ * This constructor is retained only for call-site
compatibility during transition.
+ */
+ @Deprecated
public ArrayType(Type itemType, boolean containsNull) {
this.itemType = itemType;
- this.containsNull = containsNull;
+ // Ignore the parameter — always true
+ this.containsNull = true;
}
public Type getItemType() {
return itemType;
}
+ /**
+ * Always returns {@code true}. Array elements are always nullable in
Doris.
+ */
public boolean getContainsNull() {
- return containsNull;
+ // Always true — array elements are always nullable
+ return true;
}
@Override
@@ -83,10 +101,7 @@ public class ArrayType extends Type {
return false;
}
- if (((ArrayType) t).getContainsNull() != getContainsNull()) {
- return false;
- }
-
+ // containsNull is always true, no need to compare
return itemType.matchesType(((ArrayType) t).itemType);
}
@@ -94,24 +109,27 @@ public class ArrayType extends Type {
return new ArrayType();
}
+ public static ArrayType create(Type type) {
+ return new ArrayType(type);
+ }
+
+ /**
+ * @deprecated Array elements are always nullable. Use {@link
#create(Type)} instead.
+ */
+ @Deprecated
public static ArrayType create(Type type, boolean containsNull) {
- return new ArrayType(type, containsNull);
+ return new ArrayType(type);
}
@Override
public String toSql(int depth) {
- StringBuilder typeStr = new StringBuilder();
- typeStr.append("array<").append(itemType.toSql(depth + 1));
- if (!containsNull) {
- typeStr.append(" not null");
- }
- typeStr.append(">");
- return typeStr.toString();
+ // Array elements are always nullable, no "not null" suffix needed
+ return "array<" + itemType.toSql(depth + 1) + ">";
}
@Override
public int hashCode() {
- return Objects.hash(itemType, containsNull);
+ return Objects.hash(itemType);
}
@Override
@@ -120,7 +138,7 @@ public class ArrayType extends Type {
return false;
}
ArrayType otherArrayType = (ArrayType) other;
- return otherArrayType.itemType.equals(itemType) &&
otherArrayType.containsNull == containsNull;
+ return otherArrayType.itemType.equals(itemType);
}
@Override
@@ -129,8 +147,9 @@ public class ArrayType extends Type {
container.types.add(node);
Preconditions.checkNotNull(itemType);
node.setType(TTypeNodeType.ARRAY);
- node.setContainsNull(containsNull);
- node.setContainsNulls(Lists.newArrayList(containsNull));
+ // Array elements are always nullable
+ node.setContainsNull(true);
+ node.setContainsNulls(Lists.newArrayList(true));
itemType.toThrift(container);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 53035261347..a157819e341 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -5239,7 +5239,7 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
return ParserUtils.withOrigin(ctx, () -> {
switch (ctx.complex.getType()) {
case DorisParser.ARRAY:
- return ArrayType.of(typedVisit(ctx.dataType(0)), true);
+ return ArrayType.of(typedVisit(ctx.dataType(0)));
case DorisParser.MAP:
return MapType.of(typedVisit(ctx.dataType(0)),
typedVisit(ctx.dataType(1)));
case DorisParser.STRUCT:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
index 4568e641646..331d70aac8e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -628,13 +628,13 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
private static Pair<DataType, Integer>
convertToNereidsType(List<PTypeNode> typeNodes, int start) {
PScalarType pScalarType = typeNodes.get(start).getScalarType();
- boolean containsNull = typeNodes.get(start).getContainsNull();
TPrimitiveType tPrimitiveType =
TPrimitiveType.findByValue(pScalarType.getType());
DataType type;
int parsedNodes;
if (tPrimitiveType == TPrimitiveType.ARRAY) {
Pair<DataType, Integer> itemType = convertToNereidsType(typeNodes,
start + 1);
- type = ArrayType.of(itemType.key(), containsNull);
+ // Array elements are always nullable
+ type = ArrayType.of(itemType.key());
parsedNodes = 1 + itemType.value();
} else if (tPrimitiveType == TPrimitiveType.MAP) {
Pair<DataType, Integer> keyType = convertToNereidsType(typeNodes,
start + 1);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
index 4bd08510741..22344210a1a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
@@ -344,8 +344,7 @@ public class NestedColumnPruning implements CustomRewriter {
children.values().iterator().next().pruneCastType(
origin.children.values().iterator().next(),
cast.children.values().iterator().next()
- ),
- ((ArrayType) cast.type).containsNull()
+ )
);
} else if (type instanceof MapType) {
return MapType.of(
@@ -535,7 +534,7 @@ public class NestedColumnPruning implements CustomRewriter {
}
return new StructType(newFields);
} else if (dataType instanceof ArrayType) {
- return ArrayType.of(newChildrenTypes.get(0).second,
((ArrayType) dataType).containsNull());
+ return ArrayType.of(newChildrenTypes.get(0).second);
} else if (dataType instanceof MapType) {
return MapType.of(newChildrenTypes.get(0).second,
newChildrenTypes.get(1).second);
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
index b763d6741ab..bfbb184ec30 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ArrayItemReference.java
@@ -77,7 +77,8 @@ public class ArrayItemReference extends NamedExpression
implements ExpectsInputT
@Override
public boolean nullable() {
- return ((ArrayType)
(this.children.get(0).getDataType())).containsNull();
+ // Array elements are always nullable
+ return true;
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java
index bf7ba61039d..436e8aafabd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/AggCombinerFunctionBuilder.java
@@ -95,7 +95,8 @@ public class AggCombinerFunctionBuilder extends
FunctionBuilder {
"foreach must be input array type: '" + nestedName);
}
DataType itemType = ((ArrayType) arrayType).getItemType();
- return new SlotReference("mocked", itemType, (((ArrayType)
arrayType).containsNull()));
+ // Array elements are always nullable
+ return new SlotReference("mocked", itemType, true);
}).collect(Collectors.toList());
return (AggregateFunction) nestedBuilder.build(nestedName,
forEachargs).first;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java
index c8629fb7b84..0d0abe92789 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelper.java
@@ -483,7 +483,7 @@ public class ComputeSignatureHelper {
return signature;
}
ArrayType arrayType = (ArrayType) signature.returnType;
- return signature.withReturnType(ArrayType.of(arrayType.getItemType(),
true));
+ return signature.withReturnType(ArrayType.of(arrayType.getItemType()));
}
// for time type with precision(now are DateTimeV2Type and TimeV2Type),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java
index be52d59b62c..3c5fce06159 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/combinator/ForEachCombinator.java
@@ -119,7 +119,7 @@ public class ForEachCombinator extends
NullableAggregateFunction
@Override
public DataType getDataType() {
- return ArrayType.of(nested.getDataType(), true);
+ return ArrayType.of(nested.getDataType());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
index 1a1a246cea6..1ef0e996d0d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
@@ -68,7 +68,7 @@ public class ArrayMap extends ScalarFunction
public DataType getDataType() {
Preconditions.checkArgument(children.get(0) instanceof Lambda,
"The first arg of array_map must be lambda");
- return ArrayType.of(((Lambda) children.get(0)).getRetType(), true);
+ return ArrayType.of(((Lambda) children.get(0)).getRetType());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java
index 983d3038152..3a905016da0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArraySort.java
@@ -90,11 +90,11 @@ public class ArraySort extends ScalarFunction
ArrayItemReference argRef = lambda.getLambdaArguments().get(0);
Expression arrayExpr = argRef.getArrayExpression();
ArrayType arrayType = (ArrayType) arrayExpr.getDataType();
- return ArrayType.of(arrayType.getItemType(), true);
+ return ArrayType.of(arrayType.getItemType());
} else if (children.get(0).getDataType() instanceof ArrayType) {
Expression arrayExpr = children.get(0);
ArrayType arrayType = (ArrayType) arrayExpr.getDataType();
- return ArrayType.of(arrayType.getItemType(), true);
+ return ArrayType.of(arrayType.getItemType());
} else {
throw new AnalysisException("The first arg of array_sort must be
lambda or array");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
index c189154fb42..015690e6434 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/ArrayType.java
@@ -25,45 +25,39 @@ import java.util.Objects;
/**
* Array type in Nereids.
+ *
+ * <p>Note: Array elements are always nullable in Doris. The NOT NULL
constraint on array elements
+ * (e.g., ARRAY<INT NOT NULL>) is not supported. This simplification
aligns with the actual
+ * behavior where both FE planning and BE execution always treat array
elements as nullable.</p>
*/
public class ArrayType extends DataType implements ComplexDataType,
NestedColumnPrunable {
- public static final ArrayType SYSTEM_DEFAULT = new
ArrayType(NullType.INSTANCE, true);
+ public static final ArrayType SYSTEM_DEFAULT = new
ArrayType(NullType.INSTANCE);
public static final int WIDTH = 64;
private final DataType itemType;
- private final boolean containsNull;
- private ArrayType(DataType itemType, boolean containsNull) {
+ private ArrayType(DataType itemType) {
this.itemType = Objects.requireNonNull(itemType, "itemType can not be
null");
- this.containsNull = containsNull;
}
public static ArrayType of(DataType itemType) {
- return of(itemType, true);
- }
-
- public static ArrayType of(DataType itemType, boolean containsNull) {
if (itemType.equals(NullType.INSTANCE)) {
return SYSTEM_DEFAULT;
}
- return new ArrayType(itemType, containsNull);
+ return new ArrayType(itemType);
}
@Override
public DataType conversion() {
- return new ArrayType(itemType.conversion(), containsNull);
+ return new ArrayType(itemType.conversion());
}
public DataType getItemType() {
return itemType;
}
- public boolean containsNull() {
- return containsNull;
- }
-
@Override
public boolean isInjectiveCastTo(DataType target) {
if (target instanceof ArrayType) {
@@ -74,7 +68,8 @@ public class ArrayType extends DataType implements
ComplexDataType, NestedColumn
@Override
public Type toCatalogDataType() {
- return new
org.apache.doris.catalog.ArrayType(itemType.toCatalogDataType(), containsNull);
+ // Catalog ArrayType defaults containsNull to true via single-arg
constructor
+ return new
org.apache.doris.catalog.ArrayType(itemType.toCatalogDataType());
}
@Override
@@ -94,8 +89,7 @@ public class ArrayType extends DataType implements
ComplexDataType, NestedColumn
return false;
}
ArrayType arrayType = (ArrayType) o;
- return Objects.equals(itemType, arrayType.itemType)
- && Objects.equals(containsNull, arrayType.containsNull);
+ return Objects.equals(itemType, arrayType.itemType);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 4e48e5aa484..a2adc9a9fcc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -475,7 +475,7 @@ public abstract class DataType {
return MapType.of(fromCatalogType(mapType.getKeyType()),
fromCatalogType(mapType.getValueType()));
} else if (type.isArrayType()) {
org.apache.doris.catalog.ArrayType arrayType =
(org.apache.doris.catalog.ArrayType) type;
- return ArrayType.of(fromCatalogType(arrayType.getItemType()),
arrayType.getContainsNull());
+ return ArrayType.of(fromCatalogType(arrayType.getItemType()));
} else if (type.isVariantType()) {
// In the past, variant metadata used the ScalarType type.
// Now, we use VariantType, which inherits from ScalarType, as the
new metadata storage.
@@ -802,7 +802,7 @@ public abstract class DataType {
return arrayType.getItemType()
.getAllPromotions()
.stream()
- .map(promotionType -> ArrayType.of(promotionType,
arrayType.containsNull()))
+ .map(promotionType -> ArrayType.of(promotionType))
.collect(ImmutableList.toImmutableList());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
index 4ed05adab2b..9049a2065d6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
@@ -389,8 +389,7 @@ public class TypeCoercionUtils {
public static DataType replaceSpecifiedType(DataType dataType,
Class<? extends DataType> specifiedType, DataType newType) {
if (dataType instanceof ArrayType) {
- return ArrayType.of(replaceSpecifiedType(((ArrayType)
dataType).getItemType(), specifiedType, newType),
- ((ArrayType) dataType).containsNull());
+ return ArrayType.of(replaceSpecifiedType(((ArrayType)
dataType).getItemType(), specifiedType, newType));
} else if (dataType instanceof MapType) {
return MapType.of(replaceSpecifiedType(((MapType)
dataType).getKeyType(), specifiedType, newType),
replaceSpecifiedType(((MapType) dataType).getValueType(),
specifiedType, newType));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
index cb544d3d01c..44e973e851e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TypeTest.java
@@ -35,9 +35,9 @@ public class TypeTest {
ArrayType a3 = new ArrayType(new ArrayType(Type.BIGINT, true), true);
Assert.assertFalse(Type.matchExactType(a1, a3, false));
- // containsNull differs -> matchesType fails
+ // containsNull is always true now, so a4 is equivalent to a1
ArrayType a4 = new ArrayType(new ArrayType(Type.INT, true), false);
- Assert.assertFalse(Type.matchExactType(a1, a4, false));
+ Assert.assertTrue(Type.matchExactType(a1, a4, false));
// array nested decimal test
ArrayType a5 = new ArrayType(new
ArrayType(ScalarType.createDecimalV3Type(8, 2), true), true);
@@ -63,7 +63,7 @@ public class TypeTest {
Assert.assertFalse(Type.matchExactType(m1, m3, false));
Assert.assertFalse(Type.matchExactType(m1, m3, true));
- // key/value containsNull differs -> doesn't matter for matching
+ // key/value containsNull differs, but MapType.equals() ignores it ->
matches
MapType m4 = new MapType(Type.INT, arrayOfD, false, true);
Assert.assertTrue(Type.matchExactType(m1, m4, false));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
index fcb64ff0bfa..4964414e6b3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/LiteralTest.java
@@ -64,7 +64,7 @@ class LiteralTest {
for (int i = 0; i < elementsArray.length; ++i) {
elementsArray[i] = i;
}
- DataType arrayType = ArrayType.of(IntegerType.INSTANCE, true);
+ DataType arrayType = ArrayType.of(IntegerType.INSTANCE);
PGenericType.Builder childTypeBuilder = PGenericType.newBuilder();
childTypeBuilder.setId(TypeId.INT32);
PGenericType.Builder typeBuilder = PGenericType.newBuilder();
@@ -94,8 +94,8 @@ class LiteralTest {
for (int i = 0; i < elementsArray.length; ++i) {
elementsArray[i] = i;
}
- DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE, true);
- DataType outArrayType = ArrayType.of(nestedArrayType, true);
+ DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE);
+ DataType outArrayType = ArrayType.of(nestedArrayType);
PGenericType.Builder childTypeBuilder = PGenericType.newBuilder();
childTypeBuilder.setId(TypeId.INT32);
PGenericType.Builder typeBuilder = PGenericType.newBuilder();
@@ -134,8 +134,8 @@ class LiteralTest {
for (int i = 0; i < elementsArray.length; ++i) {
elementsArray[i] = i;
}
- DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE, true);
- DataType outArrayType = ArrayType.of(nestedArrayType, true);
+ DataType nestedArrayType = ArrayType.of(IntegerType.INSTANCE);
+ DataType outArrayType = ArrayType.of(nestedArrayType);
PGenericType.Builder childTypeBuilder = PGenericType.newBuilder();
childTypeBuilder.setId(TypeId.INT32);
PGenericType.Builder typeBuilder = PGenericType.newBuilder();
@@ -182,7 +182,7 @@ class LiteralTest {
elementsArray[i] = i;
nullMap[i] = (i % 2 == 1);
}
- DataType arrayType = ArrayType.of(IntegerType.INSTANCE, true);
+ DataType arrayType = ArrayType.of(IntegerType.INSTANCE);
PGenericType.Builder childTypeBuilder = PGenericType.newBuilder();
childTypeBuilder.setId(TypeId.INT32);
PGenericType.Builder typeBuilder = PGenericType.newBuilder();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
