This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch array-type
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 28ea295ec8318c88258e2644df8d8701c2bd4180
Author: Adonis Ling <adonis0...@gmail.com>
AuthorDate: Fri Feb 18 10:59:57 2022 +0800
[feature-wip][array-type] Support ArrayLiteral in SQL. (#8089)
Please refer to #8074
---
be/src/olap/column_vector.cpp | 21 ++--
be/src/olap/rowset/segment_v2/column_reader.cpp | 8 +-
be/src/olap/rowset/segment_v2/column_reader.h | 13 +++
be/src/olap/rowset/segment_v2/column_writer.cpp | 10 +-
be/src/olap/rowset/segment_v2/column_writer.h | 5 +-
be/src/runtime/collection_value.cpp | 16 ++-
be/src/runtime/mysql_result_writer.cpp | 6 +-
be/src/runtime/raw_value.cpp | 1 +
be/src/runtime/row_batch.cpp | 10 +-
be/src/runtime/tuple.cpp | 29 +++---
.../segment_v2/column_reader_writer_test.cpp | 24 +++++
fe/fe-core/src/main/cup/sql_parser.cup | 14 +++
.../org/apache/doris/analysis/ArrayLiteral.java | 28 +++--
.../org/apache/doris/analysis/CreateTableStmt.java | 10 +-
.../main/java/org/apache/doris/analysis/Expr.java | 3 +-
.../java/org/apache/doris/catalog/ArrayType.java | 4 +
.../main/java/org/apache/doris/catalog/Column.java | 13 ++-
.../main/java/org/apache/doris/catalog/Type.java | 4 +-
.../org/apache/doris/analysis/ColumnDefTest.java | 18 +++-
.../apache/doris/analysis/InsertArrayStmtTest.java | 114 +++++++++++++++++++++
.../org/apache/doris/utframe/UtFrameUtils.java | 9 ++
21 files changed, 306 insertions(+), 54 deletions(-)
diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp
index 3340628..0237b62 100644
--- a/be/src/olap/column_vector.cpp
+++ b/be/src/olap/column_vector.cpp
@@ -210,14 +210,19 @@ void ArrayColumnVectorBatch::prepare_for_read(size_t
start_idx, size_t size, boo
DCHECK(start_idx + size <= capacity());
for (size_t i = 0; i < size; ++i) {
if (!is_null_at(start_idx + i)) {
- _data[start_idx + i] = CollectionValue(
-
_elements->mutable_cell_ptr(*(_offsets->scalar_cell_ptr(start_idx + i))),
- *(_offsets->scalar_cell_ptr(start_idx + i + 1)) -
- *(_offsets->scalar_cell_ptr(start_idx + i)),
- item_has_null,
- _elements->is_nullable() ?
const_cast<bool*>(&_elements->null_signs()[*(
-
_offsets->scalar_cell_ptr(start_idx + i))])
- : nullptr);
+ auto next_offset = *(_offsets->scalar_cell_ptr(start_idx + i + 1));
+ auto offset = *(_offsets->scalar_cell_ptr(start_idx + i));
+ uint32_t length = next_offset - offset;
+ if (length == 0) {
+ _data[start_idx + i] = CollectionValue(length);
+ } else {
+ _data[start_idx + i] = CollectionValue(
+ _elements->mutable_cell_ptr(offset),
+ length,
+ item_has_null,
+ _elements->is_nullable() ?
const_cast<bool*>(&_elements->null_signs()[offset])
+ : nullptr);
+ }
}
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 20d2918..9530d80 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -127,7 +127,7 @@ Status ColumnReader::init() {
"Bad file $0: invalid column index type $1",
_path_desc.filepath, index_meta.type()));
}
}
- if (_ordinal_index_meta == nullptr) {
+ if (!is_empty() && _ordinal_index_meta == nullptr) {
return Status::Corruption(strings::Substitute(
"Bad file $0: missing ordinal index for column $1",
_path_desc.filepath, _meta.column_id()));
}
@@ -339,6 +339,10 @@ Status ColumnReader::seek_at_or_before(ordinal_t ordinal,
OrdinalPageIndexIterat
}
Status ColumnReader::new_iterator(ColumnIterator** iterator) {
+ if (is_empty()) {
+ *iterator = new EmptyFileColumnIterator();
+ return Status::OK();
+ }
if (is_scalar_type((FieldType)_meta.type())) {
*iterator = new FileColumnIterator(this);
return Status::OK();
@@ -427,7 +431,7 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
ColumnBlockView* dst, bool
// read item
size_t item_size = array_batch->get_item_size(dst->current_offset(), *n);
- if (item_size > 0) {
+ if (item_size >= 0) {
bool item_has_null = false;
ColumnVectorBatch* item_vector_batch = array_batch->elements();
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h
b/be/src/olap/rowset/segment_v2/column_reader.h
index 0c20b75..2f80b65 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -131,6 +131,8 @@ public:
PagePointer get_dict_page_pointer() const { return _meta.dict_page(); }
+ inline bool is_empty() const { return _num_rows == 0; }
+
private:
ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta,
uint64_t num_rows,
FilePathDesc path_desc);
@@ -306,6 +308,17 @@ private:
std::unique_ptr<StringRef[]> _dict_word_info;
};
+class EmptyFileColumnIterator final : public ColumnIterator {
+public:
+ Status seek_to_first() override { return Status::OK(); }
+ Status seek_to_ordinal(ordinal_t ord) override { return Status::OK(); }
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null)
override {
+ *n = 0;
+ return Status::OK();
+ }
+ ordinal_t get_current_ordinal() const override { return 0; }
+};
+
class ArrayFileColumnIterator final : public ColumnIterator {
public:
explicit ArrayFileColumnIterator(ColumnReader* reader, FileColumnIterator*
length_reader,
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp
b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 31837d0..78327be 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -524,8 +524,8 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr,
size_t num_rows) {
if (num_written <
1) { // page is full, write first item offset and update current
length page's start ordinal
RETURN_IF_ERROR(_length_writer->finish_current_page());
- _current_length_page_first_ordinal += _lengh_sum_in_cur_page;
- _lengh_sum_in_cur_page = 0;
+ _current_length_page_first_ordinal += _length_sum_in_cur_page;
+ _length_sum_in_cur_page = 0;
} else {
// write child item.
if (_item_writer->is_nullable()) {
@@ -539,7 +539,7 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr,
size_t num_rows) {
RETURN_IF_ERROR(_item_writer->append_data(reinterpret_cast<const
uint8_t**>(&data),
col_cursor->length()));
}
- _lengh_sum_in_cur_page += col_cursor->length();
+ _length_sum_in_cur_page += col_cursor->length();
}
remaining -= num_written;
col_cursor += num_written;
@@ -579,7 +579,9 @@ Status ArrayColumnWriter::write_ordinal_index() {
if (is_nullable()) {
RETURN_IF_ERROR(_null_writer->write_ordinal_index());
}
- RETURN_IF_ERROR(_item_writer->write_ordinal_index());
+ if (!has_empty_items()) {
+ RETURN_IF_ERROR(_item_writer->write_ordinal_index());
+ }
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h
b/be/src/olap/rowset/segment_v2/column_writer.h
index b98f488..26c9731 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -307,6 +307,9 @@ public:
private:
Status put_extra_info_in_page(DataPageFooterPB* header) override;
inline Status write_null_column(size_t num_rows, bool is_null); //
写入num_rows个null标记
+ inline bool has_empty_items() const {
+ return _item_writer->get_next_rowid() == 0;
+ }
private:
std::unique_ptr<ScalarColumnWriter> _length_writer;
@@ -314,7 +317,7 @@ private:
std::unique_ptr<ColumnWriter> _item_writer;
ColumnWriterOptions _opts;
ordinal_t _current_length_page_first_ordinal = 0;
- ordinal_t _lengh_sum_in_cur_page = 0;
+ ordinal_t _length_sum_in_cur_page = 0;
};
} // namespace segment_v2
diff --git a/be/src/runtime/collection_value.cpp
b/be/src/runtime/collection_value.cpp
index 9568f9b..9b7ea7d 100644
--- a/be/src/runtime/collection_value.cpp
+++ b/be/src/runtime/collection_value.cpp
@@ -23,6 +23,10 @@
namespace doris {
int sizeof_type(PrimitiveType type) {
switch (type) {
+ case TYPE_TINYINT:
+ return sizeof(int8_t);
+ case TYPE_SMALLINT:
+ return sizeof(int16_t);
case TYPE_INT:
return sizeof(int32_t);
case TYPE_CHAR:
@@ -40,6 +44,8 @@ int sizeof_type(PrimitiveType type) {
Status type_check(PrimitiveType type) {
switch (type) {
+ case TYPE_TINYINT:
+ case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_CHAR:
case TYPE_VARCHAR:
@@ -140,7 +146,7 @@ Status CollectionValue::init_collection(FunctionContext*
context, uint32_t size,
}
CollectionValue CollectionValue::from_collection_val(const CollectionVal& val)
{
- return CollectionValue(val.data, val.length, val.null_signs);
+ return CollectionValue(val.data, val.length, val.has_null, val.null_signs);
}
Status CollectionValue::set(uint32_t i, PrimitiveType type, const AnyVal*
value) {
@@ -160,6 +166,12 @@ Status CollectionValue::set(uint32_t i, PrimitiveType
type, const AnyVal* value)
}
switch (type) {
+ case TYPE_TINYINT:
+ *reinterpret_cast<int8_t*>(iter.value()) = reinterpret_cast<const
TinyIntVal*>(value)->val;
+ break;
+ case TYPE_SMALLINT:
+ *reinterpret_cast<int16_t*>(iter.value()) = reinterpret_cast<const
SmallIntVal*>(value)->val;
+ break;
case TYPE_INT:
*reinterpret_cast<int32_t*>(iter.value()) = reinterpret_cast<const
IntVal*>(value)->val;
break;
@@ -214,7 +226,7 @@ void ArrayIterator::value(AnyVal* dest) {
break;
case TYPE_SMALLINT:
- reinterpret_cast<TinyIntVal*>(dest)->val = *reinterpret_cast<const
int16_t*>(value());
+ reinterpret_cast<SmallIntVal*>(dest)->val = *reinterpret_cast<const
int16_t*>(value());
break;
case TYPE_INT:
diff --git a/be/src/runtime/mysql_result_writer.cpp
b/be/src/runtime/mysql_result_writer.cpp
index eaf1bd7..229af18 100644
--- a/be/src/runtime/mysql_result_writer.cpp
+++ b/be/src/runtime/mysql_result_writer.cpp
@@ -185,7 +185,11 @@ int MysqlResultWriter::_add_row_value(int index, const
TypeDescriptor& type, voi
buf_ret = _add_row_value(index, children_type, iter.value());
buf_ret = _row_buffer->push_string("'", 1);
} else {
- buf_ret = _add_row_value(index, children_type, iter.value());
+ if (!iter.value()) {
+ buf_ret = _row_buffer->push_string("NULL", 4);
+ } else {
+ buf_ret = _add_row_value(index, children_type,
iter.value());
+ }
}
iter.next();
diff --git a/be/src/runtime/raw_value.cpp b/be/src/runtime/raw_value.cpp
index f101233..ba51616 100644
--- a/be/src/runtime/raw_value.cpp
+++ b/be/src/runtime/raw_value.cpp
@@ -324,6 +324,7 @@ void RawValue::write(const void* value, void* dst, const
TypeDescriptor& type, M
ArrayIterator src_iter = src->iterator(children_type);
ArrayIterator val_iter = val->iterator(children_type);
+ val->set_has_null(src->has_null());
val->copy_null_signs(src);
while (src_iter.has_next() && val_iter.has_next()) {
diff --git a/be/src/runtime/row_batch.cpp b/be/src/runtime/row_batch.cpp
index cdbbebb..73fc3aa 100644
--- a/be/src/runtime/row_batch.cpp
+++ b/be/src/runtime/row_batch.cpp
@@ -179,8 +179,10 @@ RowBatch::RowBatch(const RowDescriptor& row_desc, const
PRowBatch& input_batch,
// assgin data and null_sign pointer position in tuple_data
int data_offset = convert_to<int>(array_val->data());
array_val->set_data(tuple_data + data_offset);
- int null_offset = convert_to<int>(array_val->null_signs());
- array_val->set_null_signs(convert_to<bool*>(tuple_data +
null_offset));
+ if (array_val->has_null()) {
+ int null_offset = convert_to<int>(array_val->null_signs());
+ array_val->set_null_signs(convert_to<bool*>(tuple_data +
null_offset));
+ }
const TypeDescriptor& item_type =
slot_collection->type().children.at(0);
if (!item_type.is_string_type()) {
@@ -597,7 +599,9 @@ size_t RowBatch::total_byte_size() const {
// compute data null_signs size
CollectionValue* array_val =
tuple->get_collection_slot(slot_collection->tuple_offset());
- result += array_val->length() * sizeof(bool);
+ if (array_val->has_null()) {
+ result += array_val->length() * sizeof(bool);
+ }
const TypeDescriptor& item_type =
slot_collection->type().children.at(0);
result += array_val->length() * item_type.get_slot_size();
diff --git a/be/src/runtime/tuple.cpp b/be/src/runtime/tuple.cpp
index af55261..75ae9ce 100644
--- a/be/src/runtime/tuple.cpp
+++ b/be/src/runtime/tuple.cpp
@@ -99,25 +99,20 @@ void Tuple::deep_copy(Tuple* dst, const TupleDescriptor&
desc, MemPool* pool, bo
const TypeDescriptor& item_type = slot_desc->type().children.at(0);
int coll_byte_size = cv->length() * item_type.get_slot_size();
- int nulls_size = cv->length() * sizeof(bool);
+ int nulls_size = cv->has_null() ? cv->length() * sizeof(bool) : 0;
int64_t offset = pool->total_allocated_bytes();
char* coll_data = (char*)(pool->allocate(coll_byte_size + nulls_size));
// copy data and null_signs
- if (nulls_size > 0) {
- cv->set_has_null(true);
- cv->set_null_signs(convert_to<bool*>(coll_data) + coll_byte_size);
- memory_copy(coll_data, cv->null_signs(), nulls_size);
- } else {
- cv->set_has_null(false);
- }
+ memory_copy(convert_to<bool*>(coll_data), cv->null_signs(),
nulls_size);
memory_copy(coll_data + nulls_size, cv->data(), coll_byte_size);
// assgin new null_sign and data location
- cv->set_null_signs(convert_ptrs ? convert_to<bool*>(offset) :
convert_to<bool*>(coll_data));
- cv->set_data(convert_ptrs ? convert_to<char*>(offset + nulls_size)
- : coll_data + nulls_size);
+ if (cv->has_null()) {
+ cv->set_null_signs(convert_ptrs ? convert_to<bool*>(offset) :
convert_to<bool*>(coll_data));
+ }
+ cv->set_data(convert_ptrs ? convert_to<char*>(offset + nulls_size) :
coll_data + nulls_size);
if (!item_type.is_string_type()) {
continue;
@@ -212,7 +207,7 @@ void Tuple::deep_copy(const TupleDescriptor& desc, char**
data, int64_t* offset,
const TypeDescriptor& item_type = slot_desc->type().children.at(0);
int coll_byte_size = cv->length() * item_type.get_slot_size();
- int nulls_size = cv->length() * sizeof(bool);
+ int nulls_size = cv->has_null() ? cv->length() * sizeof(bool) : 0;
// copy null_sign
memory_copy(*data, cv->null_signs(), nulls_size);
@@ -220,8 +215,9 @@ void Tuple::deep_copy(const TupleDescriptor& desc, char**
data, int64_t* offset,
memory_copy(*data + nulls_size, cv->data(), coll_byte_size);
if (!item_type.is_string_type()) {
- cv->set_null_signs(convert_ptrs ? convert_to<bool*>(*offset)
- : convert_to<bool*>(*data));
+ if (cv->has_null()) {
+ cv->set_null_signs(convert_ptrs ? convert_to<bool*>(*offset) :
convert_to<bool*>(*data));
+ }
cv->set_data(convert_ptrs ? convert_to<char*>(*offset + nulls_size)
: *data + nulls_size);
*data += coll_byte_size + nulls_size;
@@ -250,8 +246,9 @@ void Tuple::deep_copy(const TupleDescriptor& desc, char**
data, int64_t* offset,
}
}
// assgin new null_sign and data location
- cv->set_null_signs(convert_ptrs ? convert_to<bool*>(base_offset)
- : convert_to<bool*>(base_data));
+ if (cv->has_null()) {
+ cv->set_null_signs(convert_ptrs ? convert_to<bool*>(base_offset) :
convert_to<bool*>(base_data));
+ }
cv->set_data(convert_ptrs ? convert_to<char*>(base_offset + nulls_size)
: base_data + nulls_size);
}
diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
index 0808b65..aa9254b 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
@@ -821,6 +821,30 @@ TEST_F(ColumnReaderWriterTest, test_v_default_value) {
test_v_read_default_value<OLAP_FIELD_TYPE_DECIMAL>(v_decimal, &decimal);
}
+TEST_F(ColumnReaderWriterTest, test_single_empty_array) {
+ size_t num_array = 1;
+ std::unique_ptr<uint8_t[]> array_is_null(new
uint8_t[BitmapSize(num_array)]());
+ CollectionValue array(0);
+ test_array_nullable_data<OLAP_FIELD_TYPE_TINYINT, BIT_SHUFFLE,
BIT_SHUFFLE>(
+ &array, array_is_null.get(), num_array, "test_single_empty_array");
+}
+
+TEST_F(ColumnReaderWriterTest, test_mixed_empty_arrays) {
+ size_t num_array = 3;
+ std::unique_ptr<uint8_t[]> array_is_null(new
uint8_t[BitmapSize(num_array)]());
+ std::unique_ptr<CollectionValue[]> collection_values(new
CollectionValue[num_array]);
+ int data[] = {1, 2, 3};
+ for (int i = 0; i < num_array; ++ i) {
+ if (i % 2 == 1) {
+ new (&collection_values[i]) CollectionValue(0);
+ } else {
+ new (&collection_values[i]) CollectionValue(&data, 3, false,
nullptr);
+ }
+ }
+ test_array_nullable_data<OLAP_FIELD_TYPE_INT, BIT_SHUFFLE, BIT_SHUFFLE>(
+ collection_values.get(), array_is_null.get(), num_array,
"test_mixed_empty_arrays");
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index dc2ee41..e2b2e9f 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -379,6 +379,7 @@ nonterminal CaseExpr case_expr;
nonterminal ArrayList<CaseWhenClause> case_when_clause_list;
nonterminal FunctionParams function_params;
nonterminal Expr function_call_expr, array_expr;
+nonterminal ArrayLiteral array_literal;
nonterminal StructField struct_field;
nonterminal ArrayList<StructField> struct_field_list;
nonterminal AnalyticWindow opt_window_clause;
@@ -4745,6 +4746,17 @@ function_call_expr ::=
:}
;
+array_literal ::=
+ LBRACKET RBRACKET
+ {:
+ RESULT = new ArrayLiteral();
+ :}
+ | LBRACKET expr_list:list RBRACKET
+ {:
+ RESULT = new ArrayLiteral(list.toArray(new LiteralExpr[0]));
+ :}
+ ;
+
array_expr ::=
KW_ARRAY LPAREN function_params:params RPAREN
{:
@@ -4793,6 +4805,8 @@ non_pred_expr ::=
{: RESULT = l; :}
| array_expr:a
{: RESULT = a; :}
+ | array_literal:a
+ {: RESULT = a; :}
| function_call_expr:e
{: RESULT = e; :}
| KW_DATE STRING_LITERAL:l
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
index 74cb323..8a9ed1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
@@ -17,6 +17,14 @@
package org.apache.doris.analysis;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+
+import org.apache.commons.lang.StringUtils;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -24,12 +32,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
-import org.apache.doris.catalog.ArrayType;
-import org.apache.doris.catalog.Type;
-import org.apache.doris.thrift.TExprNode;
-import org.apache.doris.thrift.TExprNodeType;
-
public class ArrayLiteral extends LiteralExpr {
public ArrayLiteral() {
@@ -113,4 +115,18 @@ public class ArrayLiteral extends LiteralExpr {
public Expr clone() {
return new ArrayLiteral(this);
}
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isArrayType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ ArrayLiteral literal = new ArrayLiteral(this);
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ literal.children.set(i,
child.uncheckedCastTo(((ArrayType)targetType).getItemType()));
+ }
+ literal.setType(targetType);
+ return literal;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 5fb937e..16e0609 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -17,10 +17,6 @@
package org.apache.doris.analysis;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
@@ -38,6 +34,12 @@ import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.external.elasticsearch.EsUtil;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index c393329..a4a9b0d 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1253,7 +1253,8 @@ abstract public class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
}
public Expr checkTypeCompatibility(Type targetType) throws
AnalysisException {
- if (targetType.getPrimitiveType().equals(type.getPrimitiveType())) {
+ if (targetType.getPrimitiveType() != PrimitiveType.ARRAY &&
+ targetType.getPrimitiveType() == type.getPrimitiveType()) {
return this;
}
// bitmap must match exactly
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
index fff4f05..172bb9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -95,6 +95,10 @@ public class ArrayType extends Type {
return otherArrayType.itemType.equals(itemType);
}
+ public static boolean canCastTo(ArrayType type, ArrayType targetType) {
+ return Type.canCastTo(type.getItemType(), targetType.getItemType());
+ }
+
@Override
public void toThrift(TTypeDesc container) {
TTypeNode node = new TTypeNode();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 278055f..eb45de6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -17,9 +17,6 @@
package org.apache.doris.catalog;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.gson.annotations.SerializedName;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
@@ -33,6 +30,11 @@ import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.thrift.TColumn;
import org.apache.doris.thrift.TColumnType;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -158,6 +160,10 @@ public class Column implements Writable {
public void createChildrenColumn(Type type, Column column) {
if (type.isArrayType()) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType)
type).getItemType());
+ // TODO We always set the item type in array nullable.
+ // We may provide an alternative to configure this property of
+ // the item type in array in future.
+ c.setIsAllowNull(true);
column.addChildrenColumn(c);
}
}
@@ -366,6 +372,7 @@ public class Column implements Writable {
childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
childrenTColumn.setColumnType(childrenTColumnType);
+ childrenTColumn.setIsAllowNull(children.isAllowNull());
tColumn.setChildrenColumn(new ArrayList<>());
tColumn.children_column.add(childrenTColumn);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
index 8d51a1f..bcb72d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Type.java
@@ -384,8 +384,10 @@ public abstract class Type {
public static boolean canCastTo(Type t1, Type t2) {
if (t1.isScalarType() && t2.isScalarType()) {
return ScalarType.canCastTo((ScalarType) t1, (ScalarType) t2);
+ } else if (t1.isArrayType() && t2.isArrayType()) {
+ return ArrayType.canCastTo((ArrayType)t1, (ArrayType)t2);
}
- return false;
+ return t1.isNull();
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
index 6aa19f9..adf50b8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ColumnDefTest.java
@@ -19,9 +19,13 @@ package org.apache.doris.analysis;
import org.apache.doris.analysis.ColumnDef.DefaultValue;
import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.junit.Assert;
import org.junit.Before;
@@ -39,7 +43,6 @@ public class ColumnDefTest {
stringCol = new TypeDef(ScalarType.createChar(10));
floatCol = new TypeDef(ScalarType.createType(PrimitiveType.FLOAT));
booleanCol = new TypeDef(ScalarType.createType(PrimitiveType.BOOLEAN));
-
}
@Test
@@ -117,5 +120,16 @@ public class ColumnDefTest {
}
}
-
+ @Test
+ public void testArray() throws AnalysisException {
+ Config.enable_complex_type_support = true;
+ TypeDef typeDef = new TypeDef(new ArrayType(Type.INT));
+ ColumnDef columnDef = new ColumnDef("array", typeDef, false, null,
true, DefaultValue.NOT_SET, "");
+ Column column = columnDef.toColumn();
+ Assert.assertEquals(1, column.getChildren().size());
+ Column childColumn = column.getChildren().get(0);
+ Assert.assertEquals("item", childColumn.getName());
+ Assert.assertEquals(Type.INT, childColumn.getType());
+ Assert.assertTrue(childColumn.isAllowNull());
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java
new file mode 100644
index 0000000..ad0d832
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/InsertArrayStmtTest.java
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.ArrayType;
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.ExceptionChecker;
+import org.apache.doris.common.util.SqlParserUtils;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.thrift.TUniqueId;
+import org.apache.doris.utframe.UtFrameUtils;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+
+public class InsertArrayStmtTest {
+ private static final String RUNNING_DIR =
UtFrameUtils.generateRandomFeRunningDir(InsertArrayStmtTest.class);
+ private static ConnectContext connectContext;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Config.enable_complex_type_support = true;
+ UtFrameUtils.createDorisCluster(RUNNING_DIR);
+ connectContext = UtFrameUtils.createDefaultCtx();
+ createDatabase("create database test;");
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ UtFrameUtils.cleanDorisFeDir(RUNNING_DIR);
+ }
+
+ private static void createDatabase(String sql) throws Exception {
+ CreateDbStmt createDbStmt = (CreateDbStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
+ }
+
+ private static void createTable(String sql) throws Exception {
+ CreateTableStmt createTableStmt = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
+ }
+
+ private static InsertStmt parseAndAnalyze(String sql) throws Exception {
+ SqlParser parser = new SqlParser(new SqlScanner(
+ new StringReader(sql),
connectContext.getSessionVariable().getSqlMode()
+ ));
+ InsertStmt insertStmt = (InsertStmt)
SqlParserUtils.getFirstStmt(parser);
+ Analyzer analyzer = new Analyzer(connectContext.getCatalog(),
connectContext);
+ insertStmt.analyze(analyzer);
+ return insertStmt;
+ };
+
+ @Test
+ public void testInsertArrayStmt() throws Exception {
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.table1 (k1 INT, v1 Array<int>)
duplicate key (k1) " +
+ " distributed by hash (k1) buckets 1 properties
('replication_num' = '1');");
+ });
+
+ connectContext.setQueryId(new TUniqueId(1, 0));
+ InsertStmt insertStmt = parseAndAnalyze("insert into test.table1
values (1, [1, 2, 3]);");
+ ArrayList<Expr> row = ((SelectStmt)
insertStmt.getQueryStmt()).getValueList().getFirstRow();
+ Assert.assertEquals(2, row.size());
+ Assert.assertTrue(row.get(1) instanceof ArrayLiteral);
+ ArrayLiteral arrayLiteral = (ArrayLiteral) row.get(1);
+ Assert.assertEquals(3, arrayLiteral.getChildren().size());
+ Assert.assertTrue(arrayLiteral.isAnalyzed);
+ for (int i = 1; i <= 3; ++ i) {
+ Expr child = arrayLiteral.getChild(i - 1);
+ Assert.assertTrue(child.isAnalyzed);
+ Assert.assertSame(PrimitiveType.INT,
child.getType().getPrimitiveType());
+ Assert.assertTrue(child instanceof IntLiteral);
+ Assert.assertEquals(i, ((IntLiteral) child).getValue());
+ }
+
+ connectContext.setQueryId(new TUniqueId(2, 0));
+ insertStmt = parseAndAnalyze("insert into test.table1 values (1,
[]);");
+ row = ((SelectStmt)
insertStmt.getQueryStmt()).getValueList().getFirstRow();
+ Assert.assertEquals(2, row.size());
+ Assert.assertTrue(row.get(1) instanceof ArrayLiteral);
+ arrayLiteral = (ArrayLiteral) row.get(1);
+ Assert.assertTrue(arrayLiteral.isAnalyzed);
+ Assert.assertTrue(arrayLiteral.getChildren().isEmpty());
+ Assert.assertSame(PrimitiveType.INT, ((ArrayType)
arrayLiteral.getType()).getItemType().getPrimitiveType());
+
+ connectContext.setQueryId(new TUniqueId(3, 0));
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "type
not match", ()-> {
+ parseAndAnalyze("insert into test.table1 values (1, [[1, 2], [3,
4]]);");
+ });
+ }
+}
\ No newline at end of file
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
index fc20d57..9635130 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
@@ -61,6 +61,7 @@ import java.nio.channels.SocketChannel;
import java.nio.file.Files;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
public class UtFrameUtils {
@@ -125,6 +126,14 @@ public class UtFrameUtils {
return statementBases;
}
+ public static String generateRandomFeRunningDir(Class testSuiteClass) {
+ return generateRandomFeRunningDir(testSuiteClass.getSimpleName());
+ }
+
+ public static String generateRandomFeRunningDir(String testSuiteName) {
+ return "fe" + "/mocked/" + testSuiteName + "/" +
UUID.randomUUID().toString() + "/";
+ }
+
public static int startFEServer(String runningDir) throws
EnvVarNotSetException, IOException,
FeStartException, NotInitException, DdlException,
InterruptedException {
// get DORIS_HOME
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
