This is an automated email from the ASF dual-hosted git repository.
ashingau pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 89433f6a13 [fix](complex_type) throw error when reading complex types
in broker/stream load (#22331)
89433f6a13 is described below
commit 89433f6a13bd7d51904083e3342d700d4bef548c
Author: Ashin Gau <ashin...@users.noreply.github.com>
AuthorDate: Mon Jul 31 22:23:08 2023 +0800
[fix](complex_type) throw error when reading complex types in broker/stream
load (#22331)
Check whether there are complex types in parquet/orc reader in
broker/stream load. Broker/stream load will cast any type as string type, and
complex types will be casted wrong. This is a temporary method, and will be
replaced by tvf.
---
be/src/vec/exec/scan/vfile_scanner.cpp | 23 ++++++++++++++++++++++
be/src/vec/exec/scan/vfile_scanner.h | 1 +
.../stream_load/test_parquet_orc_case.groovy | 11 -----------
3 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp
b/be/src/vec/exec/scan/vfile_scanner.cpp
index 7d0f7fd331..ac12172221 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -280,11 +280,34 @@ Status VFileScanner::_get_block_impl(RuntimeState* state,
Block* block, bool* eo
return Status::OK();
}
+/**
+ * Check whether there are complex types in parquet/orc reader in
broker/stream load.
+ * Broker/stream load will cast any type as string type, and complex types
will be casted wrong.
+ * This is a temporary method, and will be replaced by tvf.
+ */
+Status VFileScanner::_check_output_block_types() {
+ if (_is_load) {
+ TFileFormatType::type format_type = _params->format_type;
+ if (format_type == TFileFormatType::FORMAT_PARQUET ||
+ format_type == TFileFormatType::FORMAT_ORC) {
+ for (auto slot : _output_tuple_desc->slots()) {
+ if (slot->type().is_complex_type()) {
+ return Status::InternalError(
+ "Parquet/orc doesn't support complex types in
broker/stream load, "
+ "please use tvf(table value function) to insert
complex types.");
+ }
+ }
+ }
+ }
+ return Status::OK();
+}
+
Status VFileScanner::_init_src_block(Block* block) {
if (!_is_load) {
_src_block_ptr = block;
return Status::OK();
}
+ RETURN_IF_ERROR(_check_output_block_types());
// if (_src_block_init) {
// _src_block.clear_column_data();
diff --git a/be/src/vec/exec/scan/vfile_scanner.h
b/be/src/vec/exec/scan/vfile_scanner.h
index 3518a4e733..b4c1117d44 100644
--- a/be/src/vec/exec/scan/vfile_scanner.h
+++ b/be/src/vec/exec/scan/vfile_scanner.h
@@ -182,6 +182,7 @@ private:
private:
Status _init_expr_ctxes();
Status _init_src_block(Block* block);
+ Status _check_output_block_types();
Status _cast_to_input_block(Block* block);
Status _fill_columns_from_path(size_t rows);
Status _fill_missing_columns(size_t rows);
diff --git
a/regression-test/suites/load_p0/stream_load/test_parquet_orc_case.groovy
b/regression-test/suites/load_p0/stream_load/test_parquet_orc_case.groovy
index ec9f5f2ada..698d590c0b 100644
--- a/regression-test/suites/load_p0/stream_load/test_parquet_orc_case.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_parquet_orc_case.groovy
@@ -234,15 +234,4 @@ suite("test_parquet_orc_case", "p0") {
"replication_num"="1"
);
"""
-
- streamLoad {
- table "${arrayParquetTbl}"
- set 'format', 'parquet'
- set 'columns', '`k1`, `a1`, `a2`, `a3`, `a4`, `a5`, `a6`, `a7`, `a8`,
`a9`, `a10`, `a11`, `a12`, `a13`, `a14`'
- file 'array_test.parquet'
- // time 20000 // limit inflight 10s
- }
- sql "sync"
- qt_sql_array_parquet "select * from ${arrayParquetTbl} order by k1 limit 3"
}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
