This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 228e5afad8 [Load](Sink) remove validate the column data when data is
NULL (#13919)
228e5afad8 is described below
commit 228e5afad886b9f137c2d44c7bd1b956251cc954
Author: HappenLee <[email protected]>
AuthorDate: Thu Nov 3 08:33:45 2022 +0800
[Load](Sink) remove validate the column data when data is NULL (#13919)
---
be/src/vec/sink/vtablet_sink.cpp | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp
index 47acc5cea0..e19bf5499c 100644
--- a/be/src/vec/sink/vtablet_sink.cpp
+++ b/be/src/vec/sink/vtablet_sink.cpp
@@ -606,6 +606,10 @@ Status VOlapTableSink::_validate_column(RuntimeState*
state, const TypeDescripto
auto column_ptr =
vectorized::check_and_get_column<vectorized::ColumnNullable>(*column);
auto& real_column_ptr = column_ptr == nullptr ? column :
(column_ptr->get_nested_column_ptr());
+ auto null_map = column_ptr == nullptr ? nullptr :
column_ptr->get_null_map_data().data();
+ auto need_to_validate = [&null_map, &filter_bitmap](size_t j, size_t row) {
+ return !filter_bitmap->Get(row) && (null_map == nullptr || null_map[j]
== 0);
+ };
ssize_t last_invalid_row = -1;
switch (type.type) {
@@ -625,7 +629,7 @@ Status VOlapTableSink::_validate_column(RuntimeState*
state, const TypeDescripto
if (row == last_invalid_row) {
continue;
}
- if (!filter_bitmap->Get(row)) {
+ if (need_to_validate(j, row)) {
auto str_val = column_string->get_data_at(j);
bool invalid = str_val.size > limit;
if (invalid) {
@@ -678,7 +682,7 @@ Status VOlapTableSink::_validate_column(RuntimeState*
state, const TypeDescripto
if (row == last_invalid_row) {
continue;
}
- if (!filter_bitmap->Get(row)) {
+ if (need_to_validate(j, row)) {
auto dec_val = binary_cast<vectorized::Int128, DecimalV2Value>(
column_decimal->get_data()[j]);
bool invalid = false;
@@ -740,8 +744,7 @@ Status VOlapTableSink::_validate_column(RuntimeState*
state, const TypeDescripto
// 1. column is nullable but the desc is not nullable
// 2. desc->type is BITMAP
if ((!is_nullable || type == TYPE_OBJECT) && column_ptr) {
- const auto& null_map = column_ptr->get_null_map_data();
- for (int j = 0; j < null_map.size(); ++j) {
+ for (int j = 0; j < column->size(); ++j) {
auto row = rows ? (*rows)[j] : j;
if (row == last_invalid_row) {
continue;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
