xiaokang commented on code in PR #47846:
URL: https://github.com/apache/doris/pull/47846#discussion_r1957675431
##########
be/src/olap/rowset/segment_v2/inverted_index_writer.cpp:
##########
@@ -542,7 +561,8 @@ class InvertedIndexColumnWriterImpl : public
InvertedIndexColumnWriter {
DBUG_EXECUTE_IF("InvertedIndexColumnWriterImpl::add_array_values_field_is_nullptr",
{ _field = nullptr; })
DBUG_EXECUTE_IF(
-
"InvertedIndexColumnWriterImpl::add_array_values_index_writer_is_nullptr",
+
"InvertedIndexColumnWriterImpl::add_array_values_index_writer_is_"
Review Comment:
just reformat?
##########
be/src/olap/rowset/segment_v2/inverted_index_writer.cpp:
##########
@@ -325,8 +325,26 @@ class InvertedIndexColumnWriterImpl : public
InvertedIndexColumnWriter {
return Status::OK();
}
- Status add_array_nulls(uint32_t row_id) override {
- _null_bitmap.add(row_id);
+ Status add_array_nulls(const uint8_t* null_map, size_t num_rows) override {
+ DCHECK(_rid >= num_rows);
+ if (num_rows == 0 || null_map == nullptr) {
+ return Status::OK();
+ }
+ std::vector<uint32_t> null_indices;
+ null_indices.reserve(num_rows / 8);
+
+ // because _rid is the row id in block, not segment, and we add data
before we add nulls,
+ // so we need to subtract num_rows to get the row id in segment
+ for (size_t i = 0; i < num_rows; i++) {
+ if (null_map[i] == 1) {
+ null_indices.push_back(_rid - num_rows +
static_cast<uint32_t>(i));
+ }
+ }
+
+ if (!null_indices.empty()) {
+ _null_bitmap.addMany(null_indices.size(), null_indices.data());
Review Comment:
Is it faster than just call _null_bitmap.add(_rid - num_rows +
static_cast<uint32_t>(i)) in last loop, considering the overhead of the vector
null_indices?
##########
be/src/olap/task/index_builder.cpp:
##########
@@ -638,20 +626,27 @@ Status IndexBuilder::_add_nullable(const std::string&
column_name,
DBUG_EXECUTE_IF("IndexBuilder::_add_nullable_add_array_values_error", {
_CLTHROWA(CL_ERR_IO, "debug point:
_add_nullable_add_array_values_error");
})
+
RETURN_IF_ERROR(_inverted_index_builders[index_writer_sign]->add_array_nulls(null_map,
+
num_rows));
} catch (const std::exception& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"CLuceneError occured: {}", e.what());
}
- // we should refresh nullmap for array
- for (int row_id = 0; row_id < num_rows; row_id++) {
- if (null_map && null_map[row_id] == 1) {
- RETURN_IF_ERROR(
-
_inverted_index_builders[index_writer_sign]->add_array_nulls(row_id));
- }
- }
+
return Status::OK();
}
-
+ size_t offset = 0;
+ auto next_run_step = [&]() {
Review Comment:
just move code position?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
