This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch jiangkai_2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3180053e1e29d498924b354652c14b591e147e58
Author: airborne12 <airborn...@gmail.com>
AuthorDate: Tue Nov 7 15:43:34 2023 +0800
[Feature](inverted index) support range predicate for inverted index
---
be/src/common/config.cpp | 1 +
be/src/common/config.h | 2 +
be/src/olap/block_column_predicate.cpp | 39 +-
be/src/olap/block_column_predicate.h | 6 +-
be/src/olap/column_predicate.h | 9 +
be/src/olap/comparison_predicate.h | 76 ++-
be/src/olap/in_list_predicate.h | 32 +-
be/src/olap/match_predicate.cpp | 22 +-
be/src/olap/olap_common.h | 4 +
.../inverted_index/query/disjunction_query.cpp | 3 +-
.../inverted_index/query/disjunction_query.h | 1 -
.../inverted_index/query/inverted_index_query.cpp | 314 +++++++++
.../inverted_index/query/inverted_index_query.h | 220 ++++++
.../inverted_index/query/range_query.cpp | 128 ++++
.../query/{disjunction_query.h => range_query.h} | 17 +-
.../rowset/segment_v2/inverted_index_query_type.h | 17 +
.../rowset/segment_v2/inverted_index_reader.cpp | 745 +++++++++------------
.../olap/rowset/segment_v2/inverted_index_reader.h | 126 ++--
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 50 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 4 +-
be/src/vec/exec/scan/new_olap_scan_node.cpp | 7 +
be/src/vec/exec/scan/new_olap_scan_node.h | 4 +
be/src/vec/exec/scan/new_olap_scanner.cpp | 7 +
be/src/vec/exec/scan/vscan_node.h | 2 +-
24 files changed, 1307 insertions(+), 529 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 0a7d92f5da7..f2daf3de38f 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1003,6 +1003,7 @@ DEFINE_Int32(inverted_index_read_buffer_size, "4096");
DEFINE_Int32(max_depth_in_bkd_tree, "32");
// index compaction
DEFINE_Bool(inverted_index_compaction_enable, "false");
+DEFINE_mInt32(inverted_index_max_terms, "1024");
// use num_broadcast_buffer blocks as buffer to do broadcast
DEFINE_Int32(num_broadcast_buffer, "32");
// semi-structure configs
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c3a4a4229ba..d8a2acc060c 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1040,6 +1040,8 @@ DECLARE_Int32(inverted_index_read_buffer_size);
DECLARE_Int32(max_depth_in_bkd_tree);
// index compaction
DECLARE_Bool(inverted_index_compaction_enable);
+// max inverted index terms size for range query
+DECLARE_mInt32(inverted_index_max_terms);
// use num_broadcast_buffer blocks as buffer to do broadcast
DECLARE_Int32(num_broadcast_buffer);
// semi-structure configs
diff --git a/be/src/olap/block_column_predicate.cpp
b/be/src/olap/block_column_predicate.cpp
index 8cfb89363cd..a9b1b860281 100644
--- a/be/src/olap/block_column_predicate.cpp
+++ b/be/src/olap/block_column_predicate.cpp
@@ -213,11 +213,40 @@ void
AndBlockColumnPredicate::evaluate_vec(vectorized::MutableColumns& block, ui
}
}
-Status AndBlockColumnPredicate::evaluate(const std::string& column_name,
- InvertedIndexIterator* iterator,
uint32_t num_rows,
- roaring::Roaring* bitmap) const {
- return Status::NotSupported(
- "Not Implemented evaluate with inverted index, please check the
predicate");
+Status AndBlockColumnPredicate::evaluate(const Schema& schema,
InvertedIndexIterator* iterator,
+ uint32_t num_rows, roaring::Roaring*
bitmap) const {
+ std::set<const ColumnPredicate*> predicates;
+ get_all_column_predicate(predicates);
+ std::unique_ptr<InvertedIndexQueryBase> query_value = nullptr;
+ uint32_t column_id = 0;
+ roaring::Roaring roaring;
+
+ for (const auto& pred : predicates) {
+ RETURN_IF_ERROR(pred->set_inverted_index_query_value(query_value,
schema));
+ column_id = pred->column_id();
+ }
+ if (!predicates.empty()) {
+ const auto* column_desc = schema.column(column_id);
+ std::string column_name = column_desc->name();
+ RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
query_value.get(), num_rows,
+ &roaring));
+
+ // mask out null_bitmap, since NULL cmp VALUE will produce NULL
+ // and be treated as false in WHERE
+ // keep it after query, since query will try to read null_bitmap and
put it to cache
+ InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
+ RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap_cache_handle));
+ std::shared_ptr<roaring::Roaring> null_bitmap =
null_bitmap_cache_handle.get_bitmap();
+ if (null_bitmap) {
+ *bitmap -= *null_bitmap;
+ }
+ *bitmap &= roaring;
+ return Status::OK();
+ }
+ DCHECK(false);
+
+ return Status::Error(INTERNAL_ERROR,
+ "block column predicates size = 0, please check the
predicate");
}
} // namespace doris
diff --git a/be/src/olap/block_column_predicate.h
b/be/src/olap/block_column_predicate.h
index c91dc0c3678..fee5ef46ff2 100644
--- a/be/src/olap/block_column_predicate.h
+++ b/be/src/olap/block_column_predicate.h
@@ -90,7 +90,7 @@ public:
virtual bool can_do_bloom_filter(bool ngram) const { return false; }
//evaluate predicate on inverted
- virtual Status evaluate(const std::string& column_name,
InvertedIndexIterator* iterator,
+ virtual Status evaluate(const Schema& schema, InvertedIndexIterator*
iterator,
uint32_t num_rows, roaring::Roaring* bitmap) const
{
return Status::NotSupported(
"Not Implemented evaluate with inverted index, please check
the predicate");
@@ -199,8 +199,8 @@ public:
return true;
}
- Status evaluate(const std::string& column_name, InvertedIndexIterator*
iterator,
- uint32_t num_rows, roaring::Roaring* bitmap) const
override;
+ Status evaluate(const Schema& schema, InvertedIndexIterator* iterator,
uint32_t num_rows,
+ roaring::Roaring* bitmap) const override;
};
} //namespace doris
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 05e84999a83..2f007a589cd 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -52,6 +52,7 @@ enum class PredicateType {
BF = 11, // BloomFilter
BITMAP_FILTER = 12, // BitmapFilter
MATCH = 13, // fulltext match
+ RANGE = 14, // BKD index range search
};
inline std::string type_to_string(PredicateType type) {
@@ -208,6 +209,14 @@ public:
virtual void set_page_ng_bf(std::unique_ptr<segment_v2::BloomFilter>) {
DCHECK(false) << "should not reach here";
}
+
+ virtual Status set_inverted_index_query_value(
+ std::unique_ptr<InvertedIndexQueryBase>& /*unused*/, const Schema&
/*unused*/) const {
+ DCHECK(false) << "should not reach here";
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
+ "set_inverted_index_query_value should not reach here.");
+ }
+
uint32_t column_id() const { return _column_id; }
bool opposite() const { return _opposite; }
diff --git a/be/src/olap/comparison_predicate.h
b/be/src/olap/comparison_predicate.h
index 53149ea7ed4..d950ff28427 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -81,33 +81,11 @@ public:
auto column_desc = schema.column(_column_id);
std::string column_name = column_desc->name();
- InvertedIndexQueryType query_type;
- switch (PT) {
- case PredicateType::EQ:
- query_type = InvertedIndexQueryType::EQUAL_QUERY;
- break;
- case PredicateType::NE:
- query_type = InvertedIndexQueryType::EQUAL_QUERY;
- break;
- case PredicateType::LT:
- query_type = InvertedIndexQueryType::LESS_THAN_QUERY;
- break;
- case PredicateType::LE:
- query_type = InvertedIndexQueryType::LESS_EQUAL_QUERY;
- break;
- case PredicateType::GT:
- query_type = InvertedIndexQueryType::GREATER_THAN_QUERY;
- break;
- case PredicateType::GE:
- query_type = InvertedIndexQueryType::GREATER_EQUAL_QUERY;
- break;
- default:
- return Status::InvalidArgument("invalid comparison predicate type
{}", PT);
- }
-
roaring::Roaring roaring;
- RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
&_value, query_type,
- num_rows,
&roaring));
+ std::unique_ptr<InvertedIndexQueryBase> query_value = nullptr;
+ RETURN_IF_ERROR(set_inverted_index_query_value(query_value, schema));
+ RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
query_value.get(), num_rows,
+ &roaring));
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
// and be treated as false in WHERE
@@ -305,6 +283,52 @@ public:
return PT == PredicateType::EQ && !ngram;
}
+ Status
set_inverted_index_query_value(std::unique_ptr<InvertedIndexQueryBase>&
query_value,
+ const Schema& schema) const override
{
+ if (query_value == nullptr) {
+ auto column_desc = schema.column(_column_id);
+ if constexpr (PT == PredicateType::EQ || PT == PredicateType::NE) {
+ query_value = std::make_unique<InvertedIndexPointQuery<Type,
PT>>(
+ column_desc->type_info());
+ } else {
+ query_value = std::make_unique<InvertedIndexRangeQuery<Type,
PredicateType::RANGE>>(
+ column_desc->type_info());
+ }
+ }
+ if constexpr (PT == PredicateType::EQ || PT == PredicateType::NE) {
+ auto q = static_cast<InvertedIndexPointQuery<Type,
PT>*>(query_value.get());
+ RETURN_IF_ERROR(q->add_value(_value,
InvertedIndexQueryType::EQUAL_QUERY));
+ } else {
+ InvertedIndexQueryType query_type =
InvertedIndexQueryType::UNKNOWN_QUERY;
+ switch (PT) {
+ case PredicateType::EQ:
+ query_type = InvertedIndexQueryType::EQUAL_QUERY;
+ break;
+ case PredicateType::NE:
+ query_type = InvertedIndexQueryType::EQUAL_QUERY;
+ break;
+ case PredicateType::LT:
+ query_type = InvertedIndexQueryType::LESS_THAN_QUERY;
+ break;
+ case PredicateType::LE:
+ query_type = InvertedIndexQueryType::LESS_EQUAL_QUERY;
+ break;
+ case PredicateType::GT:
+ query_type = InvertedIndexQueryType::GREATER_THAN_QUERY;
+ break;
+ case PredicateType::GE:
+ query_type = InvertedIndexQueryType::GREATER_EQUAL_QUERY;
+ break;
+ default:
+ LOG(ERROR) << Status::InvalidArgument("invalid comparison
predicate type {}", PT);
+ }
+ auto q = static_cast<InvertedIndexRangeQuery<Type,
PredicateType::RANGE>*>(
+ query_value.get());
+ RETURN_IF_ERROR(q->add_value(_value, query_type));
+ }
+ return Status::OK();
+ }
+
void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel,
uint16_t size,
bool* flags) const override {
_evaluate_bit<false>(column, sel, size, flags);
diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h
index 329c9b8dc0e..682ef3eed95 100644
--- a/be/src/olap/in_list_predicate.h
+++ b/be/src/olap/in_list_predicate.h
@@ -221,6 +221,23 @@ public:
return Status::OK();
}
+ Status
set_inverted_index_query_value(std::unique_ptr<InvertedIndexQueryBase>&
query_value,
+ const Schema& schema) const override
{
+ if (query_value == nullptr) {
+ auto column_desc = schema.column(_column_id);
+ query_value =
+ std::make_unique<InvertedIndexPointQuery<Type,
PT>>(column_desc->type_info());
+ }
+ HybridSetBase::IteratorBase* iter = _values->begin();
+ while (iter->has_next()) {
+ const T* value = reinterpret_cast<const T*>(iter->get_value());
+ auto q = static_cast<InvertedIndexPointQuery<Type,
PT>*>(query_value.get());
+ RETURN_IF_ERROR(q->add_value(*value,
InvertedIndexQueryType::EQUAL_QUERY));
+ iter->next();
+ }
+ return Status::OK();
+ }
+
Status evaluate(const Schema& schema, InvertedIndexIterator* iterator,
uint32_t num_rows,
roaring::Roaring* result) const override {
if (iterator == nullptr) {
@@ -228,18 +245,11 @@ public:
}
auto column_desc = schema.column(_column_id);
std::string column_name = column_desc->name();
+ std::unique_ptr<InvertedIndexQueryBase> query_value = nullptr;
+ RETURN_IF_ERROR(set_inverted_index_query_value(query_value, schema));
roaring::Roaring indices;
- HybridSetBase::IteratorBase* iter = _values->begin();
- while (iter->has_next()) {
- const void* value = iter->get_value();
- InvertedIndexQueryType query_type =
InvertedIndexQueryType::EQUAL_QUERY;
- roaring::Roaring index;
- RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
value, query_type,
- num_rows,
&index));
- indices |= index;
- iter->next();
- }
-
+ RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name,
query_value.get(), num_rows,
+ &indices));
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
// and be treated as false in WHERE
// keep it after query, since query will try to read null_bitmap and
put it to cache
diff --git a/be/src/olap/match_predicate.cpp b/be/src/olap/match_predicate.cpp
index 61d25723155..8e6bf2413ff 100644
--- a/be/src/olap/match_predicate.cpp
+++ b/be/src/olap/match_predicate.cpp
@@ -59,14 +59,26 @@ Status MatchPredicate::evaluate(const Schema& schema,
InvertedIndexIterator* ite
int32_t length = _value.length();
char* buffer = const_cast<char*>(_value.c_str());
match_value.replace(buffer, length); //is it safe?
- RETURN_IF_ERROR(iterator->read_from_inverted_index(
- column_desc->name(), &match_value, inverted_index_query_type,
num_rows, &roaring));
+ //TODO: need consider TYPE_CHAR/TYPE_VARCHAR
+ auto query_value =
+ std::make_unique<InvertedIndexPointQuery<TYPE_STRING,
PredicateType::MATCH>>(
+ column_desc->type_info());
+ RETURN_IF_ERROR(query_value->add_value(match_value,
inverted_index_query_type));
+
+
RETURN_IF_ERROR(iterator->read_from_inverted_index(column_desc->name(),
query_value.get(),
+ num_rows,
&roaring));
} else if (column_desc->type() == FieldType::OLAP_FIELD_TYPE_ARRAY &&
is_numeric_type(column_desc->get_sub_field(0)->type_info()->type())) {
char buf[column_desc->get_sub_field(0)->type_info()->size()];
column_desc->get_sub_field(0)->from_string(buf, _value);
- RETURN_IF_ERROR(iterator->read_from_inverted_index(
- column_desc->name(), buf, inverted_index_query_type, num_rows,
&roaring, true));
+
+ std::unique_ptr<InvertedIndexQueryBase> query_value = nullptr;
+ RETURN_IF_ERROR(
+
InvertedIndexQueryBase::create_and_add_value_from_field_type<PredicateType::MATCH>(
+ column_desc->get_sub_field(0)->type_info(), buf,
inverted_index_query_type,
+ query_value));
+
RETURN_IF_ERROR(iterator->read_from_inverted_index(column_desc->name(),
query_value.get(),
+ num_rows, &roaring,
true));
}
// mask out null_bitmap, since NULL cmp VALUE will produce NULL
@@ -126,4 +138,4 @@ bool MatchPredicate::_skip_evaluate(InvertedIndexIterator*
iterator) const {
return false;
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index e1c5717fcd0..712349cbb73 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -347,7 +347,11 @@ struct OlapReaderStatistics {
int64_t rows_inverted_index_filtered = 0;
int64_t inverted_index_filter_timer = 0;
+ int64_t inverted_index_block_column_predicate_filter_timer = 0;
+ int64_t inverted_index_column_predicate_filter_timer = 0;
+ int64_t inverted_index_try_query_timer = 0;
int64_t inverted_index_query_timer = 0;
+ int64_t inverted_index_bkd_intersect_timer = 0;
int64_t inverted_index_query_cache_hit = 0;
int64_t inverted_index_query_cache_miss = 0;
int64_t inverted_index_query_bitmap_copy_timer = 0;
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.cpp
index 07a159b3222..3487d4275e6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.cpp
@@ -36,12 +36,11 @@ DisjunctionQuery::~DisjunctionQuery() {
void DisjunctionQuery::add(const std::wstring& field_name, const
std::vector<std::string>& terms) {
if (terms.size() < 1) {
- _CLTHROWA(CL_ERR_IllegalArgument, "ConjunctionQuery::add: terms.size()
< 1");
+ _CLTHROWA(CL_ERR_IllegalArgument, "DisjunctionQuery::add: terms.size()
< 1");
}
for (auto& term : terms) {
std::wstring ws_term = StringUtil::string_to_wstring(term);
- _wsterms.emplace_back(&ws_term);
Term* t = _CLNEW Term(field_name.c_str(), ws_term.c_str());
_terms.push_back(t);
TermDocs* term_doc = _reader->termDocs(t);
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
index f42fd69dabc..bb0a837f42a 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
@@ -39,7 +39,6 @@ public:
private:
IndexReader* _reader = nullptr;
- std::vector<std::wstring*> _wsterms;
std::vector<Term*> _terms;
std::vector<TermDocs*> _term_docs;
std::vector<TermIterator> _term_iterators;
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query/inverted_index_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query/inverted_index_query.cpp
new file mode 100644
index 00000000000..10762c58e21
--- /dev/null
+++
b/be/src/olap/rowset/segment_v2/inverted_index/query/inverted_index_query.cpp
@@ -0,0 +1,314 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "inverted_index_query.h"
+
+#include <filesystem>
+#include <set>
+
+#include "io/fs/file_system.h"
+#include "olap/column_predicate.h"
+#include "olap/key_coder.h"
+#include "olap/olap_common.h"
+#include "olap/rowset/segment_v2/inverted_index_cache.h"
+#include "olap/types.h"
+#include "util/time.h"
+#include "vec/common/string_ref.h"
+
+namespace doris::segment_v2 {
+
+template <PrimitiveType Type, PredicateType PT>
+Status Helper<Type, PT>::create_and_add_value(const TypeInfo* type_info, char*
value,
+ InvertedIndexQueryType t,
+
std::unique_ptr<InvertedIndexQueryBase>& result) {
+ using CppType = typename
PredicatePrimitiveTypeTraits<Type>::PredicateFieldType;
+
+ if (is_range_query(t)) {
+ auto range_query_ptr = std::make_unique<InvertedIndexRangeQuery<Type,
PT>>(type_info);
+
RETURN_IF_ERROR(range_query_ptr->add_value(*reinterpret_cast<CppType*>(value),
t));
+ result = std::move(range_query_ptr);
+ } else {
+ auto point_query_ptr = std::make_unique<InvertedIndexPointQuery<Type,
PT>>(type_info);
+
RETURN_IF_ERROR(point_query_ptr->add_value(*reinterpret_cast<CppType*>(value),
t));
+ result = std::move(point_query_ptr);
+ }
+
+ return Status::OK();
+}
+
+template <PredicateType PT>
+Status InvertedIndexQueryBase::create_and_add_value_from_field_type(
+ const TypeInfo* type_info, char* value, InvertedIndexQueryType t,
+ std::unique_ptr<InvertedIndexQueryBase>& result) {
+ Status st;
+ switch (type_info->type()) {
+ case FieldType::OLAP_FIELD_TYPE_DATETIME: {
+ st = Helper<PrimitiveType::TYPE_DATETIME,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DATE: {
+ st = Helper<PrimitiveType::TYPE_DATE,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
+ st = Helper<PrimitiveType::TYPE_DATETIMEV2,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DATEV2: {
+ st = Helper<PrimitiveType::TYPE_DATEV2,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_TINYINT: {
+ st = Helper<PrimitiveType::TYPE_TINYINT,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_SMALLINT: {
+ st = Helper<PrimitiveType::TYPE_SMALLINT,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_INT: {
+ st = Helper<PrimitiveType::TYPE_INT,
PT>::create_and_add_value(type_info, value, t, result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_LARGEINT: {
+ st = Helper<PrimitiveType::TYPE_LARGEINT,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL32: {
+ st = Helper<PrimitiveType::TYPE_DECIMAL32,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL64: {
+ st = Helper<PrimitiveType::TYPE_DECIMAL64,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: {
+ st = Helper<PrimitiveType::TYPE_DECIMAL128I,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_DOUBLE: {
+ st = Helper<PrimitiveType::TYPE_DOUBLE,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_FLOAT: {
+ st = Helper<PrimitiveType::TYPE_FLOAT,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_BIGINT: {
+ st = Helper<PrimitiveType::TYPE_BIGINT,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_BOOL: {
+ st = Helper<PrimitiveType::TYPE_BOOLEAN,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_CHAR: {
+ st = Helper<PrimitiveType::TYPE_CHAR,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_VARCHAR: {
+ st = Helper<PrimitiveType::TYPE_VARCHAR,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ case FieldType::OLAP_FIELD_TYPE_STRING: {
+ st = Helper<PrimitiveType::TYPE_STRING,
PT>::create_and_add_value(type_info, value, t,
+
result);
+ break;
+ }
+ default:
+ return Status::NotSupported("Unsupported column type for inverted
index {}",
+ type_info->type());
+ }
+ if (!st.ok()) {
+ return st;
+ }
+ return Status::OK();
+}
+
+template Status
InvertedIndexQueryBase::create_and_add_value_from_field_type<PredicateType::MATCH>(
+ const TypeInfo*, char*, InvertedIndexQueryType,
std::unique_ptr<InvertedIndexQueryBase>&);
+
+template <PrimitiveType Type, PredicateType PT>
+InvertedIndexPointQuery<Type, PT>::InvertedIndexPointQuery(const TypeInfo*
type_info)
+ : _type_info(type_info) {
+ _value_key_coder = get_key_coder(type_info->type());
+}
+
+template <PrimitiveType Type, PredicateType PT>
+std::string InvertedIndexPointQuery<Type, PT>::to_string() {
+ std::string result;
+ if constexpr (std::is_same_v<T, StringRef>) {
+ for (const T* v : _values) {
+ result += v->to_string();
+ }
+ } else {
+ for (auto& v : _values) {
+ result += _type_info->to_string(v);
+ }
+ }
+ return result;
+}
+
+template <PrimitiveType Type, PredicateType PT>
+Status InvertedIndexPointQuery<Type, PT>::add_value(const T& value,
InvertedIndexQueryType t) {
+ if constexpr (std::is_same_v<T, StringRef>) {
+ auto act_len = strnlen(value.data, value.size);
+ std::string value_str(value.data, act_len);
+ _values_encoded.push_back(value_str);
+ } else {
+ std::string tmp;
+ _value_key_coder->full_encode_ascending(&value, &tmp);
+ _values_encoded.push_back(tmp);
+ }
+ _values.push_back(&value);
+ _type = t;
+ return Status::OK();
+}
+
+template <PrimitiveType Type, PredicateType PT>
+InvertedIndexRangeQuery<Type, PT>::InvertedIndexRangeQuery(const TypeInfo*
type_info)
+ : _type_info(type_info) {
+ _value_key_coder = get_key_coder(type_info->type());
+ auto max_v = type_limit<T>::max();
+ auto min_v = type_limit<T>::min();
+ _value_key_coder->full_encode_ascending(&max_v, &_high_value_encoded);
+ _value_key_coder->full_encode_ascending(&min_v, &_low_value_encoded);
+}
+
+template <PrimitiveType Type, PredicateType PT>
+std::string InvertedIndexRangeQuery<Type, PT>::to_string() {
+ std::string low_op = _inclusive_low ? ">=" : ">";
+ std::string high_op = _inclusive_high ? "<=" : "<";
+ std::string buffer;
+ if (_low_value != nullptr) {
+ buffer.append(_type_info->to_string(_low_value) + low_op + " ");
+ }
+ if (_high_value != nullptr) {
+ buffer.append(_type_info->to_string(_high_value) + high_op);
+ }
+ return buffer;
+};
+
+template <PrimitiveType Type, PredicateType PT>
+Status InvertedIndexRangeQuery<Type, PT>::add_value(const T& value,
InvertedIndexQueryType t) {
+ switch (t) {
+ case InvertedIndexQueryType::GREATER_THAN_QUERY: {
+ _low_value = &value;
+ _low_value_encoded.clear();
+ _value_key_coder->full_encode_ascending(&value, &_low_value_encoded);
+ break;
+ }
+
+ case InvertedIndexQueryType::GREATER_EQUAL_QUERY: {
+ _low_value = &value;
+ _inclusive_low = true;
+ _low_value_encoded.clear();
+ _value_key_coder->full_encode_ascending(&value, &_low_value_encoded);
+ break;
+ }
+
+ case InvertedIndexQueryType::LESS_THAN_QUERY: {
+ _high_value = &value;
+ _high_value_encoded.clear();
+ _value_key_coder->full_encode_ascending(&value, &_high_value_encoded);
+ break;
+ }
+
+ case InvertedIndexQueryType::LESS_EQUAL_QUERY: {
+ _high_value = &value;
+ _inclusive_high = true;
+ _high_value_encoded.clear();
+ _value_key_coder->full_encode_ascending(&value, &_high_value_encoded);
+ break;
+ }
+ case InvertedIndexQueryType::EQUAL_QUERY: {
+ _high_value = _low_value = &value;
+ _high_value_encoded.clear();
+ _value_key_coder->full_encode_ascending(&value, &_high_value_encoded);
+ _low_value_encoded.clear();
+ _value_key_coder->full_encode_ascending(&value, &_low_value_encoded);
+ break;
+ }
+ default: {
+ return Status::InternalError("Add value failed! Unsupported
PredicateType {}", PT);
+ }
+ }
+ return Status::OK();
+}
+
+#define INSTANTIATE_FOR_TYPE_AND_PREDICATE(P, C) \
+ template class C<TYPE_BOOLEAN, PredicateType::P>; \
+ template class C<TYPE_INT, PredicateType::P>; \
+ template class C<TYPE_TINYINT, PredicateType::P>; \
+ template class C<TYPE_SMALLINT, PredicateType::P>; \
+ template class C<TYPE_BIGINT, PredicateType::P>; \
+ template class C<TYPE_LARGEINT, PredicateType::P>; \
+ template class C<TYPE_FLOAT, PredicateType::P>; \
+ template class C<TYPE_DOUBLE, PredicateType::P>; \
+ template class C<TYPE_CHAR, PredicateType::P>; \
+ template class C<TYPE_STRING, PredicateType::P>; \
+ template class C<TYPE_VARCHAR, PredicateType::P>; \
+ template class C<TYPE_TIME, PredicateType::P>; \
+ template class C<TYPE_TIMEV2, PredicateType::P>; \
+ template class C<TYPE_DATE, PredicateType::P>; \
+ template class C<TYPE_DATEV2, PredicateType::P>; \
+ template class C<TYPE_DATETIME, PredicateType::P>; \
+ template class C<TYPE_DATETIMEV2, PredicateType::P>; \
+ template class C<TYPE_DECIMALV2, PredicateType::P>; \
+ template class C<TYPE_DECIMAL32, PredicateType::P>; \
+ template class C<TYPE_DECIMAL64, PredicateType::P>; \
+ template class C<TYPE_DECIMAL128I, PredicateType::P>; \
+ template class C<TYPE_DECIMAL256, PredicateType::P>; \
+ template class C<TYPE_IPV4, PredicateType::P>; \
+ template class C<TYPE_IPV6, PredicateType::P>;
+
+#define INSTANTIATE_FOR_TYPE(C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(EQ, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(NE, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(LT, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(LE, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(GT, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(GE, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(IN_LIST, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(NOT_IN_LIST, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(IS_NULL, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(IS_NOT_NULL, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(BF, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(BITMAP_FILTER, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(MATCH, C) \
+ INSTANTIATE_FOR_TYPE_AND_PREDICATE(RANGE, C)
+
+INSTANTIATE_FOR_TYPE(InvertedIndexPointQuery)
+INSTANTIATE_FOR_TYPE(InvertedIndexRangeQuery)
+
+} // namespace doris::segment_v2
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query/inverted_index_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query/inverted_index_query.h
new file mode 100644
index 00000000000..466fefeac78
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/inverted_index_query.h
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <CLucene/util/FutureArrays.h>
+#include <CLucene/util/bkd/bkd_reader.h>
+
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "common/status.h"
+#include "io/fs/file_system.h"
+#include "io/fs/path.h"
+#include "olap/inverted_index_parser.h"
+#include "olap/rowset/segment_v2/inverted_index_cache.h"
+#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_desc.h"
+#include "olap/rowset/segment_v2/inverted_index_query_type.h"
+#include "olap/tablet_schema.h"
+#include "runtime/primitive_type.h"
+#include "runtime/type_limit.h"
+
+namespace lucene {
+namespace store {
+class Directory;
+} // namespace store
+namespace util::bkd {
+class bkd_docid_set_iterator;
+} // namespace util::bkd
+} // namespace lucene
+namespace roaring {
+class Roaring;
+} // namespace roaring
+
+namespace doris {
+class KeyCoder;
+class TypeInfo;
+struct OlapReaderStatistics;
+class RuntimeState;
+enum class PredicateType;
+
+namespace segment_v2 {
+
+enum class QueryCategory { POINT_QUERY, RANGE_QUERY };
+
+class InvertedIndexQueryBase {
+public:
+ virtual std::string to_string() = 0;
+ virtual ~InvertedIndexQueryBase() = default;
+ virtual QueryCategory get_query_category() = 0;
+ [[nodiscard]] virtual PredicateType get_predicate_type() const = 0;
+ template <PredicateType PT>
+ static Status create_and_add_value_from_field_type(
+ const TypeInfo* type_info, char* value, InvertedIndexQueryType t,
+ std::unique_ptr<InvertedIndexQueryBase>& result);
+};
+
+template <PrimitiveType Type, PredicateType PT>
+struct Helper;
+
+class InvertedIndexPointQueryI : public InvertedIndexQueryBase {
+public:
+ InvertedIndexPointQueryI() = default;
+ ~InvertedIndexPointQueryI() override = default;
+ QueryCategory get_query_category() override { return
QueryCategory::POINT_QUERY; }
+ std::string to_string() override {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexPointQueryI");
+ __builtin_unreachable();
+ }
+ [[nodiscard]] PredicateType get_predicate_type() const override {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexPointQueryI");
+ __builtin_unreachable();
+ }
+ [[nodiscard]] virtual const std::vector<std::string>& get_values() const {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexPointQueryI");
+ __builtin_unreachable();
+ };
+ [[nodiscard]] virtual InvertedIndexQueryType get_query_type() const {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexPointQueryI");
+ __builtin_unreachable();
+ };
+};
+
+template <PrimitiveType Type, PredicateType PT>
+class InvertedIndexPointQuery : public InvertedIndexPointQueryI {
+public:
+ using T = typename PredicatePrimitiveTypeTraits<Type>::PredicateFieldType;
+ InvertedIndexPointQuery(const TypeInfo* type_info);
+
+ Status add_value(const T& value, InvertedIndexQueryType t);
+ std::string to_string() override;
+ [[nodiscard]] const std::vector<std::string>& get_values() const override {
+ return _values_encoded;
+ };
+ [[nodiscard]] PredicateType get_predicate_type() const override { return
PT; };
+ [[nodiscard]] InvertedIndexQueryType get_query_type() const override {
return _type; };
+
+private:
+ std::vector<std::string> _values_encoded;
+ const KeyCoder* _value_key_coder {};
+ const TypeInfo* _type_info {};
+ std::vector<const T*> _values;
+ InvertedIndexQueryType _type;
+};
+
+template <PrimitiveType Type, PredicateType PT>
+struct Helper {
+ static Status create_and_add_value(const TypeInfo* type_info, char* value,
+ InvertedIndexQueryType t,
+
std::unique_ptr<InvertedIndexQueryBase>& result);
+};
+
+class InvertedIndexRangeQueryI : public InvertedIndexQueryBase {
+public:
+ InvertedIndexRangeQueryI() = default;
+ ~InvertedIndexRangeQueryI() override = default;
+ [[nodiscard]] virtual const std::string& get_low_value() const = 0;
+ [[nodiscard]] virtual const std::string& get_high_value() const = 0;
+ virtual bool low_value_is_null() = 0;
+ virtual bool high_value_is_null() = 0;
+ QueryCategory get_query_category() override { return
QueryCategory::RANGE_QUERY; }
+ std::string to_string() override {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexRangeQueryI");
+ __builtin_unreachable();
+ };
+ [[nodiscard]] PredicateType get_predicate_type() const override {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexRangeQueryI");
+ __builtin_unreachable();
+ };
+ [[nodiscard]] virtual bool is_low_value_inclusive() const {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexRangeQueryI");
+ __builtin_unreachable();
+ }
+ [[nodiscard]] virtual bool is_high_value_inclusive() const {
+ LOG_FATAL("Execution reached an undefined behavior code path in
InvertedIndexRangeQueryI");
+ __builtin_unreachable();
+ }
+};
+
+class BinaryType {
+ ENABLE_FACTORY_CREATOR(BinaryType);
+
+public:
+ const uint8_t* _data {};
+ size_t _size {};
+
+ BinaryType() = default;
+ BinaryType(const uint8_t* data, size_t size) : _data(data), _size(size) {}
+ BinaryType(const std::string& str)
+ : _data(reinterpret_cast<const uint8_t*>(str.data())),
_size(str.size()) {}
+
+ int operator<(const BinaryType& other) const {
+ return lucene::util::FutureArrays::CompareUnsigned(_data, 0, _size,
other._data, 0,
+ other._size) < 0;
+ }
+
+ int operator<=(const BinaryType& other) const {
+ return lucene::util::FutureArrays::CompareUnsigned(_data, 0, _size,
other._data, 0,
+ other._size) <= 0;
+ }
+
+ int operator>(const BinaryType& other) const {
+ return lucene::util::FutureArrays::CompareUnsigned(_data, 0, _size,
other._data, 0,
+ other._size) > 0;
+ }
+
+ int operator>=(const BinaryType& other) const {
+ return lucene::util::FutureArrays::CompareUnsigned(_data, 0, _size,
other._data, 0,
+ other._size) >= 0;
+ }
+};
+
+template <PrimitiveType Type, PredicateType PT>
+class InvertedIndexRangeQuery : public InvertedIndexRangeQueryI {
+public:
+ using T = typename PredicatePrimitiveTypeTraits<Type>::PredicateFieldType;
+ InvertedIndexRangeQuery(const TypeInfo* type_info);
+ Status add_value(const T& value, InvertedIndexQueryType t);
+ [[nodiscard]] const std::string& get_low_value() const override { return
_low_value_encoded; };
+ [[nodiscard]] const std::string& get_high_value() const override {
+ return _high_value_encoded;
+ };
+ bool low_value_is_null() override { return _low_value == nullptr; };
+ bool high_value_is_null() override { return _high_value == nullptr; };
+ std::string to_string() override;
+ [[nodiscard]] PredicateType get_predicate_type() const override { return
PT; };
+ [[nodiscard]] bool is_low_value_inclusive() const override { return
_inclusive_low; }
+ [[nodiscard]] bool is_high_value_inclusive() const override { return
_inclusive_high; }
+
+private:
+ const T* _low_value {};
+ const T* _high_value {};
+ std::string _low_value_encoded {};
+ std::string _high_value_encoded {};
+
+ const KeyCoder* _value_key_coder {};
+ const TypeInfo* _type_info {};
+ bool _inclusive_high {};
+ bool _inclusive_low {};
+};
+} // namespace segment_v2
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/range_query.cpp
b/be/src/olap/rowset/segment_v2/inverted_index/query/range_query.cpp
new file mode 100644
index 00000000000..abf18b5d861
--- /dev/null
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/range_query.cpp
@@ -0,0 +1,128 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "range_query.h"
+
+namespace doris {
+
+RangeQuery::RangeQuery(IndexReader* reader) : _reader(reader) {}
+
+RangeQuery::~RangeQuery() {
+ for (auto& term_doc : _term_docs) {
+ if (term_doc) {
+ _CLDELETE(term_doc);
+ }
+ }
+}
+
+Status RangeQuery::add(const std::wstring& field_name,
InvertedIndexRangeQueryI* query) {
+ std::unique_ptr<lucene::index::Term, void (*)(lucene::index::Term*)>
lower_term(
+ nullptr, [](lucene::index::Term* term) { _CLDECDELETE(term); });
+ std::unique_ptr<lucene::index::Term, void (*)(lucene::index::Term*)>
upper_term(
+ nullptr, [](lucene::index::Term* term) { _CLDECDELETE(term); });
+
+ if (query->low_value_is_null() && query->high_value_is_null()) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_INVALID_PARAMETERS>(
+ "StringTypeInvertedIndexReader::handle_range_query error: both
low_value and "
+ "high_value is null");
+ }
+ auto search_low = query->get_low_value();
+ if (!query->low_value_is_null()) {
+ std::wstring search_low_ws = StringUtil::string_to_wstring(search_low);
+ lower_term.reset(_CLNEW lucene::index::Term(field_name.c_str(),
search_low_ws.c_str()));
+ } else {
+ lower_term.reset(_CLNEW Term(field_name.c_str(), L""));
+ }
+ auto search_high = query->get_high_value();
+ if (!query->high_value_is_null()) {
+ std::wstring search_high_ws =
StringUtil::string_to_wstring(search_high);
+ upper_term.reset(_CLNEW lucene::index::Term(field_name.c_str(),
search_high_ws.c_str()));
+ }
+
+ auto* _enumerator = _reader->terms(lower_term.get());
+ Term* lastTerm = nullptr;
+ try {
+ bool checkLower = false;
+ if (!query->is_low_value_inclusive()) { // make adjustments to set to
exclusive
+ checkLower = true;
+ }
+
+ do {
+ lastTerm = _enumerator->term();
+ if (lastTerm != nullptr && lastTerm->field() == field_name) {
+ if (!checkLower || _tcscmp(lastTerm->text(),
lower_term->text()) > 0) {
+ checkLower = false;
+ if (upper_term != nullptr) {
+ int compare = _tcscmp(upper_term->text(),
lastTerm->text());
+ /* if beyond the upper term, or is exclusive and
+ * this is equal to the upper term, break out */
+ if ((compare < 0) ||
(!query->is_high_value_inclusive() && compare == 0)) {
+ break;
+ }
+ }
+ TermDocs* term_doc = _reader->termDocs(lastTerm);
+ _term_docs.push_back(term_doc);
+ _term_iterators.emplace_back(term_doc);
+ }
+ } else {
+ break;
+ }
+ _CLDECDELETE(lastTerm);
+ } while (_enumerator->next());
+ } catch (CLuceneError& e) {
+ _CLDECDELETE(lastTerm);
+ _enumerator->close();
+ _CLDELETE(_enumerator);
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError occured, error msg: {}, search_str: {}",
e.what(),
+ query->to_string());
+ }
+ _CLDECDELETE(lastTerm);
+ _enumerator->close();
+ _CLDELETE(_enumerator);
+ return Status::OK();
+}
+
+void RangeQuery::search(roaring::Roaring& roaring) {
+ roaring::Roaring result;
+ auto func = [&roaring](const TermIterator& term_docs, bool first) {
+ roaring::Roaring result;
+ DocRange doc_range;
+ while (term_docs.readRange(&doc_range)) {
+ if (doc_range.type_ == DocRangeType::kMany) {
+ result.addMany(doc_range.doc_many_size_,
doc_range.doc_many->data());
+ } else {
+ result.addRange(doc_range.doc_range.first,
doc_range.doc_range.second);
+ }
+ }
+ if (first) {
+ roaring.swap(result);
+ } else {
+ roaring |= result;
+ }
+ };
+ for (int i = 0; i < _term_iterators.size(); i++) {
+ auto& iter = _term_iterators[i];
+ if (i == 0) {
+ func(iter, true);
+ } else {
+ func(iter, false);
+ }
+ }
+}
+
+} // namespace doris
\ No newline at end of file
diff --git
a/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
b/be/src/olap/rowset/segment_v2/inverted_index/query/range_query.h
similarity index 74%
copy from be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
copy to be/src/olap/rowset/segment_v2/inverted_index/query/range_query.h
index f42fd69dabc..47873d83e0b 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query/disjunction_query.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/range_query.h
@@ -23,26 +23,27 @@
#include <CLucene/index/Term.h>
#include <CLucene/search/query/TermIterator.h>
+#include "inverted_index_query.h"
#include "roaring/roaring.hh"
CL_NS_USE(index)
namespace doris {
+using namespace segment_v2;
-class DisjunctionQuery {
+class RangeQuery {
public:
- DisjunctionQuery(IndexReader* reader);
- ~DisjunctionQuery();
+ RangeQuery(IndexReader* reader);
+ ~RangeQuery();
- void add(const std::wstring& field_name, const std::vector<std::string>&
terms);
+ Status add(const std::wstring& field_name, InvertedIndexRangeQueryI*
query);
void search(roaring::Roaring& roaring);
+ [[nodiscard]] size_t get_terms_size() const { return _term_docs.size(); }
private:
IndexReader* _reader = nullptr;
- std::vector<std::wstring*> _wsterms;
- std::vector<Term*> _terms;
- std::vector<TermDocs*> _term_docs;
- std::vector<TermIterator> _term_iterators;
+ std::vector<TermDocs*> _term_docs {};
+ std::vector<TermIterator> _term_iterators {};
};
} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_query_type.h
b/be/src/olap/rowset/segment_v2/inverted_index_query_type.h
index 1ebfe635918..23a1f3b6bd9 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_query_type.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_query_type.h
@@ -32,8 +32,22 @@ enum class InvertedIndexQueryType {
MATCH_ANY_QUERY = 5,
MATCH_ALL_QUERY = 6,
MATCH_PHRASE_QUERY = 7,
+ RANGE_QUERY = 8,
};
+inline bool is_range_query(InvertedIndexQueryType query_type) {
+ return (query_type == InvertedIndexQueryType::GREATER_THAN_QUERY ||
+ query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY ||
+ query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
+ query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY);
+}
+
+inline bool is_match_query(InvertedIndexQueryType query_type) {
+ return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
+ query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY);
+}
+
inline std::string InvertedIndexQueryType_toString(InvertedIndexQueryType
query_type) {
switch (query_type) {
case InvertedIndexQueryType::UNKNOWN_QUERY: {
@@ -63,6 +77,9 @@ inline std::string
InvertedIndexQueryType_toString(InvertedIndexQueryType query_
case InvertedIndexQueryType::MATCH_PHRASE_QUERY: {
return "MPHRASE";
}
+ case InvertedIndexQueryType::RANGE_QUERY: {
+ return "RANGE";
+ }
default:
return "";
}
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 0969a7545ad..7653f557287 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -20,51 +20,49 @@
#include <CLucene/analysis/AnalysisHeader.h>
#include <CLucene/analysis/Analyzers.h>
#include <CLucene/analysis/LanguageBasedAnalyzer.h>
-#include <CLucene/analysis/standard/StandardAnalyzer.h>
-#include <CLucene/clucene-config.h>
-#include <CLucene/config/repl_wchar.h>
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wshadow-field"
+#endif
+#include "CLucene/analysis/standard95/StandardAnalyzer.h"
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
#include <CLucene/debug/error.h>
#include <CLucene/debug/mem.h>
-#include <CLucene/index/IndexReader.h>
#include <CLucene/index/Term.h>
#include <CLucene/search/IndexSearcher.h>
#include <CLucene/search/PhraseQuery.h>
#include <CLucene/search/Query.h>
-#include <CLucene/search/RangeQuery.h>
#include <CLucene/search/TermQuery.h>
#include <CLucene/store/Directory.h>
#include <CLucene/store/IndexInput.h>
#include <CLucene/util/CLStreams.h>
-#include <CLucene/util/FutureArrays.h>
#include <CLucene/util/bkd/bkd_docid_iterator.h>
#include <CLucene/util/stringUtil.h>
-#include <math.h>
-#include <string.h>
-#include <algorithm>
#include <filesystem>
#include <ostream>
#include <roaring/roaring.hh>
#include <set>
-#include "CLucene/analysis/standard95/StandardAnalyzer.h"
#include "common/config.h"
#include "common/logging.h"
-#include "io/fs/file_system.h"
+#include "common/status.h"
+#include "olap/column_predicate.h"
#include "olap/inverted_index_parser.h"
-#include "olap/key_coder.h"
#include "olap/olap_common.h"
#include
"olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h"
#include "olap/rowset/segment_v2/inverted_index/query/conjunction_query.h"
+#include "olap/rowset/segment_v2/inverted_index/query/disjunction_query.h"
+#include "olap/rowset/segment_v2/inverted_index/query/range_query.h"
#include "olap/rowset/segment_v2/inverted_index_cache.h"
#include "olap/rowset/segment_v2/inverted_index_compound_directory.h"
-#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/types.h"
#include "runtime/runtime_state.h"
#include "util/faststring.h"
#include "util/runtime_profile.h"
#include "util/time.h"
-#include "vec/common/string_ref.h"
#define FINALIZE_INPUT(x) \
if (x != nullptr) { \
@@ -77,21 +75,7 @@
} catch (...) { \
}
-namespace doris {
-namespace segment_v2 {
-
-bool InvertedIndexReader::_is_range_query(InvertedIndexQueryType query_type) {
- return (query_type == InvertedIndexQueryType::GREATER_THAN_QUERY ||
- query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY ||
- query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
- query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY);
-}
-
-bool InvertedIndexReader::_is_match_query(InvertedIndexQueryType query_type) {
- return (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY);
-}
+namespace doris::segment_v2 {
bool InvertedIndexReader::indexExists(io::Path& index_file_path) {
bool exists = false;
@@ -154,8 +138,7 @@ std::vector<std::string>
InvertedIndexReader::get_analyse_result(
while (token_stream->next(&token)) {
if (token.termLength<char>() != 0) {
- analyse_result.emplace_back(
- std::string(token.termBuffer<char>(),
token.termLength<char>()));
+ analyse_result.emplace_back(token.termBuffer<char>(),
token.termLength<char>());
}
}
@@ -185,7 +168,7 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
auto index_file_path = index_dir / index_file_name;
InvertedIndexQueryCache::CacheKey cache_key {
index_file_path, "", InvertedIndexQueryType::UNKNOWN_QUERY,
"null_bitmap"};
- auto cache = InvertedIndexQueryCache::instance();
+ auto* cache = InvertedIndexQueryCache::instance();
if (cache->lookup(cache_key, cache_handle)) {
return Status::OK();
}
@@ -209,15 +192,15 @@ Status
InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
*null_bitmap =
roaring::Roaring::read(reinterpret_cast<char*>(buf.data()), false);
null_bitmap->runOptimize();
cache->insert(cache_key, null_bitmap, cache_handle);
- FINALIZE_INPUT(null_bitmap_in);
+ FINALIZE_INPUT(null_bitmap_in)
}
if (owned_dir) {
- FINALIZE_INPUT(dir);
+ FINALIZE_INPUT(dir)
}
} catch (CLuceneError& e) {
- FINALLY_FINALIZE_INPUT(null_bitmap_in);
+ FINALLY_FINALIZE_INPUT(null_bitmap_in)
if (owned_dir) {
- FINALLY_FINALIZE_INPUT(dir);
+ FINALLY_FINALIZE_INPUT(dir)
}
return Status::Error<doris::ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"Inverted index read null bitmap error occurred, reason={}",
e.what());
@@ -232,31 +215,16 @@ Status
FullTextIndexReader::new_iterator(OlapReaderStatistics* stats, RuntimeSta
return Status::OK();
}
-Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
- const std::string& column_name, const void*
query_value,
- InvertedIndexQueryType query_type,
roaring::Roaring* bit_map) {
- SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
-
- std::string search_str = reinterpret_cast<const
StringRef*>(query_value)->to_string();
+Status FullTextIndexReader::_query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
+ const std::string& column_name,
std::string& search_str,
+ InvertedIndexQueryType query_type,
roaring::Roaring* bit_map) {
LOG(INFO) << column_name << " begin to search the fulltext index from
clucene, query_str ["
<< search_str << "]";
- io::Path path(_path);
- auto index_dir = path.parent_path();
- auto index_file_name =
- InvertedIndexDescriptor::get_index_file_name(path.filename(),
_index_meta.index_id());
- auto index_file_path = index_dir / index_file_name;
- InvertedIndexCtxSPtr inverted_index_ctx =
std::make_shared<InvertedIndexCtx>();
- inverted_index_ctx->parser_type =
get_inverted_index_parser_type_from_string(
- get_parser_string_from_properties(_index_meta.properties()));
- inverted_index_ctx->parser_mode =
- get_parser_mode_string_from_properties(_index_meta.properties());
- inverted_index_ctx->char_filter_map =
-
get_parser_char_filter_map_from_properties(_index_meta.properties());
try {
- auto analyzer = create_analyzer(inverted_index_ctx.get());
- auto reader = create_reader(inverted_index_ctx.get(), search_str);
- inverted_index_ctx->analyzer = analyzer.get();
+ auto analyzer = create_analyzer(_inverted_index_ctx.get());
+ auto reader = create_reader(_inverted_index_ctx.get(), search_str);
+ _inverted_index_ctx->analyzer = analyzer.get();
std::vector<std::string> analyse_result =
get_analyse_result(reader.get(), analyzer.get(), column_name,
query_type);
@@ -270,21 +238,22 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) {
LOG(WARNING) << msg;
return Status::OK();
- } else {
- return Status::Error<ErrorCode::INVERTED_INDEX_NO_TERMS>(msg);
}
+ return Status::Error<ErrorCode::INVERTED_INDEX_NO_TERMS>(msg);
}
// check index file existence
- if (!indexExists(index_file_path)) {
+ if (!indexExists(_file_full_path)) {
return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
- "inverted index path: {} not exist.",
index_file_path.string());
+ "inverted index path: {} not exist.",
_file_full_path.string());
}
- InvertedIndexCacheHandle inverted_index_cache_handle;
- InvertedIndexSearcherCache::instance()->get_index_searcher(
- _fs, index_dir.c_str(), index_file_name,
&inverted_index_cache_handle, stats);
- auto index_searcher = inverted_index_cache_handle.get_index_searcher();
+ auto get_index_search = [this, &stats]() {
+ InvertedIndexCacheHandle inverted_index_cache_handle;
+
static_cast<void>(InvertedIndexSearcherCache::instance()->get_index_searcher(
+ _fs, _file_dir.c_str(), _file_name,
&inverted_index_cache_handle, stats));
+ return inverted_index_cache_handle.get_index_searcher();
+ };
std::unique_ptr<lucene::search::Query> query;
std::wstring field_ws = std::wstring(column_name.begin(),
column_name.end());
@@ -299,76 +268,65 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
str_tokens += token;
str_tokens += " ";
}
-
- auto cache = InvertedIndexQueryCache::instance();
- InvertedIndexQueryCache::CacheKey cache_key;
- cache_key.index_path = index_file_path;
- cache_key.column_name = column_name;
- cache_key.query_type = query_type;
- //auto str_tokens = lucene_wcstoutf8string(wstr_tokens.c_str(),
wstr_tokens.length());
- cache_key.value.swap(str_tokens);
- InvertedIndexQueryCacheHandle cache_handle;
std::shared_ptr<roaring::Roaring> term_match_bitmap = nullptr;
- if (cache->lookup(cache_key, &cache_handle)) {
- stats->inverted_index_query_cache_hit++;
- term_match_bitmap = cache_handle.get_bitmap();
- } else {
- stats->inverted_index_query_cache_miss++;
-
- term_match_bitmap = std::make_shared<roaring::Roaring>();
-
- Status res = Status::OK();
- if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) {
- auto* phrase_query = new lucene::search::PhraseQuery();
- for (auto& token : analyse_result) {
- std::wstring wtoken =
StringUtil::string_to_wstring(token);
- auto* term = _CLNEW
lucene::index::Term(field_ws.c_str(), wtoken.c_str());
- phrase_query->add(term);
- _CLDECDELETE(term);
- }
- query.reset(phrase_query);
- res = normal_index_search(stats, query_type,
index_searcher,
- null_bitmap_already_read, query,
term_match_bitmap);
- } else {
- res = match_all_index_search(stats, runtime_state,
field_ws, analyse_result,
- index_searcher,
term_match_bitmap);
- }
- if (!res.ok()) {
- return res;
+ InvertedIndexQueryCache::CacheKey cache_key {_file_full_path,
column_name, query_type,
+ str_tokens};
+ auto* cache = InvertedIndexQueryCache::instance();
+ InvertedIndexQueryCacheHandle cache_handler;
+ auto cache_status = handle_cache(cache, cache_key, &cache_handler,
stats, bit_map);
+ if (cache_status.ok()) {
+ return Status::OK();
+ }
+ auto index_searcher = get_index_search();
+ term_match_bitmap = std::make_shared<roaring::Roaring>();
+ Status res = Status::OK();
+ if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) {
+ auto* phrase_query = new lucene::search::PhraseQuery();
+ for (auto& token : analyse_result) {
+ std::wstring wtoken = StringUtil::string_to_wstring(token);
+ auto* term = _CLNEW lucene::index::Term(field_ws.c_str(),
wtoken.c_str());
+ phrase_query->add(term);
+ _CLDECDELETE(term)
}
-
- // add to cache
- term_match_bitmap->runOptimize();
- cache->insert(cache_key, term_match_bitmap, &cache_handle);
+ query.reset(phrase_query);
+ res = normal_index_search(stats, query_type, index_searcher,
+ null_bitmap_already_read, query,
term_match_bitmap);
+ } else {
+ res = match_all_index_search(stats, runtime_state, field_ws,
analyse_result,
+ index_searcher,
term_match_bitmap);
}
+ if (!res.ok()) {
+ return res;
+ }
+
+ // add to cache
+ term_match_bitmap->runOptimize();
+ cache->insert(cache_key, term_match_bitmap, &cache_handler);
query_match_bitmap = *term_match_bitmap;
+
} else {
bool first = true;
for (auto token : analyse_result) {
std::shared_ptr<roaring::Roaring> term_match_bitmap = nullptr;
// try to get term bitmap match result from cache to avoid
query index on cache hit
- auto cache = InvertedIndexQueryCache::instance();
+ auto* cache = InvertedIndexQueryCache::instance();
// use EQUAL_QUERY type here since cache is for each term/token
- //auto token = lucene_wcstoutf8string(token_ws.c_str(),
token_ws.length());
std::wstring token_ws = StringUtil::string_to_wstring(token);
-
InvertedIndexQueryCache::CacheKey cache_key {
- index_file_path, column_name,
InvertedIndexQueryType::EQUAL_QUERY, token};
- VLOG_DEBUG << "cache_key:" << cache_key.encode();
- InvertedIndexQueryCacheHandle cache_handle;
- if (cache->lookup(cache_key, &cache_handle)) {
- stats->inverted_index_query_cache_hit++;
- term_match_bitmap = cache_handle.get_bitmap();
+ _file_full_path, column_name,
InvertedIndexQueryType::EQUAL_QUERY, token};
+ InvertedIndexQueryCacheHandle cache_handler;
+ auto cache_status = handle_cache(cache, cache_key,
&cache_handler, stats, bit_map);
+ if (cache_status.ok()) {
+ term_match_bitmap = cache_handler.get_bitmap();
} else {
- stats->inverted_index_query_cache_miss++;
-
+ auto index_searcher = get_index_search();
term_match_bitmap = std::make_shared<roaring::Roaring>();
// unique_ptr with custom deleter
std::unique_ptr<lucene::index::Term, void
(*)(lucene::index::Term*)> term {
_CLNEW lucene::index::Term(field_ws.c_str(),
token_ws.c_str()),
- [](lucene::index::Term* term) {
_CLDECDELETE(term); }};
- query.reset(new lucene::search::TermQuery(term.get()));
+ [](lucene::index::Term* term) { _CLDECDELETE(term)
}};
+ query =
std::make_unique<lucene::search::TermQuery>(term.get());
Status res =
normal_index_search(stats, query_type,
index_searcher,
@@ -379,7 +337,7 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
// add to cache
term_match_bitmap->runOptimize();
- cache->insert(cache_key, term_match_bitmap, &cache_handle);
+ cache->insert(cache_key, term_match_bitmap,
&cache_handler);
}
// add to query_match_bitmap
@@ -412,6 +370,27 @@ Status FullTextIndexReader::query(OlapReaderStatistics*
stats, RuntimeState* run
}
}
+Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
+ const std::string& column_name,
+ InvertedIndexQueryBase* query_value,
roaring::Roaring* bit_map) {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
+ const auto& tmp = static_cast<InvertedIndexPointQueryI*>(query_value);
+ auto values = tmp->get_values();
+ auto query_type = tmp->get_query_type();
+ auto query_bitmap = std::make_shared<roaring::Roaring>();
+
+ for (auto it = values.begin(); it != values.end(); ++it) {
+ RETURN_IF_ERROR(
+ _query(stats, runtime_state, column_name, *it, query_type,
query_bitmap.get()));
+ if (it == values.begin()) {
+ *bit_map = *query_bitmap;
+ } else {
+ *bit_map |= *query_bitmap;
+ }
+ }
+ return Status::OK();
+}
+
Status FullTextIndexReader::normal_index_search(
OlapReaderStatistics* stats, InvertedIndexQueryType query_type,
const IndexSearcherPtr& index_searcher, bool& null_bitmap_already_read,
@@ -487,87 +466,38 @@ Status StringTypeInvertedIndexReader::new_iterator(
return Status::OK();
}
-Status StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
- RuntimeState* runtime_state,
- const std::string& column_name,
const void* query_value,
- InvertedIndexQueryType query_type,
- roaring::Roaring* bit_map) {
- SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
-
- const StringRef* search_query = reinterpret_cast<const
StringRef*>(query_value);
- auto act_len = strnlen(search_query->data, search_query->size);
- std::string search_str(search_query->data, act_len);
+Status StringTypeInvertedIndexReader::handle_range_query(const std::string&
column_name,
+ OlapReaderStatistics*
stats,
+
InvertedIndexRangeQueryI* query,
+ roaring::Roaring*
bit_map) {
// std::string search_str = reinterpret_cast<const
StringRef*>(query_value)->to_string();
VLOG_DEBUG << "begin to query the inverted index from clucene"
- << ", column_name: " << column_name << ", search_str: " <<
search_str;
- std::wstring column_name_ws = std::wstring(column_name.begin(),
column_name.end());
- std::wstring search_str_ws = StringUtil::string_to_wstring(search_str);
- // unique_ptr with custom deleter
- std::unique_ptr<lucene::index::Term, void (*)(lucene::index::Term*)> term {
- _CLNEW lucene::index::Term(column_name_ws.c_str(),
search_str_ws.c_str()),
- [](lucene::index::Term* term) { _CLDECDELETE(term); }};
- std::unique_ptr<lucene::search::Query> query;
-
- io::Path path(_path);
- auto index_dir = path.parent_path();
- auto index_file_name =
- InvertedIndexDescriptor::get_index_file_name(path.filename(),
_index_meta.index_id());
- auto index_file_path = index_dir / index_file_name;
-
- // try to get query bitmap result from cache and return immediately on
cache hit
- InvertedIndexQueryCache::CacheKey cache_key {index_file_path, column_name,
query_type,
- search_str};
- auto cache = InvertedIndexQueryCache::instance();
- InvertedIndexQueryCacheHandle cache_handle;
- if (cache->lookup(cache_key, &cache_handle)) {
- stats->inverted_index_query_cache_hit++;
- SCOPED_RAW_TIMER(&stats->inverted_index_query_bitmap_copy_timer);
- *bit_map = *cache_handle.get_bitmap();
- return Status::OK();
- } else {
- stats->inverted_index_query_cache_miss++;
- }
+ << ", column_name: " << column_name << ", search_str: " <<
query->to_string();
- // check index file existence
- if (!indexExists(index_file_path)) {
- return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
- "inverted index path: {} not exist.",
index_file_path.string());
- }
-
- switch (query_type) {
- case InvertedIndexQueryType::MATCH_ANY_QUERY:
- case InvertedIndexQueryType::MATCH_ALL_QUERY:
- case InvertedIndexQueryType::MATCH_PHRASE_QUERY:
- case InvertedIndexQueryType::EQUAL_QUERY: {
- query.reset(new lucene::search::TermQuery(term.get()));
- break;
- }
- case InvertedIndexQueryType::LESS_THAN_QUERY: {
- query.reset(new lucene::search::RangeQuery(nullptr, term.get(),
false));
- break;
- }
- case InvertedIndexQueryType::LESS_EQUAL_QUERY: {
- query.reset(new lucene::search::RangeQuery(nullptr, term.get(), true));
- break;
- }
- case InvertedIndexQueryType::GREATER_THAN_QUERY: {
- query.reset(new lucene::search::RangeQuery(term.get(), nullptr,
false));
- break;
- }
- case InvertedIndexQueryType::GREATER_EQUAL_QUERY: {
- query.reset(new lucene::search::RangeQuery(term.get(), nullptr, true));
- break;
- }
- default:
- return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
- "invalid query type when query untokenized inverted index");
+ InvertedIndexQueryCache::CacheKey cache_key {
+ _file_full_path, column_name, InvertedIndexQueryType::RANGE_QUERY,
query->to_string()};
+ auto* cache = InvertedIndexQueryCache::instance();
+ InvertedIndexQueryCacheHandle cache_handler;
+ auto cache_status = handle_cache(cache, cache_key, &cache_handler, stats,
bit_map);
+ if (cache_status.ok()) {
+ return Status::OK();
}
roaring::Roaring result;
InvertedIndexCacheHandle inverted_index_cache_handle;
- InvertedIndexSearcherCache::instance()->get_index_searcher(
- _fs, index_dir.c_str(), index_file_name,
&inverted_index_cache_handle, stats);
+
static_cast<void>(InvertedIndexSearcherCache::instance()->get_index_searcher(
+ _fs, _file_dir, _file_name, &inverted_index_cache_handle, stats));
auto index_searcher = inverted_index_cache_handle.get_index_searcher();
+ RangeQuery range_query(index_searcher->getReader());
+ std::wstring column_name_ws = std::wstring(column_name.begin(),
column_name.end());
+
+ RETURN_IF_ERROR(range_query.add(column_name_ws, query));
+ if (range_query.get_terms_size() > config::inverted_index_max_terms) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
+ "range query term exceeds limits, try to downgrade from
inverted index, column "
+ "name:{}, search_str:{}",
+ column_name, query->to_string());
+ }
// try to reuse index_searcher's directory to read null_bitmap to cache
// to avoid open directory additionally for null_bitmap
@@ -575,48 +505,102 @@ Status
StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
read_null_bitmap(&null_bitmap_cache_handle,
index_searcher->getReader()->directory());
try {
- if (query_type == InvertedIndexQueryType::MATCH_ANY_QUERY ||
- query_type == InvertedIndexQueryType::MATCH_ALL_QUERY ||
- query_type == InvertedIndexQueryType::EQUAL_QUERY) {
- SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
- index_searcher->_search(query.get(), [&result](DocRange*
doc_range) {
- if (doc_range->type_ == DocRangeType::kMany) {
- result.addMany(doc_range->doc_many_size_,
doc_range->doc_many->data());
- } else {
- result.addRange(doc_range->doc_range.first,
doc_range->doc_range.second);
- }
- });
- } else {
- SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
- index_searcher->_search(query.get(),
- [&result](const int32_t docid, const
float_t /*score*/) {
- // docid equal to rowid in segment
- result.add(docid);
- });
- }
+ SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
+ range_query.search(result);
+
} catch (const CLuceneError& e) {
- if (_is_range_query(query_type) && e.number() ==
CL_ERR_TooManyClauses) {
+ if (e.number() == CL_ERR_TooManyClauses) {
return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
"range query term exceeds limits, try to downgrade from
inverted index, column "
"name:{}, search_str:{}",
- column_name, search_str);
- } else {
- return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
- "CLuceneError occured, error msg: {}, column name: {},
search_str: {}",
- e.what(), column_name, search_str);
+ column_name, query->to_string());
}
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError occured, error msg: {}, column name: {},
search_str: {}", e.what(),
+ column_name, query->to_string());
}
+ // add to cache
+ std::shared_ptr<roaring::Roaring> range_query_bitmap =
+ std::make_shared<roaring::Roaring>(result);
+ range_query_bitmap->runOptimize();
+ cache->insert(cache_key, range_query_bitmap, &cache_handler);
+
+ bit_map->swap(result);
+ return Status::OK();
+}
+
+Status StringTypeInvertedIndexReader::handle_point_query(const std::string&
column_name,
+ OlapReaderStatistics*
stats,
+
InvertedIndexPointQueryI* query,
+ roaring::Roaring*
bit_map) {
+ std::wstring column_name_ws = std::wstring(column_name.begin(),
column_name.end());
+ auto values = query->get_values();
+
+ InvertedIndexQueryCache::CacheKey cache_key {_file_full_path, column_name,
+ query->get_query_type(),
query->to_string()};
+ auto* cache = InvertedIndexQueryCache::instance();
+ InvertedIndexQueryCacheHandle cache_handler;
+ auto cache_status = handle_cache(cache, cache_key, &cache_handler, stats,
bit_map);
+ if (cache_status.ok()) {
+ return Status::OK();
+ }
+
+ roaring::Roaring result;
+ InvertedIndexCacheHandle inverted_index_cache_handle;
+
static_cast<void>(InvertedIndexSearcherCache::instance()->get_index_searcher(
+ _fs, _file_dir, _file_name, &inverted_index_cache_handle, stats));
+ auto index_searcher = inverted_index_cache_handle.get_index_searcher();
+ DisjunctionQuery dis_query(index_searcher->getReader());
+ dis_query.add(column_name_ws, values);
+ try {
+ SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer);
+ dis_query.search(result);
+ } catch (const CLuceneError& e) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "CLuceneError occured, error msg: {}, column name: {},
search_str: {}", e.what(),
+ column_name, query->to_string());
+ }
+
+ // try to reuse index_searcher's directory to read null_bitmap to cache
+ // to avoid open directory additionally for null_bitmap
+ InvertedIndexQueryCacheHandle null_bitmap_cache_handle;
+ static_cast<void>(
+ read_null_bitmap(&null_bitmap_cache_handle,
index_searcher->getReader()->directory()));
+
// add to cache
std::shared_ptr<roaring::Roaring> term_match_bitmap =
std::make_shared<roaring::Roaring>(result);
term_match_bitmap->runOptimize();
- cache->insert(cache_key, term_match_bitmap, &cache_handle);
+ cache->insert(cache_key, term_match_bitmap, &cache_handler);
bit_map->swap(result);
return Status::OK();
}
+Status StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats,
+ RuntimeState* runtime_state,
+ const std::string& column_name,
+ InvertedIndexQueryBase*
query_value,
+ roaring::Roaring* bit_map) {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
+ // check index file existence
+ if (!indexExists(_file_full_path)) {
+ return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
+ "inverted index path: {} not exist.",
_file_full_path.string());
+ }
+ if (query_value->get_query_category() == QueryCategory::RANGE_QUERY) {
+ const auto& tmp = static_cast<InvertedIndexRangeQueryI*>(query_value);
+ return handle_range_query(column_name, stats, tmp, bit_map);
+ }
+ if (query_value->get_query_category() == QueryCategory::POINT_QUERY) {
+ const auto& tmp = static_cast<InvertedIndexPointQueryI*>(query_value);
+ return handle_point_query(column_name, stats, tmp, bit_map);
+ }
+ return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
+ "bkd index query type not support:{}",
query_value->get_query_category());
+}
+
InvertedIndexReaderType StringTypeInvertedIndexReader::type() {
return InvertedIndexReaderType::STRING_TYPE;
}
@@ -624,20 +608,12 @@ InvertedIndexReaderType
StringTypeInvertedIndexReader::type() {
BkdIndexReader::BkdIndexReader(io::FileSystemSPtr fs, const std::string& path,
const TabletIndex* index_meta)
: InvertedIndexReader(fs, path, index_meta), _compoundReader(nullptr) {
- io::Path io_path(_path);
- auto index_dir = io_path.parent_path();
- auto index_file_name =
InvertedIndexDescriptor::get_index_file_name(io_path.filename(),
-
index_meta->index_id());
-
- // check index file existence
- auto index_file = index_dir / index_file_name;
- if (!indexExists(index_file)) {
- LOG(WARNING) << "bkd index: " << index_file.string() << " not exist.";
+ if (!indexExists(_file_full_path)) {
+ LOG(WARNING) << "bkd index: " << _file_full_path.string() << " not
exist.";
return;
}
- _file_full_path = index_file;
_compoundReader = std::make_unique<DorisCompoundReader>(
- DorisCompoundDirectory::getDirectory(fs, index_dir.c_str()),
index_file_name.c_str(),
+ DorisCompoundDirectory::getDirectory(fs, _file_dir.c_str()),
_file_name.c_str(),
config::inverted_index_read_buffer_size);
}
@@ -647,51 +623,22 @@ Status BkdIndexReader::new_iterator(OlapReaderStatistics*
stats, RuntimeState* r
return Status::OK();
}
-Status BkdIndexReader::bkd_query(OlapReaderStatistics* stats, const
std::string& column_name,
- const void* query_value,
InvertedIndexQueryType query_type,
-
std::shared_ptr<lucene::util::bkd::bkd_reader> r,
- InvertedIndexVisitor* visitor) {
- char tmp[r->bytes_per_dim_];
- switch (query_type) {
- case InvertedIndexQueryType::EQUAL_QUERY: {
- _value_key_coder->full_encode_ascending(query_value,
&visitor->query_max);
- _value_key_coder->full_encode_ascending(query_value,
&visitor->query_min);
- break;
- }
- case InvertedIndexQueryType::LESS_THAN_QUERY:
- case InvertedIndexQueryType::LESS_EQUAL_QUERY: {
- _value_key_coder->full_encode_ascending(query_value,
&visitor->query_max);
- _type_info->set_to_min(tmp);
- _value_key_coder->full_encode_ascending(tmp, &visitor->query_min);
- break;
- }
- case InvertedIndexQueryType::GREATER_THAN_QUERY:
- case InvertedIndexQueryType::GREATER_EQUAL_QUERY: {
- _value_key_coder->full_encode_ascending(query_value,
&visitor->query_min);
- _type_info->set_to_max(tmp);
- _value_key_coder->full_encode_ascending(tmp, &visitor->query_max);
- break;
- }
- default:
- return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
- "invalid query type when query bkd index");
- }
- visitor->set_reader(r.get());
- return Status::OK();
-}
-
Status BkdIndexReader::try_query(OlapReaderStatistics* stats, const
std::string& column_name,
- const void* query_value,
InvertedIndexQueryType query_type,
- uint32_t* count) {
- auto visitor = std::make_unique<InvertedIndexVisitor>(nullptr, query_type,
true);
+ InvertedIndexQueryBase* query_value,
uint32_t* count) {
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
+ SCOPED_RAW_TIMER(&stats->inverted_index_try_query_timer);
+ auto visitor = std::make_unique<InvertedIndexVisitor>(nullptr,
query_value, true);
std::shared_ptr<lucene::util::bkd::bkd_reader> r;
RETURN_IF_ERROR(get_bkd_reader(&r));
- std::string query_str;
- _value_key_coder->full_encode_ascending(query_value, &query_str);
+ std::string query_str = query_value->to_string();
+ auto query_type =
+ (query_value->get_query_category() == QueryCategory::RANGE_QUERY)
+ ? InvertedIndexQueryType::RANGE_QUERY
+ :
static_cast<InvertedIndexPointQueryI*>(query_value)->get_query_type();
InvertedIndexQueryCache::CacheKey cache_key {_file_full_path, column_name,
query_type,
- query_str};
- auto cache = InvertedIndexQueryCache::instance();
+ query_str + "__TRY__"};
+ auto* cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCacheHandle cache_handler;
roaring::Roaring bit_map;
auto cache_status = handle_cache(cache, cache_key, &cache_handler, stats,
&bit_map);
@@ -700,54 +647,37 @@ Status BkdIndexReader::try_query(OlapReaderStatistics*
stats, const std::string&
return Status::OK();
}
try {
- auto st = bkd_query(stats, column_name, query_value, query_type, r,
visitor.get());
- if (!st.ok()) {
- if (st.code() == ErrorCode::END_OF_FILE) {
- return Status::OK();
- }
- LOG(WARNING) << "bkd_query for column " << column_name << "
failed: " << st;
- return st;
- }
+ visitor->set_reader(r.get());
*count = r->estimate_point_count(visitor.get());
+ bit_map.addRange(0, *count - 1);
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"BKD Query CLuceneError Occurred, error msg: {}", e.what());
}
-
+ std::shared_ptr<roaring::Roaring> query_bitmap =
std::make_shared<roaring::Roaring>(bit_map);
+ query_bitmap->runOptimize();
+ cache->insert(cache_key, query_bitmap, &cache_handler);
VLOG_DEBUG << "BKD index try search column: " << column_name << " result:
" << *count;
return Status::OK();
}
-Status BkdIndexReader::handle_cache(InvertedIndexQueryCache* cache,
- const InvertedIndexQueryCache::CacheKey&
cache_key,
- InvertedIndexQueryCacheHandle*
cache_handler,
- OlapReaderStatistics* stats,
roaring::Roaring* bit_map) {
- if (cache->lookup(cache_key, cache_handler)) {
- stats->inverted_index_query_cache_hit++;
- SCOPED_RAW_TIMER(&stats->inverted_index_query_bitmap_copy_timer);
- *bit_map = *cache_handler->get_bitmap();
- return Status::OK();
- } else {
- stats->inverted_index_query_cache_miss++;
- return Status::Error<ErrorCode::KEY_NOT_FOUND>("cache miss");
- }
-}
-
Status BkdIndexReader::query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
- const std::string& column_name, const void*
query_value,
- InvertedIndexQueryType query_type,
roaring::Roaring* bit_map) {
+ const std::string& column_name,
InvertedIndexQueryBase* query_value,
+ roaring::Roaring* bit_map) {
SCOPED_RAW_TIMER(&stats->inverted_index_query_timer);
-
- auto visitor = std::make_unique<InvertedIndexVisitor>(bit_map, query_type);
+ auto visitor = std::make_unique<InvertedIndexVisitor>(bit_map,
query_value);
std::shared_ptr<lucene::util::bkd::bkd_reader> r;
RETURN_IF_ERROR(get_bkd_reader(&r));
- std::string query_str;
- _value_key_coder->full_encode_ascending(query_value, &query_str);
+ std::string query_str = query_value->to_string();
+ auto query_type =
+ (query_value->get_query_category() == QueryCategory::RANGE_QUERY)
+ ? InvertedIndexQueryType::RANGE_QUERY
+ :
static_cast<InvertedIndexPointQueryI*>(query_value)->get_query_type();
InvertedIndexQueryCache::CacheKey cache_key {_file_full_path, column_name,
query_type,
query_str};
- auto cache = InvertedIndexQueryCache::instance();
+ auto* cache = InvertedIndexQueryCache::instance();
InvertedIndexQueryCacheHandle cache_handler;
auto cache_status = handle_cache(cache, cache_key, &cache_handler, stats,
bit_map);
if (cache_status.ok()) {
@@ -755,14 +685,8 @@ Status BkdIndexReader::query(OlapReaderStatistics* stats,
RuntimeState* runtime_
}
try {
- auto st = bkd_query(stats, column_name, query_value, query_type, r,
visitor.get());
- if (!st.ok()) {
- if (st.code() == ErrorCode::END_OF_FILE) {
- return Status::OK();
- }
- LOG(WARNING) << "bkd_query for column " << column_name << "
failed: " << st;
- return st;
- }
+ visitor->set_reader(r.get());
+ SCOPED_RAW_TIMER(&stats->inverted_index_bkd_intersect_timer);
r->intersect(visitor.get());
} catch (const CLuceneError& e) {
return Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
@@ -816,7 +740,6 @@ Status
BkdIndexReader::get_bkd_reader(std::shared_ptr<lucene::util::bkd::bkd_rea
return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
"unsupported typeinfo, type={}", (*bkdReader)->type);
}
- _value_key_coder = get_key_coder(_type_info->type());
return Status::OK();
}
@@ -824,47 +747,59 @@ InvertedIndexReaderType BkdIndexReader::type() {
return InvertedIndexReaderType::BKD;
}
-InvertedIndexVisitor::InvertedIndexVisitor(roaring::Roaring* h,
InvertedIndexQueryType query_type,
- bool only_count)
- : _hits(h), _num_hits(0), _only_count(only_count),
_query_type(query_type) {}
+InvertedIndexVisitor::InvertedIndexVisitor(roaring::Roaring* hits,
+ InvertedIndexQueryBase*
query_value, bool only_count)
+ : _hits(hits),
+ _num_hits(0),
+ _only_count(only_count),
+ _low_op(PredicateType::GT),
+ _high_op(PredicateType::LT) {
+ if (query_value->get_query_category() == QueryCategory::RANGE_QUERY) {
+ auto* range_query =
reinterpret_cast<InvertedIndexRangeQueryI*>(query_value);
+ query_max = range_query->get_high_value();
+ query_min = range_query->get_low_value();
+ if (range_query->is_low_value_inclusive()) {
+ _low_op = PredicateType::GE;
+ }
+ if (range_query->is_high_value_inclusive()) {
+ _high_op = PredicateType::LE;
+ }
+ } else if (query_value->get_query_category() ==
QueryCategory::POINT_QUERY) {
+ auto* point_query =
reinterpret_cast<InvertedIndexPointQueryI*>(query_value);
+ for (const std::string& v : point_query->get_values()) {
+ query_points.emplace_back(v);
+ }
+ // =1 equals 1<= && >=1
+ _low_op = PredicateType::GE;
+ _high_op = PredicateType::LE;
+ }
+}
+
+bool InvertedIndexVisitor::_matches(const BinaryType& packed_value, const
BinaryType& qmax,
+ const BinaryType& qmin) {
+ bool minInside = (_low_op == PredicateType::GE ? packed_value >= qmin :
packed_value > qmin);
+ bool maxInside = (_high_op == PredicateType::LE ? packed_value <= qmax :
packed_value < qmax);
+ return minInside && maxInside;
+}
bool InvertedIndexVisitor::matches(uint8_t* packed_value) {
- for (int dim = 0; dim < _reader->num_data_dims_; dim++) {
- int offset = dim * _reader->bytes_per_dim_;
- if (_query_type == InvertedIndexQueryType::LESS_THAN_QUERY) {
- if (lucene::util::FutureArrays::CompareUnsigned(
- packed_value, offset, offset + _reader->bytes_per_dim_,
- (const uint8_t*)query_max.c_str(), offset,
- offset + _reader->bytes_per_dim_) >= 0) {
- // Doc's value is too high, in this dimension
- return false;
- }
- } else if (_query_type == InvertedIndexQueryType::GREATER_THAN_QUERY) {
- if (lucene::util::FutureArrays::CompareUnsigned(
- packed_value, offset, offset + _reader->bytes_per_dim_,
- (const uint8_t*)query_min.c_str(), offset,
- offset + _reader->bytes_per_dim_) <= 0) {
- // Doc's value is too high, in this dimension
- return false;
- }
- } else {
- if (lucene::util::FutureArrays::CompareUnsigned(
- packed_value, offset, offset + _reader->bytes_per_dim_,
- (const uint8_t*)query_min.c_str(), offset,
- offset + _reader->bytes_per_dim_) < 0) {
- // Doc's value is too low, in this dimension
- return false;
- }
- if (lucene::util::FutureArrays::CompareUnsigned(
- packed_value, offset, offset + _reader->bytes_per_dim_,
- (const uint8_t*)query_max.c_str(), offset,
- offset + _reader->bytes_per_dim_) > 0) {
- // Doc's value is too high, in this dimension
+ auto dim_match = [&](const BinaryType& qmax, const BinaryType& qmin) ->
bool {
+ for (int dim = 0; dim < _reader->num_data_dims_; dim++) {
+ int offset = dim * _reader->bytes_per_dim_;
+ if (!_matches(BinaryType(packed_value + offset,
_reader->bytes_per_dim_),
+ BinaryType(qmax._data + offset,
_reader->bytes_per_dim_),
+ BinaryType(qmin._data + offset,
_reader->bytes_per_dim_))) {
return false;
}
}
+ return true;
+ };
+ if (!query_points.empty()) {
+ return std::ranges::any_of(query_points, [&dim_match](const
BinaryType& query_point) {
+ return dim_match(query_point, query_point);
+ });
}
- return true;
+ return dim_match(query_max, query_min);
}
void InvertedIndexVisitor::visit(std::vector<char>& doc_id,
std::vector<uint8_t>& packed_value) {
@@ -931,77 +866,72 @@ void InvertedIndexVisitor::visit(int row_id,
std::vector<uint8_t>& packed_value)
}
}
-lucene::util::bkd::relation
InvertedIndexVisitor::compare(std::vector<uint8_t>& min_packed,
-
std::vector<uint8_t>& max_packed) {
- bool crosses = false;
+lucene::util::bkd::relation InvertedIndexVisitor::_compare(const BinaryType&
min_packed,
+ const BinaryType&
max_packed,
+ const BinaryType&
qmax,
+ const BinaryType&
qmin) {
+ bool minOutside = (_high_op == PredicateType::LE ? min_packed > qmax :
min_packed >= qmax);
+ bool maxOutside = (_low_op == PredicateType::GE ? max_packed < qmin :
max_packed <= qmin);
+ bool minInside = (_low_op == PredicateType::GE ? min_packed >= qmin :
min_packed > qmin);
+ bool maxInside = (_high_op == PredicateType::LE ? max_packed <= qmax :
max_packed < qmax);
- for (int dim = 0; dim < _reader->num_data_dims_; dim++) {
- int offset = dim * _reader->bytes_per_dim_;
+ if (minOutside || maxOutside) {
+ return lucene::util::bkd::relation::CELL_OUTSIDE_QUERY;
+ }
- if (_query_type == InvertedIndexQueryType::LESS_THAN_QUERY) {
- if (lucene::util::FutureArrays::CompareUnsigned(
- min_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_max.c_str(), offset,
- offset + _reader->bytes_per_dim_) >= 0) {
- return lucene::util::bkd::relation::CELL_OUTSIDE_QUERY;
- }
- } else if (_query_type == InvertedIndexQueryType::GREATER_THAN_QUERY) {
- if (lucene::util::FutureArrays::CompareUnsigned(
- max_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_min.c_str(), offset,
- offset + _reader->bytes_per_dim_) <= 0) {
- return lucene::util::bkd::relation::CELL_OUTSIDE_QUERY;
+ if (minInside && maxInside) {
+ return lucene::util::bkd::relation::CELL_INSIDE_QUERY;
+ }
+
+ return lucene::util::bkd::relation::CELL_CROSSES_QUERY;
+}
+
+lucene::util::bkd::relation
InvertedIndexVisitor::compare(std::vector<uint8_t>& min_packed,
+
std::vector<uint8_t>& max_packed) {
+ auto dim_compare = [&](const BinaryType& qmax,
+ const BinaryType& qmin) ->
lucene::util::bkd::relation {
+ lucene::util::bkd::relation final_relation =
lucene::util::bkd::relation::CELL_INSIDE_QUERY;
+ for (int dim = 0; dim < _reader->num_data_dims_; dim++) {
+ int offset = dim * _reader->bytes_per_dim_;
+ auto relation =
+ _compare(BinaryType(min_packed.data() + offset,
_reader->bytes_per_dim_),
+ BinaryType(max_packed.data() + offset,
_reader->bytes_per_dim_),
+ BinaryType(qmax._data + offset,
_reader->bytes_per_dim_),
+ BinaryType(qmin._data + offset,
_reader->bytes_per_dim_));
+ if (relation == lucene::util::bkd::relation::CELL_OUTSIDE_QUERY) {
+ return relation;
}
- } else {
- if (lucene::util::FutureArrays::CompareUnsigned(
- min_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_max.c_str(), offset,
- offset + _reader->bytes_per_dim_) > 0 ||
- lucene::util::FutureArrays::CompareUnsigned(
- max_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_min.c_str(), offset,
- offset + _reader->bytes_per_dim_) < 0) {
- return lucene::util::bkd::relation::CELL_OUTSIDE_QUERY;
+ if (relation == lucene::util::bkd::relation::CELL_CROSSES_QUERY) {
+ final_relation =
lucene::util::bkd::relation::CELL_CROSSES_QUERY;
}
}
- if (_query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
- _query_type == InvertedIndexQueryType::GREATER_THAN_QUERY) {
- crosses |= lucene::util::FutureArrays::CompareUnsigned(
- min_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_min.c_str(), offset,
- offset + _reader->bytes_per_dim_) <= 0 ||
- lucene::util::FutureArrays::CompareUnsigned(
- max_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_max.c_str(), offset,
- offset + _reader->bytes_per_dim_) >= 0;
- } else {
- crosses |= lucene::util::FutureArrays::CompareUnsigned(
- min_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_min.c_str(), offset,
- offset + _reader->bytes_per_dim_) < 0 ||
- lucene::util::FutureArrays::CompareUnsigned(
- max_packed.data(), offset, offset +
_reader->bytes_per_dim_,
- (const uint8_t*)query_max.c_str(), offset,
- offset + _reader->bytes_per_dim_) > 0;
+ return final_relation;
+ };
+ if (!query_points.empty()) {
+ lucene::util::bkd::relation final_relation =
+ lucene::util::bkd::relation::CELL_OUTSIDE_QUERY;
+ for (auto query_point : query_points) {
+ lucene::util::bkd::relation relation = dim_compare(query_point,
query_point);
+ if (relation == lucene::util::bkd::relation::CELL_INSIDE_QUERY) {
+ return relation;
+ }
+ if (relation == lucene::util::bkd::relation::CELL_CROSSES_QUERY) {
+ final_relation =
lucene::util::bkd::relation::CELL_CROSSES_QUERY;
+ }
}
+ return final_relation;
}
- if (crosses) {
- return lucene::util::bkd::relation::CELL_CROSSES_QUERY;
- } else {
- return lucene::util::bkd::relation::CELL_INSIDE_QUERY;
- }
+ return dim_compare(query_max, query_min);
}
Status InvertedIndexIterator::read_from_inverted_index(const std::string&
column_name,
- const void* query_value,
- InvertedIndexQueryType
query_type,
+ InvertedIndexQueryBase*
query_value,
uint32_t
segment_num_rows,
roaring::Roaring*
bit_map, bool skip_try) {
if (!skip_try && _reader->type() == InvertedIndexReaderType::BKD) {
auto query_bkd_limit_percent =
config::query_bkd_inverted_index_limit_percent;
uint32_t hit_count = 0;
- RETURN_IF_ERROR(
- try_read_from_inverted_index(column_name, query_value,
query_type, &hit_count));
+ RETURN_IF_ERROR(try_read_from_inverted_index(column_name, query_value,
&hit_count));
if (hit_count > segment_num_rows * query_bkd_limit_percent / 100) {
return Status::Error<ErrorCode::INVERTED_INDEX_BYPASS>(
"hit count: {}, bkd inverted reached limit {}%, segment
num rows:{}", hit_count,
@@ -1009,22 +939,16 @@ Status
InvertedIndexIterator::read_from_inverted_index(const std::string& column
}
}
- RETURN_IF_ERROR(
- _reader->query(_stats, _runtime_state, column_name, query_value,
query_type, bit_map));
+ RETURN_IF_ERROR(_reader->query(_stats, _runtime_state, column_name,
query_value, bit_map));
return Status::OK();
}
Status InvertedIndexIterator::try_read_from_inverted_index(const std::string&
column_name,
- const void*
query_value,
-
InvertedIndexQueryType query_type,
+
InvertedIndexQueryBase* query_value,
uint32_t* count) {
// NOTE: only bkd index support try read now.
- if (query_type == InvertedIndexQueryType::GREATER_EQUAL_QUERY ||
- query_type == InvertedIndexQueryType::GREATER_THAN_QUERY ||
- query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY ||
- query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
- query_type == InvertedIndexQueryType::EQUAL_QUERY) {
- RETURN_IF_ERROR(_reader->try_query(_stats, column_name, query_value,
query_type, count));
+ if (_reader->type() == InvertedIndexReaderType::BKD) {
+ RETURN_IF_ERROR(_reader->try_query(_stats, column_name, query_value,
count));
}
return Status::OK();
}
@@ -1037,5 +961,4 @@ const std::map<string, string>&
InvertedIndexIterator::get_index_properties() co
return _reader->get_index_properties();
}
-} // namespace segment_v2
-} // namespace doris
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
index 20c5c731f9e..bd133dec801 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h
@@ -18,7 +18,6 @@
#pragma once
#include <CLucene/util/bkd/bkd_reader.h>
-#include <stdint.h>
#include <memory>
#include <string>
@@ -29,20 +28,21 @@
#include "io/fs/file_system.h"
#include "io/fs/path.h"
#include "olap/inverted_index_parser.h"
+#include "olap/rowset/segment_v2/inverted_index/query/inverted_index_query.h"
#include "olap/rowset/segment_v2/inverted_index_cache.h"
#include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
+#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/rowset/segment_v2/inverted_index_query_type.h"
#include "olap/tablet_schema.h"
+#include "runtime/type_limit.h"
namespace lucene {
namespace store {
class Directory;
} // namespace store
-namespace util {
-namespace bkd {
+namespace util::bkd {
class bkd_docid_set_iterator;
-} // namespace bkd
-} // namespace util
+} // namespace util::bkd
} // namespace lucene
namespace roaring {
class Roaring;
@@ -53,6 +53,7 @@ class KeyCoder;
class TypeInfo;
struct OlapReaderStatistics;
class RuntimeState;
+enum class PredicateType;
namespace segment_v2 {
@@ -72,18 +73,39 @@ class InvertedIndexReader : public
std::enable_shared_from_this<InvertedIndexRea
public:
explicit InvertedIndexReader(io::FileSystemSPtr fs, const std::string&
path,
const TabletIndex* index_meta)
- : _fs(fs), _path(path), _index_meta(*index_meta) {}
+ : _fs(std::move(fs)), _path(path), _index_meta(*index_meta) {
+ io::Path io_path(_path);
+ auto index_dir = io_path.parent_path();
+ auto index_file_name =
InvertedIndexDescriptor::get_index_file_name(io_path.filename(),
+
index_meta->index_id());
+ auto index_file_path = index_dir / index_file_name;
+ _file_full_path = index_file_path;
+ _file_name = index_file_name;
+ _file_dir = index_dir.c_str();
+ }
+ virtual Status handle_cache(InvertedIndexQueryCache* cache,
+ const InvertedIndexQueryCache::CacheKey&
cache_key,
+ InvertedIndexQueryCacheHandle* cache_handler,
+ OlapReaderStatistics* stats, roaring::Roaring*
bit_map) {
+ if (cache->lookup(cache_key, cache_handler)) {
+ stats->inverted_index_query_cache_hit++;
+ SCOPED_RAW_TIMER(&stats->inverted_index_query_bitmap_copy_timer);
+ *bit_map = *cache_handler->get_bitmap();
+ return Status::OK();
+ }
+ stats->inverted_index_query_cache_miss++;
+ return Status::Error<ErrorCode::KEY_NOT_FOUND>("cache miss");
+ }
virtual ~InvertedIndexReader() = default;
// create a new column iterator. Client should delete returned iterator
virtual Status new_iterator(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
std::unique_ptr<InvertedIndexIterator>*
iterator) = 0;
virtual Status query(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
- const std::string& column_name, const void*
query_value,
- InvertedIndexQueryType query_type, roaring::Roaring*
bit_map) = 0;
+ const std::string& column_name,
InvertedIndexQueryBase* query_value,
+ roaring::Roaring* bit_map) = 0;
virtual Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
- const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) = 0;
+ InvertedIndexQueryBase* query_value, uint32_t*
count) = 0;
Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr);
@@ -108,31 +130,42 @@ public:
InvertedIndexCtx* inverted_index_ctx);
protected:
- bool _is_range_query(InvertedIndexQueryType query_type);
- bool _is_match_query(InvertedIndexQueryType query_type);
friend class InvertedIndexIterator;
io::FileSystemSPtr _fs;
const std::string& _path;
TabletIndex _index_meta;
+ io::Path _file_full_path;
+ std::string _file_name;
+ std::string _file_dir;
};
class FullTextIndexReader : public InvertedIndexReader {
ENABLE_FACTORY_CREATOR(FullTextIndexReader);
+private:
+ InvertedIndexCtxSPtr _inverted_index_ctx {};
+
public:
explicit FullTextIndexReader(io::FileSystemSPtr fs, const std::string&
path,
const TabletIndex* index_meta)
- : InvertedIndexReader(fs, path, index_meta) {}
+ : InvertedIndexReader(fs, path, index_meta) {
+ _inverted_index_ctx = std::make_shared<InvertedIndexCtx>();
+ _inverted_index_ctx->parser_type =
get_inverted_index_parser_type_from_string(
+ get_parser_string_from_properties(_index_meta.properties()));
+ _inverted_index_ctx->parser_mode =
+
get_parser_mode_string_from_properties(_index_meta.properties());
+ _inverted_index_ctx->char_filter_map =
+
get_parser_char_filter_map_from_properties(_index_meta.properties());
+ }
~FullTextIndexReader() override = default;
Status new_iterator(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
std::unique_ptr<InvertedIndexIterator>* iterator)
override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
- const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, roaring::Roaring* bit_map)
override;
+ const std::string& column_name, InvertedIndexQueryBase*
query_value,
+ roaring::Roaring* bit_map) override;
Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
- const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) override {
+ InvertedIndexQueryBase* query_value, uint32_t* count)
override {
return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
"FullTextIndexReader not support try_query");
}
@@ -153,6 +186,9 @@ private:
const std::shared_ptr<roaring::Roaring>&
term_match_bitmap);
void check_null_bitmap(const IndexSearcherPtr& index_searcher, bool&
null_bitmap_already_read);
+ Status _query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
+ const std::string& column_name, std::string& search_str,
+ InvertedIndexQueryType query_type, roaring::Roaring*
bit_map);
};
class StringTypeInvertedIndexReader : public InvertedIndexReader {
@@ -167,14 +203,18 @@ public:
Status new_iterator(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
std::unique_ptr<InvertedIndexIterator>* iterator)
override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
- const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, roaring::Roaring* bit_map)
override;
+ const std::string& column_name, InvertedIndexQueryBase*
query_value,
+ roaring::Roaring* bit_map) override;
Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
- const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) override {
+ InvertedIndexQueryBase* query_value, uint32_t* count)
override {
return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>(
"StringTypeInvertedIndexReader not support try_query");
}
+ Status handle_range_query(const std::string& column_name,
OlapReaderStatistics* stats,
+ InvertedIndexRangeQueryI* query,
roaring::Roaring* bit_map);
+ Status handle_point_query(const std::string& column_name,
OlapReaderStatistics* stats,
+ InvertedIndexPointQueryI* query,
roaring::Roaring* bit_map);
+
InvertedIndexReaderType type() override;
};
@@ -184,19 +224,20 @@ private:
uint32_t _num_hits;
bool _only_count;
lucene::util::bkd::bkd_reader* _reader;
- InvertedIndexQueryType _query_type;
+ PredicateType _low_op;
+ PredicateType _high_op;
public:
- std::string query_min;
- std::string query_max;
+ BinaryType query_min;
+ BinaryType query_max;
+ std::vector<BinaryType> query_points;
public:
- InvertedIndexVisitor(roaring::Roaring* hits, InvertedIndexQueryType
query_type,
+ InvertedIndexVisitor(roaring::Roaring* hits, InvertedIndexQueryBase*
query_value,
bool only_count = false);
~InvertedIndexVisitor() override = default;
void set_reader(lucene::util::bkd::bkd_reader* r) { _reader = r; }
- lucene::util::bkd::bkd_reader* get_reader() { return _reader; }
void visit(int row_id) override;
void visit(roaring::Roaring& r) override;
@@ -207,17 +248,17 @@ public:
void visit(lucene::util::bkd::bkd_docid_set_iterator* iter,
std::vector<uint8_t>& packed_value) override;
bool matches(uint8_t* packed_value);
+ bool _matches(const BinaryType& packed_value, const BinaryType& qmax,
const BinaryType& qmin);
lucene::util::bkd::relation compare(std::vector<uint8_t>& min_packed,
std::vector<uint8_t>& max_packed)
override;
+ lucene::util::bkd::relation _compare(const BinaryType& min_packed, const
BinaryType& max_packed,
+ const BinaryType& qmax, const
BinaryType& qmin);
uint32_t get_num_hits() const { return _num_hits; }
};
class BkdIndexReader : public InvertedIndexReader {
ENABLE_FACTORY_CREATOR(BkdIndexReader);
-private:
- std::string _file_full_path;
-
public:
explicit BkdIndexReader(io::FileSystemSPtr fs, const std::string& path,
const TabletIndex* index_meta);
@@ -240,27 +281,16 @@ public:
std::unique_ptr<InvertedIndexIterator>* iterator)
override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
- const std::string& column_name, const void* query_value,
- InvertedIndexQueryType query_type, roaring::Roaring* bit_map)
override;
+ const std::string& column_name, InvertedIndexQueryBase*
query_value,
+ roaring::Roaring* bit_map) override;
Status try_query(OlapReaderStatistics* stats, const std::string&
column_name,
- const void* query_value, InvertedIndexQueryType
query_type,
- uint32_t* count) override;
- Status bkd_query(OlapReaderStatistics* stats, const std::string&
column_name,
- const void* query_value, InvertedIndexQueryType
query_type,
- std::shared_ptr<lucene::util::bkd::bkd_reader> r,
- InvertedIndexVisitor* visitor);
-
- Status handle_cache(InvertedIndexQueryCache* cache,
- const InvertedIndexQueryCache::CacheKey& cache_key,
- InvertedIndexQueryCacheHandle* cache_handler,
OlapReaderStatistics* stats,
- roaring::Roaring* bit_map);
+ InvertedIndexQueryBase* query_value, uint32_t* count)
override;
InvertedIndexReaderType type() override;
Status get_bkd_reader(std::shared_ptr<lucene::util::bkd::bkd_reader>*
reader);
private:
const TypeInfo* _type_info {};
- const KeyCoder* _value_key_coder {};
std::unique_ptr<DorisCompoundReader> _compoundReader;
};
@@ -270,13 +300,13 @@ class InvertedIndexIterator {
public:
InvertedIndexIterator(OlapReaderStatistics* stats, RuntimeState*
runtime_state,
std::shared_ptr<InvertedIndexReader> reader)
- : _stats(stats), _runtime_state(runtime_state), _reader(reader) {}
+ : _stats(stats), _runtime_state(runtime_state),
_reader(std::move(reader)) {}
- Status read_from_inverted_index(const std::string& column_name, const
void* query_value,
- InvertedIndexQueryType query_type,
uint32_t segment_num_rows,
+ Status read_from_inverted_index(const std::string& column_name,
+ InvertedIndexQueryBase* query_value,
uint32_t segment_num_rows,
roaring::Roaring* bit_map, bool skip_try =
false);
- Status try_read_from_inverted_index(const std::string& column_name, const
void* query_value,
- InvertedIndexQueryType query_type,
uint32_t* count);
+ Status try_read_from_inverted_index(const std::string& column_name,
+ InvertedIndexQueryBase* query_value,
uint32_t* count);
Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr) {
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 28274752b05..ab6cc36cb9c 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -234,8 +234,10 @@ Status SegmentIterator::_init_impl(const
StorageReadOptions& opts) {
predicate->clone(&cloned);
_pool->add(cloned);
_col_predicates.emplace_back(cloned);
+ origin_to_clone_predicates[predicate] = cloned;
} else {
_col_predicates.emplace_back(predicate);
+ origin_to_clone_predicates[predicate] = predicate;
}
}
_tablet_id = opts.tablet_id;
@@ -866,6 +868,7 @@ inline bool
SegmentIterator::_inverted_index_not_support_pred_type(const Predica
Status SegmentIterator::_apply_inverted_index_on_column_predicate(
ColumnPredicate* pred, std::vector<ColumnPredicate*>&
remaining_predicates,
bool* continue_apply) {
+
SCOPED_RAW_TIMER(&_opts.stats->inverted_index_column_predicate_filter_timer);
if (!_check_apply_by_inverted_index(pred)) {
remaining_predicates.emplace_back(pred);
} else {
@@ -916,7 +919,8 @@ Status
SegmentIterator::_apply_inverted_index_on_column_predicate(
Status SegmentIterator::_apply_inverted_index_on_block_column_predicate(
ColumnId column_id, MutilColumnBlockPredicate* pred,
std::set<const ColumnPredicate*>& no_need_to_pass_column_predicate_set,
- bool* continue_apply) {
+ std::vector<ColumnPredicate*>& remaining_predicates, bool*
continue_apply) {
+
SCOPED_RAW_TIMER(&_opts.stats->inverted_index_block_column_predicate_filter_timer);
bool handle_by_fulltext = _column_has_fulltext_index(column_id);
std::set<const ColumnPredicate*> predicate_set {};
@@ -933,15 +937,29 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
std::string column_name = _schema->column(column_id)->name();
- auto res = pred->evaluate(column_name,
_inverted_index_iterators[column_id].get(),
- num_rows(), &output_result);
+ auto process_predicate_set = [&](const auto& predicate_set) {
+ for (auto& orig_pred : predicate_set) {
+ if (origin_to_clone_predicates.contains(orig_pred)) {
+ auto& cloned_pred = origin_to_clone_predicates[orig_pred];
+ no_need_to_pass_column_predicate_set.emplace(cloned_pred);
+ } else {
+ LOG(ERROR)
+ << "column:" << column_name << " pred:" <<
orig_pred->debug_string()
+ << " is not in origin_to_clone_predicates when
process_predicate_set";
+ }
+ }
+ };
+
+ auto res = pred->evaluate(*_schema,
_inverted_index_iterators[column_id].get(), num_rows(),
+ &output_result);
if (res.ok()) {
if (_check_column_pred_all_push_down(column_name) &&
!all_predicates_are_marked_by_runtime_filter(predicate_set)) {
_need_read_data_indices[column_id] = false;
}
- no_need_to_pass_column_predicate_set.insert(predicate_set.begin(),
predicate_set.end());
+ process_predicate_set(predicate_set);
+
//no_need_to_pass_column_predicate_set.insert(predicate_set.begin(),
predicate_set.end());
_row_bitmap &= output_result;
if (_row_bitmap.isEmpty()) {
// all rows have been pruned, no need to process further
predicates
@@ -949,8 +967,25 @@ Status
SegmentIterator::_apply_inverted_index_on_block_column_predicate(
}
return res;
} else {
- //TODO:mock until AndBlockColumnPredicate evaluate is ok.
- if (res.code() == ErrorCode::NOT_IMPLEMENTED_ERROR) {
+ // because column predicate only process LE/LT/GT/GE predicate
type, need_remaining_after_evaluate only support in_or_list
+ bool need_remaining_after_evaluate = false;
+ if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
+ // downgrade without index query
+ //process_predicate_set(predicate_set);
+ // need to pass non-index evaluate after
+ for (auto& orig_pred : predicate_set) {
+ if (origin_to_clone_predicates.contains(orig_pred)) {
+ auto& cloned_pred =
origin_to_clone_predicates[orig_pred];
+ remaining_predicates.push_back(cloned_pred);
+
no_need_to_pass_column_predicate_set.emplace(cloned_pred);
+ } else {
+ LOG(ERROR)
+ << "column:" << column_name << " pred:" <<
orig_pred->debug_string()
+ << " is not in origin_to_clone_predicates when
"
+ "_downgrade_without_index";
+ }
+ }
+ _not_apply_index_pred.insert(column_id);
return Status::OK();
}
LOG(WARNING) << "failed to evaluate index"
@@ -1008,7 +1043,8 @@ Status SegmentIterator::_apply_inverted_index() {
auto pred = entry.second;
bool continue_apply = true;
RETURN_IF_ERROR(_apply_inverted_index_on_block_column_predicate(
- column_id, pred.get(), no_need_to_pass_column_predicate_set,
&continue_apply));
+ column_id, pred.get(), no_need_to_pass_column_predicate_set,
remaining_predicates,
+ &continue_apply));
if (!continue_apply) {
break;
}
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h
b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 33d3a3f5f9c..3559dfbaaf4 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -181,7 +181,7 @@ private:
[[nodiscard]] Status _apply_inverted_index_on_block_column_predicate(
ColumnId column_id, MutilColumnBlockPredicate* pred,
std::set<const ColumnPredicate*>&
no_need_to_pass_column_predicate_set,
- bool* continue_apply);
+ std::vector<ColumnPredicate*>& remaining_predicates, bool*
continue_apply);
[[nodiscard]] Status _apply_index_except_leafnode_of_andnode();
[[nodiscard]] Status _apply_bitmap_index_except_leafnode_of_andnode(
ColumnPredicate* pred, roaring::Roaring* output_result);
@@ -392,6 +392,8 @@ private:
// make a copy of `_opts.column_predicates` in order to make local changes
std::vector<ColumnPredicate*> _col_predicates;
std::vector<ColumnPredicate*> _col_preds_except_leafnode_of_andnode;
+ // comparison predicate will be cloned when segment_iterator init, we need
to store the pair in order to find clone predicate directly.
+ std::unordered_map<const ColumnPredicate*, ColumnPredicate*>
origin_to_clone_predicates;
vectorized::VExprContextSPtrs _common_expr_ctxs_push_down;
bool _enable_common_expr_pushdown = false;
std::vector<vectorized::VExprSPtr> _remaining_conjunct_roots;
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp
b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index 5aa179d3006..698b50923a9 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -165,11 +165,18 @@ Status NewOlapScanNode::_init_profile() {
_inverted_index_filter_counter =
ADD_COUNTER(_segment_profile, "RowsInvertedIndexFiltered",
TUnit::UNIT);
_inverted_index_filter_timer = ADD_TIMER(_segment_profile,
"InvertedIndexFilterTime");
+ _inverted_index_block_column_predicate_filter_timer =
+ ADD_TIMER(_segment_profile,
"InvertedIndexBlockColumnPredicateFilterTime");
+ _inverted_index_column_predicate_filter_timer =
+ ADD_TIMER(_segment_profile,
"InvertedIndexColumnPredicateFilterTime");
_inverted_index_query_cache_hit_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexQueryCacheHit",
TUnit::UNIT);
_inverted_index_query_cache_miss_counter =
ADD_COUNTER(_segment_profile, "InvertedIndexQueryCacheMiss",
TUnit::UNIT);
_inverted_index_query_timer = ADD_TIMER(_segment_profile,
"InvertedIndexQueryTime");
+ _inverted_index_try_query_timer = ADD_TIMER(_segment_profile,
"InvertedIndexTryQueryTime");
+ _inverted_index_bkd_intersect_timer =
+ ADD_TIMER(_segment_profile, "InvertedIndexBKDIntersectTime");
_inverted_index_query_bitmap_copy_timer =
ADD_TIMER(_segment_profile, "InvertedIndexQueryBitmapCopyTime");
_inverted_index_query_bitmap_op_timer =
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h
b/be/src/vec/exec/scan/new_olap_scan_node.h
index 0725c37cf5e..ff64ed6d567 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.h
+++ b/be/src/vec/exec/scan/new_olap_scan_node.h
@@ -182,9 +182,13 @@ private:
RuntimeProfile::Counter* _inverted_index_filter_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_filter_timer = nullptr;
+ RuntimeProfile::Counter*
_inverted_index_block_column_predicate_filter_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_column_predicate_filter_timer =
nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr;
RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter =
nullptr;
RuntimeProfile::Counter* _inverted_index_query_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_try_query_timer = nullptr;
+ RuntimeProfile::Counter* _inverted_index_bkd_intersect_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_query_bitmap_op_timer = nullptr;
RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr;
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index ea4187e556a..d26241532b2 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -585,6 +585,13 @@ void NewOlapScanner::_update_counters_before_close() {
stats.inverted_index_searcher_open_timer);
COUNTER_UPDATE(olap_parent->_inverted_index_searcher_search_timer,
stats.inverted_index_searcher_search_timer);
+
COUNTER_UPDATE(olap_parent->_inverted_index_block_column_predicate_filter_timer,
+ stats.inverted_index_block_column_predicate_filter_timer);
+ COUNTER_UPDATE(olap_parent->_inverted_index_column_predicate_filter_timer,
+ stats.inverted_index_column_predicate_filter_timer);
+ COUNTER_UPDATE(olap_parent->_inverted_index_bkd_intersect_timer,
+ stats.inverted_index_bkd_intersect_timer);
+ COUNTER_UPDATE(olap_parent->_inverted_index_try_query_timer,
stats.inverted_index_try_query_timer);
if (config::enable_file_cache) {
io::FileCacheProfileReporter
cache_profile(olap_parent->_segment_profile.get());
diff --git a/be/src/vec/exec/scan/vscan_node.h
b/be/src/vec/exec/scan/vscan_node.h
index 7600189a251..f81753efb6e 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -277,7 +277,7 @@ protected:
std::unordered_map<std::string, ColumnValueRangeType>
_colname_to_value_range;
/**
* _colname_to_value_range only store the leaf of and in the conjunct expr
tree,
- * we use _compound_value_ranges to store conresponding value ranges
+ * we use _compound_value_ranges to store corresponding value ranges
* in the one compound relationship except the leaf of and node,
* such as `where a > 1 or b > 10 and c < 200`, the expr tree like:
* or
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
