This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new d0ec1f6b0ff branch-4.2: [fix](be) Isolate runtime filter wrappers
shared with local consumers (#70623)
d0ec1f6b0ff is described below
commit d0ec1f6b0ff709420a54fde7e79c24bc0704eba4
Author: Jerry Hu <[email protected]>
AuthorDate: Fri Nov 20 01:19:37 2026 +0800
branch-4.2: [fix](be) Isolate runtime filter wrappers shared with local
consumers (#70623)
## Summary
Backport #68684 to branch-4.2.
- Keep the runtime-filter merger from mutating a producer wrapper that
plain local consumers or broadcast producers may still read. Clone
non-exclusive inputs, while retaining direct ownership transfer for
exclusive inputs.
- Deep-copy IN, MIN/MAX, and BLOOM filter state. When an IN_OR_BLOOM
filter transitions to BLOOM, copy a shared input Bloom filter before
merging into it.
- Preserve branch-4.2 behavior and APIs; this branch does not have the
bucket-pruning cache used by the source change.
## Testing
- [x] BE ASAN UT: `./run-be-ut.sh --run -j 64
--filter='RuntimeFilterMergerTest.*:RuntimeFilterProducerTest.*:RuntimeFilterWrapperTest.*'`
(41 tests passed).
- [x] `./build-support/check-format.sh`.
- [x] `clang-format --dry-run --Werror` on the changed C++ files.
- [x] `git diff --check` for the committed change.
- [ ] SQL regression was not run; the source change includes no SQL
regression cases.
Issue Number: None
---
be/src/exec/runtime_filter/runtime_filter_merger.h | 36 +-
.../runtime_filter/runtime_filter_producer.cpp | 8 +-
.../exec/runtime_filter/runtime_filter_wrapper.cpp | 31 +-
.../exec/runtime_filter/runtime_filter_wrapper.h | 6 +-
be/src/exprs/bloom_filter_func.h | 34 ++
be/src/exprs/filter_base.h | 1 +
be/src/exprs/minmax_predicate.h | 11 +
.../runtime_filter/runtime_filter_merger_test.cpp | 242 ++++++++++
.../runtime_filter_producer_test.cpp | 519 ++++++++++++++++++++-
.../runtime_filter/runtime_filter_wrapper_test.cpp | 341 ++++++++++++++
10 files changed, 1214 insertions(+), 15 deletions(-)
diff --git a/be/src/exec/runtime_filter/runtime_filter_merger.h
b/be/src/exec/runtime_filter/runtime_filter_merger.h
index 4e1d19dabc3..292cd00efad 100644
--- a/be/src/exec/runtime_filter/runtime_filter_merger.h
+++ b/be/src/exec/runtime_filter/runtime_filter_merger.h
@@ -57,25 +57,41 @@ public:
_received_rf_size_num, _received_sum_size);
}
- // If input is a disabled predicate, the final result is a disabled
predicate.
+ // Keep a private copy of the first input: later merges must not change a
producer wrapper
+ // which a local consumer may already be probing. A disabled input
disables the result.
// Returns true only for the call that makes the merger ready.
Status merge_from(const RuntimeFilter* other, bool* ready) {
std::unique_lock<std::recursive_mutex> l(_rmtx);
- _received_producer_num++;
- if (_expected_producer_num < _received_producer_num) {
+ *ready = false;
+ if (_expected_producer_num <= _received_producer_num) {
return Status::InternalError(
"runtime filter merger input product more than expected,
{}", debug_string());
}
+ Status status;
+ if (_wrapper->get_state() == RuntimeFilterWrapper::State::UNINITED) {
+ // For example, consider a join whose probe side has two scan
targets:
+ // SELECT p.k FROM build_table b JOIN
+ // (SELECT k FROM local_probe UNION ALL SELECT k FROM
merged_probe) p ON b.k = p.k;
+ // Depending on the physical plan, the same RF can be routed as
follows:
+ // producer 1 RF {5} -----> Scan(local_probe) [direct, no merge]
+ // |-- copy --> merger <-- producer 2 RF {6}
+ // |
+ // +----> Scan(merged_probe) [merged: {5,
6}]
+ // The merger must clone producer 1's wrapper: adopting it would
let the
+ // later merge change the direct scan's filter from {5} to {5, 6}.
+ status = other->_wrapper->clone(&_wrapper);
+ } else {
+ status = _wrapper->merge(other->_wrapper.get());
+ }
+ RETURN_IF_ERROR(status);
+ // A failed copy or merge must not count as an arrival: otherwise
later inputs could
+ // publish a filter that does not contain the failed producer's values.
+ ++_received_producer_num;
*ready = _received_producer_num == _expected_producer_num;
- if (_received_producer_num == _expected_producer_num) {
+ if (*ready) {
_rf_state = State::READY;
}
- if (_wrapper->get_state() == RuntimeFilterWrapper::State::UNINITED) {
- _wrapper = other->_wrapper;
- return Status::OK();
- }
- auto st = _wrapper->merge(other->_wrapper.get());
- return st;
+ return Status::OK();
}
// Only raise the expected producer count. RuntimeFilterMgr may compute the
diff --git a/be/src/exec/runtime_filter/runtime_filter_producer.cpp
b/be/src/exec/runtime_filter/runtime_filter_producer.cpp
index b3684a774ab..350ee069e4a 100644
--- a/be/src/exec/runtime_filter/runtime_filter_producer.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_producer.cpp
@@ -27,6 +27,7 @@
namespace doris {
#include "common/compile_check_begin.h"
+
Status RuntimeFilterProducer::_send_to_remote_targets(RuntimeState* state,
RuntimeFilter*
merger_filter) {
TNetworkAddress addr;
@@ -45,6 +46,9 @@ Status
RuntimeFilterProducer::_send_to_local_targets(RuntimeState* state, Runtim
return Status::OK();
};
+// Pre-existing complexity debt from the target/broadcast/remote branching
below is not addressed
+// here to keep this change focused.
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
Status RuntimeFilterProducer::publish(RuntimeState* state, bool
build_hash_table) {
std::unique_lock<std::recursive_mutex> l(_rmtx);
_check_state({State::READY_TO_PUBLISH});
@@ -76,6 +80,8 @@ Status RuntimeFilterProducer::publish(RuntimeState* state,
bool build_hash_table
if (!_has_remote_target) {
// A runtime filter may have multiple targets and some of those are
local-merge RF and others are not.
// So for all runtime filters' producers, `publish` should notify all
consumers in global RF mgr which manages local-merge RF and local RF mgr which
manages others.
+ // The merger copies this wrapper before later merges, so plain local
consumers can read
+ // the producer wrapper independently, including with mixed local and
merge targets.
RETURN_IF_ERROR(do_merge());
RETURN_IF_ERROR(_send_to_local_targets(state, this, false));
} else if (build_hash_table) {
@@ -88,7 +94,7 @@ Status RuntimeFilterProducer::publish(RuntimeState* state,
bool build_hash_table
DCHECK(_is_broadcast_join);
}
- // wrapper may moved to rf merger, release wrapper here to make sure
thread safe
+ // Release the producer reference after consumers and the merger hold
their own references.
_wrapper.reset();
set_state(State::PUBLISHED);
return Status::OK();
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
index 079d87ad487..01d064a835d 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp
@@ -124,6 +124,9 @@ bool RuntimeFilterWrapper::build_bf_by_runtime_size() const
{
return _bloom_filter_func ? _bloom_filter_func->build_bf_by_runtime_size()
: false;
}
+// Pre-existing size/complexity debt from the per-filter-type switch below,
already over
+// threshold before this change; not addressed here to keep this diff focused.
+//
NOLINTNEXTLINE(readability-function-size,readability-function-cognitive-complexity)
Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) {
if (_state == State::DISABLED) {
return Status::OK();
@@ -179,8 +182,9 @@ Status RuntimeFilterWrapper::merge(const
RuntimeFilterWrapper* other) {
RETURN_IF_ERROR(_change_to_bloom_filter());
}
} else {
- // case1&case2: use input bf directly and insert hybrid set
data into bf
- _bloom_filter_func = other->_bloom_filter_func;
+ // Use a private copy before adding this IN set; the input
Bloom may still be
+ // read by its own consumers.
+
RETURN_IF_ERROR(other->_bloom_filter_func->clone(&_bloom_filter_func, true));
RETURN_IF_ERROR(_change_to_bloom_filter());
}
} else {
@@ -201,6 +205,29 @@ Status RuntimeFilterWrapper::merge(const
RuntimeFilterWrapper* other) {
return Status::OK();
}
+Status RuntimeFilterWrapper::clone(std::shared_ptr<RuntimeFilterWrapper>* res)
const {
+ auto cloned = std::make_shared<RuntimeFilterWrapper>(_column_return_type,
_filter_type,
+ _filter_id,
_state.load(), _max_in_num);
+ cloned->_disable_always_true_logic = _disable_always_true_logic;
+ cloned->_reason.update(_reason.status());
+ if (_hybrid_set) {
+ cloned->_hybrid_set.reset(create_set(_column_return_type,
_hybrid_set->null_aware()));
+ cloned->_hybrid_set->insert(_hybrid_set.get());
+ }
+ if (_minmax_func) {
+ cloned->_minmax_func.reset(_minmax_func->clone());
+ }
+ if (_bloom_filter_func) {
+ // An IN_OR_BLOOM filter which is still an IN filter never reads its
bloom filter: a
+ // later merge which changes the copy to a bloom filter either
re-initializes the bloom
+ // filter or clones the input one first. So do not copy the unused
bloom filter.
+ RETURN_IF_ERROR(_bloom_filter_func->clone(
+ &cloned->_bloom_filter_func, get_real_type() ==
RuntimeFilterType::BLOOM_FILTER));
+ }
+ *res = std::move(cloned);
+ return Status::OK();
+}
+
Status RuntimeFilterWrapper::_assign(const PInFilter& in_filter, bool
contain_null) {
if (contain_null) {
_hybrid_set->insert((const void*)nullptr);
diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h
b/be/src/exec/runtime_filter/runtime_filter_wrapper.h
index 290599b3e82..772722bbb7a 100644
--- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h
+++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h
@@ -53,7 +53,11 @@ public:
Status init(const size_t runtime_size);
Status insert(const ColumnPtr& column, size_t start);
- Status merge(const RuntimeFilterWrapper* wrapper);
+ // Merge into this wrapper without modifying the input.
+ Status merge(const RuntimeFilterWrapper* other);
+ // Deep copy of the filter. The result shares no mutable data with this
wrapper, so the
+ // merger can keep merging into the copy while this one is already used by
consumers.
+ Status clone(std::shared_ptr<RuntimeFilterWrapper>* res) const;
template <class T>
Status assign(const T& request, butil::IOBufAsZeroCopyInputStream* data);
diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h
index 32e6cc26f1a..fb1fc5a960f 100644
--- a/be/src/exprs/bloom_filter_func.h
+++ b/be/src/exprs/bloom_filter_func.h
@@ -133,6 +133,12 @@ public:
_bloom_filter = other_func->_bloom_filter;
}
+ // Deep copy. The returned filter owns a separately allocated bloom
filter. Only the
+ // parameters are copied if `copy_bloom_filter` is false, e.g. the filter
is still used as
+ // an IN filter, so its bloom filter is never read.
+ virtual Status clone(std::shared_ptr<BloomFilterFuncBase>* res,
+ bool copy_bloom_filter) const = 0;
+
virtual void insert_set(std::shared_ptr<HybridSetBase> set) = 0;
virtual void insert_fixed_len(const ColumnPtr& column, size_t start) = 0;
@@ -167,6 +173,26 @@ private:
}
protected:
+ Status _deep_copy_from(const BloomFilterFuncBase& other, bool
copy_bloom_filter) {
+ _bloom_filter_length = other._bloom_filter_length;
+ _runtime_bloom_filter_min_size = other._runtime_bloom_filter_min_size;
+ _runtime_bloom_filter_max_size = other._runtime_bloom_filter_max_size;
+ _build_bf_by_runtime_size = other._build_bf_by_runtime_size;
+ _bloom_filter_size_calculated_by_ndv =
other._bloom_filter_size_calculated_by_ndv;
+ if (!copy_bloom_filter) {
+ return Status::OK();
+ }
+ if (other._bloom_filter == nullptr) {
+ // `other` is not initialized yet, e.g. the producer is disabled
before it is built.
+ return Status::OK();
+ }
+ _bloom_filter_alloced = other._bloom_filter_alloced;
+ _bloom_filter.reset(BloomFilterAdaptor::create(_null_aware));
+ RETURN_IF_ERROR(_bloom_filter->init(_bloom_filter_alloced));
+ _bloom_filter->set_contain_null(other.contain_null());
+ return _bloom_filter->merge(other._bloom_filter.get());
+ }
+
int64_t _bloom_filter_alloced = 0;
std::shared_ptr<BloomFilterAdaptor> _bloom_filter;
int64_t _bloom_filter_length;
@@ -180,6 +206,14 @@ template <PrimitiveType type>
class BloomFilterFunc final : public BloomFilterFuncBase {
public:
BloomFilterFunc(bool null_aware) : BloomFilterFuncBase(null_aware) {}
+
+ Status clone(std::shared_ptr<BloomFilterFuncBase>* res, bool
copy_bloom_filter) const override {
+ auto cloned = std::make_shared<BloomFilterFunc<type>>(_null_aware);
+ RETURN_IF_ERROR(cloned->_deep_copy_from(*this, copy_bloom_filter));
+ *res = std::move(cloned);
+ return Status::OK();
+ }
+
void insert_set(std::shared_ptr<HybridSetBase> set) override {
OpV2::insert_set(*_bloom_filter, set);
_bloom_filter->set_contain_null(set->contain_null());
diff --git a/be/src/exprs/filter_base.h b/be/src/exprs/filter_base.h
index 25778059886..ca808b310ca 100644
--- a/be/src/exprs/filter_base.h
+++ b/be/src/exprs/filter_base.h
@@ -25,6 +25,7 @@ class FilterBase {
public:
FilterBase(bool null_aware) : _null_aware(null_aware) {}
bool contain_null() const { return _null_aware && _contain_null; }
+ bool null_aware() const { return _null_aware; }
void set_contain_null(bool contain_null) { _contain_null |= contain_null; }
diff --git a/be/src/exprs/minmax_predicate.h b/be/src/exprs/minmax_predicate.h
index 3855da4a863..0885a5a5bb3 100644
--- a/be/src/exprs/minmax_predicate.h
+++ b/be/src/exprs/minmax_predicate.h
@@ -37,6 +37,8 @@ public:
virtual Status assign(void* min_data, void* max_data) = 0;
// merge from other minmax_func
virtual Status merge(MinMaxFuncBase* minmax_func) = 0;
+ // Deep copy. The returned object shares nothing with this one.
+ virtual MinMaxFuncBase* clone() const = 0;
virtual ~MinMaxFuncBase() = default;
virtual void to_pb(PMinMaxFilter* filter) = 0;
@@ -87,6 +89,15 @@ public:
return Status::OK();
}
+ MinMaxFuncBase* clone() const override {
+ auto* cloned = new MinMaxNumFunc(_null_aware);
+ cloned->_max = _max;
+ cloned->_min = _min;
+ cloned->_min_value_set = _min_value_set;
+ cloned->_contain_null = _contain_null;
+ return cloned;
+ }
+
void* get_max() override { return &_max; }
void* get_min() override { return &_min; }
diff --git a/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp
index 3d0bb701c8b..b569148cb7a 100644
--- a/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp
@@ -20,8 +20,14 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
+#include <memory>
+
+#include "core/data_type/data_type_number.h"
#include "exec/runtime_filter/runtime_filter_producer.h"
#include "exec/runtime_filter/runtime_filter_test_utils.h"
+#include "exprs/bloom_filter_func.h"
+#include "exprs/hybrid_set.h"
+#include "testutil/column_helper.h"
namespace doris {
@@ -95,6 +101,242 @@ public:
}
};
+// The merger merges into a private copy of the first wrapper and never writes
a producer's
+// wrapper, which may be shared with consumers in local RF mgr or with other
producers.
+TEST_F(RuntimeFilterMergerTest, merge_from_never_writes_producer_wrapper) {
+ std::shared_ptr<RuntimeFilterMerger> merger;
+ auto desc = TRuntimeFilterDescBuilder().build();
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(RuntimeFilterMerger::create(_query_ctx.get(),
&desc, &merger));
+ merger->increase_expected_producer_num(3);
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ auto wrapper = producer->wrapper();
+
+ bool ready = false;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(producer.get(),
&ready));
+ ASSERT_FALSE(ready);
+ ASSERT_NE(merger->_wrapper, wrapper);
+ ASSERT_NE(merger->_wrapper->hybrid_set(), wrapper->hybrid_set());
+ ASSERT_EQ(merger->_wrapper->_state, RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(merger->_wrapper->hybrid_set()->size(), 1);
+
+ // The same wrapper published by another producer of a broadcast join
changes nothing.
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+ producer2->set_wrapper(wrapper);
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(producer2.get(),
&ready));
+ ASSERT_FALSE(ready);
+ ASSERT_EQ(merger->_wrapper->hybrid_set()->size(), 1);
+
+ // A disabled producer (e.g. of an early terminated instance) disables the
merger's copy only.
+ auto terminated_mgr = std::make_unique<RuntimeFilterMgr>(false);
+ auto terminated_state =
+ RuntimeState::create_unique(TUniqueId(), 0, _query_options,
_query_ctx->query_globals,
+ ExecEnv::GetInstance(),
_query_ctx.get());
+ terminated_state->set_runtime_filter_mgr(terminated_mgr.get());
+ terminated_state->set_desc_tbl(&_tbl);
+ std::shared_ptr<RuntimeFilterProducer> producer3;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ terminated_state->register_producer_runtime_filter(desc,
&producer3));
+
producer3->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::DISABLED,
+ "skip all rf process");
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(producer3.get(),
&ready));
+ ASSERT_TRUE(ready);
+ ASSERT_EQ(merger->_wrapper->_state, RuntimeFilterWrapper::State::DISABLED);
+ ASSERT_NE(merger->_wrapper->_reason.status().msg().find("skip all rf
process"),
+ std::string::npos);
+
+ int32_t five = 5;
+ ASSERT_EQ(wrapper->_state, RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(wrapper->hybrid_set()->size(), 1);
+ ASSERT_TRUE(wrapper->hybrid_set()->find(&five));
+}
+
+// A deserialized RPC input is short lived; the merger must keep a private
copy after it dies.
+TEST_F(RuntimeFilterMergerTest, merge_from_rpc_input_uses_private_copy) {
+ std::shared_ptr<RuntimeFilterMerger> merger;
+ auto desc = TRuntimeFilterDescBuilder().build();
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(RuntimeFilterMerger::create(_query_ctx.get(),
&desc, &merger));
+ merger->increase_expected_producer_num(2);
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ PMergeFilterRequest request;
+ void* data = nullptr;
+ int len = 0;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->serialize(&request, &data,
&len));
+ butil::IOBuf buf;
+ buf.append(data, len);
+ butil::IOBufAsZeroCopyInputStream stream(buf);
+ std::shared_ptr<RuntimeFilterProducer> rpc_input;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(_query_ctx.get(), &desc,
&rpc_input));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(rpc_input->assign(request, &stream));
+ auto wrapper = rpc_input->wrapper();
+
+ bool ready = false;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(rpc_input.get(),
&ready));
+ ASSERT_FALSE(ready);
+ ASSERT_NE(merger->_wrapper, wrapper);
+ ASSERT_NE(merger->_wrapper->hybrid_set(), wrapper->hybrid_set());
+ auto merged_wrapper = merger->_wrapper;
+ rpc_input.reset();
+
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer2->insert(ColumnHelper::create_column<DataTypeInt32>({6}),
0));
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ request.Clear();
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->serialize(&request, &data,
&len));
+ butil::IOBuf buf2;
+ buf2.append(data, len);
+ butil::IOBufAsZeroCopyInputStream stream2(buf2);
+ std::shared_ptr<RuntimeFilterProducer> rpc_input2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(_query_ctx.get(), &desc,
&rpc_input2));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(rpc_input2->assign(request, &stream2));
+ auto wrapper2 = rpc_input2->wrapper();
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(rpc_input2.get(),
&ready));
+ ASSERT_TRUE(ready);
+ ASSERT_EQ(merger->_wrapper, merged_wrapper);
+ ASSERT_EQ(merger->_wrapper->hybrid_set()->size(), 2);
+ int32_t five = 5;
+ int32_t six = 6;
+ ASSERT_TRUE(merged_wrapper->hybrid_set()->find(&five));
+ ASSERT_TRUE(merged_wrapper->hybrid_set()->find(&six));
+ ASSERT_EQ(wrapper->hybrid_set()->size(), 1);
+ ASSERT_TRUE(wrapper->hybrid_set()->find(&five));
+ ASSERT_FALSE(wrapper->hybrid_set()->find(&six));
+ ASSERT_EQ(wrapper2->hybrid_set()->size(), 1);
+ ASSERT_FALSE(wrapper2->hybrid_set()->find(&five));
+}
+
+TEST_F(RuntimeFilterMergerTest,
failed_rpc_merge_does_not_publish_partial_filter) {
+ auto options = _query_options;
+ options.__set_runtime_filter_max_in_num(1024);
+ options.__set_runtime_bloom_filter_min_size(64);
+ options.__set_runtime_bloom_filter_max_size(16384);
+ options.__set_ignore_runtime_filter_error(true);
+ TNetworkAddress fe_address;
+ fe_address.hostname = LOCALHOST;
+ fe_address.port = DUMMY_PORT;
+ auto query_ctx = QueryContext::create(TUniqueId(), ExecEnv::GetInstance(),
options, fe_address,
+ true, fe_address,
QuerySource::INTERNAL_FRONTEND);
+ ASSERT_TRUE(query_ctx->ignore_runtime_filter_error());
+
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_type(TRuntimeFilterType::IN_OR_BLOOM)
+ .set_build_bf_by_runtime_size(false)
+ .set_bloom_filter_size_bytes(4096)
+ .set_mode(false)
+ .build();
+ std::shared_ptr<RuntimeFilterMerger> merger;
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(RuntimeFilterMerger::create(query_ctx.get(),
&desc, &merger));
+ merger->increase_expected_producer_num(3);
+
+ std::shared_ptr<RuntimeFilterProducer> bloom_source;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(query_ctx.get(), &desc,
&bloom_source));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(bloom_source->init(2000));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+
bloom_source->insert(ColumnHelper::create_column<DataTypeInt32>({5}), 0));
+
bloom_source->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(bloom_source->wrapper()->get_real_type(),
RuntimeFilterType::BLOOM_FILTER);
+
+ PMergeFilterRequest request;
+ void* data = nullptr;
+ int len = 0;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(bloom_source->serialize(&request, &data,
&len));
+ butil::IOBuf buf;
+ buf.append(data, len);
+ butil::IOBufAsZeroCopyInputStream stream(buf);
+ std::shared_ptr<RuntimeFilterProducer> rpc_bloom;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(query_ctx.get(), &desc, &rpc_bloom));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(rpc_bloom->assign(request, &stream));
+ ASSERT_EQ(rpc_bloom->wrapper()->get_real_type(),
RuntimeFilterType::BLOOM_FILTER);
+
+ bool ready = false;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(rpc_bloom.get(),
&ready));
+ ASSERT_FALSE(ready);
+ ASSERT_FALSE(merger->ready());
+ auto merged_wrapper = merger->_wrapper;
+ ASSERT_NE(merged_wrapper, rpc_bloom->wrapper());
+ rpc_bloom.reset();
+
+ // An RPC Bloom with a different directory size cannot be merged. The
failed arrival
+ // must not make a later successful input publish an incomplete filter.
+ auto mismatched_desc = TRuntimeFilterDescBuilder()
+ .set_type(TRuntimeFilterType::IN_OR_BLOOM)
+ .set_build_bf_by_runtime_size(false)
+ .set_bloom_filter_size_bytes(8192)
+ .set_mode(false)
+ .build();
+ std::shared_ptr<RuntimeFilterProducer> mismatched_source;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(query_ctx.get(), &mismatched_desc,
&mismatched_source));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(mismatched_source->init(2000));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+
mismatched_source->insert(ColumnHelper::create_column<DataTypeInt32>({7}), 0));
+
mismatched_source->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ request.Clear();
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(mismatched_source->serialize(&request,
&data, &len));
+ butil::IOBuf mismatched_buf;
+ mismatched_buf.append(data, len);
+ butil::IOBufAsZeroCopyInputStream mismatched_stream(mismatched_buf);
+ std::shared_ptr<RuntimeFilterProducer> mismatched_rpc;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(query_ctx.get(), &mismatched_desc,
&mismatched_rpc));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(mismatched_rpc->assign(request,
&mismatched_stream));
+ ASSERT_NE(merged_wrapper->bloom_filter_func()->get_size(),
+ mismatched_rpc->wrapper()->bloom_filter_func()->get_size());
+ auto failed_status = merger->merge_from(mismatched_rpc.get(), &ready);
+ ASSERT_EQ(failed_status.code(), ErrorCode::INTERNAL_ERROR) <<
failed_status.to_string();
+ ASSERT_FALSE(ready);
+ ASSERT_FALSE(merger->ready());
+ ASSERT_EQ(merger->_wrapper, merged_wrapper);
+
+ std::shared_ptr<RuntimeFilterProducer> in_source;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(query_ctx.get(), &desc, &in_source));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(in_source->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ in_source->insert(ColumnHelper::create_column<DataTypeInt32>({6}),
0));
+
in_source->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ request.Clear();
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(in_source->serialize(&request, &data,
&len));
+ butil::IOBuf empty_buf;
+ butil::IOBufAsZeroCopyInputStream empty_stream(empty_buf);
+ std::shared_ptr<RuntimeFilterProducer> rpc_in;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ RuntimeFilterProducer::create(query_ctx.get(), &desc, &rpc_in));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(rpc_in->assign(request, &empty_stream));
+ ASSERT_EQ(rpc_in->wrapper()->get_real_type(),
RuntimeFilterType::IN_FILTER);
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(rpc_in.get(), &ready));
+ ASSERT_FALSE(ready);
+ ASSERT_FALSE(merger->ready());
+ ASSERT_EQ(merger->_wrapper, merged_wrapper);
+ ASSERT_EQ(merger->_wrapper->get_real_type(),
RuntimeFilterType::BLOOM_FILTER);
+ ASSERT_NE(merger->debug_string().find("received_producer_num: 2"),
std::string::npos);
+}
+
TEST_F(RuntimeFilterMergerTest, basic) {
test_merge_from(RuntimeFilterWrapper::State::READY,
RuntimeFilterWrapper::State::READY,
RuntimeFilterWrapper::State::READY,
RuntimeFilterWrapper::State::READY);
diff --git a/be/test/exec/runtime_filter/runtime_filter_producer_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_producer_test.cpp
index aa0f93e95ed..93692119dcf 100644
--- a/be/test/exec/runtime_filter/runtime_filter_producer_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_producer_test.cpp
@@ -20,8 +20,19 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
+#include <algorithm>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "core/data_type/data_type_number.h"
#include "exec/runtime_filter/runtime_filter_consumer.h"
+#include "exec/runtime_filter/runtime_filter_merger.h"
#include "exec/runtime_filter/runtime_filter_test_utils.h"
+#include "exprs/bloom_filter_func.h"
+#include "exprs/hybrid_set.h"
+#include "testutil/column_helper.h"
namespace doris {
@@ -175,6 +186,512 @@ TEST_F(RuntimeFilterProducerTest, set_disable) {
ASSERT_EQ(consumer->_wrapper->_state,
RuntimeFilterWrapper::State::DISABLED);
}
+// A runtime filter whose consumers are partly in local RF mgr and partly in
global RF mgr.
+TEST_F(RuntimeFilterProducerTest,
publish_mixed_targets_local_consumer_gets_private_wrapper) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(false)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, false,
0, &local_consumer));
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, false,
0, &local_consumer2));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer2->insert(ColumnHelper::create_column<DataTypeInt32>({6}),
0));
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+ ASSERT_EQ(local_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(merge_consumer->_rf_state,
RuntimeFilterConsumer::State::NOT_READY);
+
+ // The local consumer materializes its predicate before the merger
receives producer2.
+ std::vector<VRuntimeFilterPtr> exprs;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(local_consumer->acquire_expr(exprs));
+ ASSERT_EQ(exprs.size(), 1);
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(local_consumer2->_rf_state, RuntimeFilterConsumer::State::READY);
+
+ int32_t five = 5;
+ int32_t six = 6;
+ auto merged_set = merge_consumer->_wrapper->hybrid_set();
+ ASSERT_EQ(merged_set->size(), 2);
+ ASSERT_TRUE(merged_set->find(&five));
+ ASSERT_TRUE(merged_set->find(&six));
+
+ auto local_set = local_consumer->_wrapper->hybrid_set();
+ ASSERT_EQ(local_set->size(), 1);
+ ASSERT_TRUE(local_set->find(&five));
+ ASSERT_FALSE(local_set->find(&six));
+ auto local_set2 = local_consumer2->_wrapper->hybrid_set();
+ ASSERT_EQ(local_set2->size(), 1);
+ ASSERT_TRUE(local_set2->find(&six));
+ ASSERT_FALSE(local_set2->find(&five));
+
+ ASSERT_NE(local_consumer->_wrapper, merge_consumer->_wrapper);
+ ASSERT_NE(local_consumer2->_wrapper, merge_consumer->_wrapper);
+ ASSERT_NE(local_consumer->_wrapper, local_consumer2->_wrapper);
+ ASSERT_NE(local_set, merged_set);
+ ASSERT_NE(local_set2, merged_set);
+ ASSERT_NE(local_consumer->_wrapper->bloom_filter_func(),
+ merge_consumer->_wrapper->bloom_filter_func());
+
+ std::vector<VRuntimeFilterPtr> merged_exprs;
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merge_consumer->acquire_expr(merged_exprs));
+ ASSERT_EQ(merged_exprs.size(), 1);
+}
+
+TEST_F(RuntimeFilterProducerTest,
publish_mixed_targets_merge_overflow_keeps_local_filter) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(false)
+ .set_type(TRuntimeFilterType::IN)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ // Only the first instance has a consumer in local RF mgr.
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, false,
0, &local_consumer));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ // Each producer is within runtime_filter_max_in_num, but the merged one
is not.
+ std::vector<int32_t> data(600);
+ std::iota(data.begin(), data.end(), 0);
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(data.size()));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>(data),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ std::iota(data.begin(), data.end(), 600);
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->init(data.size()));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+
producer2->insert(ColumnHelper::create_column<DataTypeInt32>(data), 0));
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(merge_consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::DISABLED);
+
+ ASSERT_EQ(local_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(local_consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::READY);
+ auto local_set = local_consumer->_wrapper->hybrid_set();
+ ASSERT_EQ(local_set->size(), 600);
+ int32_t value = 0;
+ ASSERT_TRUE(local_set->find(&value));
+ value = 599;
+ ASSERT_TRUE(local_set->find(&value));
+ value = 600;
+ ASSERT_FALSE(local_set->find(&value));
+}
+
+// Broadcast join producers which share one hash table also share one wrapper.
+TEST_F(RuntimeFilterProducerTest, publish_mixed_targets_shared_wrapper) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(true)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, false,
0, &local_consumer));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+ producer2->set_wrapper(producer->wrapper());
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+ std::vector<VRuntimeFilterPtr> exprs;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(local_consumer->acquire_expr(exprs));
+ ASSERT_EQ(exprs.size(), 1);
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ auto merged_set = merge_consumer->_wrapper->hybrid_set();
+ ASSERT_EQ(merged_set->size(), 1);
+ int32_t five = 5;
+ ASSERT_TRUE(merged_set->find(&five));
+ ASSERT_NE(local_consumer->_wrapper, merge_consumer->_wrapper);
+}
+
+// An early terminated instance of a broadcast join publishes its own disabled
wrapper, which must
+// not disable the shared wrapper in place: the local consumers of other
instances still use it.
+TEST_F(RuntimeFilterProducerTest,
publish_mixed_targets_shared_wrapper_with_disabled_instance) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(true)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ auto terminated_mgr = std::make_unique<RuntimeFilterMgr>(false);
+ auto terminated_state =
+ RuntimeState::create_unique(TUniqueId(), 0, _query_options,
_query_ctx->query_globals,
+ ExecEnv::GetInstance(),
_query_ctx.get());
+ terminated_state->set_runtime_filter_mgr(terminated_mgr.get());
+ terminated_state->set_desc_tbl(&_tbl);
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+ std::shared_ptr<RuntimeFilterProducer> terminated_producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ terminated_state->register_producer_runtime_filter(desc,
&terminated_producer));
+
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, false,
0, &local_consumer));
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, false,
0, &local_consumer2));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+ auto shared_wrapper = producer->wrapper();
+ producer2->set_wrapper(shared_wrapper);
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ terminated_producer->set_wrapper_state_and_ready_to_publish(
+ RuntimeFilterWrapper::State::DISABLED, "skip all rf process");
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(terminated_producer->publish(terminated_state.get(),
true));
+ int32_t five = 5;
+ ASSERT_EQ(shared_wrapper->get_state(), RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(shared_wrapper->hybrid_set()->size(), 1);
+ ASSERT_TRUE(shared_wrapper->hybrid_set()->find(&five));
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+ ASSERT_EQ(shared_wrapper->get_state(), RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(shared_wrapper->hybrid_set()->size(), 1);
+ for (const auto& consumer : {local_consumer, local_consumer2}) {
+ ASSERT_EQ(consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(consumer->_wrapper, shared_wrapper);
+ ASSERT_EQ(consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(consumer->_wrapper->hybrid_set()->size(), 1);
+ ASSERT_TRUE(consumer->_wrapper->hybrid_set()->find(&five));
+ }
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_NE(merge_consumer->_wrapper, shared_wrapper);
+ ASSERT_EQ(merge_consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::DISABLED);
+ ASSERT_NE(merge_consumer->_wrapper->_reason.status().msg().find("skip all
rf process"),
+ std::string::npos);
+}
+
+TEST_F(RuntimeFilterProducerTest, publish_mixed_targets_disabled) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(false)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, false,
0, &local_consumer));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ // Producers are disabled before the filters are built.
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::DISABLED);
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::DISABLED);
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+
+ ASSERT_EQ(local_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(local_consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::DISABLED);
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(merge_consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::DISABLED);
+ ASSERT_NE(local_consumer->_wrapper, merge_consumer->_wrapper);
+}
+
+// Even without a plain local consumer, the merger keeps its own copy of each
input.
+TEST_F(RuntimeFilterProducerTest,
+ publish_local_merge_only_uses_private_copy_without_local_consumer) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(false)
+ .add_planId_to_target_expr(0)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ std::shared_ptr<RuntimeFilterConsumer> consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
0, &consumer));
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer2->insert(ColumnHelper::create_column<DataTypeInt32>({6}),
0));
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
+ auto wrapper = producer->wrapper();
+ auto wrapper2 = producer2->wrapper();
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+ std::shared_ptr<LocalMergeContext> context;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+
_query_ctx->runtime_filter_mgr()->get_local_merge_context(desc.filter_id, 0,
&context));
+ ASSERT_NE(context->merger->_wrapper, wrapper);
+ ASSERT_NE(context->merger->_wrapper->hybrid_set(), wrapper->hybrid_set());
+ ASSERT_EQ(context->merger->_wrapper->hybrid_set()->size(), 1);
+ ASSERT_FALSE(context->merger->ready());
+ auto merged_wrapper = context->merger->_wrapper;
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+ ASSERT_EQ(consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(consumer->_wrapper, merged_wrapper);
+ ASSERT_EQ(context->merger->_wrapper, merged_wrapper);
+ ASSERT_EQ(consumer->_wrapper->hybrid_set()->size(), 2);
+ int32_t five = 5;
+ int32_t six = 6;
+ ASSERT_TRUE(merged_wrapper->hybrid_set()->find(&five));
+ ASSERT_TRUE(merged_wrapper->hybrid_set()->find(&six));
+ ASSERT_TRUE(wrapper->hybrid_set()->find(&five));
+ ASSERT_FALSE(wrapper->hybrid_set()->find(&six));
+ ASSERT_EQ(wrapper->hybrid_set()->size(), 1);
+ ASSERT_NE(merged_wrapper->hybrid_set(), wrapper2->hybrid_set());
+ ASSERT_TRUE(wrapper2->hybrid_set()->find(&six));
+ ASSERT_FALSE(wrapper2->hybrid_set()->find(&five));
+ ASSERT_EQ(wrapper2->hybrid_set()->size(), 1);
+ ASSERT_EQ(wrapper->get_state(), RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(wrapper2->get_state(), RuntimeFilterWrapper::State::READY);
+}
+
+// The local consumer retains the producer wrapper; the merger owns a distinct
copy even with one
+// expected producer.
+TEST_F(RuntimeFilterProducerTest,
publish_single_producer_with_local_consumer_uses_private_copy) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(false)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, false,
0, &local_consumer));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
+ auto wrapper = producer->wrapper();
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+
+ ASSERT_EQ(local_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(local_consumer->_wrapper, wrapper);
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_NE(merge_consumer->_wrapper, wrapper);
+ ASSERT_NE(merge_consumer->_wrapper->hybrid_set(), wrapper->hybrid_set());
+ int32_t five = 5;
+ ASSERT_TRUE(wrapper->hybrid_set()->find(&five));
+ ASSERT_TRUE(merge_consumer->_wrapper->hybrid_set()->find(&five));
+ ASSERT_EQ(wrapper->hybrid_set()->size(), 1);
+ ASSERT_EQ(merge_consumer->_wrapper->hybrid_set()->size(), 1);
+}
+
+// A remote-target producer does not expose its wrapper to a local consumer,
but the local merger
+// still makes a private copy before the producer releases its reference.
+TEST_F(RuntimeFilterProducerTest,
publish_remote_target_merge_uses_private_copy) {
+ auto desc =
TRuntimeFilterDescBuilder().set_mode(false).set_is_broadcast_join(false).build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+
+ auto wrapper = producer->wrapper();
+ // Only the first of the two expected producers has published, so the
merger is not ready
+ // yet and publish() never reaches `_send_to_remote_targets()`.
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+ ASSERT_EQ(producer->_wrapper, nullptr);
+
+ std::shared_ptr<LocalMergeContext> context;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+
_query_ctx->runtime_filter_mgr()->get_local_merge_context(desc.filter_id, 0,
&context));
+ ASSERT_NE(context->merger->_wrapper, wrapper);
+ ASSERT_NE(context->merger->_wrapper->hybrid_set(), wrapper->hybrid_set());
+ ASSERT_EQ(wrapper->hybrid_set()->size(), 1);
+ int32_t five = 5;
+ int32_t six = 6;
+ ASSERT_TRUE(wrapper->hybrid_set()->find(&five));
+ auto merged_wrapper = context->merger->_wrapper;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer2->insert(ColumnHelper::create_column<DataTypeInt32>({6}),
0));
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ auto wrapper2 = producer2->wrapper();
+ bool ready = false;
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(context->merger->merge_from(producer2.get(),
&ready));
+ ASSERT_TRUE(ready);
+ ASSERT_EQ(context->merger->_wrapper, merged_wrapper);
+ ASSERT_TRUE(merged_wrapper->hybrid_set()->find(&five));
+ ASSERT_TRUE(merged_wrapper->hybrid_set()->find(&six));
+ ASSERT_EQ(merged_wrapper->hybrid_set()->size(), 2);
+ ASSERT_EQ(wrapper->hybrid_set()->size(), 1);
+ ASSERT_FALSE(wrapper->hybrid_set()->find(&six));
+ ASSERT_EQ(wrapper2->hybrid_set()->size(), 1);
+ ASSERT_FALSE(wrapper2->hybrid_set()->find(&five));
+}
+
+// An IN_OR_BLOOM merger which is still an IN filter takes the bloom filter of
a producer which
+// already changed to a bloom filter. It must take a copy, the producer's own
local consumers are
+// probing the original.
+TEST_F(RuntimeFilterProducerTest,
publish_mixed_targets_in_or_bloom_merge_copies_bloom_filter) {
+ auto desc = TRuntimeFilterDescBuilder()
+ .set_build_bf_by_runtime_size(false)
+ .set_is_broadcast_join(false)
+ .add_planId_to_target_expr(0)
+ .add_planId_to_target_expr(1)
+ .build();
+
+ std::shared_ptr<RuntimeFilterProducer> producer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[0]->register_producer_runtime_filter(desc,
&producer));
+ std::shared_ptr<RuntimeFilterProducer> producer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_producer_runtime_filter(desc,
&producer2));
+
+ // Only the second instance has a consumer in local RF mgr.
+ std::shared_ptr<RuntimeFilterConsumer> local_consumer2;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, false,
0, &local_consumer2));
+ std::shared_ptr<RuntimeFilterConsumer> merge_consumer;
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ _runtime_states[1]->register_consumer_runtime_filter(desc, true,
1, &merge_consumer));
+
+ // The first producer stays an IN filter, the second one exceeds
runtime_filter_max_in_num
+ // and changes to a bloom filter.
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->init(1));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer->insert(ColumnHelper::create_column<DataTypeInt32>({5}),
0));
+
producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(producer->wrapper()->get_real_type(),
RuntimeFilterType::IN_FILTER);
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->init(2000));
+ FAIL_IF_ERROR_OR_CATCH_EXCEPTION(
+ producer2->insert(ColumnHelper::create_column<DataTypeInt32>({6,
7}), 0));
+
producer2->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY);
+ auto bloom_wrapper = producer2->wrapper();
+ ASSERT_EQ(bloom_wrapper->get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ char* bloom_data = nullptr;
+ int bloom_len = 0;
+ bloom_wrapper->bloom_filter_func()->get_data(&bloom_data, &bloom_len);
+ ASSERT_GT(bloom_len, 0);
+ const std::string snapshot(bloom_data, bloom_len);
+
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer->publish(_runtime_states[0].get(),
true));
+
FAIL_IF_ERROR_OR_CATCH_EXCEPTION(producer2->publish(_runtime_states[1].get(),
true));
+
+ ASSERT_EQ(local_consumer2->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(local_consumer2->_wrapper, bloom_wrapper);
+ ASSERT_EQ(merge_consumer->_rf_state, RuntimeFilterConsumer::State::READY);
+ ASSERT_EQ(merge_consumer->_wrapper->get_state(),
RuntimeFilterWrapper::State::READY);
+ ASSERT_EQ(merge_consumer->_wrapper->get_real_type(),
RuntimeFilterType::BLOOM_FILTER);
+ ASSERT_NE(merge_consumer->_wrapper->bloom_filter_func(),
bloom_wrapper->bloom_filter_func());
+
+ // The merged filter contains the values of both producers.
+ std::vector<uint8_t> found(3);
+ merge_consumer->_wrapper->bloom_filter_func()->find_fixed_len(
+ ColumnHelper::create_column<DataTypeInt32>({5, 6, 7}),
found.data());
+ ASSERT_TRUE(std::all_of(found.begin(), found.end(), [](uint8_t i) -> bool
{ return i; }));
+
+ // The bloom filter of the second producer is not written by the merge.
+ char* merged_data = nullptr;
+ int merged_len = 0;
+ merge_consumer->_wrapper->bloom_filter_func()->get_data(&merged_data,
&merged_len);
+ ASSERT_NE(merged_data, bloom_data);
+ ASSERT_EQ(bloom_len, merged_len);
+ ASSERT_EQ(snapshot, std::string(bloom_data, bloom_len));
+ ASSERT_NE(snapshot, std::string(merged_data, merged_len));
+}
+
TEST_F(RuntimeFilterProducerTest, publish_release_wrapper) {
auto desc = TRuntimeFilterDescBuilder()
.set_build_bf_by_runtime_size(false)
@@ -191,4 +708,4 @@ TEST_F(RuntimeFilterProducerTest, publish_release_wrapper) {
ASSERT_EQ(producer->_wrapper, nullptr);
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
index 842bce360da..0dbdb20aa88 100644
--- a/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
+++ b/be/test/exec/runtime_filter/runtime_filter_wrapper_test.cpp
@@ -20,8 +20,11 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
+#include <algorithm>
+#include <numeric>
#include <string>
#include <utility>
+#include <vector>
#include "core/data_type/data_type_date.h"
#include "core/data_type/data_type_date_time.h"
@@ -1309,6 +1312,344 @@ TEST_F(RuntimeFilterWrapperTest, TestInOrBloom) {
EXPECT_EQ(wrapper->contain_null(), false);
}
+// Merging a bloom filter into an IN_OR_BLOOM filter which is still an IN
filter takes a copy of
+// the input bloom filter, the input may still be probed by its own consumers.
+TEST_F(RuntimeFilterWrapperTest, TestMergeInWithBloomCopiesBloomFilter) {
+ using DataType = DataTypeInt32;
+ auto make_params = [](int32_t max_in_num) {
+ return RuntimeFilterParams {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::IN_OR_BLOOM_FILTER,
+ .column_return_type =
PrimitiveType::TYPE_INT,
+ .null_aware = false,
+ .max_in_num = max_in_num,
+ .runtime_bloom_filter_min_size = 64,
+ .runtime_bloom_filter_max_size = 128,
+ .bloom_filter_size = 64,
+ .build_bf_by_runtime_size = false,
+ .bloom_filter_size_calculated_by_ndv =
false};
+ };
+ auto in_params = make_params(18);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(&in_params);
+ ASSERT_TRUE(wrapper->init(16).ok());
+ ASSERT_EQ(wrapper->get_real_type(), RuntimeFilterType::IN_FILTER);
+ std::vector<int> data_vector(10);
+ std::iota(data_vector.begin(), data_vector.end(), 0);
+
ASSERT_TRUE(wrapper->insert(ColumnHelper::create_column<DataType>(data_vector),
0).ok());
+
+ auto bloom_params = make_params(8);
+ auto bloom_wrapper = std::make_shared<RuntimeFilterWrapper>(&bloom_params);
+ ASSERT_TRUE(bloom_wrapper->init(16).ok());
+ ASSERT_EQ(bloom_wrapper->get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ std::vector<int> bloom_data_vector(10);
+ std::iota(bloom_data_vector.begin(), bloom_data_vector.end(), 10);
+
ASSERT_TRUE(bloom_wrapper->insert(ColumnHelper::create_column<DataType>(bloom_data_vector),
0)
+ .ok());
+ bloom_wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ char* bloom_data = nullptr;
+ int bloom_len = 0;
+ bloom_wrapper->bloom_filter_func()->get_data(&bloom_data, &bloom_len);
+ ASSERT_GT(bloom_len, 0);
+ const std::string snapshot(bloom_data, bloom_len);
+
+ ASSERT_TRUE(wrapper->merge(bloom_wrapper.get()).ok());
+ EXPECT_EQ(wrapper->get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ EXPECT_NE(wrapper->bloom_filter_func(),
bloom_wrapper->bloom_filter_func());
+ char* merged_data = nullptr;
+ int merged_len = 0;
+ wrapper->bloom_filter_func()->get_data(&merged_data, &merged_len);
+ EXPECT_NE(merged_data, bloom_data);
+ EXPECT_EQ(merged_len, bloom_len);
+ // The input bloom filter is unchanged, the merged one has the IN values
added.
+ EXPECT_EQ(snapshot, std::string(bloom_data, bloom_len));
+ EXPECT_NE(snapshot, std::string(merged_data, merged_len));
+
+ std::vector<int> final_data_vector(20);
+ std::iota(final_data_vector.begin(), final_data_vector.end(), 0);
+ std::vector<uint8_t> final_res(20);
+ wrapper->bloom_filter_func()->find_fixed_len(
+ ColumnHelper::create_column<DataType>(final_data_vector),
final_res.data());
+ EXPECT_TRUE(
+ std::all_of(final_res.begin(), final_res.end(), [](uint8_t i) ->
bool { return i; }));
+}
+
+// A null-aware IN input must be inserted into a private copy of the Bloom
input.
+TEST_F(RuntimeFilterWrapperTest, TestMergeInWithBloomCopiesNullAwareInput) {
+ using DataType = DataTypeInt32;
+ auto make_params = [](int32_t max_in_num) {
+ return RuntimeFilterParams {.filter_id = 0,
+ .filter_type =
RuntimeFilterType::IN_OR_BLOOM_FILTER,
+ .column_return_type =
PrimitiveType::TYPE_INT,
+ .null_aware = true,
+ .max_in_num = max_in_num,
+ .runtime_bloom_filter_min_size = 64,
+ .runtime_bloom_filter_max_size = 128,
+ .bloom_filter_size = 64,
+ .build_bf_by_runtime_size = false,
+ .bloom_filter_size_calculated_by_ndv =
false};
+ };
+ auto in_params = make_params(18);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(&in_params);
+ ASSERT_TRUE(wrapper->init(16).ok());
+ ASSERT_EQ(wrapper->get_real_type(), RuntimeFilterType::IN_FILTER);
+ std::vector<int> data_vector(10);
+ std::iota(data_vector.begin(), data_vector.end(), 0);
+
ASSERT_TRUE(wrapper->insert(ColumnHelper::create_column<DataType>(data_vector),
0).ok());
+ wrapper->hybrid_set()->insert((const void*)nullptr);
+ ASSERT_TRUE(wrapper->contain_null());
+
+ auto bloom_params = make_params(8);
+ auto bloom_wrapper = std::make_shared<RuntimeFilterWrapper>(&bloom_params);
+ ASSERT_TRUE(bloom_wrapper->init(16).ok());
+ ASSERT_EQ(bloom_wrapper->get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ std::vector<int> bloom_data_vector(10);
+ std::iota(bloom_data_vector.begin(), bloom_data_vector.end(), 10);
+
ASSERT_TRUE(bloom_wrapper->insert(ColumnHelper::create_column<DataType>(bloom_data_vector),
0)
+ .ok());
+ bloom_wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ auto bloom_filter_func_before_merge = bloom_wrapper->bloom_filter_func();
+ char* bloom_data = nullptr;
+ int bloom_len = 0;
+ bloom_filter_func_before_merge->get_data(&bloom_data, &bloom_len);
+ ASSERT_GT(bloom_len, 0);
+ const std::string snapshot(bloom_data, bloom_len);
+ ASSERT_FALSE(bloom_wrapper->contain_null());
+
+ ASSERT_TRUE(wrapper->merge(bloom_wrapper.get()).ok());
+ EXPECT_EQ(wrapper->get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ EXPECT_NE(wrapper->bloom_filter_func(), bloom_filter_func_before_merge);
+ char* merged_data = nullptr;
+ int merged_len = 0;
+ wrapper->bloom_filter_func()->get_data(&merged_data, &merged_len);
+ EXPECT_NE(merged_data, bloom_data);
+ EXPECT_EQ(merged_len, bloom_len);
+ EXPECT_EQ(snapshot, std::string(bloom_data, bloom_len));
+ EXPECT_FALSE(bloom_wrapper->contain_null());
+ EXPECT_TRUE(wrapper->contain_null());
+
+ std::vector<int> final_data_vector(20);
+ std::iota(final_data_vector.begin(), final_data_vector.end(), 0);
+ std::vector<uint8_t> final_res(20);
+ wrapper->bloom_filter_func()->find_fixed_len(
+ ColumnHelper::create_column<DataType>(final_data_vector),
final_res.data());
+ EXPECT_TRUE(
+ std::all_of(final_res.begin(), final_res.end(), [](uint8_t i) ->
bool { return i; }));
+}
+
+TEST_F(RuntimeFilterWrapperTest, TestClone) {
+ using DataType = DataTypeInt32;
+ auto make_params = [](RuntimeFilterType filter_type, bool null_aware,
int32_t max_in_num,
+ PrimitiveType column_type = PrimitiveType::TYPE_INT)
{
+ return RuntimeFilterParams {.filter_id = 3,
+ .filter_type = filter_type,
+ .column_return_type = column_type,
+ .null_aware = null_aware,
+ .max_in_num = max_in_num,
+ .runtime_bloom_filter_min_size = 64,
+ .runtime_bloom_filter_max_size = 128,
+ .bloom_filter_size = 64,
+ .build_bf_by_runtime_size = false,
+ .bloom_filter_size_calculated_by_ndv =
true};
+ };
+ {
+ // An initialized empty IN filter still needs a separately mutable set.
+ auto params = make_params(RuntimeFilterType::IN_FILTER, false, 8);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(wrapper->init(0).ok());
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_NE(cloned, wrapper);
+ EXPECT_NE(cloned->hybrid_set(), wrapper->hybrid_set());
+ EXPECT_EQ(cloned->hybrid_set()->size(), 0);
+ EXPECT_FALSE(cloned->contain_null());
+ int32_t one = 1;
+ cloned->hybrid_set()->insert(&one);
+ EXPECT_EQ(wrapper->hybrid_set()->size(), 0);
+ EXPECT_EQ(cloned->hybrid_set()->size(), 1);
+ }
+ {
+ // In filter, the copy owns its hybrid set independently.
+ auto params = make_params(RuntimeFilterType::IN_FILTER, true, 8);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(wrapper->init(3).ok());
+ ASSERT_TRUE(wrapper->insert(ColumnHelper::create_column<DataType>({1,
2}), 0).ok());
+ wrapper->hybrid_set()->insert((const void*)nullptr);
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_NE(cloned, wrapper);
+ EXPECT_EQ(cloned->get_state(), RuntimeFilterWrapper::State::READY);
+ EXPECT_EQ(cloned->filter_id(), wrapper->filter_id());
+ EXPECT_EQ(cloned->get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(cloned->_max_in_num, wrapper->_max_in_num);
+ EXPECT_NE(cloned->hybrid_set(), wrapper->hybrid_set());
+ EXPECT_EQ(cloned->hybrid_set()->size(), 2);
+ int32_t value = 1;
+ EXPECT_TRUE(cloned->hybrid_set()->find(&value));
+ value = 2;
+ EXPECT_TRUE(cloned->hybrid_set()->find(&value));
+ EXPECT_TRUE(cloned->contain_null());
+
+ value = 3;
+ cloned->hybrid_set()->insert(&value);
+ EXPECT_EQ(wrapper->hybrid_set()->size(), 2);
+ EXPECT_FALSE(wrapper->hybrid_set()->find(&value));
+ value = 4;
+ wrapper->hybrid_set()->insert(&value);
+ EXPECT_EQ(cloned->hybrid_set()->size(), 3);
+ EXPECT_FALSE(cloned->hybrid_set()->find(&value));
+ }
+ {
+ // MinMax filter
+ auto params = make_params(RuntimeFilterType::MINMAX_FILTER, false, 0);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(wrapper->insert(ColumnHelper::create_column<DataType>({3,
7}), 0).ok());
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_state(), RuntimeFilterWrapper::State::READY);
+ EXPECT_NE(cloned->minmax_func(), wrapper->minmax_func());
+ EXPECT_EQ(*(int32_t*)cloned->minmax_func()->get_min(), 3);
+ EXPECT_EQ(*(int32_t*)cloned->minmax_func()->get_max(), 7);
+
+ ASSERT_TRUE(cloned->insert(ColumnHelper::create_column<DataType>({1,
9}), 0).ok());
+ EXPECT_EQ(*(int32_t*)wrapper->minmax_func()->get_min(), 3);
+ EXPECT_EQ(*(int32_t*)wrapper->minmax_func()->get_max(), 7);
+ }
+ {
+ // Bloom filter
+ auto params = make_params(RuntimeFilterType::BLOOM_FILTER, false, 0);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(wrapper->init(80).ok());
+ std::vector<int32_t> data(10);
+ std::iota(data.begin(), data.end(), 0);
+
ASSERT_TRUE(wrapper->insert(ColumnHelper::create_column<DataType>(data),
0).ok());
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_state(), RuntimeFilterWrapper::State::READY);
+ EXPECT_NE(cloned->bloom_filter_func(), wrapper->bloom_filter_func());
+ ASSERT_EQ(cloned->bloom_filter_func()->get_size(),
+ wrapper->bloom_filter_func()->get_size());
+ char* origin_data = nullptr;
+ int origin_len = 0;
+ wrapper->bloom_filter_func()->get_data(&origin_data, &origin_len);
+ char* cloned_data = nullptr;
+ int cloned_len = 0;
+ cloned->bloom_filter_func()->get_data(&cloned_data, &cloned_len);
+ ASSERT_EQ(cloned_len, origin_len);
+ EXPECT_NE(cloned_data, origin_data);
+ EXPECT_EQ(memcmp(cloned_data, origin_data, origin_len), 0);
+
+ std::string snapshot(origin_data, origin_len);
+ ASSERT_TRUE(cloned->insert(ColumnHelper::create_column<DataType>({100,
200, 300}), 0).ok());
+ EXPECT_EQ(memcmp(snapshot.data(), origin_data, origin_len), 0);
+ }
+ {
+ // In or bloom filter keeps its real type.
+ auto params = make_params(RuntimeFilterType::IN_OR_BLOOM_FILTER,
false, 2);
+ auto in_wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(in_wrapper->init(1).ok());
+
ASSERT_TRUE(in_wrapper->insert(ColumnHelper::create_column<DataType>({1}),
0).ok());
+ in_wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ // `init()` leaves the bloom filter of an IN filter unallocated, so
allocate it by hand:
+ // the copy must skip the unused bloom filter, not merely find none to
copy.
+
ASSERT_TRUE(in_wrapper->bloom_filter_func()->init_with_fixed_length(1).ok());
+ ASSERT_GT(in_wrapper->bloom_filter_func()->get_size(), 0);
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(in_wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_real_type(), RuntimeFilterType::IN_FILTER);
+ EXPECT_EQ(cloned->hybrid_set()->size(), 1);
+ EXPECT_NE(cloned->bloom_filter_func(),
in_wrapper->bloom_filter_func());
+ // The unused bloom filter of an IN filter is not copied.
+ EXPECT_EQ(cloned->bloom_filter_func()->get_size(), 0);
+
+ auto bloom_wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(bloom_wrapper->init(3).ok());
+ ASSERT_TRUE(
+
bloom_wrapper->insert(ColumnHelper::create_column<DataType>({1, 2, 3}),
0).ok());
+ bloom_wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ ASSERT_TRUE(bloom_wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_real_type(), RuntimeFilterType::BLOOM_FILTER);
+ EXPECT_EQ(cloned->hybrid_set(), nullptr);
+ EXPECT_EQ(cloned->bloom_filter_func()->get_size(),
+ bloom_wrapper->bloom_filter_func()->get_size());
+ }
+ {
+ // A disabled filter which is never initialized has no bloom filter
allocated.
+ auto params = make_params(RuntimeFilterType::IN_OR_BLOOM_FILTER,
false, 2);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ wrapper->set_state(RuntimeFilterWrapper::State::DISABLED, "some
reason");
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_state(), RuntimeFilterWrapper::State::DISABLED);
+ EXPECT_NE(cloned->_reason.status().msg().find("some reason"),
std::string::npos);
+ EXPECT_EQ(cloned->bloom_filter_func()->get_size(), 0);
+ }
+ {
+ // String in filter, the copy owns its strings.
+ auto params =
+ make_params(RuntimeFilterType::IN_FILTER, false, 8,
PrimitiveType::TYPE_STRING);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(wrapper->init(2).ok());
+ const std::string long_value(64, 'x');
+ ASSERT_TRUE(
+
wrapper->insert(ColumnHelper::create_column<DataTypeString>({"a", long_value}),
0)
+ .ok());
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ wrapper->hybrid_set()->clear();
+ wrapper.reset();
+ EXPECT_EQ(cloned->hybrid_set()->size(), 2);
+ StringRef value("a");
+ EXPECT_TRUE(cloned->hybrid_set()->find(&value));
+ value = StringRef(long_value);
+ EXPECT_TRUE(cloned->hybrid_set()->find(&value));
+ }
+ {
+ // Null aware bloom filter and minmax filter keep `contain_null`.
+ auto column = ColumnHelper::create_nullable_column<DataType>({1, 2,
0}, {0, 0, 1});
+ for (auto filter_type :
+ {RuntimeFilterType::BLOOM_FILTER,
RuntimeFilterType::MINMAX_FILTER}) {
+ auto params = make_params(filter_type, true, 0);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+ ASSERT_TRUE(wrapper->init(3).ok());
+ ASSERT_TRUE(wrapper->insert(column, 0).ok());
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ ASSERT_TRUE(wrapper->contain_null());
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_real_type(), filter_type);
+ EXPECT_TRUE(cloned->contain_null());
+ }
+ }
+ {
+ // Min filter and max filter
+ for (auto filter_type : {RuntimeFilterType::MIN_FILTER,
RuntimeFilterType::MAX_FILTER}) {
+ auto params = make_params(filter_type, false, 0);
+ auto wrapper = std::make_shared<RuntimeFilterWrapper>(¶ms);
+
ASSERT_TRUE(wrapper->insert(ColumnHelper::create_column<DataType>({3, 7}),
0).ok());
+ wrapper->set_state(RuntimeFilterWrapper::State::READY);
+ std::shared_ptr<RuntimeFilterWrapper> cloned;
+ ASSERT_TRUE(wrapper->clone(&cloned).ok());
+ EXPECT_EQ(cloned->get_real_type(), filter_type);
+ EXPECT_NE(cloned->minmax_func(), wrapper->minmax_func());
+
ASSERT_TRUE(cloned->insert(ColumnHelper::create_column<DataType>({1, 9}),
0).ok());
+ if (filter_type == RuntimeFilterType::MIN_FILTER) {
+ EXPECT_EQ(*(int32_t*)cloned->minmax_func()->get_min(), 1);
+ EXPECT_EQ(*(int32_t*)wrapper->minmax_func()->get_min(), 3);
+ } else {
+ EXPECT_EQ(*(int32_t*)cloned->minmax_func()->get_max(), 9);
+ EXPECT_EQ(*(int32_t*)wrapper->minmax_func()->get_max(), 7);
+ }
+ }
+ }
+}
+
TEST_F(RuntimeFilterWrapperTest, TestErrorPath) {
using DataType = DataTypeInt32;
int32_t filter_id = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]