This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d39e68c81ed [fix](be) Preserve floating-point equality in Parquet
pruning (#66470)
d39e68c81ed is described below
commit d39e68c81ed94ebbe64600239a8916b87f770965
Author: Gabriel <[email protected]>
AuthorDate: Fri Aug 7 12:58:19 2026 +0800
[fix](be) Preserve floating-point equality in Parquet pruning (#66470)
## Summary
- backport the floating-point pruning correctness fixes to branch-4.1
for File Scanner V2
- preserve Doris NaN and signed-zero equality semantics in V2 Parquet
Bloom, min/max, and IN-predicate pruning
- keep pruning conservative when Parquet statistics omit NaNs or a Bloom
encoding cannot safely prove absence
- leave the legacy File Scanner V1 path unchanged
## Testing
- clang-format 16 check on all affected C/C++ files
- compiled every affected production and test translation unit after
rebasing onto branch-4.1
- 121 focused BE unit tests passed, including expression, hybrid-set,
and native V2 Parquet pruning coverage
---
be/src/core/field.h | 13 +
be/src/exprs/expr_zonemap_filter.cpp | 37 ++-
be/src/exprs/expr_zonemap_filter.h | 4 +-
be/src/exprs/function/functions_comparison.h | 17 +-
be/src/exprs/hybrid_set.h | 18 +-
be/src/exprs/vdirect_in_predicate.h | 5 +-
be/src/exprs/vin_predicate.cpp | 8 +-
be/src/exprs/vin_predicate.h | 1 +
be/src/format_v2/parquet/parquet_statistics.cpp | 3 +
.../index/zone_map/zonemap_eval_context.cpp | 5 +
.../storage/index/zone_map/zonemap_eval_context.h | 3 +
be/test/core/field_test.cpp | 11 +
be/test/exprs/expr_zonemap_filter_test.cpp | 223 +++++++++++++-
be/test/exprs/hybrid_set_test.cpp | 29 ++
be/test/format_v2/parquet/parquet_scan_test.cpp | 241 +++++++++++++++
.../format_v2/parquet/parquet_statistics_test.cpp | 330 +++++++++++++++++++++
16 files changed, 921 insertions(+), 27 deletions(-)
diff --git a/be/src/core/field.h b/be/src/core/field.h
index d39d8208777..f418283d4e5 100644
--- a/be/src/core/field.h
+++ b/be/src/core/field.h
@@ -25,6 +25,7 @@
#include <algorithm>
#include <cassert>
+#include <cmath>
#include <cstring>
#include <map>
#include <string>
@@ -271,6 +272,18 @@ public:
template <PrimitiveType T>
const typename PrimitiveTypeTraits<T>::CppType& get() const;
+ bool is_nan() const {
+ // Keep type dispatch with the value so callers cannot reinterpret
Field storage using a
+ // mismatched PrimitiveType.
+ if (type == PrimitiveType::TYPE_FLOAT) {
+ return std::isnan(get<TYPE_FLOAT>());
+ }
+ if (type == PrimitiveType::TYPE_DOUBLE) {
+ return std::isnan(get<TYPE_DOUBLE>());
+ }
+ return false;
+ }
+
bool operator==(const Field& rhs) const {
return operator<=>(rhs) == std::strong_ordering::equal;
}
diff --git a/be/src/exprs/expr_zonemap_filter.cpp
b/be/src/exprs/expr_zonemap_filter.cpp
index 95502a761b0..608f9604397 100644
--- a/be/src/exprs/expr_zonemap_filter.cpp
+++ b/be/src/exprs/expr_zonemap_filter.cpp
@@ -18,7 +18,9 @@
#include "exprs/expr_zonemap_filter.h"
#include <algorithm>
+#include <cmath>
#include <set>
+#include <type_traits>
#include <utility>
#include "common/logging.h"
@@ -59,6 +61,24 @@ bool dictionary_contains(const
DictionaryEvalContext::SlotDictionary& dictionary
});
}
+template <typename T>
+bool floating_point_bloom_filter_may_contain(const segment_v2::BloomFilter&
bloom_filter, T value) {
+ static_assert(std::is_floating_point_v<T>);
+ // Doris equality collapses NaN payloads and signed zeros, while Parquet
Bloom hashes physical
+ // bytes. A negative probe is safe only after covering the entire
Doris-equivalent class.
+ if (std::isnan(value)) {
+ return true;
+ }
+ const auto test_value = [&](T candidate) {
+ return bloom_filter.test_bytes(reinterpret_cast<const
char*>(&candidate),
+ sizeof(candidate));
+ };
+ if (test_value(value)) {
+ return true;
+ }
+ return value == T {0} && test_value(-value);
+}
+
bool bloom_filter_may_contain(const BloomFilterEvalContext::SlotBloomFilter&
slot_filter,
const Field& value) {
DORIS_CHECK(slot_filter.data_type != nullptr);
@@ -83,13 +103,11 @@ bool bloom_filter_may_contain(const
BloomFilterEvalContext::SlotBloomFilter& slo
}
case TYPE_FLOAT: {
const float typed_value = value.get<TYPE_FLOAT>();
- return slot_filter.bloom_filter->test_bytes(reinterpret_cast<const
char*>(&typed_value),
- sizeof(typed_value));
+ return
floating_point_bloom_filter_may_contain(*slot_filter.bloom_filter, typed_value);
}
case TYPE_DOUBLE: {
const double typed_value = value.get<TYPE_DOUBLE>();
- return slot_filter.bloom_filter->test_bytes(reinterpret_cast<const
char*>(&typed_value),
- sizeof(typed_value));
+ return
floating_point_bloom_filter_may_contain(*slot_filter.bloom_filter, typed_value);
}
case TYPE_CHAR:
case TYPE_VARCHAR:
@@ -150,6 +168,7 @@ Status
materialize_hybrid_set_for_zonemap_filter(HybridSetBase& set, const DataT
DORIS_CHECK(value_type != nullptr);
result->contains_null = set.contain_null();
+ result->contains_nan = false;
result->values.clear();
result->min_value = Field();
result->max_value = Field();
@@ -164,6 +183,7 @@ Status
materialize_hybrid_set_for_zonemap_filter(HybridSetBase& set, const DataT
auto literal = VLiteral::create_shared(literal_node);
Field field;
literal->get_column_ptr()->get(0, field);
+ result->contains_nan |= field.is_nan();
result->values.emplace_back(std::move(field));
}
iterator->next();
@@ -243,7 +263,8 @@ ZoneMapFilterResult eval_null_zonemap(const
ZoneMapEvalContext& ctx, const VExpr
ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const
VExprSPtr& slot_expr,
bool is_not_in, const std::vector<Field>&
values,
- const Field& min_value, const Field&
max_value) {
+ bool contains_nan, const Field& min_value,
+ const Field& max_value) {
auto slot = std::dynamic_pointer_cast<VSlotRef>(slot_expr);
DORIS_CHECK(slot != nullptr);
// Empty IN has no candidate values, while NOT IN with an empty set cannot
filter anything.
@@ -276,6 +297,12 @@ ZoneMapFilterResult eval_in_zonemap(const
ZoneMapEvalContext& ctx, const VExprSP
return ZoneMapFilterResult::kNoMatch;
}
+ if (ctx.floating_nan_count_unknown(slot->column_id()) &&
+ ((!is_not_in && contains_nan) || (is_not_in && !contains_nan))) {
+ // Hidden Parquet NaNs can satisfy IN only when queried, and NOT IN
only when omitted.
+ return unsupported_zonemap_filter(ctx);
+ }
+
if (!range_stats_usable_for_zonemap(zone_map, slot_type)) {
return unsupported_zonemap_filter(ctx);
}
diff --git a/be/src/exprs/expr_zonemap_filter.h
b/be/src/exprs/expr_zonemap_filter.h
index a068a1e5147..b87d4e0030e 100644
--- a/be/src/exprs/expr_zonemap_filter.h
+++ b/be/src/exprs/expr_zonemap_filter.h
@@ -45,6 +45,7 @@ namespace doris::expr_zonemap {
struct InZonemapMaterializedSet {
bool contains_null = false;
+ bool contains_nan = false;
std::vector<Field> values;
Field min_value;
Field max_value;
@@ -142,7 +143,8 @@ ZoneMapFilterResult eval_null_zonemap(const
ZoneMapEvalContext& ctx, const VExpr
ZoneMapFilterResult eval_in_zonemap(const ZoneMapEvalContext& ctx, const
VExprSPtr& slot_expr,
bool is_not_in, const std::vector<Field>&
values,
- const Field& min_value, const Field&
max_value);
+ bool contains_nan, const Field& min_value,
+ const Field& max_value);
ZoneMapFilterResult eval_eq_dictionary(const DictionaryEvalContext& ctx,
const SlotLiteral& slot_literal);
diff --git a/be/src/exprs/function/functions_comparison.h
b/be/src/exprs/function/functions_comparison.h
index 1910bd845c7..3487bf9778b 100644
--- a/be/src/exprs/function/functions_comparison.h
+++ b/be/src/exprs/function/functions_comparison.h
@@ -322,6 +322,15 @@ inline ZoneMapFilterResult evaluate(const
ZoneMapEvalContext& ctx, const VExprSP
const auto effective_op = slot_literal->literal_on_left ? symmetric_op(op)
: op;
const auto& literal = slot_literal->literal;
+ const bool literal_is_nan = literal.is_nan();
+ const bool hidden_nan_can_match = (effective_op == Op::EQ &&
literal_is_nan) ||
+ (effective_op == Op::NE &&
!literal_is_nan) ||
+ (effective_op == Op::GT &&
!literal_is_nan) ||
+ effective_op == Op::GE;
+ if (ctx.floating_nan_count_unknown(slot_literal->slot_index) &&
hidden_nan_can_match) {
+ // Parquet bounds omit NaNs, so only operators that cannot match a
hidden NaN may prune.
+ return unsupported_zonemap_filter(ctx);
+ }
switch (effective_op) {
case Op::EQ:
return literal < zone_map.min_value || zone_map.max_value < literal
@@ -375,7 +384,13 @@ inline bool can_evaluate(const VExprSPtrs& arguments) {
}
inline bool can_evaluate_equality(const VExprSPtrs& arguments, Op op) {
- return op == Op::EQ && can_evaluate(arguments);
+ if (op != Op::EQ || !can_evaluate(arguments)) {
+ return false;
+ }
+ const auto slot_literal =
expr_zonemap::extract_slot_and_literal(arguments);
+ DORIS_CHECK(slot_literal.has_value());
+ // Bloom membership cannot disprove Doris NaN equality across different
physical encodings.
+ return !slot_literal->literal.is_nan();
}
inline bool dictionary_value_matches(const Field& value, const Field& literal,
Op op) {
diff --git a/be/src/exprs/hybrid_set.h b/be/src/exprs/hybrid_set.h
index f542968786b..6a408bc1a57 100644
--- a/be/src/exprs/hybrid_set.h
+++ b/be/src/exprs/hybrid_set.h
@@ -138,6 +138,19 @@ struct IsFixedContainer : std::false_type {};
template <typename T, size_t N>
struct IsFixedContainer<FixedContainer<T, N>> : std::true_type {};
+template <typename T>
+struct DynamicContainerHash {
+ size_t operator()(const T& value) const {
+ if constexpr (std::is_floating_point_v<T>) {
+ T normalized = value;
+ // The hash must collapse NaN payloads and signed zeros exactly as
Doris equality does.
+ NormalizeFloat(normalized);
+ return phmap::Hash<T> {}(normalized);
+ }
+ return phmap::Hash<T> {}(value);
+ }
+};
+
/**
* Dynamic Container uses phmap::flat_hash_set.
* @tparam T Element Type
@@ -146,7 +159,8 @@ template <typename T>
class DynamicContainer {
public:
using Self = DynamicContainer;
- using Iterator = typename flat_hash_set<T>::iterator;
+ using Set = flat_hash_set<T, DynamicContainerHash<T>>;
+ using Iterator = typename Set::iterator;
using ElementType = T;
DynamicContainer() = default;
@@ -167,7 +181,7 @@ public:
size_t size() const { return _set.size(); }
private:
- flat_hash_set<T> _set;
+ Set _set;
};
// TODO Maybe change void* parameter to template parameter better.
diff --git a/be/src/exprs/vdirect_in_predicate.h
b/be/src/exprs/vdirect_in_predicate.h
index b272c816edb..ace2ac513a2 100644
--- a/be/src/exprs/vdirect_in_predicate.h
+++ b/be/src/exprs/vdirect_in_predicate.h
@@ -42,6 +42,7 @@ class VDirectInPredicate final : public VExpr {
std::once_flag materialize_once;
Status materialization_status;
bool zonemap_materialized = false;
+ bool seg_filter_contains_nan = false;
std::vector<Field> seg_filter_values;
Field seg_filter_min;
Field seg_filter_max;
@@ -102,7 +103,8 @@ public:
ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx)
const override {
return expr_zonemap::eval_in_zonemap(
ctx, get_child(0), false, _pruning_state->seg_filter_values,
- _pruning_state->seg_filter_min,
_pruning_state->seg_filter_max);
+ _pruning_state->seg_filter_contains_nan,
_pruning_state->seg_filter_min,
+ _pruning_state->seg_filter_max);
}
bool can_evaluate_zonemap_filter() const override {
@@ -332,6 +334,7 @@ private:
return;
}
pruning_state->seg_filter_values = std::move(materialized.values);
+ pruning_state->seg_filter_contains_nan = materialized.contains_nan;
pruning_state->seg_filter_min = std::move(materialized.min_value);
pruning_state->seg_filter_max = std::move(materialized.max_value);
pruning_state->zonemap_materialized = true;
diff --git a/be/src/exprs/vin_predicate.cpp b/be/src/exprs/vin_predicate.cpp
index 00f355add51..b5151e9c656 100644
--- a/be/src/exprs/vin_predicate.cpp
+++ b/be/src/exprs/vin_predicate.cpp
@@ -158,6 +158,7 @@ Status VInPredicate::evaluate_inverted_index(VExprContext*
context, uint32_t seg
Status VInPredicate::_materialize_for_zonemap_filter(VExprContext* context) {
_seg_filter_values.clear();
_seg_filter_contains_null = false;
+ _seg_filter_contains_nan = false;
_zonemap_materialized = false;
_direct_filter_set.reset();
if (_children.size() < 2 || !_children[0]->is_slot_ref()) {
@@ -184,6 +185,7 @@ Status
VInPredicate::_materialize_for_zonemap_filter(VExprContext* context) {
RETURN_IF_ERROR(expr_zonemap::materialize_hybrid_set_for_zonemap_filter(
*in_state->hybrid_set, data_type, &materialized));
_seg_filter_contains_null = materialized.contains_null;
+ _seg_filter_contains_nan = materialized.contains_nan;
_seg_filter_values = std::move(materialized.values);
_seg_filter_min = std::move(materialized.min_value);
_seg_filter_max = std::move(materialized.max_value);
@@ -196,7 +198,8 @@ ZoneMapFilterResult
VInPredicate::evaluate_zonemap_filter(const ZoneMapEvalConte
return ZoneMapFilterResult::kNoMatch;
}
return expr_zonemap::eval_in_zonemap(ctx, get_child(0), _is_not_in,
_seg_filter_values,
- _seg_filter_min, _seg_filter_max);
+ _seg_filter_contains_nan,
_seg_filter_min,
+ _seg_filter_max);
}
bool VInPredicate::can_evaluate_zonemap_filter() const {
@@ -218,7 +221,8 @@ ZoneMapFilterResult
VInPredicate::evaluate_bloom_filter(const BloomFilterEvalCon
}
bool VInPredicate::can_evaluate_bloom_filter() const {
- return _zonemap_materialized && !_is_not_in &&
+ // A NaN member forces conservative retention regardless of the remaining
finite probes.
+ return _zonemap_materialized && !_is_not_in && !_seg_filter_contains_nan &&
std::dynamic_pointer_cast<VSlotRef>(get_child(0)) != nullptr;
}
diff --git a/be/src/exprs/vin_predicate.h b/be/src/exprs/vin_predicate.h
index 245c1439be0..2fbbf8393f4 100644
--- a/be/src/exprs/vin_predicate.h
+++ b/be/src/exprs/vin_predicate.h
@@ -102,6 +102,7 @@ private:
bool _is_args_all_constant = false;
bool _zonemap_materialized = false;
bool _seg_filter_contains_null = false;
+ bool _seg_filter_contains_nan = false;
std::shared_ptr<HybridSetBase> _direct_filter_set;
std::vector<Field> _seg_filter_values;
Field _seg_filter_min;
diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp
b/be/src/format_v2/parquet/parquet_statistics.cpp
index 63b880f7c5e..26b76ab4c9c 100644
--- a/be/src/format_v2/parquet/parquet_statistics.cpp
+++ b/be/src/format_v2/parquet/parquet_statistics.cpp
@@ -838,6 +838,9 @@ void add_slot_zonemap(ZoneMapEvalContext* ctx, int
slot_index, const DataTypePtr
ZoneMapEvalContext::SlotZoneMap slot_zone_map;
slot_zone_map.data_type = data_type;
slot_zone_map.zone_map = std::move(zone_map);
+ const auto primitive_type =
remove_nullable(data_type)->get_primitive_type();
+ slot_zone_map.floating_nan_count_unknown =
+ primitive_type == TYPE_FLOAT || primitive_type == TYPE_DOUBLE;
ctx->slots.emplace(slot_index, std::move(slot_zone_map));
}
diff --git a/be/src/storage/index/zone_map/zonemap_eval_context.cpp
b/be/src/storage/index/zone_map/zonemap_eval_context.cpp
index b65d4400e8a..2fb6864349c 100644
--- a/be/src/storage/index/zone_map/zonemap_eval_context.cpp
+++ b/be/src/storage/index/zone_map/zonemap_eval_context.cpp
@@ -37,6 +37,11 @@ DataTypePtr ZoneMapEvalContext::data_type(int slot_index)
const {
return it->second.data_type;
}
+bool ZoneMapEvalContext::floating_nan_count_unknown(int slot_index) const {
+ auto it = slots.find(slot_index);
+ return it != slots.end() && it->second.floating_nan_count_unknown;
+}
+
void ZoneMapEvalStats::merge_page_eval_stats(const ZoneMapEvalStats& src) {
// Page-level evaluation repeats the same conjuncts for many pages. Keep
structural
// diagnostics once per column, while operation counters still reflect
actual page checks.
diff --git a/be/src/storage/index/zone_map/zonemap_eval_context.h
b/be/src/storage/index/zone_map/zonemap_eval_context.h
index c5a750b0591..7c1afab8780 100644
--- a/be/src/storage/index/zone_map/zonemap_eval_context.h
+++ b/be/src/storage/index/zone_map/zonemap_eval_context.h
@@ -52,10 +52,13 @@ public:
struct SlotZoneMap {
DataTypePtr data_type;
std::shared_ptr<const segment_v2::ZoneMap> zone_map;
+ // Parquet min/max does not expose whether a floating chunk also
contains NaNs.
+ bool floating_nan_count_unknown = false;
};
std::shared_ptr<const segment_v2::ZoneMap> zone_map(int slot_index) const;
DataTypePtr data_type(int slot_index) const;
+ bool floating_nan_count_unknown(int slot_index) const;
phmap::flat_hash_map<int, SlotZoneMap> slots;
diff --git a/be/test/core/field_test.cpp b/be/test/core/field_test.cpp
index 694b292049c..efe94cef968 100644
--- a/be/test/core/field_test.cpp
+++ b/be/test/core/field_test.cpp
@@ -20,6 +20,7 @@
#include <gtest/gtest-message.h>
#include <gtest/gtest-test-part.h>
+#include <limits>
#include <string>
#include "core/column/column_string.h"
@@ -32,6 +33,16 @@
#include "gtest/gtest_pred_impl.h" // IWYU pragma: keep
namespace doris {
+TEST(VFieldTest, detects_floating_point_nan) {
+
EXPECT_TRUE(Field::create_field<TYPE_FLOAT>(std::numeric_limits<float>::quiet_NaN()).is_nan());
+ EXPECT_TRUE(
+
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN()).is_nan());
+ EXPECT_FALSE(Field::create_field<TYPE_FLOAT>(0.0F).is_nan());
+ EXPECT_FALSE(Field::create_field<TYPE_DOUBLE>(-0.0).is_nan());
+ EXPECT_FALSE(Field::create_field<TYPE_INT>(0).is_nan());
+ EXPECT_FALSE(Field().is_nan());
+}
+
TEST(VFieldTest, field_string) {
Field f;
diff --git a/be/test/exprs/expr_zonemap_filter_test.cpp
b/be/test/exprs/expr_zonemap_filter_test.cpp
index 668a528b6a1..d5d8d62a100 100644
--- a/be/test/exprs/expr_zonemap_filter_test.cpp
+++ b/be/test/exprs/expr_zonemap_filter_test.cpp
@@ -20,6 +20,9 @@
#include <gtest/gtest.h>
#include <array>
+#include <bit>
+#include <cstdint>
+#include <limits>
#include <map>
#include <memory>
#include <mutex>
@@ -428,6 +431,193 @@ TEST(ExprZonemapFilterTest,
ComparisonDictionarySupportsTypedRangesWhileBloomUse
{string_slot,
make_string_literal("delta")}));
}
+TEST(ExprZonemapFilterTest, FloatingPointNanBloomProbeIsConservative) {
+ auto bloom_filter = std::make_unique<segment_v2::BlockSplitBloomFilter>();
+
ASSERT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok());
+ const double finite_value = 1.0;
+ bloom_filter->add_bytes(reinterpret_cast<const char*>(&finite_value),
sizeof(finite_value));
+
+ FunctionComparison<EqualsOp, NameEquals> equals;
+ const auto check_type = [&](const DataTypePtr& type, Field nan_field) {
+ auto slot = make_slot(0, type);
+ auto literal = std::make_shared<VLiteral>(create_texpr_node_from(
+ nan_field, remove_nullable(type)->get_primitive_type(), 0, 0));
+ auto bloom_ctx = make_bloom_filter_context(bloom_filter.get(), type);
+
+ EXPECT_FALSE(equals.can_evaluate_bloom_filter({slot, literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
+ equals.evaluate_bloom_filter(bloom_ctx, {slot, literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
+ expr_zonemap::eval_in_bloom_filter(bloom_ctx, slot, false,
{nan_field}));
+
+ const auto primitive_type =
remove_nullable(type)->get_primitive_type();
+ const Field absent_finite = primitive_type == TYPE_FLOAT
+ ?
Field::create_field<TYPE_FLOAT>(2.0F)
+ :
Field::create_field<TYPE_DOUBLE>(2.0);
+ auto finite_literal = std::make_shared<VLiteral>(
+ create_texpr_node_from(absent_finite, primitive_type, 0, 0));
+ EXPECT_TRUE(equals.can_evaluate_bloom_filter({slot, finite_literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
+ equals.evaluate_bloom_filter(bloom_ctx, {slot,
finite_literal}));
+ };
+
+ check_type(std::make_shared<DataTypeFloat32>(),
+
Field::create_field<TYPE_FLOAT>(std::numeric_limits<float>::quiet_NaN()));
+ check_type(std::make_shared<DataTypeFloat64>(),
+
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN()));
+}
+
+TEST(ExprZonemapFilterTest, FloatingPointInWithNanIsNotBloomEligible) {
+ auto type = std::make_shared<DataTypeFloat64>();
+ auto predicate =
std::make_shared<VInPredicate>(make_in_predicate_node(false, 2));
+ predicate->add_child(make_slot(0, type));
+ predicate->_zonemap_materialized = true;
+ predicate->_seg_filter_contains_nan = true;
+ predicate->_seg_filter_values = {
+ Field::create_field<TYPE_DOUBLE>(1.0),
+
Field::create_field<TYPE_DOUBLE>(std::numeric_limits<double>::quiet_NaN())};
+
+ EXPECT_FALSE(predicate->can_evaluate_bloom_filter());
+ predicate->_seg_filter_contains_nan = false;
+ EXPECT_TRUE(predicate->can_evaluate_bloom_filter());
+}
+
+TEST(ExprZonemapFilterTest,
FloatingPointNanEqualityIgnoresFiniteOnlyRangeBounds) {
+ const auto check_type = []<PrimitiveType Type, typename DataType, typename
UInt>(
+ UInt nan_bits) {
+ using T = typename PrimitiveTypeTraits<Type>::CppType;
+ auto type = std::make_shared<DataType>();
+ auto slot = make_slot(0, type);
+ const auto nan_field =
Field::create_field<Type>(std::bit_cast<T>(nan_bits));
+ auto nan_literal =
+ std::make_shared<VLiteral>(create_texpr_node_from(nan_field,
Type, 0, 0));
+
+ segment_v2::ZoneMap zone_map;
+ zone_map.min_value = Field::create_field<Type>(T {0});
+ zone_map.max_value = Field::create_field<Type>(T {0});
+ zone_map.has_not_null = true;
+ auto ctx = make_context(std::move(zone_map), type);
+ ctx.slots.at(0).floating_nan_count_unknown = true;
+
+ FunctionComparison<EqualsOp, NameEquals> equals;
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ equals.evaluate_zonemap_filter(ctx, {slot, nan_literal}));
+
+ const auto finite_field = Field::create_field<Type>(T {10});
+ const auto zero_field = Field::create_field<Type>(T {0});
+ const auto one_field = Field::create_field<Type>(T {1});
+ auto zero_literal =
+ std::make_shared<VLiteral>(create_texpr_node_from(zero_field,
Type, 0, 0));
+ auto one_literal =
+ std::make_shared<VLiteral>(create_texpr_node_from(one_field,
Type, 0, 0));
+ FunctionComparison<NotEqualsOp, NameNotEquals> not_equals;
+ FunctionComparison<GreaterOp, NameGreater> greater;
+ FunctionComparison<GreaterOrEqualsOp, NameGreaterOrEquals>
greater_equal;
+ FunctionComparison<LessOp, NameLess> less;
+ FunctionComparison<LessOrEqualsOp, NameLessOrEquals> less_equal;
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ not_equals.evaluate_zonemap_filter(ctx, {slot,
zero_literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ greater.evaluate_zonemap_filter(ctx, {slot, one_literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ greater_equal.evaluate_zonemap_filter(ctx, {slot,
one_literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ less.evaluate_zonemap_filter(ctx, {one_literal, slot}));
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ less_equal.evaluate_zonemap_filter(ctx, {one_literal,
slot}));
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ expr_zonemap::eval_in_zonemap(ctx, slot, false,
{finite_field, nan_field}, true,
+ finite_field, nan_field));
+ EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
+ expr_zonemap::eval_in_zonemap(ctx, slot, true, {zero_field},
false, zero_field,
+ zero_field));
+
+ segment_v2::ZoneMap all_null_zone_map;
+ all_null_zone_map.min_value = zero_field;
+ all_null_zone_map.max_value = zero_field;
+ auto all_null_ctx = make_context(std::move(all_null_zone_map), type);
+ all_null_ctx.slots.at(0).floating_nan_count_unknown = true;
+ EXPECT_EQ(
+ ZoneMapFilterResult::kNoMatch,
+ expr_zonemap::eval_in_zonemap(all_null_ctx, slot, false,
{finite_field, nan_field},
+ true, finite_field, nan_field));
+
+ ctx.slots.at(0).floating_nan_count_unknown = false;
+ EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
+ equals.evaluate_zonemap_filter(ctx, {slot, nan_literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
+ expr_zonemap::eval_in_zonemap(ctx, slot, false,
{finite_field, nan_field}, true,
+ finite_field, nan_field));
+ };
+
+ check_type.template operator()<TYPE_FLOAT, DataTypeFloat32>(uint32_t
{0x7fc00002U});
+ check_type.template operator()<TYPE_DOUBLE, DataTypeFloat64>(uint64_t
{0x7ff8000000000002ULL});
+}
+
+TEST(ExprZonemapFilterTest, DirectInRawFixedKeepsEqualNanPayloadFromLargeSet) {
+ const auto check_type = []<PrimitiveType Type, typename DataType, typename
UInt>(
+ UInt stored_bits, UInt probe_bits) {
+ using T = typename PrimitiveTypeTraits<Type>::CppType;
+ auto type = std::make_shared<DataType>();
+ std::shared_ptr<HybridSetBase> filter(create_set(Type, false));
+ for (int value = 0; value < FIXED_CONTAINER_MAX_SIZE; ++value) {
+ T finite = static_cast<T>(value);
+ filter->insert(&finite);
+ }
+ const T stored_nan = std::bit_cast<T>(stored_bits);
+ filter->insert(&stored_nan);
+ ASSERT_EQ(FIXED_CONTAINER_MAX_SIZE + 1, filter->size());
+
+ VDirectInPredicate predicate(make_in_predicate_node(false, 1), filter,
true);
+ predicate.add_child(make_slot(0, type));
+ ASSERT_TRUE(predicate.can_execute_on_raw_fixed_values(type, 0));
+
+ const T probe_nan = std::bit_cast<T>(probe_bits);
+ uint8_t match = 1;
+ ASSERT_TRUE(
+ predicate
+ .execute_on_raw_fixed_values(reinterpret_cast<const
uint8_t*>(&probe_nan),
+ 1, sizeof(T), type, 0,
&match)
+ .ok());
+ EXPECT_EQ(1, match);
+ };
+
+ check_type.template operator()<TYPE_FLOAT, DataTypeFloat32>(uint32_t
{0x7fc00001U},
+ uint32_t
{0x7fc00002U});
+ check_type.template operator()<TYPE_DOUBLE, DataTypeFloat64>(uint64_t
{0x7ff8000000000001ULL},
+ uint64_t
{0x7ff8000000000002ULL});
+}
+
+TEST(ExprZonemapFilterTest,
FloatingPointSignedZeroBloomProbeChecksBothEncodings) {
+ FunctionComparison<EqualsOp, NameEquals> equals;
+ const auto check_type = [&]<PrimitiveType Type>(
+ const DataTypePtr& type,
+ typename
PrimitiveTypeTraits<Type>::CppType stored_value,
+ typename
PrimitiveTypeTraits<Type>::CppType predicate_value) {
+ auto bloom_filter =
std::make_unique<segment_v2::BlockSplitBloomFilter>();
+
ASSERT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok());
+ bloom_filter->add_bytes(reinterpret_cast<const char*>(&stored_value),
sizeof(stored_value));
+ ASSERT_FALSE(bloom_filter->test_bytes(reinterpret_cast<const
char*>(&predicate_value),
+ sizeof(predicate_value)));
+
+ auto slot = make_slot(0, type);
+ const auto field = Field::create_field<Type>(predicate_value);
+ auto literal =
std::make_shared<VLiteral>(create_texpr_node_from(field, Type, 0, 0));
+ auto bloom_ctx = make_bloom_filter_context(bloom_filter.get(), type);
+ EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
+ equals.evaluate_bloom_filter(bloom_ctx, {slot, literal}));
+ EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
+ expr_zonemap::eval_in_bloom_filter(bloom_ctx, slot, false,
{field}));
+ };
+
+ const auto float_type = std::make_shared<DataTypeFloat32>();
+ check_type.template operator()<TYPE_FLOAT>(float_type, -0.0F, 0.0F);
+ check_type.template operator()<TYPE_FLOAT>(float_type, 0.0F, -0.0F);
+ const auto double_type = std::make_shared<DataTypeFloat64>();
+ check_type.template operator()<TYPE_DOUBLE>(double_type, -0.0, 0.0);
+ check_type.template operator()<TYPE_DOUBLE>(double_type, 0.0, -0.0);
+}
+
TEST(ExprZonemapFilterTest,
DefaultFunctionForwardsDictionaryAndBloomEvaluation) {
auto type = int_type();
auto slot = make_slot(0, type);
@@ -479,7 +669,7 @@ TEST(ExprZonemapFilterTest,
MissingSlotTypeCountsUnsupportedZonemapEvalOnce) {
std::vector<Field> values {int_field(10)};
ZoneMapEvalContext in_ctx;
EXPECT_EQ(ZoneMapFilterResult::kUnsupported,
- expr_zonemap::eval_in_zonemap(in_ctx, slot, false, values,
int_field(10),
+ expr_zonemap::eval_in_zonemap(in_ctx, slot, false, values,
false, int_field(10),
int_field(10)));
EXPECT_EQ(1, in_ctx.stats.unusable_zonemap_eval_count);
}
@@ -542,20 +732,20 @@ TEST(ExprZonemapFilterTest,
InZonemapSkipsZonesWithoutNonNullValues) {
segment_v2::ZoneMap empty_zone;
auto empty_ctx = make_context(empty_zone, type);
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(empty_ctx, slot, false, values,
int_field(10),
+ expr_zonemap::eval_in_zonemap(empty_ctx, slot, false, values,
false, int_field(10),
int_field(10)));
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(empty_ctx, slot, true, values,
int_field(10),
+ expr_zonemap::eval_in_zonemap(empty_ctx, slot, true, values,
false, int_field(10),
int_field(10)));
segment_v2::ZoneMap only_null_zone;
only_null_zone.has_null = true;
auto only_null_ctx = make_context(only_null_zone, type);
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(only_null_ctx, slot, false,
values, int_field(10),
- int_field(10)));
+ expr_zonemap::eval_in_zonemap(only_null_ctx, slot, false,
values, false,
+ int_field(10), int_field(10)));
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(only_null_ctx, slot, true, values,
int_field(10),
+ expr_zonemap::eval_in_zonemap(only_null_ctx, slot, true, values,
false, int_field(10),
int_field(10)));
}
@@ -645,8 +835,9 @@ TEST(ExprZonemapFilterTest,
CharZonemapUsesTrimmedLogicalBounds) {
auto in_value = Field::create_field<TYPE_STRING>("gamma");
std::vector<Field> values {in_value};
auto in_ctx = make_context(zone_map, char_type);
- EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(in_ctx, slot, false, values,
in_value, in_value));
+ EXPECT_EQ(
+ ZoneMapFilterResult::kNoMatch,
+ expr_zonemap::eval_in_zonemap(in_ctx, slot, false, values, false,
in_value, in_value));
}
TEST(ExprZonemapFilterTest, InZonemapFallsBackToRangeWhenPointListIsLarge) {
@@ -659,7 +850,8 @@ TEST(ExprZonemapFilterTest,
InZonemapFallsBackToRangeWhenPointListIsLarge) {
values.emplace_back(int_field(value));
}
EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
- expr_zonemap::eval_in_zonemap(ctx, slot, false, values,
int_field(1), int_field(65)));
+ expr_zonemap::eval_in_zonemap(ctx, slot, false, values, false,
int_field(1),
+ int_field(65)));
EXPECT_EQ(0, ctx.stats.in_zonemap_point_check_count);
EXPECT_EQ(1, ctx.stats.in_zonemap_range_only_count);
@@ -672,7 +864,8 @@ TEST(ExprZonemapFilterTest,
InZonemapUsesPointChecksUnderThreshold) {
std::vector<Field> values {int_field(1), int_field(30)};
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(ctx, slot, false, values,
int_field(1), int_field(30)));
+ expr_zonemap::eval_in_zonemap(ctx, slot, false, values, false,
int_field(1),
+ int_field(30)));
EXPECT_EQ(1, ctx.stats.in_zonemap_point_check_count);
}
@@ -683,19 +876,19 @@ TEST(ExprZonemapFilterTest,
InZonemapHandlesEmptyListAndNotInSingleValueRange) {
std::vector<Field> empty_values;
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(ctx, slot, false, empty_values,
{}, {}));
+ expr_zonemap::eval_in_zonemap(ctx, slot, false, empty_values,
false, {}, {}));
EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
- expr_zonemap::eval_in_zonemap(ctx, slot, true, empty_values, {},
{}));
+ expr_zonemap::eval_in_zonemap(ctx, slot, true, empty_values,
false, {}, {}));
auto single_value_ctx = make_context(make_int_zonemap(10, 10), type);
std::vector<Field> values {int_field(10)};
EXPECT_EQ(ZoneMapFilterResult::kNoMatch,
- expr_zonemap::eval_in_zonemap(single_value_ctx, slot, true,
values, int_field(10),
- int_field(10)));
+ expr_zonemap::eval_in_zonemap(single_value_ctx, slot, true,
values, false,
+ int_field(10), int_field(10)));
std::vector<Field> other_values {int_field(11)};
EXPECT_EQ(ZoneMapFilterResult::kMayMatch,
- expr_zonemap::eval_in_zonemap(single_value_ctx, slot, true,
other_values,
+ expr_zonemap::eval_in_zonemap(single_value_ctx, slot, true,
other_values, false,
int_field(11), int_field(11)));
}
diff --git a/be/test/exprs/hybrid_set_test.cpp
b/be/test/exprs/hybrid_set_test.cpp
index 5f14dc924eb..9c7611d13b0 100644
--- a/be/test/exprs/hybrid_set_test.cpp
+++ b/be/test/exprs/hybrid_set_test.cpp
@@ -19,6 +19,8 @@
#include <gtest/gtest.h>
+#include <bit>
+#include <cstdint>
#include <memory>
#include <string>
@@ -391,6 +393,33 @@ TEST_F(HybridSetTest, double) {
a = 5.1;
EXPECT_FALSE(set->find(&a));
}
+
+TEST_F(HybridSetTest, DynamicFloatingSetFindsDorisEqualNanPayload) {
+ const auto check_type = []<PrimitiveType Type, typename UInt>(UInt
stored_bits,
+ UInt
probe_bits) {
+ using T = typename PrimitiveTypeTraits<Type>::CppType;
+ std::unique_ptr<HybridSetBase> set(create_set(Type, false));
+ for (int value = 0; value < FIXED_CONTAINER_MAX_SIZE; ++value) {
+ T finite = static_cast<T>(value);
+ set->insert(&finite);
+ }
+ const T stored_nan = std::bit_cast<T>(stored_bits);
+ set->insert(&stored_nan);
+ ASSERT_EQ(FIXED_CONTAINER_MAX_SIZE + 1, set->size());
+
+ const T probe_nan = std::bit_cast<T>(probe_bits);
+ EXPECT_TRUE(set->find(&probe_nan));
+ uint8_t match = 1;
+ set->find_batch_raw_fixed(reinterpret_cast<const
uint8_t*>(&probe_nan), 1, sizeof(T),
+ &match);
+ EXPECT_EQ(1, match);
+ };
+
+ check_type.template operator()<TYPE_FLOAT>(uint32_t {0x7fc00001U},
uint32_t {0x7fc00002U});
+ check_type.template operator()<TYPE_DOUBLE>(uint64_t
{0x7ff8000000000001ULL},
+ uint64_t
{0x7ff8000000000002ULL});
+}
+
TEST_F(HybridSetTest, string) {
std::unique_ptr<HybridSetBase> set(create_set(PrimitiveType::TYPE_VARCHAR,
false));
StringRef a;
diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp
b/be/test/format_v2/parquet/parquet_scan_test.cpp
index c3532c051f0..68775a74f20 100644
--- a/be/test/format_v2/parquet/parquet_scan_test.cpp
+++ b/be/test/format_v2/parquet/parquet_scan_test.cpp
@@ -25,6 +25,7 @@
#include <parquet/arrow/writer.h>
#include <parquet/encoding.h>
+#include <bit>
#include <cstring>
#include <filesystem>
#include <fstream>
@@ -66,6 +67,7 @@
#include "format_v2/file_reader.h"
#include "format_v2/parquet/parquet_column_schema.h"
#include "format_v2/parquet/parquet_reader.h"
+#include "format_v2/parquet/reader/native/block_split_bloom_filter.h"
#include "format_v2/parquet/reader/native_column_reader.h"
#include "gen_cpp/PlanNodes_types.h"
#include "gen_cpp/Types_types.h"
@@ -570,6 +572,47 @@ VExprContextSPtr create_string_in_conjunct(int column_id,
const std::vector<std:
return VExprContext::create_shared(std::move(root));
}
+DataTypePtr floating_data_type(PrimitiveType type) {
+ if (type == TYPE_FLOAT) {
+ return std::make_shared<DataTypeFloat32>();
+ }
+ DORIS_CHECK(type == TYPE_DOUBLE);
+ return std::make_shared<DataTypeFloat64>();
+}
+
+VExprContextSPtr create_floating_function_conjunct(int column_id,
PrimitiveType type,
+ const std::string&
function_name,
+ TExprOpcode::type opcode,
const Field& value) {
+ const auto data_type = floating_data_type(type);
+ auto root = create_binary_predicate(
+ function_name, opcode,
+ VSlotRef::create_shared(column_id, column_id, -1,
make_nullable(data_type),
+ "floating_key"),
+ VLiteral::create_shared(data_type, value));
+ return VExprContext::create_shared(std::move(root));
+}
+
+VExprContextSPtr create_floating_in_conjunct(int column_id, PrimitiveType type,
+ const std::vector<Field>& values,
bool is_not_in) {
+ const auto data_type = floating_data_type(type);
+ const auto result_type = make_nullable(std::make_shared<DataTypeUInt8>());
+ TExprNode node;
+ node.__set_node_type(TExprNodeType::IN_PRED);
+ node.__set_type(result_type->to_thrift());
+ node.__set_num_children(static_cast<int16_t>(values.size() + 1));
+ node.__set_is_nullable(true);
+ TInPredicate in_predicate;
+ in_predicate.__set_is_not_in(is_not_in);
+ node.__set_in_predicate(in_predicate);
+ auto root = VInPredicate::create_shared(node);
+ root->add_child(VSlotRef::create_shared(column_id, column_id, -1,
make_nullable(data_type),
+ "floating_key"));
+ for (const auto& value : values) {
+ root->add_child(VLiteral::create_shared(data_type, value));
+ }
+ return VExprContext::create_shared(std::move(root));
+}
+
VExprContextSPtr create_null_conjunct(int column_id, const DataTypePtr&
data_type, bool is_null) {
const auto nullable_type = make_nullable(remove_nullable(data_type));
const auto result_type = std::make_shared<DataTypeUInt8>();
@@ -1173,6 +1216,22 @@ std::shared_ptr<arrow::Array> build_int64_array(const
std::vector<int64_t>& valu
return finish_array(&builder);
}
+std::shared_ptr<arrow::Array> build_float_array(const std::vector<float>&
values) {
+ arrow::FloatBuilder builder;
+ for (const auto value : values) {
+ EXPECT_TRUE(builder.Append(value).ok());
+ }
+ return finish_array(&builder);
+}
+
+std::shared_ptr<arrow::Array> build_double_array(const std::vector<double>&
values) {
+ arrow::DoubleBuilder builder;
+ for (const auto value : values) {
+ EXPECT_TRUE(builder.Append(value).ok());
+ }
+ return finish_array(&builder);
+}
+
std::shared_ptr<arrow::Array> build_int8_array(const std::vector<int8_t>&
values) {
arrow::Int8Builder builder;
for (const auto value : values) {
@@ -1297,6 +1356,111 @@ void write_table(const std::string& file_path, const
std::shared_ptr<arrow::Tabl
row_group_size,
builder.build()));
}
+template <typename T>
+std::vector<uint8_t> build_parquet_bloom_filter(const T* values, size_t count)
{
+ format::parquet::native::BlockSplitBloomFilter bloom_filter;
+ DORIS_CHECK(bloom_filter
+ .init(segment_v2::BloomFilter::MINIMUM_BYTES,
+ segment_v2::HashStrategyPB::XX_HASH_64)
+ .ok());
+ for (size_t index = 0; index < count; ++index) {
+ bloom_filter.add_bytes(reinterpret_cast<const char*>(&values[index]),
sizeof(T));
+ }
+
+ tparquet::BloomFilterAlgorithm algorithm;
+ algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm());
+ tparquet::BloomFilterHash hash;
+ hash.__set_XXHASH(tparquet::XxHash());
+ tparquet::BloomFilterCompression compression;
+ compression.__set_UNCOMPRESSED(tparquet::Uncompressed());
+ tparquet::BloomFilterHeader header;
+ header.__set_numBytes(static_cast<int32_t>(bloom_filter.size()));
+ header.__set_algorithm(algorithm);
+ header.__set_hash(hash);
+ header.__set_compression(compression);
+
+ std::vector<uint8_t> bytes;
+ ThriftSerializer serializer(/*compact=*/true, 64);
+ DORIS_CHECK(serializer.serialize(&header, &bytes).ok());
+ bytes.insert(bytes.end(), bloom_filter.data(), bloom_filter.data() +
bloom_filter.size());
+ return bytes;
+}
+
+void append_floating_bloom_filters(const std::string& file_path,
+ const std::vector<float>& float_values,
+ const std::vector<double>& double_values) {
+ DORIS_CHECK(float_values.size() == double_values.size());
+ std::ifstream input(file_path, std::ios::binary | std::ios::ate);
+ DORIS_CHECK(input.good());
+ const auto input_size = static_cast<std::streamoff>(input.tellg());
+ DORIS_CHECK(input_size >= static_cast<std::streamoff>(8));
+ std::vector<uint8_t> file_bytes(cast_set<size_t>(input_size));
+ input.seekg(0);
+ input.read(reinterpret_cast<char*>(file_bytes.data()),
cast_set<std::streamsize>(input_size));
+ DORIS_CHECK(input.good());
+ DORIS_CHECK(memcmp(file_bytes.data() + file_bytes.size() - 4, "PAR1", 4)
== 0);
+
+ const uint32_t footer_size = decode_fixed32_le(file_bytes.data() +
file_bytes.size() - 8);
+ DORIS_CHECK(footer_size <= file_bytes.size() - 8);
+ const size_t footer_offset = file_bytes.size() - 8 - footer_size;
+ uint32_t thrift_size = footer_size;
+ tparquet::FileMetaData metadata;
+ DORIS_CHECK(
+ deserialize_thrift_msg(file_bytes.data() + footer_offset,
&thrift_size, true, &metadata)
+ .ok());
+
+ file_bytes.resize(footer_offset);
+ size_t first_row = 0;
+ for (auto& row_group : metadata.row_groups) {
+ const size_t row_count = cast_set<size_t>(row_group.num_rows);
+ DORIS_CHECK(row_group.columns.size() >= 2);
+ DORIS_CHECK(first_row + row_count <= float_values.size());
+ const auto append_column_bloom = [&]<typename T>(size_t column_id,
+ const std::vector<T>&
values) {
+ auto bytes = build_parquet_bloom_filter(values.data() + first_row,
row_count);
+ auto& column = row_group.columns[column_id].meta_data;
+
column.__set_bloom_filter_offset(cast_set<int64_t>(file_bytes.size()));
+ column.__set_bloom_filter_length(cast_set<int32_t>(bytes.size()));
+ file_bytes.insert(file_bytes.end(), bytes.begin(), bytes.end());
+ };
+ append_column_bloom(0, float_values);
+ append_column_bloom(1, double_values);
+ first_row += row_count;
+ }
+ DORIS_CHECK(first_row == float_values.size());
+
+ std::vector<uint8_t> footer;
+ ThriftSerializer serializer(/*compact=*/true, 1024);
+ DORIS_CHECK(serializer.serialize(&metadata, &footer).ok());
+ file_bytes.insert(file_bytes.end(), footer.begin(), footer.end());
+ std::array<uint8_t, sizeof(uint32_t)> encoded_footer_size {};
+ encode_fixed32_le(encoded_footer_size.data(),
cast_set<uint32_t>(footer.size()));
+ file_bytes.insert(file_bytes.end(), encoded_footer_size.begin(),
encoded_footer_size.end());
+ file_bytes.insert(file_bytes.end(), {'P', 'A', 'R', '1'});
+
+ std::ofstream output(file_path, std::ios::binary | std::ios::trunc);
+ output.write(reinterpret_cast<const char*>(file_bytes.data()),
file_bytes.size());
+ output.close();
+ DORIS_CHECK(output.good());
+}
+
+void write_floating_parquet_file_with_bloom_filters(const std::string&
file_path) {
+ const std::vector<float> float_values {
+ -1.0F, -0.0F, 1.0F, 0.5F, std::bit_cast<float>(uint32_t
{0x7fc00001U}),
+ -2.0F, 2.0F, 4.0F};
+ const std::vector<double> double_values {
+ -1.0, -0.0, 1.0, 0.5, std::bit_cast<double>(uint64_t
{0x7ff8000000000001ULL}),
+ -2.0, 2.0, 4.0};
+ auto schema = arrow::schema({arrow::field("float_value", arrow::float32(),
false),
+ arrow::field("double_value",
arrow::float64(), false),
+ arrow::field("id", arrow::int32(), false)});
+ auto table = arrow::Table::Make(
+ schema, {build_float_array(float_values),
build_double_array(double_values),
+ build_int32_array({0, 1, 2, 3, 4, 5, 6, 7})});
+ write_table(file_path, table, 4);
+ append_floating_bloom_filters(file_path, float_values, double_values);
+}
+
void write_required_adjusted_time_parquet_file(const std::string& file_path) {
auto file_result = arrow::io::FileOutputStream::Open(file_path);
ASSERT_TRUE(file_result.ok()) << file_result.status();
@@ -1781,6 +1945,83 @@ protected:
std::string _file_path;
};
+TEST_F(ParquetScanTest,
FloatingPredicatesPreserveDorisSemanticsWithRealBloomFilters) {
+ write_floating_parquet_file_with_bloom_filters(_file_path);
+
+ const auto expect_ids = [&](int column_id, const VExprContextSPtr&
conjunct,
+ const ColumnInt32::Container& expected,
+ bool expect_bloom_pruning = false) {
+ RuntimeProfile profile("profile");
+ auto reader = create_reader(0, -1, &profile);
+ TQueryOptions options;
+ options.__set_enable_parquet_filter_by_bloom_filter(true);
+ RuntimeState state {options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ auto request = std::make_shared<format::FileScanRequest>();
+ use_schema_order_positions(request.get(), schema);
+ format::FileScanRequestBuilder request_builder(request.get());
+
ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(column_id)).ok());
+
ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok());
+
request->predicate_only_columns.push_back(format::LocalColumnId(column_id));
+ ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok());
+ ASSERT_TRUE(conjunct->open(&state).ok());
+ request->conjuncts.push_back(conjunct);
+ ASSERT_TRUE(reader->open(request).ok());
+
+ ColumnInt32::Container actual;
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block(schema);
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ const auto& ids =
int32_data_column(*block.get_by_position(2).column).get_data();
+ actual.insert(actual.end(), ids.begin(), ids.end());
+ }
+ EXPECT_EQ(actual, expected);
+ if (expect_bloom_pruning) {
+ EXPECT_EQ(counter_value(profile,
"RowGroupsFilteredByBloomFilter"), 1);
+ }
+ conjunct->close();
+ };
+
+ const auto check_type = [&]<PrimitiveType Type>(int column_id) {
+ using CppType = typename PrimitiveTypeTraits<Type>::CppType;
+ const CppType query_nan = [] {
+ if constexpr (Type == TYPE_FLOAT) {
+ return std::bit_cast<float>(uint32_t {0x7fc00002U});
+ } else {
+ return std::bit_cast<double>(uint64_t {0x7ff8000000000002ULL});
+ }
+ }();
+ const auto nan = Field::create_field<Type>(query_nan);
+ const auto zero = Field::create_field<Type>(CppType {0});
+ const auto one = Field::create_field<Type>(CppType {1});
+ const auto absent = Field::create_field<Type>(CppType {10});
+
+ expect_ids(column_id,
+ create_floating_function_conjunct(column_id, Type, "eq",
TExprOpcode::EQ, nan),
+ {4});
+ expect_ids(column_id, create_floating_in_conjunct(column_id, Type,
{absent, nan}, false),
+ {4});
+ expect_ids(column_id, create_floating_in_conjunct(column_id, Type,
{zero}, true),
+ {0, 2, 3, 4, 5, 6, 7});
+ expect_ids(column_id,
+ create_floating_function_conjunct(column_id, Type, "eq",
TExprOpcode::EQ, zero),
+ {1}, true);
+ expect_ids(column_id,
+ create_floating_function_conjunct(column_id, Type, "gt",
TExprOpcode::GT, one),
+ {4, 6, 7});
+ };
+
+ // The file stores different NaN payloads and -0.0, so these scans
exercise both semantic
+ // equivalence classes through the real V2 footer-statistics and
Bloom-filter path.
+ check_type.template operator()<TYPE_FLOAT>(0);
+ check_type.template operator()<TYPE_DOUBLE>(1);
+}
+
TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) {
format::parquet::SelectionVector selection(4);
selection.set_index(0, 0);
diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp
b/be/test/format_v2/parquet/parquet_statistics_test.cpp
index 8f7e8c2b285..47b56730bea 100644
--- a/be/test/format_v2/parquet/parquet_statistics_test.cpp
+++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp
@@ -19,6 +19,7 @@
#include <gtest/gtest.h>
+#include <bit>
#include <cstdint>
#include <cstring>
#include <limits>
@@ -40,6 +41,7 @@
#include "core/data_type/data_type_variant_v2.h"
#include "core/field.h"
#include "exprs/expr_zonemap_filter.h"
+#include "exprs/function/functions_comparison.h"
#include "exprs/vcompound_pred.h"
#include "exprs/vexpr.h"
#include "exprs/vexpr_context.h"
@@ -115,6 +117,115 @@ private:
const std::string _expr_name = "BloomInExpr";
};
+class BloomEqExpr final : public VExpr {
+public:
+ BloomEqExpr(int column_id, DataTypePtr data_type, Field value)
+ : VExpr(std::make_shared<DataTypeUInt8>(), false),
+ _slot(VSlotRef::create_shared(0, column_id, -1,
std::move(data_type), "c0")),
+ _value(std::move(value)) {}
+
+ const std::string& expr_name() const override { return _expr_name; }
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
+ ColumnPtr&) const override {
+ return Status::InternalError("BloomEqExpr is only used by parquet
statistics tests");
+ }
+ bool can_evaluate_bloom_filter() const override { return true; }
+ ZoneMapFilterResult evaluate_bloom_filter(const BloomFilterEvalContext&
ctx) const override {
+ return expr_zonemap::eval_eq_bloom_filter(
+ ctx, expr_zonemap::SlotLiteral {.slot_index =
_slot->column_id(),
+ .slot_type =
_slot->data_type(),
+ .literal = _value,
+ .literal_type =
_slot->data_type(),
+ .literal_on_left = false});
+ }
+ void collect_slot_column_ids(std::set<int>& column_ids) const override {
+ _slot->collect_slot_column_ids(column_ids);
+ }
+
+private:
+ std::shared_ptr<VSlotRef> _slot;
+ Field _value;
+ const std::string _expr_name = "BloomEqExpr";
+};
+
+class MetadataFloatingEqualityExpr final : public VExpr {
+public:
+ enum class Mode { EQ, IN, NE, GT, GE, REVERSED_LT, REVERSED_LE, NOT_IN };
+
+ MetadataFloatingEqualityExpr(int column_id, DataTypePtr data_type, Field
nan_value, Mode mode)
+ : VExpr(std::make_shared<DataTypeUInt8>(), false),
+ _slot(VSlotRef::create_shared(0, column_id, -1, data_type,
"c0")),
+ _nan_literal(VLiteral::create_shared(create_texpr_node_from(
+ nan_value,
remove_nullable(data_type)->get_primitive_type(), 0, 0))),
+ _mode(mode),
+ _values {Field::create_field<TYPE_DOUBLE>(10.0),
std::move(nan_value)} {
+ const auto primitive_type =
remove_nullable(data_type)->get_primitive_type();
+ if (primitive_type == TYPE_FLOAT) {
+ _values[0] = Field::create_field<TYPE_FLOAT>(10.0F);
+ _zero_literal = VLiteral::create_shared(create_texpr_node_from(
+ Field::create_field<TYPE_FLOAT>(0.0F), TYPE_FLOAT, 0, 0));
+ _one_literal = VLiteral::create_shared(create_texpr_node_from(
+ Field::create_field<TYPE_FLOAT>(1.0F), TYPE_FLOAT, 0, 0));
+ _not_in_values = {Field::create_field<TYPE_FLOAT>(0.0F)};
+ } else {
+ _zero_literal = VLiteral::create_shared(create_texpr_node_from(
+ Field::create_field<TYPE_DOUBLE>(0.0), TYPE_DOUBLE, 0, 0));
+ _one_literal = VLiteral::create_shared(create_texpr_node_from(
+ Field::create_field<TYPE_DOUBLE>(1.0), TYPE_DOUBLE, 0, 0));
+ _not_in_values = {Field::create_field<TYPE_DOUBLE>(0.0)};
+ }
+ }
+
+ const std::string& expr_name() const override { return _expr_name; }
+ Status execute_column_impl(VExprContext*, const Block*, const Selector*,
size_t,
+ ColumnPtr&) const override {
+ return Status::InternalError("MetadataFloatingEqualityExpr is
metadata-only");
+ }
+ bool can_evaluate_zonemap_filter() const override { return true; }
+ ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx)
const override {
+ switch (_mode) {
+ case Mode::EQ:
+ return comparison_zonemap_detail::evaluate(ctx, {_slot,
_nan_literal},
+
comparison_zonemap_detail::Op::EQ);
+ case Mode::IN:
+ return expr_zonemap::eval_in_zonemap(ctx, _slot, false, _values,
true, _values[0],
+ _values[1]);
+ case Mode::NE:
+ return comparison_zonemap_detail::evaluate(ctx, {_slot,
_zero_literal},
+
comparison_zonemap_detail::Op::NE);
+ case Mode::GT:
+ return comparison_zonemap_detail::evaluate(ctx, {_slot,
_one_literal},
+
comparison_zonemap_detail::Op::GT);
+ case Mode::GE:
+ return comparison_zonemap_detail::evaluate(ctx, {_slot,
_one_literal},
+
comparison_zonemap_detail::Op::GE);
+ case Mode::REVERSED_LT:
+ return comparison_zonemap_detail::evaluate(ctx, {_one_literal,
_slot},
+
comparison_zonemap_detail::Op::LT);
+ case Mode::REVERSED_LE:
+ return comparison_zonemap_detail::evaluate(ctx, {_one_literal,
_slot},
+
comparison_zonemap_detail::Op::LE);
+ case Mode::NOT_IN:
+ return expr_zonemap::eval_in_zonemap(ctx, _slot, true,
_not_in_values, false,
+ _not_in_values[0],
_not_in_values[0]);
+ }
+ __builtin_unreachable();
+ }
+ void collect_slot_column_ids(std::set<int>& column_ids) const override {
+ _slot->collect_slot_column_ids(column_ids);
+ }
+
+private:
+ VExprSPtr _slot;
+ VExprSPtr _nan_literal;
+ VExprSPtr _zero_literal;
+ VExprSPtr _one_literal;
+ Mode _mode;
+ std::vector<Field> _values;
+ std::vector<Field> _not_in_values;
+ const std::string _expr_name = "MetadataFloatingEqualityExpr";
+};
+
class DictionaryStringInExpr final : public VExpr {
public:
DictionaryStringInExpr() : VExpr(std::make_shared<DataTypeUInt8>(), false)
{}
@@ -340,6 +451,11 @@ VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type,
std::vector<Field> valu
std::make_shared<BloomInExpr>(0, std::move(data_type),
std::move(values)))};
}
+VExprContextSPtrs bloom_eq_conjunct(DataTypePtr data_type, Field value) {
+ return {VExprContext::create_shared(
+ std::make_shared<BloomEqExpr>(0, std::move(data_type),
std::move(value)))};
+}
+
format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type,
std::vector<Field> values)
{
format::FileScanRequest request;
@@ -376,6 +492,96 @@ TEST(NativeParquetStatisticsTest,
InvalidNullableDateBoundsDisableMinMax) {
EXPECT_FALSE(result.has_min_max);
}
+TEST(NativeParquetStatisticsTest,
FloatingNanEqualityKeepsFiniteOnlyFooterAndPageRanges) {
+ const auto check_type = []<PrimitiveType Type, typename DataType, typename
UInt>(
+ tparquet::Type::type physical_type, UInt
nan_bits) {
+ using T = typename PrimitiveTypeTraits<Type>::CppType;
+ auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ column_schema->kind =
format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
+ column_schema->local_id = 0;
+ column_schema->leaf_column_id = 0;
+ column_schema->type = std::make_shared<DataType>();
+ column_schema->type_descriptor.doris_type = column_schema->type;
+ column_schema->type_descriptor.physical_type = physical_type;
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>>
schema;
+ schema.push_back(std::move(column_schema));
+
+ const T finite_bound = T {0};
+ const std::string encoded_bound(reinterpret_cast<const
char*>(&finite_bound), sizeof(T));
+ tparquet::Statistics statistics;
+ statistics.__set_min_value(encoded_bound);
+ statistics.__set_max_value(encoded_bound);
+ statistics.__set_null_count(0);
+ tparquet::ColumnMetaData column_metadata;
+ column_metadata.__set_type(physical_type);
+ column_metadata.__set_num_values(2);
+ column_metadata.__set_total_compressed_size(0);
+ column_metadata.__set_statistics(statistics);
+ tparquet::ColumnChunk chunk;
+ chunk.__set_meta_data(column_metadata);
+ tparquet::RowGroup row_group;
+ row_group.__set_columns({chunk});
+ row_group.__set_num_rows(2);
+ tparquet::ColumnOrder order;
+ order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
+ tparquet::FileMetaData metadata;
+ metadata.__set_column_orders({order});
+ metadata.__set_row_groups({row_group});
+
+ format::parquet::NativeParquetPageIndex page_index;
+ page_index.column_index.__set_min_values({encoded_bound});
+ page_index.column_index.__set_max_values({encoded_bound});
+ page_index.column_index.__set_null_pages({false});
+ page_index.column_index.__set_null_counts({0});
+ tparquet::PageLocation location;
+ location.__set_offset(0);
+ location.__set_compressed_page_size(10);
+ location.__set_first_row_index(0);
+ page_index.offset_index.__set_page_locations({location});
+ std::unordered_map<int, format::parquet::NativeParquetPageIndex>
page_indexes;
+ page_indexes.emplace(0, std::move(page_index));
+
+ const auto nan_field =
Field::create_field<Type>(std::bit_cast<T>(nan_bits));
+ for (const auto mode :
+ {MetadataFloatingEqualityExpr::Mode::EQ,
MetadataFloatingEqualityExpr::Mode::IN,
+ MetadataFloatingEqualityExpr::Mode::NE,
MetadataFloatingEqualityExpr::Mode::GT,
+ MetadataFloatingEqualityExpr::Mode::GE,
+ MetadataFloatingEqualityExpr::Mode::REVERSED_LT,
+ MetadataFloatingEqualityExpr::Mode::REVERSED_LE,
+ MetadataFloatingEqualityExpr::Mode::NOT_IN}) {
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ request.predicate_columns = {
+
format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
+ request.conjuncts = {
+
VExprContext::create_shared(std::make_shared<MetadataFloatingEqualityExpr>(
+ 0, schema[0]->type, nan_field, mode))};
+
+ std::vector<int> selected_row_groups;
+ ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(
+ metadata, schema, request, nullptr,
&selected_row_groups, false,
+ nullptr)
+ .ok());
+ EXPECT_EQ(selected_row_groups, std::vector<int>({0}));
+
+ std::vector<format::parquet::RowRange> selected_ranges;
+ std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
+
ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
+ metadata, metadata.row_groups[0],
page_indexes, schema, request, 2,
+ &selected_ranges, &skip_plans, nullptr)
+ .ok());
+ ASSERT_EQ(1, selected_ranges.size());
+ EXPECT_EQ(0, selected_ranges[0].start);
+ EXPECT_EQ(2, selected_ranges[0].length);
+ }
+ };
+
+ check_type.template operator()<TYPE_FLOAT,
DataTypeFloat32>(tparquet::Type::FLOAT,
+ uint32_t
{0x7fc00002U});
+ check_type.template operator()<TYPE_DOUBLE,
DataTypeFloat64>(tparquet::Type::DOUBLE,
+ uint64_t
{0x7ff8000000000002ULL});
+}
+
TEST(NativeParquetStatisticsTest, InvalidNullableDecimalBoundsDisableMinMax) {
format::parquet::ParquetColumnSchema column_schema;
column_schema.type = make_nullable(std::make_shared<DataTypeDecimal32>(2,
0));
@@ -650,6 +856,130 @@ TEST(ParquetBloomFilterPruningTest,
NativeUint32BloomUsesPhysicalInt32Hash) {
bloom_filter));
}
+TEST(ParquetBloomFilterPruningTest, NativeFloatingBloomPreservesDorisEquality)
{
+ const auto check_type = []<PrimitiveType Type, typename DataType>(
+ tparquet::Type::type physical_type,
+ typename
PrimitiveTypeTraits<Type>::CppType stored_value,
+ typename
PrimitiveTypeTraits<Type>::CppType predicate_value) {
+ format::parquet::ParquetColumnSchema column_schema;
+ column_schema.type = std::make_shared<DataType>();
+ column_schema.type_descriptor.doris_type = column_schema.type;
+ column_schema.type_descriptor.physical_type = physical_type;
+
+ format::parquet::native::BlockSplitBloomFilter bloom_filter;
+ ASSERT_TRUE(bloom_filter
+ .init(segment_v2::BloomFilter::MINIMUM_BYTES,
+ segment_v2::HashStrategyPB::XX_HASH_64)
+ .ok());
+ bloom_filter.add_bytes(reinterpret_cast<const char*>(&stored_value),
sizeof(stored_value));
+ ASSERT_FALSE(bloom_filter.test_bytes(reinterpret_cast<const
char*>(&predicate_value),
+ sizeof(predicate_value)));
+ const auto field = Field::create_field<Type>(predicate_value);
+
+
EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes(
+ column_schema, 0, bloom_eq_conjunct(column_schema.type,
field), bloom_filter));
+
EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes(
+ column_schema, 0, bloom_conjuncts(column_schema.type,
{field}), bloom_filter));
+ };
+
+ check_type.template operator()<TYPE_FLOAT,
DataTypeFloat32>(tparquet::Type::FLOAT, -0.0F, 0.0F);
+ check_type.template operator()<TYPE_FLOAT,
DataTypeFloat32>(tparquet::Type::FLOAT, 0.0F, -0.0F);
+ check_type.template operator()<TYPE_DOUBLE,
DataTypeFloat64>(tparquet::Type::DOUBLE, -0.0, 0.0);
+ check_type.template operator()<TYPE_DOUBLE,
DataTypeFloat64>(tparquet::Type::DOUBLE, 0.0, -0.0);
+ check_type.template operator()<TYPE_FLOAT, DataTypeFloat32>(
+ tparquet::Type::FLOAT, std::bit_cast<float>(uint32_t
{0x7fc00001U}),
+ std::bit_cast<float>(uint32_t {0x7fc00002U}));
+ check_type.template operator()<TYPE_DOUBLE, DataTypeFloat64>(
+ tparquet::Type::DOUBLE, std::bit_cast<double>(uint64_t
{0x7ff8000000000001ULL}),
+ std::bit_cast<double>(uint64_t {0x7ff8000000000002ULL}));
+}
+
+TEST(ParquetBloomFilterPruningTest,
NativeRowGroupKeepsDorisEqualFloatingValues) {
+ const auto check_type = []<PrimitiveType Type, typename DataType>(
+ tparquet::Type::type physical_type,
+ typename
PrimitiveTypeTraits<Type>::CppType stored_value,
+ typename
PrimitiveTypeTraits<Type>::CppType predicate_value) {
+ format::parquet::native::BlockSplitBloomFilter bloom_filter;
+ ASSERT_TRUE(bloom_filter
+ .init(segment_v2::BloomFilter::MINIMUM_BYTES,
+ segment_v2::HashStrategyPB::XX_HASH_64)
+ .ok());
+ bloom_filter.add_bytes(reinterpret_cast<const char*>(&stored_value),
sizeof(stored_value));
+
+ tparquet::BloomFilterAlgorithm algorithm;
+ algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm());
+ tparquet::BloomFilterHash hash;
+ hash.__set_XXHASH(tparquet::XxHash());
+ tparquet::BloomFilterCompression compression;
+ compression.__set_UNCOMPRESSED(tparquet::Uncompressed());
+ tparquet::BloomFilterHeader bloom_header;
+ bloom_header.__set_numBytes(static_cast<int32_t>(bloom_filter.size()));
+ bloom_header.__set_algorithm(algorithm);
+ bloom_header.__set_hash(hash);
+ bloom_header.__set_compression(compression);
+ std::vector<uint8_t> bloom_bytes;
+ ThriftSerializer serializer(/*compact=*/true, 64);
+ ASSERT_TRUE(serializer.serialize(&bloom_header, &bloom_bytes).ok());
+ bloom_bytes.insert(bloom_bytes.end(), bloom_filter.data(),
+ bloom_filter.data() + bloom_filter.size());
+
+ tparquet::ColumnMetaData column_metadata;
+ column_metadata.__set_type(physical_type);
+ column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED);
+ column_metadata.__set_num_values(1);
+ column_metadata.__set_total_compressed_size(0);
+ column_metadata.__set_data_page_offset(0);
+ column_metadata.__set_bloom_filter_offset(0);
+
column_metadata.__set_bloom_filter_length(static_cast<int32_t>(bloom_bytes.size()));
+ tparquet::ColumnChunk chunk;
+ chunk.__set_meta_data(column_metadata);
+ tparquet::RowGroup row_group;
+ row_group.__set_columns({chunk});
+ row_group.__set_total_byte_size(0);
+ row_group.__set_num_rows(1);
+ tparquet::FileMetaData metadata;
+ metadata.__set_version(1);
+ metadata.__set_num_rows(1);
+ metadata.__set_row_groups({row_group});
+
+ const auto field = Field::create_field<Type>(predicate_value);
+ for (const bool use_eq : {true, false}) {
+ auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>();
+ column_schema->type = std::make_shared<DataType>();
+ column_schema->type_descriptor.doris_type = column_schema->type;
+ column_schema->type_descriptor.physical_type = physical_type;
+ column_schema->local_id = 0;
+ column_schema->leaf_column_id = 0;
+
+ format::FileScanRequest request;
+ request.local_positions.emplace(format::LocalColumnId(0),
format::LocalIndex(0));
+ request.conjuncts = use_eq ?
bloom_eq_conjunct(column_schema->type, field)
+ : bloom_conjuncts(column_schema->type,
{field});
+ std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>>
schema;
+ schema.push_back(std::move(column_schema));
+ format::parquet::ParquetFileContext file_context;
+ file_context.native_file =
std::make_shared<StatisticsMemoryFileReader>(bloom_bytes);
+ std::vector<int> selected_row_groups;
+ format::parquet::ParquetPruningStats pruning_stats;
+ ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(
+ metadata, schema, request, nullptr,
&selected_row_groups, true,
+ &pruning_stats, nullptr, nullptr,
&file_context)
+ .ok());
+ EXPECT_EQ(selected_row_groups, std::vector<int>({0}));
+ EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0);
+ }
+ };
+
+ check_type.template operator()<TYPE_FLOAT,
DataTypeFloat32>(tparquet::Type::FLOAT, -0.0F, 0.0F);
+ check_type.template operator()<TYPE_DOUBLE,
DataTypeFloat64>(tparquet::Type::DOUBLE, 0.0, -0.0);
+ check_type.template operator()<TYPE_FLOAT, DataTypeFloat32>(
+ tparquet::Type::FLOAT, std::bit_cast<float>(uint32_t
{0x7fc00001U}),
+ std::bit_cast<float>(uint32_t {0x7fc00002U}));
+ check_type.template operator()<TYPE_DOUBLE, DataTypeFloat64>(
+ tparquet::Type::DOUBLE, std::bit_cast<double>(uint64_t
{0x7ff8000000000001ULL}),
+ std::bit_cast<double>(uint64_t {0x7ff8000000000002ULL}));
+}
+
TEST(ParquetBloomFilterPruningTest,
NativeRowGroupKeepsPresentUint32AboveInt32Max) {
auto column_schema =
std::make_unique<format::parquet::ParquetColumnSchema>(uint32_parquet_bloom_schema());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
