This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ec716229c84 [fix](be) Fix ORC V2 pushdown correctness and fallback
(#65562)
ec716229c84 is described below
commit ec716229c84bfd9a67c18c2e61913f651e02e1c4
Author: Chenjunwei <[email protected]>
AuthorDate: Wed Jul 15 11:23:23 2026 +0800
[fix](be) Fix ORC V2 pushdown correctness and fallback (#65562)
---
be/src/format_v2/orc/orc_reader.cpp | 78 +++-
be/src/format_v2/orc/orc_search_argument.cpp | 589 +++++++++++++-----------
be/src/format_v2/orc/orc_search_argument.h | 7 +-
be/test/format_v2/orc/orc_reader_test.cpp | 646 ++++++++++++++++++++++++++-
4 files changed, 1047 insertions(+), 273 deletions(-)
diff --git a/be/src/format_v2/orc/orc_reader.cpp
b/be/src/format_v2/orc/orc_reader.cpp
index b728c6fbedb..9a2aeb53897 100644
--- a/be/src/format_v2/orc/orc_reader.cpp
+++ b/be/src/format_v2/orc/orc_reader.cpp
@@ -30,6 +30,7 @@
#include <list>
#include <map>
#include <memory>
+#include <new>
#include <optional>
#include <orc/OrcFile.hh>
#include <orc/Vector.hh>
@@ -47,6 +48,7 @@
#include "common/config.h"
#include "common/consts.h"
#include "common/exception.h"
+#include "common/logging.h"
#include "core/block/block.h"
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
@@ -78,6 +80,7 @@
#include "storage/index/zone_map/zone_map_index.h"
#include "storage/segment/condition_cache.h"
#include "storage/utils.h"
+#include "util/debug_points.h"
#include "util/slice.h"
#include "util/timezone_utils.h"
@@ -139,6 +142,27 @@ bool is_orc_stop(const io::IOContext* io_ctx, const
std::exception& e) {
return io_ctx != nullptr && io_ctx->should_stop &&
std::string_view(e.what()) == "stop";
}
+Status fall_back_without_orc_sarg(const char* operation, const std::string&
file_path,
+ ::orc::RowReaderOptions* row_reader_options,
+ const std::exception& e) {
+ LOG(WARNING) << "Failed to " << operation << " ORC search argument for
file " << file_path
+ << "; falling back to scan without SARG: " << e.what();
+ row_reader_options->searchArgument(nullptr);
+ return Status::OK();
+}
+
+Status handle_orc_sarg_exception(const char* operation, const std::string&
file_path,
+ ::orc::RowReaderOptions* row_reader_options,
const Exception& e) {
+ if (e.code() == ErrorCode::MEM_ALLOC_FAILED) {
+ return Status::MemoryLimitExceeded("Failed to {} ORC search argument:
{}", operation,
+ e.what());
+ }
+ if (e.code() == ErrorCode::MEM_LIMIT_EXCEEDED) {
+ return e.to_status();
+ }
+ return fall_back_without_orc_sarg(operation, file_path,
row_reader_options, e);
+}
+
bool is_hour_offset_timezone(std::string_view timezone) {
return timezone.size() == 6 && (timezone[0] == '+' || timezone[0] == '-')
&&
std::isdigit(static_cast<unsigned char>(timezone[1])) &&
@@ -605,8 +629,12 @@ bool build_zone_map_from_orc_statistics(const ::orc::Type&
type,
return set_string_zone_map(type, statistics, zone_map);
case ::orc::TypeKind::DATE:
return set_date_zone_map(statistics, zone_map);
- case ::orc::TypeKind::TIMESTAMP:
- return set_timestamp_zone_map(statistics, timezone, false, zone_map);
+ case ::orc::TypeKind::TIMESTAMP: {
+ // ORC stores timestamp statistics as wall-clock values in UTC
coordinates. Restore them
+ // with UTC so the session timezone does not shift the civil time.
+ static const auto utc_time_zone = cctz::utc_time_zone();
+ return set_timestamp_zone_map(statistics, utc_time_zone, false,
zone_map);
+ }
case ::orc::TypeKind::TIMESTAMP_INSTANT:
return set_timestamp_zone_map(statistics, timezone,
enable_mapping_timestamp_tz, zone_map);
case ::orc::TypeKind::DECIMAL:
@@ -720,6 +748,7 @@ struct OrcReaderScanState {
bool enable_filter_by_min_max = true;
bool orc_lazy_read_enabled = false;
bool orc_lazy_selection_valid = false;
+ OrcSargBuildOptions sarg_build_options;
std::vector<StripeRange> selected_stripe_ranges;
size_t current_stripe_range = 0;
@@ -853,6 +882,10 @@ Status OrcReader::init(RuntimeState* state) {
_state->enable_filter_by_min_max =
state->query_options().enable_orc_filter_by_min_max;
_state->timezone = state->timezone();
_state->timezone_obj = state->timezone_obj();
+ if (state->query_options().__isset.max_pushdown_conditions_per_column)
{
+ _state->sarg_build_options.max_pushdown_conditions_per_column =
+ state->query_options().max_pushdown_conditions_per_column;
+ }
}
::orc::ReaderOptions options;
@@ -1350,18 +1383,26 @@ Status
OrcReader::_init_search_argument_from_local_filters() {
if (conjunct == nullptr) {
continue;
}
- has_pushdown =
- build_orc_search_argument(*_request, *_state->root_type,
_state->timezone_obj,
- conjunct->root(), builder) ||
- has_pushdown;
+ has_pushdown = build_orc_search_argument(
+ *_request, *_state->root_type,
_state->timezone_obj,
+ _state->sarg_build_options,
conjunct->root(), builder) ||
+ has_pushdown;
}
if (!has_pushdown) {
return Status::OK();
}
builder->end();
+
DBUG_EXECUTE_IF("OrcReader._init_search_argument_from_local_filters.inject_failure",
+ DBUG_RUN_CALLBACK());
_state->row_reader_options.searchArgument(builder->build());
+ } catch (const Exception& e) {
+ return handle_orc_sarg_exception("build", _file_description->path,
+ &_state->row_reader_options, e);
+ } catch (const std::bad_alloc& e) {
+ return Status::MemoryLimitExceeded("Failed to build ORC search
argument: {}", e.what());
} catch (const std::exception& e) {
- return Status::InternalError("Failed to build ORC search argument:
{}", e.what());
+ return fall_back_without_orc_sarg("build", _file_description->path,
+ &_state->row_reader_options, e);
}
return Status::OK();
}
@@ -1421,6 +1462,8 @@ void OrcReader::_apply_split_range() {
// ORC RowReader ranges are continuous, so non-adjacent surviving stripes are
// compacted into separate ranges.
+// Keep stripe selection and range state transitions atomic.
+// NOLINTNEXTLINE(readability-function-size)
Status OrcReader::_select_stripe_ranges_by_statistics() {
_state->selected_stripe_ranges.clear();
_state->current_stripe_range = 0;
@@ -1439,8 +1482,20 @@ Status OrcReader::_select_stripe_ranges_by_statistics() {
std::vector<int> sarg_needed_stripes;
try {
sarg_needed_stripes =
_state->reader->getNeedReadStripes(_state->row_reader_options);
+ } catch (const Exception& e) {
+ if (is_orc_stop(_io_ctx.get(), e)) {
+ return Status::EndOfFile("stop");
+ }
+ return handle_orc_sarg_exception("evaluate", _file_description->path,
+ &_state->row_reader_options, e);
+ } catch (const std::bad_alloc& e) {
+ return Status::MemoryLimitExceeded("Failed to evaluate ORC search
argument: {}", e.what());
} catch (const std::exception& e) {
- return Status::InternalError("Failed to evaluate ORC search argument:
{}", e.what());
+ if (is_orc_stop(_io_ctx.get(), e)) {
+ return Status::EndOfFile("stop");
+ }
+ return fall_back_without_orc_sarg("evaluate", _file_description->path,
+ &_state->row_reader_options, e);
}
std::vector<uint64_t> selected_stripes;
@@ -1941,6 +1996,8 @@ Status OrcReader::get_block(Block* file_block, size_t*
rows, bool* eof) {
return Status::OK();
}
+// COUNT and MIN/MAX share selected-stripe metadata and the same conservative
fallback contract.
+// NOLINTNEXTLINE(readability-function-size)
Status OrcReader::get_aggregate_result(const format::FileAggregateRequest&
request,
format::FileAggregateResult* result) {
DORIS_CHECK(result != nullptr);
@@ -2049,6 +2106,11 @@ Status OrcReader::get_aggregate_result(const
format::FileAggregateRequest& reque
RETURN_IF_ERROR(find_projected_minmax_leaf(*_state->root_type,
request_column.projection,
&leaf_type));
DORIS_CHECK(leaf_type != nullptr);
+ if (leaf_type->getKind() == ::orc::TypeKind::TIMESTAMP &&
+ _state->reader->getWriterVersion() < ::orc::WriterVersion_ORC_135)
{
+ return Status::NotSupported(
+ "ORC TIMESTAMP min/max statistics are unsafe before writer
version ORC-135");
+ }
auto& aggregate_column = result->columns[column_idx];
aggregate_column.projection = request_column.projection;
diff --git a/be/src/format_v2/orc/orc_search_argument.cpp
b/be/src/format_v2/orc/orc_search_argument.cpp
index 4766fa83045..244109c12ce 100644
--- a/be/src/format_v2/orc/orc_search_argument.cpp
+++ b/be/src/format_v2/orc/orc_search_argument.cpp
@@ -29,6 +29,7 @@
#include <orc/sargs/Literal.hh>
#include <orc/sargs/SearchArgument.hh>
#include <string>
+#include <unordered_map>
#include <utility>
#include <vector>
@@ -46,6 +47,9 @@
#include "exprs/vslot_ref.h"
#include "exprs/vtopn_pred.h"
#include "format_v2/timestamp_statistics.h"
+#ifdef BE_TEST
+#include "util/debug_points.h"
+#endif
namespace doris::format::orc {
namespace {
@@ -95,6 +99,24 @@ struct OrcSargComparison {
TExprOpcode::type normalized_op = TExprOpcode::INVALID_OPCODE;
};
+enum class OrcSargNodeKind {
+ AND,
+ OR,
+ NOT,
+ LESS_THAN,
+ LESS_THAN_EQUALS,
+ EQUALS,
+ IN,
+ IS_NULL,
+};
+
+struct OrcSargNode {
+ OrcSargNodeKind kind;
+ std::optional<OrcSargColumn> column;
+ std::vector<::orc::Literal> literals;
+ std::vector<OrcSargNode> children;
+};
+
std::optional<format::LocalColumnId> file_column_id_for_slot_position(
const format::FileScanRequest& request, int slot_column_id) {
if (slot_column_id < 0) {
@@ -743,12 +765,8 @@ bool is_null_literal(const VExprSPtr& literal_expr) {
literal->get_column_ptr()->is_null_at(0);
}
-// Buildability is checked before emission so the builder path can assume the
-// expression shape was validated and use DORIS_CHECK for impossible branches.
-bool can_build_search_argument(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const cctz::time_zone& timezone, const
VExprSPtr& expr);
-
-std::optional<VExprSPtr> expression_for_search_argument(const VExprSPtr& expr)
{
+std::optional<VExprSPtr> expression_for_search_argument(const VExprSPtr& expr,
+ const
OrcSargBuildOptions& options) {
if (expr == nullptr) {
return std::nullopt;
}
@@ -766,6 +784,10 @@ std::optional<VExprSPtr>
expression_for_search_argument(const VExprSPtr& expr) {
}
if (const auto* direct_in = dynamic_cast<const
VDirectInPredicate*>(impl.get());
direct_in != nullptr) {
+ const auto filter_size = direct_in->get_set_func()->size();
+ if (filter_size == 0 || filter_size >
options.max_pushdown_conditions_per_column) {
+ return std::nullopt;
+ }
VExprSPtr in_expr;
if (!direct_in->get_slot_in_expr(in_expr)) {
return std::nullopt;
@@ -965,6 +987,9 @@ std::optional<OrcSargComparisonLiteral>
make_comparison_literal_for_sarg(
std::optional<std::vector<::orc::Literal>> make_in_literals_for_sarg(
const OrcSargColumn& column, const VExprSPtr& source_expr,
const std::vector<VExprSPtr>& children, const cctz::time_zone&
timezone) {
+#ifdef BE_TEST
+ DBUG_EXECUTE_IF("OrcSearchArgument.make_in_literals_for_sarg",
DBUG_RUN_CALLBACK());
+#endif
if (children.size() < 2) {
return std::nullopt;
}
@@ -1080,35 +1105,6 @@ std::optional<OrcSargComparison>
sarg_comparison_for_expr(const format::FileScan
};
}
-bool can_build_slot_literal_predicate(const format::FileScanRequest& request,
- const ::orc::Type& root_type, const
cctz::time_zone& timezone,
- const VExprSPtr& expr) {
- if (expr == nullptr || expr->children().size() != 2) {
- return false;
- }
- return sarg_comparison_for_expr(request, root_type, timezone,
expr).has_value();
-}
-
-bool can_build_in_predicate(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const cctz::time_zone& timezone, const VExprSPtr&
expr) {
- if (expr == nullptr || expr->children().size() < 2) {
- return false;
- }
- const auto sarg_column =
- sarg_column_for_slot_or_safe_cast(request, root_type,
expr->children()[0]);
- if (!sarg_column.has_value()) {
- return false;
- }
- return make_in_literals_for_sarg(*sarg_column, expr->children()[0],
expr->children(), timezone)
- .has_value();
-}
-
-bool can_build_is_null_predicate(const format::FileScanRequest& request,
- const ::orc::Type& root_type, const
VExprSPtr& expr) {
- return expr != nullptr && expr->children().size() == 1 &&
- sarg_column_for_slot_or_safe_cast(request, root_type,
expr->children()[0]).has_value();
-}
-
std::optional<OrcSargColumn> sarg_column_for_null_safe_equal_null(
const format::FileScanRequest& request, const ::orc::Type& root_type,
const VExprSPtr& expr) {
@@ -1170,263 +1166,356 @@ std::optional<OrcSargComparison>
sarg_comparison_for_null_safe_equal_literal(
return comparison_if_other_child_is_non_null_literal(1, 0);
}
-bool can_build_null_safe_equal_predicate(const format::FileScanRequest&
request,
- const ::orc::Type& root_type,
- const cctz::time_zone& timezone,
const VExprSPtr& expr) {
- return sarg_column_for_null_safe_equal_null(request, root_type,
expr).has_value() ||
- sarg_comparison_for_null_safe_equal_literal(request, root_type,
timezone, expr)
- .has_value();
-}
-
-bool contains_null_safe_equal(const VExprSPtr& expr) {
- const auto sarg_expr = expression_for_search_argument(expr);
- if (!sarg_expr.has_value() || *sarg_expr == nullptr) {
- return false;
- }
- if (sarg_expr->get() != expr.get()) {
- return contains_null_safe_equal(*sarg_expr);
- }
- if ((*sarg_expr)->op() == TExprOpcode::EQ_FOR_NULL) {
- return true;
- }
- return std::ranges::any_of((*sarg_expr)->children(),
contains_null_safe_equal);
+OrcSargNode make_not_node(OrcSargNode child) {
+ std::vector<OrcSargNode> children;
+ children.push_back(std::move(child));
+ return OrcSargNode {
+ .kind = OrcSargNodeKind::NOT,
+ .column = std::nullopt,
+ .literals = {},
+ .children = std::move(children),
+ };
}
-bool can_build_search_argument(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const cctz::time_zone& timezone, const
VExprSPtr& expr) {
- const auto sarg_expr = expression_for_search_argument(expr);
- if (!sarg_expr.has_value()) {
- return false;
- }
- if (*sarg_expr == nullptr) {
- return false;
- }
- if (sarg_expr->get() != expr.get()) {
- return can_build_search_argument(request, root_type, timezone,
*sarg_expr);
- }
-
- switch ((*sarg_expr)->op()) {
- case TExprOpcode::COMPOUND_AND:
- return std::ranges::any_of((*sarg_expr)->children(), [&](const auto&
child) {
- return can_build_search_argument(request, root_type, timezone,
child);
- });
- case TExprOpcode::COMPOUND_OR:
- if (contains_null_safe_equal(*sarg_expr)) {
- return false;
- }
- return !(*sarg_expr)->children().empty() &&
- std::ranges::all_of((*sarg_expr)->children(), [&](const auto&
child) {
- return can_build_search_argument(request, root_type,
timezone, child);
- });
- case TExprOpcode::COMPOUND_NOT:
- if (contains_null_safe_equal(*sarg_expr)) {
- return false;
- }
- return (*sarg_expr)->children().size() == 1 &&
- can_build_search_argument(request, root_type, timezone,
(*sarg_expr)->children()[0]);
+OrcSargNode make_comparison_node(OrcSargComparison comparison) {
+ OrcSargNodeKind kind;
+ bool negate = false;
+ switch (comparison.normalized_op) {
case TExprOpcode::GE:
+ kind = OrcSargNodeKind::LESS_THAN;
+ negate = true;
+ break;
case TExprOpcode::GT:
+ kind = OrcSargNodeKind::LESS_THAN_EQUALS;
+ negate = true;
+ break;
case TExprOpcode::LE:
+ kind = OrcSargNodeKind::LESS_THAN_EQUALS;
+ break;
case TExprOpcode::LT:
+ kind = OrcSargNodeKind::LESS_THAN;
+ break;
case TExprOpcode::EQ:
+ kind = OrcSargNodeKind::EQUALS;
+ break;
case TExprOpcode::NE:
- return (*sarg_expr)->node_type() !=
TExprNodeType::NULL_AWARE_BINARY_PRED &&
- can_build_slot_literal_predicate(request, root_type, timezone,
*sarg_expr);
- case TExprOpcode::EQ_FOR_NULL:
- return can_build_null_safe_equal_predicate(request, root_type,
timezone, *sarg_expr);
- case TExprOpcode::FILTER_IN:
- case TExprOpcode::FILTER_NOT_IN:
- return (*sarg_expr)->node_type() != TExprNodeType::NULL_AWARE_IN_PRED
&&
- can_build_in_predicate(request, root_type, timezone,
*sarg_expr);
- case TExprOpcode::INVALID_OPCODE:
- return (*sarg_expr)->node_type() == TExprNodeType::FUNCTION_CALL &&
- ((*sarg_expr)->fn().name.function_name == "is_null_pred" ||
- (*sarg_expr)->fn().name.function_name == "is_not_null_pred") &&
- can_build_is_null_predicate(request, root_type, *sarg_expr);
+ kind = OrcSargNodeKind::EQUALS;
+ negate = true;
+ break;
default:
- return false;
+ DORIS_CHECK(false) << "Unsupported normalized ORC SARG comparison op "
+ << comparison.normalized_op;
+ __builtin_unreachable();
}
-}
-void build_less_than(const OrcSargColumn& column, const ::orc::Literal&
literal,
- std::unique_ptr<::orc::SearchArgumentBuilder>& builder) {
- builder->lessThan(column.column_id, column.predicate_type, literal);
+ std::vector<::orc::Literal> literals;
+ literals.push_back(std::move(comparison.literal));
+ OrcSargNode node {
+ .kind = kind,
+ .column = comparison.column,
+ .literals = std::move(literals),
+ .children = {},
+ };
+ if (negate) {
+ return make_not_node(std::move(node));
+ }
+ return node;
}
-void build_less_than(const OrcSargComparison& comparison,
- std::unique_ptr<::orc::SearchArgumentBuilder>& builder) {
- build_less_than(comparison.column, comparison.literal, builder);
+OrcSargNode make_in_node(OrcSargColumn column, std::vector<::orc::Literal>
literals) {
+ DORIS_CHECK(!literals.empty());
+ return OrcSargNode {
+ .kind = literals.size() == 1 ? OrcSargNodeKind::EQUALS :
OrcSargNodeKind::IN,
+ .column = column,
+ .literals = std::move(literals),
+ .children = {},
+ };
}
-void build_less_than_equals(const OrcSargColumn& column, const ::orc::Literal&
literal,
- std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- builder->lessThanEquals(column.column_id, column.predicate_type, literal);
+OrcSargNode make_is_null_node(OrcSargColumn column) {
+ return OrcSargNode {
+ .kind = OrcSargNodeKind::IS_NULL,
+ .column = column,
+ .literals = {},
+ .children = {},
+ };
}
-void build_less_than_equals(const OrcSargComparison& comparison,
- std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- build_less_than_equals(comparison.column, comparison.literal, builder);
-}
+class OrcSargCompiler {
+public:
+ OrcSargCompiler(const format::FileScanRequest& request, const ::orc::Type&
root_type,
+ const cctz::time_zone& timezone, const
OrcSargBuildOptions& options)
+ : _request(request), _root_type(root_type), _timezone(timezone),
_options(options) {}
-void build_equals(const OrcSargColumn& column, const ::orc::Literal& literal,
- std::unique_ptr<::orc::SearchArgumentBuilder>& builder) {
- builder->equals(column.column_id, column.predicate_type, literal);
-}
+ std::optional<OrcSargNode> compile(const VExprSPtr& expr) { return
compile_node(expr, false); }
-void build_equals(const OrcSargComparison& comparison,
- std::unique_ptr<::orc::SearchArgumentBuilder>& builder) {
- build_equals(comparison.column, comparison.literal, builder);
-}
+private:
+ std::optional<VExprSPtr> normalized_expression(const VExprSPtr& expr) {
+ if (expr == nullptr) {
+ return std::nullopt;
+ }
+ const auto found = _normalized_expressions.find(expr.get());
+ if (found != _normalized_expressions.end()) {
+ return found->second;
+ }
+ auto normalized = expression_for_search_argument(expr, _options);
+ _normalized_expressions.emplace(expr.get(), normalized);
+ return normalized;
+ }
-void build_comparison_predicate(const format::FileScanRequest& request,
- const ::orc::Type& root_type, const VExprSPtr&
expr,
- const cctz::time_zone& timezone,
- std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- const auto comparison = *sarg_comparison_for_expr(request, root_type,
timezone, expr);
- switch (comparison.normalized_op) {
- case TExprOpcode::GE:
- builder->startNot();
- build_less_than(comparison, builder);
- builder->end();
- return;
- case TExprOpcode::GT:
- builder->startNot();
- build_less_than_equals(comparison, builder);
- builder->end();
- return;
- case TExprOpcode::LE:
- build_less_than_equals(comparison, builder);
- return;
- case TExprOpcode::LT:
- build_less_than(comparison, builder);
- return;
- case TExprOpcode::EQ:
- build_equals(comparison, builder);
- return;
- case TExprOpcode::NE:
- builder->startNot();
- build_equals(comparison, builder);
- builder->end();
- return;
- default:
- DORIS_CHECK(false) << "Unsupported normalized ORC SARG comparison op "
- << comparison.normalized_op;
+ bool contains_null_safe_equal(const VExprSPtr& expr) {
+ const auto normalized = normalized_expression(expr);
+ if (!normalized.has_value() || *normalized == nullptr) {
+ return false;
+ }
+ if (normalized->get() != expr.get()) {
+ return contains_null_safe_equal(*normalized);
+ }
+ if ((*normalized)->op() == TExprOpcode::EQ_FOR_NULL) {
+ return true;
+ }
+ // The probe expression may be complex, but valid IN values are
literals. Avoid walking a
+ // potentially huge literal list only for null-safe-equality metadata.
+ if ((*normalized)->op() == TExprOpcode::FILTER_IN ||
+ (*normalized)->op() == TExprOpcode::FILTER_NOT_IN) {
+ return !(*normalized)->children().empty() &&
+ contains_null_safe_equal((*normalized)->children().front());
+ }
+ return std::ranges::any_of((*normalized)->children(), [this](const
auto& child) {
+ return contains_null_safe_equal(child);
+ });
}
-}
-void build_in_predicate(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const VExprSPtr& expr, const cctz::time_zone& timezone,
- std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- const auto sarg_column =
- *sarg_column_for_slot_or_safe_cast(request, root_type,
expr->children()[0]);
- auto literals = *make_in_literals_for_sarg(sarg_column,
expr->children()[0], expr->children(),
- timezone);
- DORIS_CHECK(!literals.empty());
- if (literals.size() == 1) {
- builder->equals(sarg_column.column_id, sarg_column.predicate_type,
literals.front());
- return;
+ std::optional<OrcSargNode> compile_node(const VExprSPtr& expr, bool
is_negated) {
+ const auto normalized = normalized_expression(expr);
+ if (!normalized.has_value() || *normalized == nullptr) {
+ return std::nullopt;
+ }
+ if (normalized->get() != expr.get()) {
+ return compile_node(*normalized, is_negated);
+ }
+
+ switch ((*normalized)->op()) {
+ case TExprOpcode::COMPOUND_AND:
+ return compile_and(*normalized, is_negated);
+ case TExprOpcode::COMPOUND_OR:
+ return compile_or(*normalized, is_negated);
+ case TExprOpcode::COMPOUND_NOT:
+ return compile_not(*normalized, is_negated);
+ case TExprOpcode::GE:
+ case TExprOpcode::GT:
+ case TExprOpcode::LE:
+ case TExprOpcode::LT:
+ case TExprOpcode::EQ:
+ case TExprOpcode::NE:
+ return compile_comparison(*normalized);
+ case TExprOpcode::EQ_FOR_NULL:
+ return compile_null_safe_equal(*normalized);
+ case TExprOpcode::FILTER_IN:
+ case TExprOpcode::FILTER_NOT_IN:
+ return compile_in(*normalized);
+ case TExprOpcode::INVALID_OPCODE:
+ return compile_is_null(*normalized);
+ default:
+ return std::nullopt;
+ }
}
- builder->in(sarg_column.column_id, sarg_column.predicate_type, literals);
-}
-void build_is_null(const format::FileScanRequest& request, const ::orc::Type&
root_type,
- const VExprSPtr& expr,
std::unique_ptr<::orc::SearchArgumentBuilder>& builder) {
- const auto sarg_column =
- *sarg_column_for_slot_or_safe_cast(request, root_type,
expr->children()[0]);
- builder->isNull(sarg_column.column_id, sarg_column.predicate_type);
-}
+ std::optional<OrcSargNode> compile_and(const VExprSPtr& expr, bool
is_negated) {
+ const auto& source_children = expr->children();
+ if (source_children.empty()) {
+ return std::nullopt;
+ }
+ std::vector<OrcSargNode> children;
+ children.reserve(source_children.size());
+ bool all_children_compiled = true;
+ for (const auto& child : source_children) {
+ auto compiled_child = compile_node(child, is_negated);
+ if (compiled_child.has_value()) {
+ children.push_back(std::move(*compiled_child));
+ } else {
+ all_children_compiled = false;
+ }
+ }
+ // Omitting an AND child weakens the SARG only in positive context.
Under an odd number of
+ // NOTs it would strengthen the SARG and could incorrectly prune
matching stripes.
+ if (children.empty() || (is_negated && !all_children_compiled)) {
+ return std::nullopt;
+ }
+ return OrcSargNode {
+ .kind = OrcSargNodeKind::AND,
+ .column = std::nullopt,
+ .literals = {},
+ .children = std::move(children),
+ };
+ }
-void build_null_safe_equal(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const VExprSPtr& expr, const cctz::time_zone&
timezone,
- std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- const auto sarg_column = sarg_column_for_null_safe_equal_null(request,
root_type, expr);
- if (sarg_column.has_value()) {
- builder->isNull(sarg_column->column_id, sarg_column->predicate_type);
- return;
+ std::optional<OrcSargNode> compile_or(const VExprSPtr& expr, bool
is_negated) {
+ const auto& source_children = expr->children();
+ if (source_children.empty() || contains_null_safe_equal(expr)) {
+ return std::nullopt;
+ }
+ std::vector<OrcSargNode> children;
+ children.reserve(source_children.size());
+ for (const auto& child : source_children) {
+ auto compiled_child = compile_node(child, is_negated);
+ if (!compiled_child.has_value()) {
+ return std::nullopt;
+ }
+ children.push_back(std::move(*compiled_child));
+ }
+ return OrcSargNode {
+ .kind = OrcSargNodeKind::OR,
+ .column = std::nullopt,
+ .literals = {},
+ .children = std::move(children),
+ };
}
- const auto comparison =
- *sarg_comparison_for_null_safe_equal_literal(request, root_type,
timezone, expr);
- build_equals(comparison, builder);
-}
-bool build_search_argument(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const cctz::time_zone& timezone, const VExprSPtr&
expr,
- std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- const auto sarg_expr = expression_for_search_argument(expr);
- if (!sarg_expr.has_value() || *sarg_expr == nullptr) {
- return false;
+ std::optional<OrcSargNode> compile_not(const VExprSPtr& expr, bool
is_negated) {
+ if (expr->children().size() != 1 || contains_null_safe_equal(expr)) {
+ return std::nullopt;
+ }
+ auto child = compile_node(expr->children()[0], !is_negated);
+ if (!child.has_value()) {
+ return std::nullopt;
+ }
+ return make_not_node(std::move(*child));
}
- if (sarg_expr->get() != expr.get()) {
- return build_search_argument(request, root_type, timezone, *sarg_expr,
builder);
+
+ std::optional<OrcSargNode> compile_comparison(const VExprSPtr& expr) const
{
+ if (expr->node_type() == TExprNodeType::NULL_AWARE_BINARY_PRED) {
+ return std::nullopt;
+ }
+ auto comparison = sarg_comparison_for_expr(_request, _root_type,
_timezone, expr);
+ if (!comparison.has_value()) {
+ return std::nullopt;
+ }
+ return make_comparison_node(std::move(*comparison));
}
- if (!can_build_search_argument(request, root_type, timezone, *sarg_expr)) {
- return false;
+
+ std::optional<OrcSargNode> compile_null_safe_equal(const VExprSPtr& expr)
const {
+ const auto column = sarg_column_for_null_safe_equal_null(_request,
_root_type, expr);
+ if (column.has_value()) {
+ return make_is_null_node(*column);
+ }
+ auto comparison =
+ sarg_comparison_for_null_safe_equal_literal(_request,
_root_type, _timezone, expr);
+ if (!comparison.has_value()) {
+ return std::nullopt;
+ }
+ return make_comparison_node(std::move(*comparison));
}
- switch ((*sarg_expr)->op()) {
- case TExprOpcode::COMPOUND_AND:
- builder->startAnd();
- for (const auto& child : (*sarg_expr)->children()) {
- static_cast<void>(build_search_argument(request, root_type,
timezone, child, builder));
+ std::optional<OrcSargNode> compile_in(const VExprSPtr& expr) const {
+ if (expr->node_type() == TExprNodeType::NULL_AWARE_IN_PRED ||
expr->children().size() < 2) {
+ return std::nullopt;
}
- builder->end();
- return true;
- case TExprOpcode::COMPOUND_OR:
- builder->startOr();
- for (const auto& child : (*sarg_expr)->children()) {
- const auto built = build_search_argument(request, root_type,
timezone, child, builder);
- DORIS_CHECK(built);
+ const auto column =
+ sarg_column_for_slot_or_safe_cast(_request, _root_type,
expr->children()[0]);
+ if (!column.has_value()) {
+ return std::nullopt;
}
- builder->end();
- return true;
- case TExprOpcode::COMPOUND_NOT:
- builder->startNot();
- DORIS_CHECK(build_search_argument(request, root_type, timezone,
(*sarg_expr)->children()[0],
- builder));
- builder->end();
- return true;
- case TExprOpcode::GE:
- case TExprOpcode::GT:
- case TExprOpcode::LE:
- case TExprOpcode::LT:
- case TExprOpcode::EQ:
- case TExprOpcode::NE:
- build_comparison_predicate(request, root_type, *sarg_expr, timezone,
builder);
- return true;
- case TExprOpcode::EQ_FOR_NULL:
- build_null_safe_equal(request, root_type, *sarg_expr, timezone,
builder);
- return true;
- case TExprOpcode::FILTER_IN:
- build_in_predicate(request, root_type, *sarg_expr, timezone, builder);
- return true;
- case TExprOpcode::FILTER_NOT_IN:
- builder->startNot();
- build_in_predicate(request, root_type, *sarg_expr, timezone, builder);
- builder->end();
- return true;
- case TExprOpcode::INVALID_OPCODE:
- if ((*sarg_expr)->fn().name.function_name == "is_null_pred") {
- build_is_null(request, root_type, *sarg_expr, builder);
- return true;
+ auto literals = make_in_literals_for_sarg(*column,
expr->children()[0], expr->children(),
+ _timezone);
+ if (!literals.has_value()) {
+ return std::nullopt;
}
- if ((*sarg_expr)->fn().name.function_name == "is_not_null_pred") {
- builder->startNot();
- build_is_null(request, root_type, *sarg_expr, builder);
- builder->end();
- return true;
+ auto node = make_in_node(*column, std::move(*literals));
+ if (expr->op() == TExprOpcode::FILTER_NOT_IN) {
+ node = make_not_node(std::move(node));
}
- return false;
- default:
- return false;
+ return node;
+ }
+
+ std::optional<OrcSargNode> compile_is_null(const VExprSPtr& expr) const {
+ if (expr->node_type() != TExprNodeType::FUNCTION_CALL ||
expr->children().size() != 1) {
+ return std::nullopt;
+ }
+ const auto& function_name = expr->fn().name.function_name;
+ if (function_name != "is_null_pred" && function_name !=
"is_not_null_pred") {
+ return std::nullopt;
+ }
+ const auto column =
+ sarg_column_for_slot_or_safe_cast(_request, _root_type,
expr->children()[0]);
+ if (!column.has_value()) {
+ return std::nullopt;
+ }
+ auto node = make_is_null_node(*column);
+ if (function_name == "is_not_null_pred") {
+ node = make_not_node(std::move(node));
+ }
+ return node;
+ }
+
+ const format::FileScanRequest& _request;
+ const ::orc::Type& _root_type;
+ const cctz::time_zone& _timezone;
+ const OrcSargBuildOptions& _options;
+ std::unordered_map<const VExpr*, std::optional<VExprSPtr>>
_normalized_expressions;
+};
+
+void emit_search_argument(const OrcSargNode& node,
::orc::SearchArgumentBuilder& builder) {
+ switch (node.kind) {
+ case OrcSargNodeKind::AND:
+ DORIS_CHECK(!node.children.empty());
+ builder.startAnd();
+ for (const auto& child : node.children) {
+ emit_search_argument(child, builder);
+ }
+ builder.end();
+ return;
+ case OrcSargNodeKind::OR:
+ DORIS_CHECK(!node.children.empty());
+ builder.startOr();
+ for (const auto& child : node.children) {
+ emit_search_argument(child, builder);
+ }
+ builder.end();
+ return;
+ case OrcSargNodeKind::NOT:
+ DORIS_CHECK(node.children.size() == 1);
+ builder.startNot();
+ emit_search_argument(node.children.front(), builder);
+ builder.end();
+ return;
+ case OrcSargNodeKind::LESS_THAN:
+ DORIS_CHECK(node.column.has_value() && node.literals.size() == 1);
+ builder.lessThan(node.column->column_id, node.column->predicate_type,
+ node.literals.front());
+ return;
+ case OrcSargNodeKind::LESS_THAN_EQUALS:
+ DORIS_CHECK(node.column.has_value() && node.literals.size() == 1);
+ builder.lessThanEquals(node.column->column_id,
node.column->predicate_type,
+ node.literals.front());
+ return;
+ case OrcSargNodeKind::EQUALS:
+ DORIS_CHECK(node.column.has_value() && node.literals.size() == 1);
+ builder.equals(node.column->column_id, node.column->predicate_type,
node.literals.front());
+ return;
+ case OrcSargNodeKind::IN:
+ DORIS_CHECK(node.column.has_value() && node.literals.size() > 1);
+ builder.in(node.column->column_id, node.column->predicate_type,
node.literals);
+ return;
+ case OrcSargNodeKind::IS_NULL:
+ DORIS_CHECK(node.column.has_value() && node.literals.empty());
+ builder.isNull(node.column->column_id, node.column->predicate_type);
+ return;
}
}
} // namespace
bool build_orc_search_argument(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const cctz::time_zone& timezone, const
VExprSPtr& expr,
+ const cctz::time_zone& timezone, const
OrcSargBuildOptions& options,
+ const VExprSPtr& expr,
std::unique_ptr<::orc::SearchArgumentBuilder>&
builder) {
- return build_search_argument(request, root_type, timezone, expr, builder);
+ auto node = OrcSargCompiler(request, root_type, timezone,
options).compile(expr);
+ if (!node.has_value()) {
+ return false;
+ }
+ emit_search_argument(*node, *builder);
+ return true;
}
} // namespace doris::format::orc
diff --git a/be/src/format_v2/orc/orc_search_argument.h
b/be/src/format_v2/orc/orc_search_argument.h
index ad922915e6e..8cf3e23ac2a 100644
--- a/be/src/format_v2/orc/orc_search_argument.h
+++ b/be/src/format_v2/orc/orc_search_argument.h
@@ -31,11 +31,16 @@ class Type;
namespace doris::format::orc {
+struct OrcSargBuildOptions {
+ int max_pushdown_conditions_per_column = 1024;
+};
+
// Lower already-localized Doris file filters to ORC SearchArgument predicates.
// TableColumnMapper owns table-schema -> file-local localization; this module
// owns the ORC-specific type-id/literal lowering needed by the ORC C++
library.
bool build_orc_search_argument(const format::FileScanRequest& request, const
::orc::Type& root_type,
- const cctz::time_zone& timezone, const
VExprSPtr& expr,
+ const cctz::time_zone& timezone, const
OrcSargBuildOptions& options,
+ const VExprSPtr& expr,
std::unique_ptr<::orc::SearchArgumentBuilder>&
builder);
} // namespace doris::format::orc
diff --git a/be/test/format_v2/orc/orc_reader_test.cpp
b/be/test/format_v2/orc/orc_reader_test.cpp
index b98e2bf14e6..6809186f115 100644
--- a/be/test/format_v2/orc/orc_reader_test.cpp
+++ b/be/test/format_v2/orc/orc_reader_test.cpp
@@ -29,17 +29,20 @@
#include <cstring>
#include <filesystem>
#include <fstream>
+#include <functional>
#include <limits>
#include <memory>
#include <optional>
#include <orc/OrcFile.hh>
#include <set>
+#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "common/config.h"
+#include "common/exception.h"
#include "core/assert_cast.h"
#include "core/block/block.h"
#include "core/column/column_array.h"
@@ -77,11 +80,30 @@
#include "runtime/runtime_state.h"
#include "storage/segment/condition_cache.h"
#include "storage/utils.h"
+#include "util/debug_points.h"
#include "util/timezone_utils.h"
namespace doris {
namespace {
+class ScopedDebugPoint {
+public:
+ ScopedDebugPoint(std::string name, std::function<void()> callback)
+ : _name(std::move(name)),
_enable_debug_points(config::enable_debug_points) {
+ config::enable_debug_points = true;
+ DebugPoints::instance()->add_with_callback(_name, std::move(callback));
+ }
+
+ ~ScopedDebugPoint() {
+ DebugPoints::instance()->remove(_name);
+ config::enable_debug_points = _enable_debug_points;
+ }
+
+private:
+ std::string _name;
+ bool _enable_debug_points;
+};
+
std::filesystem::path unique_test_dir(std::string_view prefix) {
static std::atomic<uint64_t> next_dir_id {0};
std::string name(prefix);
@@ -582,9 +604,7 @@ private:
class CompoundPredicateExpr final : public VExpr {
public:
CompoundPredicateExpr(TExprOpcode::type opcode, VExprSPtrs children)
- : VExpr(std::make_shared<DataTypeUInt8>(), false),
- _expr_name(opcode == TExprOpcode::COMPOUND_OR ?
"CompoundOrPredicateExpr"
- :
"CompoundNotPredicateExpr") {
+ : VExpr(std::make_shared<DataTypeUInt8>(), false),
_expr_name("CompoundPredicateExpr") {
_node_type = TExprNodeType::COMPOUND_PRED;
_opcode = opcode;
set_children(std::move(children));
@@ -607,7 +627,7 @@ public:
return Status::OK();
}
- DORIS_CHECK(_opcode == TExprOpcode::COMPOUND_OR);
+ DORIS_CHECK(_opcode == TExprOpcode::COMPOUND_AND || _opcode ==
TExprOpcode::COMPOUND_OR);
DORIS_CHECK(!_children.empty());
std::vector<ColumnPtr> child_columns;
child_columns.reserve(_children.size());
@@ -617,12 +637,17 @@ public:
child_columns.push_back(std::move(child_column));
}
for (size_t row = 0; row < count; ++row) {
- result_data[row] = 0;
+ result_data[row] = _opcode == TExprOpcode::COMPOUND_AND;
for (const auto& child_column : child_columns) {
- if (bool_value(*child_column, row)) {
+ const auto child_value = bool_value(*child_column, row);
+ if (_opcode == TExprOpcode::COMPOUND_OR && child_value) {
result_data[row] = 1;
break;
}
+ if (_opcode == TExprOpcode::COMPOUND_AND && !child_value) {
+ result_data[row] = 0;
+ break;
+ }
}
}
result_column = std::move(result);
@@ -674,6 +699,22 @@ private:
const std::string _expr_name;
};
+class CountingIntHybridSet final : public HybridSet<TYPE_INT> {
+public:
+ using HybridSet<TYPE_INT>::HybridSet;
+
+ HybridSetBase::IteratorBase* begin() override {
+ ++_begin_calls;
+ return HybridSet<TYPE_INT>::begin();
+ }
+
+ void reset_begin_calls() { _begin_calls = 0; }
+ size_t begin_calls() const { return _begin_calls; }
+
+private:
+ size_t _begin_calls = 0;
+};
+
class NullableInt32CastToInt64GreaterThanExpr final : public VExpr {
public:
NullableInt32CastToInt64GreaterThanExpr(int column_id, int64_t value)
@@ -4274,7 +4315,8 @@ void write_two_stripe_constant_date_file(const
std::string& file_path, int64_t f
void write_two_stripe_constant_timestamp_file(const std::string& file_path,
int64_t first_timestamp_second,
- int64_t second_timestamp_second)
{
+ int64_t second_timestamp_second,
+ std::string_view writer_timezone
= "GMT") {
auto type = std::unique_ptr<::orc::Type>(
::orc::Type::buildTypeFromString("struct<timestamp_col:timestamp,payload:string>"));
@@ -4282,6 +4324,7 @@ void write_two_stripe_constant_timestamp_file(const
std::string& file_path,
::orc::WriterOptions options;
options.setCompression(::orc::CompressionKind_NONE);
options.setMemoryPool(::orc::getDefaultPool());
+ options.setTimezoneName(std::string(writer_timezone));
options.setStripeSize(1);
options.setDictionaryKeySizeThreshold(0);
auto writer = ::orc::createWriter(*type, &memory_stream, options);
@@ -4358,6 +4401,40 @@ std::vector<OrcStripeLayout> get_orc_stripe_layout(const
std::string& file_path)
return layout;
}
+bool corrupt_orc_stripe_statistics(const std::string& file_path) {
+ std::ifstream in(file_path, std::ios::binary | std::ios::ate);
+ const auto file_size = in.tellg();
+ if (file_size <= 0) {
+ return false;
+ }
+ in.seekg(0);
+ std::vector<char> data(static_cast<size_t>(file_size));
+ in.read(data.data(), static_cast<std::streamsize>(data.size()));
+ if (!in.good()) {
+ return false;
+ }
+
+ ::orc::ReaderOptions options;
+ options.setMemoryPool(*::orc::getDefaultPool());
+ auto input_stream = std::make_unique<MemoryInputStream>(data.data(),
data.size());
+ auto reader = ::orc::createReader(std::move(input_stream), options);
+ const auto metadata_length = reader->getStripeStatisticsLength();
+ const auto footer_length = reader->getFileFooterLength();
+ const auto postscript_length = reader->getFilePostscriptLength();
+ const auto orc_file_length = reader->getFileLength();
+ const auto trailing_length = metadata_length + footer_length +
postscript_length + 1;
+ if (metadata_length == 0 || orc_file_length != data.size() ||
+ trailing_length > orc_file_length) {
+ return false;
+ }
+
+ const auto metadata_offset = orc_file_length - trailing_length;
+ std::fill_n(data.data() + metadata_offset, metadata_length,
static_cast<char>(0xff));
+ std::ofstream out(file_path, std::ios::binary | std::ios::trunc);
+ out.write(data.data(), static_cast<std::streamsize>(data.size()));
+ return out.good();
+}
+
Block build_file_block(const std::vector<format::ColumnDefinition>& schema) {
Block block;
for (const auto& field : schema) {
@@ -4393,6 +4470,21 @@ format::LocalColumnIndex struct_child_projection(int32_t
root_field_id, int32_t
class NewOrcReaderTest : public testing::Test {
protected:
+ enum class DirectInPredicateShape {
+ ROOT,
+ AND,
+ OR,
+ OR_WITH_NULL_SAFE_EQUAL,
+ };
+
+ struct DirectInScanResult {
+ std::vector<int32_t> ids;
+ size_t materialize_calls = 0;
+ size_t prepare_literals_calls = 0;
+ int64_t filtered_row_groups = 0;
+ int64_t filtered_row_groups_by_min_max = 0;
+ };
+
void SetUp() override {
_test_dir = unique_test_dir("doris_new_orc_reader_test");
std::filesystem::remove_all(_test_dir);
@@ -4447,6 +4539,88 @@ protected:
nullptr, profile,
std::nullopt);
}
+ Status scan_direct_in_filter(const std::string& file_path, int32_t
in_list_size,
+ bool enable_sarg, int
max_pushdown_conditions_per_column,
+ DirectInPredicateShape shape,
DirectInScanResult* result) const {
+ DORIS_CHECK(result != nullptr);
+ *result = {};
+
+ auto reader = create_reader_for_path(file_path);
+ TQueryOptions query_options;
+ query_options.__set_enable_orc_filter_by_min_max(enable_sarg);
+
query_options.__set_max_pushdown_conditions_per_column(max_pushdown_conditions_per_column);
+ RuntimeState state {query_options, TQueryGlobals()};
+ RETURN_IF_ERROR(reader->init(&state));
+
+ std::vector<format::ColumnDefinition> schema;
+ RETURN_IF_ERROR(reader->get_schema(&schema));
+ DORIS_CHECK(schema.size() == 2);
+
+ constexpr int32_t in_list_start = 1000;
+ auto filter = std::make_shared<CountingIntHybridSet>(false);
+ for (int32_t value = in_list_start; value < in_list_start +
in_list_size; ++value) {
+ filter->insert(&value);
+ }
+
+ auto node = make_filter_in_node(TExprNodeType::IN_PRED);
+ auto direct_in = VDirectInPredicate::create_shared(node, filter, true);
+ direct_in->add_child(TableSlotRef::create_shared(0, 0, -1,
schema[0].type, "id"));
+ VExprSPtr predicate = RuntimeFilterExpr::create_shared(node,
direct_in, 0.0, false, 7);
+ if (shape == DirectInPredicateShape::AND) {
+ predicate = std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_AND,
+ VExprSPtrs {std::move(predicate),
+
std::make_shared<NullableInt32GreaterThanExpr>(0, 500)});
+ } else if (shape == DirectInPredicateShape::OR) {
+ predicate = std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_OR,
+ VExprSPtrs {std::move(predicate),
+ std::make_shared<NullableInt32LessThanExpr>(0,
0)});
+ } else if (shape == DirectInPredicateShape::OR_WITH_NULL_SAFE_EQUAL) {
+ predicate = std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_OR,
+ VExprSPtrs {std::move(predicate),
+
std::make_shared<NullableInt32NullSafeEqualsLiteralExpr>(0, -1,
+
false)});
+ } else {
+ DORIS_CHECK(shape == DirectInPredicateShape::ROOT);
+ }
+ auto context = VExprContext::create_shared(std::move(predicate));
+ RETURN_IF_ERROR(context->prepare(&state, RowDescriptor()));
+ RETURN_IF_ERROR(context->open(&state));
+
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ request->conjuncts.push_back(std::move(context));
+
+ ScopedDebugPoint
debug_point("OrcSearchArgument.make_in_literals_for_sarg",
+ [&]() { ++result->prepare_literals_calls;
});
+ filter->reset_begin_calls();
+ RETURN_IF_ERROR(reader->open(request));
+
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block({schema[0]});
+ size_t rows = 0;
+ RETURN_IF_ERROR(reader->get_block(&block, &rows, &eof));
+ if (rows == 0) {
+ continue;
+ }
+ const auto& ids_nullable =
+ assert_cast<const
ColumnNullable&>(*block.get_by_position(0).column);
+ const auto& ids = assert_cast<const
ColumnInt32&>(ids_nullable.get_nested_column());
+ for (size_t row = 0; row < rows; ++row) {
+ result->ids.push_back(ids.get_element(row));
+ }
+ }
+
+ result->materialize_calls = filter->begin_calls();
+ result->filtered_row_groups =
reader->reader_statistics().filtered_row_groups;
+ result->filtered_row_groups_by_min_max =
+ reader->reader_statistics().filtered_row_groups_by_min_max;
+ return Status::OK();
+ }
+
std::filesystem::path _test_dir;
std::string _file_path;
};
@@ -4641,13 +4815,48 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownUsesPrunedStripes) {
EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 1);
}
-TEST_F(NewOrcReaderTest, AggregatePushdownTimestampMinMaxUsesSessionTimezone) {
+TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxMatchesScanInSessionTimezone) {
const auto multi_stripe_file_path = (_test_dir /
"aggregate_timestamp_timezone.orc").string();
- write_two_stripe_constant_timestamp_file(multi_stripe_file_path, 0, 3600);
+ write_two_stripe_constant_timestamp_file(multi_stripe_file_path,
1609430400, 1609434000,
+ "Asia/Shanghai");
- auto reader = create_reader_for_path(multi_stripe_file_path);
RuntimeState state {TQueryOptions(), TQueryGlobals()};
state.set_timezone("Asia/Shanghai");
+
+ auto scan_reader = create_reader_for_path(multi_stripe_file_path);
+ ASSERT_TRUE(scan_reader->init(&state).ok());
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(scan_reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+ auto scan_request = std::make_shared<format::FileScanRequest>();
+ scan_request->non_predicate_columns = {field_projection(0)};
+ ASSERT_TRUE(scan_reader->open(scan_request).ok());
+
+ std::optional<DateV2Value<DateTimeV2ValueType>> scan_min;
+ std::optional<DateV2Value<DateTimeV2ValueType>> scan_max;
+ size_t scan_rows = 0;
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block({schema[0]});
+ size_t rows = 0;
+ ASSERT_TRUE(scan_reader->get_block(&block, &rows, &eof).ok());
+ scan_rows += rows;
+ const auto& nullable = assert_cast<const
ColumnNullable&>(*block.get_by_position(0).column);
+ const auto& values = assert_cast<const
ColumnDateTimeV2&>(nullable.get_nested_column());
+ for (size_t row = 0; row < rows; ++row) {
+ ASSERT_FALSE(nullable.is_null_at(row));
+ const auto value = values.get_element(row);
+ scan_min = !scan_min.has_value() || value < *scan_min ? value :
*scan_min;
+ scan_max = !scan_max.has_value() || *scan_max < value ? value :
*scan_max;
+ }
+ }
+ ASSERT_EQ(scan_rows, 400);
+ ASSERT_TRUE(scan_min.has_value());
+ ASSERT_TRUE(scan_max.has_value());
+ EXPECT_EQ(*scan_min, make_datetime_v2(2021, 1, 1, 0, 0, 0, 123000));
+ EXPECT_EQ(*scan_max, make_datetime_v2(2021, 1, 1, 1, 0, 0, 123000));
+
+ auto reader = create_reader_for_path(multi_stripe_file_path);
ASSERT_TRUE(reader->init(&state).ok());
auto request = std::make_shared<format::FileScanRequest>();
@@ -4665,10 +4874,87 @@ TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxUsesSessionTimezone) {
EXPECT_EQ(aggregate_result.count, 400);
EXPECT_TRUE(aggregate_result.columns[0].has_min);
EXPECT_TRUE(aggregate_result.columns[0].has_max);
- EXPECT_EQ(aggregate_result.columns[0].min_value.get<TYPE_DATETIMEV2>(),
- make_datetime_v2(1970, 1, 1, 8, 0, 0, 123000));
- EXPECT_EQ(aggregate_result.columns[0].max_value.get<TYPE_DATETIMEV2>(),
- make_datetime_v2(1970, 1, 1, 9, 0, 0, 123000));
+ EXPECT_EQ(aggregate_result.columns[0].min_value.get<TYPE_DATETIMEV2>(),
*scan_min);
+ EXPECT_EQ(aggregate_result.columns[0].max_value.get<TYPE_DATETIMEV2>(),
*scan_max);
+}
+
+TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampMinMaxFallsBackForPreOrc135Writer) {
+ const auto file_path = find_repo_file(
+
"contrib/apache-orc/java/core/src/test/resources/orc-file-no-timezone.orc");
+ ASSERT_TRUE(std::filesystem::exists(file_path)) << file_path;
+
+ std::ifstream in(file_path, std::ios::binary | std::ios::ate);
+ ASSERT_TRUE(in.good());
+ const auto file_size = in.tellg();
+ ASSERT_GT(file_size, 0);
+ in.seekg(0);
+ std::vector<char> data(static_cast<size_t>(file_size));
+ in.read(data.data(), static_cast<std::streamsize>(data.size()));
+ ASSERT_TRUE(in.good());
+
+ ::orc::ReaderOptions options;
+ options.setMemoryPool(*::orc::getDefaultPool());
+ auto input_stream = std::make_unique<MemoryInputStream>(data.data(),
data.size());
+ auto metadata_reader = ::orc::createReader(std::move(input_stream),
options);
+ ASSERT_EQ(metadata_reader->getWriterVersion(),
::orc::WriterVersion_HIVE_8732);
+ auto stripe_statistics = metadata_reader->getStripeStatistics(0);
+ ASSERT_NE(stripe_statistics, nullptr);
+ const auto* timestamp_statistics = dynamic_cast<const
::orc::TimestampColumnStatistics*>(
+ stripe_statistics->getColumnStatistics(1));
+ ASSERT_NE(timestamp_statistics, nullptr);
+ ASSERT_TRUE(timestamp_statistics->hasMinimum());
+ ASSERT_TRUE(timestamp_statistics->hasMaximum());
+
+ auto reader = create_reader_for_path(file_path.string());
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ state.set_timezone("UTC");
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 1);
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->non_predicate_columns = {field_projection(0)};
+ ASSERT_TRUE(reader->open(request).ok());
+
+ format::FileAggregateRequest aggregate_request;
+ aggregate_request.agg_type = TPushAggOp::type::MINMAX;
+ aggregate_request.columns.push_back(
+ {.projection =
format::LocalColumnIndex::top_level(format::LocalColumnId(0))});
+ format::FileAggregateResult aggregate_result;
+ const auto aggregate_status =
+ reader->get_aggregate_result(aggregate_request, &aggregate_result);
+ ASSERT_TRUE(aggregate_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
aggregate_status;
+
+ const auto read_values = [&](format::orc::OrcReader* scan_reader,
+ std::vector<std::string>* values) -> Status {
+ DORIS_CHECK(scan_reader != nullptr);
+ DORIS_CHECK(values != nullptr);
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block(schema);
+ size_t rows = 0;
+ RETURN_IF_ERROR(scan_reader->get_block(&block, &rows, &eof));
+ for (size_t row = 0; row < rows; ++row) {
+
values->push_back(schema[0].type->to_string(*block.get_by_position(0).column,
row));
+ }
+ }
+ return Status::OK();
+ };
+
+ std::vector<std::string> fallback_values;
+ ASSERT_TRUE(read_values(reader.get(), &fallback_values).ok());
+
+ auto baseline_reader = create_reader_for_path(file_path.string());
+ ASSERT_TRUE(baseline_reader->init(&state).ok());
+ auto baseline_request = std::make_shared<format::FileScanRequest>();
+ baseline_request->non_predicate_columns = {field_projection(0)};
+ ASSERT_TRUE(baseline_reader->open(baseline_request).ok());
+
+ std::vector<std::string> baseline_values;
+ ASSERT_TRUE(read_values(baseline_reader.get(), &baseline_values).ok());
+ ASSERT_EQ(baseline_values.size(), 2);
+ EXPECT_EQ(fallback_values, baseline_values);
}
TEST_F(NewOrcReaderTest,
AggregatePushdownTimestampInstantMinMaxUsesTimestampTzWhenMapped) {
@@ -6632,6 +6918,99 @@ TEST_F(NewOrcReaderTest,
SargRuntimeFilterWrapperConjunctPrunesStripes) {
EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 200);
}
+TEST_F(NewOrcReaderTest, SargDirectInCompoundMaterializesLiteralsOnce) {
+ const auto multi_stripe_file_path = (_test_dir /
"sarg_direct_in_prepared_once.orc").string();
+ write_multi_stripe_orc_int_file(multi_stripe_file_path);
+ ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2);
+
+ constexpr int32_t in_list_size = 1024;
+ for (const auto shape : {DirectInPredicateShape::AND,
DirectInPredicateShape::OR}) {
+ SCOPED_TRACE(shape == DirectInPredicateShape::AND ? "AND" : "OR");
+ DirectInScanResult enabled;
+ DirectInScanResult disabled;
+ ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path,
in_list_size, true, in_list_size,
+ shape, &enabled)
+ .ok());
+ ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path,
in_list_size, false, in_list_size,
+ shape, &disabled)
+ .ok());
+
+ EXPECT_EQ(enabled.materialize_calls, 1);
+ EXPECT_EQ(enabled.prepare_literals_calls, 1);
+ EXPECT_EQ(enabled.filtered_row_groups, 1);
+ EXPECT_EQ(enabled.filtered_row_groups_by_min_max, 1);
+ EXPECT_EQ(disabled.materialize_calls, 0);
+ EXPECT_EQ(disabled.prepare_literals_calls, 0);
+ EXPECT_EQ(disabled.filtered_row_groups, 0);
+ EXPECT_EQ(disabled.filtered_row_groups_by_min_max, 0);
+ ASSERT_EQ(enabled.ids, disabled.ids);
+ ASSERT_EQ(enabled.ids.size(), 200);
+ EXPECT_EQ(enabled.ids.front(), 1000);
+ EXPECT_EQ(enabled.ids.back(), 1199);
+ }
+}
+
+TEST_F(NewOrcReaderTest,
SargDirectInNullSafeOrFallsBackBeforeLiteralConversion) {
+ const auto multi_stripe_file_path = (_test_dir /
"sarg_direct_in_null_safe_or.orc").string();
+ write_multi_stripe_orc_int_file(multi_stripe_file_path);
+ ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2);
+
+ constexpr int32_t in_list_size = 1024;
+ DirectInScanResult enabled;
+ DirectInScanResult disabled;
+ ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size,
true, in_list_size,
+
DirectInPredicateShape::OR_WITH_NULL_SAFE_EQUAL, &enabled)
+ .ok());
+ ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size,
false, in_list_size,
+
DirectInPredicateShape::OR_WITH_NULL_SAFE_EQUAL, &disabled)
+ .ok());
+
+ EXPECT_EQ(enabled.materialize_calls, 1);
+ EXPECT_EQ(enabled.prepare_literals_calls, 0);
+ EXPECT_EQ(enabled.filtered_row_groups, 0);
+ EXPECT_EQ(enabled.filtered_row_groups_by_min_max, 0);
+ EXPECT_EQ(disabled.materialize_calls, 0);
+ EXPECT_EQ(disabled.prepare_literals_calls, 0);
+ EXPECT_EQ(disabled.filtered_row_groups, 0);
+ EXPECT_EQ(disabled.filtered_row_groups_by_min_max, 0);
+ ASSERT_EQ(enabled.ids, disabled.ids);
+ ASSERT_EQ(enabled.ids.size(), 200);
+ EXPECT_EQ(enabled.ids.front(), 1000);
+ EXPECT_EQ(enabled.ids.back(), 1199);
+}
+
+TEST_F(NewOrcReaderTest, SargDirectInOverLimitFallsBackBeforeMaterialization) {
+ const auto multi_stripe_file_path = (_test_dir /
"sarg_direct_in_over_limit.orc").string();
+ write_multi_stripe_orc_int_file(multi_stripe_file_path);
+ ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2);
+
+ constexpr int32_t max_pushdown_conditions_per_column = 1024;
+ constexpr int32_t in_list_size = max_pushdown_conditions_per_column + 1;
+ DirectInScanResult enabled;
+ DirectInScanResult disabled;
+ ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size,
true,
+ max_pushdown_conditions_per_column,
+ DirectInPredicateShape::ROOT, &enabled)
+ .ok());
+ ASSERT_TRUE(scan_direct_in_filter(multi_stripe_file_path, in_list_size,
false,
+ max_pushdown_conditions_per_column,
+ DirectInPredicateShape::ROOT, &disabled)
+ .ok());
+
+ EXPECT_EQ(enabled.materialize_calls, 0);
+ EXPECT_EQ(enabled.prepare_literals_calls, 0);
+ EXPECT_EQ(enabled.filtered_row_groups, 0);
+ EXPECT_EQ(enabled.filtered_row_groups_by_min_max, 0);
+ EXPECT_EQ(disabled.materialize_calls, 0);
+ EXPECT_EQ(disabled.prepare_literals_calls, 0);
+ EXPECT_EQ(disabled.filtered_row_groups, 0);
+ EXPECT_EQ(disabled.filtered_row_groups_by_min_max, 0);
+ ASSERT_EQ(enabled.ids, disabled.ids);
+ ASSERT_EQ(enabled.ids.size(), 200);
+ EXPECT_EQ(enabled.ids.front(), 1000);
+ EXPECT_EQ(enabled.ids.back(), 1199);
+}
+
TEST_F(NewOrcReaderTest, SargNullAwareRuntimeFilterDoesNotPruneNullStripe) {
const auto multi_stripe_file_path = (_test_dir /
"sarg_null_aware_runtime_filter.orc").string();
write_two_stripe_orc_nullable_int_file(multi_stripe_file_path);
@@ -6936,6 +7315,113 @@ TEST_F(NewOrcReaderTest,
SargSlotToSlotNullSafeEqualDoesNotPruneStripes) {
EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0);
}
+TEST_F(NewOrcReaderTest, SargPartialAndRespectsNegationContext) {
+ const auto multi_stripe_file_path = (_test_dir /
"sarg_partial_and_not.orc").string();
+ write_multi_stripe_orc_int_file(multi_stripe_file_path);
+ ASSERT_EQ(get_orc_stripe_count(multi_stripe_file_path), 2);
+
+ auto partial_and = []() -> VExprSPtr {
+ return std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_AND,
+ VExprSPtrs {std::make_shared<NullableInt32GreaterThanExpr>(0,
500),
+
std::make_shared<NullableInt32CastToFloatGreaterThanExpr>(0, 1100.5F)});
+ };
+ auto verify = [&](const auto& conjunct_factory, const
std::vector<int32_t>& expected_ids,
+ int64_t expected_filtered_row_groups) {
+ std::array<std::vector<int32_t>, 2> result_ids;
+ for (size_t run = 0; run < result_ids.size(); ++run) {
+ const bool enable_filter_by_min_max = run == 0;
+ auto reader = create_reader_for_path(multi_stripe_file_path);
+ TQueryOptions query_options;
+
query_options.__set_enable_orc_filter_by_min_max(enable_filter_by_min_max);
+ RuntimeState state {query_options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+
request->conjuncts.push_back(VExprContext::create_shared(conjunct_factory()));
+ ASSERT_TRUE(reader->open(request).ok());
+
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block({schema[0]});
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ if (eof || rows == 0) {
+ continue;
+ }
+ const auto& ids_nullable =
+ assert_cast<const
ColumnNullable&>(*block.get_by_position(0).column);
+ const auto& ids = assert_cast<const
ColumnInt32&>(ids_nullable.get_nested_column());
+ for (size_t row = 0; row < rows; ++row) {
+ ASSERT_FALSE(ids_nullable.is_null_at(row));
+ result_ids[run].push_back(ids.get_element(row));
+ }
+ }
+
+ const auto filtered_row_groups =
+ enable_filter_by_min_max ? expected_filtered_row_groups :
0;
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups,
filtered_row_groups);
+
EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max,
+ filtered_row_groups);
+ EXPECT_EQ(reader->reader_statistics().filtered_group_rows,
filtered_row_groups * 200);
+ }
+
+ EXPECT_EQ(result_ids[0], expected_ids);
+ EXPECT_EQ(result_ids[0], result_ids[1]);
+ };
+
+ std::vector<int32_t> expected_not_ids;
+ for (int32_t id = 1; id <= 200; ++id) {
+ expected_not_ids.push_back(id);
+ }
+ for (int32_t id = 1000; id <= 1100; ++id) {
+ expected_not_ids.push_back(id);
+ }
+ verify(
+ [&]() -> VExprSPtr {
+ return
std::make_shared<CompoundPredicateExpr>(TExprOpcode::COMPOUND_NOT,
+ VExprSPtrs
{partial_and()});
+ },
+ expected_not_ids, 0);
+
+ std::vector<int32_t> expected_and_ids;
+ for (int32_t id = 1101; id <= 1199; ++id) {
+ expected_and_ids.push_back(id);
+ }
+ verify(partial_and, expected_and_ids, 1);
+ verify(
+ [&]() -> VExprSPtr {
+ auto inner_not = std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_NOT, VExprSPtrs {partial_and()});
+ return
std::make_shared<CompoundPredicateExpr>(TExprOpcode::COMPOUND_NOT,
+ VExprSPtrs
{std::move(inner_not)});
+ },
+ expected_and_ids, 1);
+
+ std::vector<int32_t> expected_nested_ids;
+ expected_nested_ids.insert(expected_nested_ids.end(),
expected_not_ids.begin(),
+ expected_not_ids.begin() + 200);
+ expected_nested_ids.insert(expected_nested_ids.end(),
expected_and_ids.begin(),
+ expected_and_ids.end());
+ verify(
+ [&]() -> VExprSPtr {
+ auto inner_not = std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_NOT, VExprSPtrs {partial_and()});
+ auto outer_and = std::make_shared<CompoundPredicateExpr>(
+ TExprOpcode::COMPOUND_AND,
+ VExprSPtrs {std::move(inner_not),
+
std::make_shared<NullableInt32GreaterThanExpr>(0, 500)});
+ return
std::make_shared<CompoundPredicateExpr>(TExprOpcode::COMPOUND_NOT,
+ VExprSPtrs
{std::move(outer_and)});
+ },
+ expected_nested_ids, 0);
+}
+
TEST_F(NewOrcReaderTest, SargNullSafeEqualInsideOrDoesNotPruneStripes) {
const auto multi_stripe_file_path = (_test_dir /
"sarg_null_safe_equal_inside_or.orc").string();
write_multi_stripe_orc_int_file(multi_stripe_file_path);
@@ -7155,6 +7641,138 @@ TEST_F(NewOrcReaderTest,
SargBinaryVarbinaryLiteralConjunctPrunesRowGroups) {
EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0);
}
+TEST_F(NewOrcReaderTest, SargExceptionsFallBackToResidualScan) {
+ const auto valid_file_path = (_test_dir /
"sarg_build_exception.orc").string();
+ const auto corrupt_statistics_file_path = (_test_dir /
"sarg_corrupt_statistics.orc").string();
+ write_multi_stripe_orc_binary_file(valid_file_path);
+ write_multi_stripe_orc_binary_file(corrupt_statistics_file_path);
+ const auto valid_stripe_layout = get_orc_stripe_layout(valid_file_path);
+ const auto corrupt_statistics_stripe_layout =
+ get_orc_stripe_layout(corrupt_statistics_file_path);
+ ASSERT_EQ(valid_stripe_layout.size(), 2);
+ ASSERT_EQ(corrupt_statistics_stripe_layout.size(), 2);
+ ASSERT_TRUE(corrupt_orc_stripe_statistics(corrupt_statistics_file_path));
+
+ auto scan_first_stripe = [&](const std::string& file_path, const
OrcStripeLayout& stripe,
+ bool enable_filter_by_min_max, bool
add_real_residual_filter,
+ std::vector<std::string>* result_values,
Status* open_status) {
+ auto reader = create_reader_with_range(file_path,
static_cast<int64_t>(stripe.offset),
+
static_cast<int64_t>(stripe.length));
+ TQueryOptions query_options;
+
query_options.__set_enable_orc_filter_by_min_max(enable_filter_by_min_max);
+ RuntimeState state {query_options, TQueryGlobals()};
+ ASSERT_TRUE(reader->init(&state).ok());
+
+ std::vector<format::ColumnDefinition> schema;
+ ASSERT_TRUE(reader->get_schema(&schema).ok());
+ ASSERT_EQ(schema.size(), 2);
+
+ auto request = std::make_shared<format::FileScanRequest>();
+ request->predicate_columns = {field_projection(0)};
+ request->conjuncts.push_back(VExprContext::create_shared(
+ std::make_shared<SargOnlyStringEqualsVarbinaryLiteralExpr>(
+ 0, std::make_shared<DataTypeVarbinary>(), "zzz_1000",
"value")));
+ if (add_real_residual_filter) {
+ request->conjuncts.push_back(
+
VExprContext::create_shared(std::make_shared<NullableStringEqualsExpr>(
+ 0, std::make_shared<DataTypeString>(), "aaa_1050",
"value")));
+ }
+ *open_status = reader->open(request);
+ if (!open_status->ok()) {
+ return;
+ }
+
+ bool eof = false;
+ while (!eof) {
+ Block block = build_file_block({schema[0]});
+ size_t rows = 0;
+ ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok());
+ if (eof || rows == 0) {
+ continue;
+ }
+ const auto& values_nullable =
+ assert_cast<const
ColumnNullable&>(*block.get_by_position(0).column);
+ const auto& values =
+ assert_cast<const
ColumnString&>(values_nullable.get_nested_column());
+ for (size_t row = 0; row < rows; ++row) {
+ ASSERT_FALSE(values_nullable.is_null_at(row));
+ result_values->push_back(values.get_data_at(row).to_string());
+ }
+ }
+
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups, 0);
+ EXPECT_EQ(reader->reader_statistics().filtered_row_groups_by_min_max,
0);
+ EXPECT_EQ(reader->reader_statistics().filtered_group_rows, 0);
+ ASSERT_TRUE(reader->close().ok());
+ };
+
+ std::vector<std::string> without_sarg;
+ Status without_sarg_open_status;
+ scan_first_stripe(corrupt_statistics_file_path,
corrupt_statistics_stripe_layout[0], false,
+ false, &without_sarg, &without_sarg_open_status);
+ ASSERT_TRUE(without_sarg_open_status.ok()) << without_sarg_open_status;
+ ASSERT_EQ(without_sarg.size(), 200);
+ EXPECT_EQ(without_sarg.front(), "aaa_1000");
+ EXPECT_EQ(without_sarg.back(), "aaa_1199");
+
+ int injection_count = 0;
+ std::vector<std::string> after_build_failure;
+ Status build_failure_open_status;
+ {
+ ScopedDebugPoint debug_point(
+
"OrcReader._init_search_argument_from_local_filters.inject_failure", [&]() {
+ ++injection_count;
+ throw std::runtime_error("injected ORC SARG build
failure");
+ });
+ scan_first_stripe(valid_file_path, valid_stripe_layout[0], true, false,
+ &after_build_failure, &build_failure_open_status);
+ }
+ EXPECT_EQ(injection_count, 1);
+ ASSERT_TRUE(build_failure_open_status.ok()) << build_failure_open_status;
+ EXPECT_EQ(after_build_failure, without_sarg);
+
+ std::vector<std::string> after_evaluation_failure;
+ Status evaluation_failure_open_status;
+ scan_first_stripe(corrupt_statistics_file_path,
corrupt_statistics_stripe_layout[0], true,
+ false, &after_evaluation_failure,
&evaluation_failure_open_status);
+ ASSERT_TRUE(evaluation_failure_open_status.ok()) <<
evaluation_failure_open_status;
+ EXPECT_EQ(after_evaluation_failure, without_sarg);
+
+ std::vector<std::string> residual_without_sarg;
+ Status residual_without_sarg_open_status;
+ scan_first_stripe(corrupt_statistics_file_path,
corrupt_statistics_stripe_layout[0], false,
+ true, &residual_without_sarg,
&residual_without_sarg_open_status);
+ ASSERT_TRUE(residual_without_sarg_open_status.ok()) <<
residual_without_sarg_open_status;
+ const std::vector<std::string> expected_residual_values = {"aaa_1050"};
+ EXPECT_EQ(residual_without_sarg, expected_residual_values);
+
+ std::vector<std::string> residual_after_evaluation_failure;
+ Status residual_evaluation_failure_open_status;
+ scan_first_stripe(corrupt_statistics_file_path,
corrupt_statistics_stripe_layout[0], true, true,
+ &residual_after_evaluation_failure,
&residual_evaluation_failure_open_status);
+ ASSERT_TRUE(residual_evaluation_failure_open_status.ok())
+ << residual_evaluation_failure_open_status;
+ EXPECT_EQ(residual_after_evaluation_failure, expected_residual_values);
+
+ int memory_failure_injection_count = 0;
+ std::vector<std::string> after_memory_failure;
+ Status memory_failure_open_status;
+ {
+ ScopedDebugPoint debug_point(
+
"OrcReader._init_search_argument_from_local_filters.inject_failure", [&]() {
+ ++memory_failure_injection_count;
+ throw Exception(ErrorCode::MEM_ALLOC_FAILED,
+ "injected ORC SARG allocation failure");
+ });
+ scan_first_stripe(valid_file_path, valid_stripe_layout[0], true, false,
+ &after_memory_failure, &memory_failure_open_status);
+ }
+ EXPECT_EQ(memory_failure_injection_count, 1);
+ EXPECT_TRUE(memory_failure_open_status.is<ErrorCode::MEM_LIMIT_EXCEEDED>())
+ << memory_failure_open_status;
+ EXPECT_TRUE(after_memory_failure.empty());
+}
+
TEST_F(NewOrcReaderTest,
SargBinaryCastFromVarbinaryToStringConjunctPrunesRowGroups) {
const auto multi_stripe_file_path =
(_test_dir / "binary_varbinary_to_string_cast_sarg.orc").string();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
