This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new a63f3729c82 [fix](top rf) fix external table top rf lack of
consideration of nulls first conditions. (#56392) (#56489)
a63f3729c82 is described below
commit a63f3729c8278e10be5b0dd74391218881a844f3
Author: daidai <[email protected]>
AuthorDate: Fri Sep 26 17:39:24 2025 +0800
[fix](top rf) fix external table top rf lack of consideration of nulls
first conditions. (#56392) (#56489)
### What problem does this PR solve?
bp #56392
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/vec/exec/scan/file_scanner.cpp | 11 +-
be/src/vec/exprs/vtopn_pred.h | 45 ++
.../scripts/create_preinstalled_scripts/run81.hql | 48 ++
.../hive/test_hive_topn_rf_null.out | 489 +++++++++++++++++++++
.../hive/test_hive_topn_rf_null.groovy | 79 ++++
5 files changed, 670 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/exec/scan/file_scanner.cpp
b/be/src/vec/exec/scan/file_scanner.cpp
index eb30354fe3b..82fe714b55d 100644
--- a/be/src/vec/exec/scan/file_scanner.cpp
+++ b/be/src/vec/exec/scan/file_scanner.cpp
@@ -154,11 +154,11 @@ Status FileScanner::init(RuntimeState* state, const
VExprContextSPtrs& conjuncts
ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(),
"FileNumber", TUnit::UNIT, 1);
_file_read_bytes_counter =
ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(),
- "FileReadBytes",
TUnit::BYTES, 1);
+ FileReadBytesProfile,
TUnit::BYTES, 1);
_file_read_calls_counter =
ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(),
"FileReadCalls",
TUnit::UNIT, 1);
_file_read_time_counter =
- ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(),
"FileReadTime", 1);
+ ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(),
FileReadTimeProfile, 1);
_runtime_filter_partition_pruned_range_counter =
ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(),
@@ -1427,6 +1427,10 @@ Status FileScanner::prepare_for_read_lines(const
TFileRangeDesc& range) {
_file_cache_statistics.reset(new io::FileCacheStatistics());
_file_reader_stats.reset(new io::FileReaderStats());
+ _file_read_bytes_counter =
+ ADD_COUNTER_WITH_LEVEL(_profile, FileReadBytesProfile,
TUnit::BYTES, 1);
+ _file_read_time_counter = ADD_TIMER_WITH_LEVEL(_profile,
FileReadTimeProfile, 1);
+
RETURN_IF_ERROR(_init_io_ctx());
_io_ctx->file_cache_stats = _file_cache_statistics.get();
_io_ctx->file_reader_stats = _file_reader_stats.get();
@@ -1508,6 +1512,9 @@ Status FileScanner::read_lines_from_range(const
TFileRangeDesc& range,
_cur_reader->collect_profile_before_close();
RETURN_IF_ERROR(_cur_reader->close());
+
+ COUNTER_UPDATE(_file_read_bytes_counter, _file_reader_stats->read_bytes);
+ COUNTER_UPDATE(_file_read_time_counter, _file_reader_stats->read_time_ns);
return Status::OK();
}
diff --git a/be/src/vec/exprs/vtopn_pred.h b/be/src/vec/exprs/vtopn_pred.h
index c22ba2dbd01..5a4de2fa82a 100644
--- a/be/src/vec/exprs/vtopn_pred.h
+++ b/be/src/vec/exprs/vtopn_pred.h
@@ -170,6 +170,51 @@ public:
slot_data_type->get_scale());
new_root->add_child(VLiteral::create_shared(node));
}
+
+ // Since the normal greater than or less than relationship does not
consider the relationship of null values, the generated `col >=/<= xxx OR col
is null.`
+ if (_predicate->nulls_first()) {
+ VExprSPtr col_is_null_node;
+ {
+ TFunction fn;
+ TFunctionName fn_name;
+ fn_name.__set_db_name("");
+ fn_name.__set_function_name("is_null_pred");
+ fn.__set_name(fn_name);
+ fn.__set_binary_type(TFunctionBinaryType::BUILTIN);
+ std::vector<TTypeDesc> arg_types;
+
arg_types.push_back(create_type_desc(slot_data_type->get_primitive_type(),
+
slot_data_type->get_precision(),
+
slot_data_type->get_scale()));
+ fn.__set_arg_types(arg_types);
+
fn.__set_ret_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ fn.__set_has_var_args(false);
+
+ TExprNode texpr_node;
+
texpr_node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ texpr_node.__set_node_type(TExprNodeType::FUNCTION_CALL);
+ texpr_node.__set_fn(fn);
+ texpr_node.__set_num_children(1);
+ col_is_null_node = VectorizedFnCall::create_shared(texpr_node);
+
+ // add slot.
+ col_is_null_node->add_child(children().at(0));
+ }
+
+ VExprSPtr or_node;
+ {
+ TExprNode texpr_node;
+
texpr_node.__set_type(create_type_desc(PrimitiveType::TYPE_BOOLEAN));
+ texpr_node.__set_node_type(TExprNodeType::COMPOUND_PRED);
+ texpr_node.__set_opcode(TExprOpcode::COMPOUND_OR);
+ texpr_node.__set_num_children(2);
+ or_node = VectorizedFnCall::create_shared(texpr_node);
+ }
+
+ or_node->add_child(col_is_null_node);
+ or_node->add_child(new_root);
+ new_root = or_node;
+ }
+
return true;
}
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run81.hql
b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run81.hql
new file mode 100644
index 00000000000..3f261636be3
--- /dev/null
+++
b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run81.hql
@@ -0,0 +1,48 @@
+use `default`;
+
+SET hive.merge.mapfiles=false;
+SET hive.merge.mapredfiles=false;
+
+
+
+CREATE TABLE test_topn_rf_null_parquet (
+ id INT,
+ value INT,
+ name STRING
+)
+STORED AS PARQUET;
+
+INSERT INTO test_topn_rf_null_parquet VALUES (1, 100, 'Alice');
+INSERT INTO test_topn_rf_null_parquet VALUES (2, 200, null);
+INSERT INTO test_topn_rf_null_parquet VALUES (3, 300, 'Charlie');
+INSERT INTO test_topn_rf_null_parquet VALUES (4, 400, 'David');
+INSERT INTO test_topn_rf_null_parquet VALUES (5, null, null);
+INSERT INTO test_topn_rf_null_parquet VALUES (6, 600, 'Frank');
+INSERT INTO test_topn_rf_null_parquet VALUES (7, null, 'Grace');
+INSERT INTO test_topn_rf_null_parquet VALUES (8, 800, null);
+INSERT INTO test_topn_rf_null_parquet VALUES (9, null, 'Ivan');
+INSERT INTO test_topn_rf_null_parquet VALUES (10, 1000, 'Judy');
+
+
+
+CREATE TABLE test_topn_rf_null_orc (
+ id INT,
+ value INT,
+ name STRING
+)
+STORED AS ORC;
+
+INSERT INTO test_topn_rf_null_orc VALUES (1, 100, 'Alice');
+INSERT INTO test_topn_rf_null_orc VALUES (2, 200, null);
+INSERT INTO test_topn_rf_null_orc VALUES (3, 300, 'Charlie');
+INSERT INTO test_topn_rf_null_orc VALUES (4, 400, 'David');
+INSERT INTO test_topn_rf_null_orc VALUES (5, null, null);
+INSERT INTO test_topn_rf_null_orc VALUES (6, 600, 'Frank');
+INSERT INTO test_topn_rf_null_orc VALUES (7, null, 'Grace');
+INSERT INTO test_topn_rf_null_orc VALUES (8, 800, null);
+INSERT INTO test_topn_rf_null_orc VALUES (9, null, 'Ivan');
+INSERT INTO test_topn_rf_null_orc VALUES (10, 1000, 'Judy');
+
+
+
+
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_topn_rf_null.out
b/regression-test/data/external_table_p0/hive/test_hive_topn_rf_null.out
new file mode 100644
index 00000000000..6cccfa3d1dc
--- /dev/null
+++ b/regression-test/data/external_table_p0/hive/test_hive_topn_rf_null.out
@@ -0,0 +1,489 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql_test_1 --
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+5 \N \N
+
+-- !sql_test_2 --
+10 1000
+9 \N
+8 800
+
+-- !sql_test_3 --
+\N Ivan
+\N Grace
+\N \N
+100 Alice
+200 \N
+
+-- !sql_test_4 --
+10
+8
+6
+4
+3
+
+-- !sql_test_5 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_6 --
+10 Judy
+9 Ivan
+7 Grace
+6 Frank
+4 David
+
+-- !sql_test_7 --
+1000
+800
+600
+
+-- !sql_test_8 --
+1 Alice
+2 \N
+3 Charlie
+4 David
+5 \N
+6 Frank
+7 Grace
+
+-- !sql_test_9 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+6 600 Frank
+8 800 \N
+10 1000 Judy
+
+-- !sql_test_10 --
+\N
+\N
+
+-- !sql_test_11 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+
+-- !sql_test_12 --
+1 100
+2 200
+3 300
+4 400
+6 600
+
+-- !sql_test_13 --
+\N Ivan
+\N Grace
+\N \N
+1000 Judy
+800 \N
+
+-- !sql_test_14 --
+10
+8
+6
+4
+3
+
+-- !sql_test_15 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_16 --
+1 Alice
+3 Charlie
+4 David
+6 Frank
+7 Grace
+
+-- !sql_test_17 --
+200
+\N
+800
+
+-- !sql_test_18 --
+10 1000 Judy
+9 \N Ivan
+7 \N Grace
+
+-- !sql_test_1 --
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+5 \N \N
+
+-- !sql_test_2 --
+10 1000
+9 \N
+8 800
+
+-- !sql_test_3 --
+\N Ivan
+\N Grace
+\N \N
+100 Alice
+200 \N
+
+-- !sql_test_4 --
+10
+8
+6
+4
+3
+
+-- !sql_test_5 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_6 --
+10 Judy
+9 Ivan
+7 Grace
+6 Frank
+4 David
+
+-- !sql_test_7 --
+1000
+800
+600
+
+-- !sql_test_8 --
+1 Alice
+2 \N
+3 Charlie
+4 David
+5 \N
+6 Frank
+7 Grace
+
+-- !sql_test_9 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+6 600 Frank
+8 800 \N
+10 1000 Judy
+
+-- !sql_test_10 --
+\N
+\N
+
+-- !sql_test_11 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+
+-- !sql_test_12 --
+1 100
+2 200
+3 300
+4 400
+6 600
+
+-- !sql_test_13 --
+\N Ivan
+\N Grace
+\N \N
+1000 Judy
+800 \N
+
+-- !sql_test_14 --
+10
+8
+6
+4
+3
+
+-- !sql_test_15 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_16 --
+1 Alice
+3 Charlie
+4 David
+6 Frank
+7 Grace
+
+-- !sql_test_17 --
+200
+\N
+800
+
+-- !sql_test_18 --
+10 1000 Judy
+9 \N Ivan
+7 \N Grace
+
+-- !sql_test_1 --
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+5 \N \N
+
+-- !sql_test_2 --
+10 1000
+9 \N
+8 800
+
+-- !sql_test_3 --
+\N Ivan
+\N Grace
+\N \N
+100 Alice
+200 \N
+
+-- !sql_test_4 --
+10
+8
+6
+4
+3
+
+-- !sql_test_5 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_6 --
+10 Judy
+9 Ivan
+7 Grace
+6 Frank
+4 David
+
+-- !sql_test_7 --
+1000
+800
+600
+
+-- !sql_test_8 --
+1 Alice
+2 \N
+3 Charlie
+4 David
+5 \N
+6 Frank
+7 Grace
+
+-- !sql_test_9 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+6 600 Frank
+8 800 \N
+10 1000 Judy
+
+-- !sql_test_10 --
+\N
+\N
+
+-- !sql_test_11 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+
+-- !sql_test_12 --
+1 100
+2 200
+3 300
+4 400
+6 600
+
+-- !sql_test_13 --
+\N Ivan
+\N Grace
+\N \N
+1000 Judy
+800 \N
+
+-- !sql_test_14 --
+10
+8
+6
+4
+3
+
+-- !sql_test_15 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_16 --
+1 Alice
+3 Charlie
+4 David
+6 Frank
+7 Grace
+
+-- !sql_test_17 --
+200
+\N
+800
+
+-- !sql_test_18 --
+10 1000 Judy
+9 \N Ivan
+7 \N Grace
+
+-- !sql_test_1 --
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+5 \N \N
+
+-- !sql_test_2 --
+10 1000
+9 \N
+8 800
+
+-- !sql_test_3 --
+\N Ivan
+\N Grace
+\N \N
+100 Alice
+200 \N
+
+-- !sql_test_4 --
+10
+8
+6
+4
+3
+
+-- !sql_test_5 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_6 --
+10 Judy
+9 Ivan
+7 Grace
+6 Frank
+4 David
+
+-- !sql_test_7 --
+1000
+800
+600
+
+-- !sql_test_8 --
+1 Alice
+2 \N
+3 Charlie
+4 David
+5 \N
+6 Frank
+7 Grace
+
+-- !sql_test_9 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+3 300 Charlie
+4 400 David
+6 600 Frank
+8 800 \N
+10 1000 Judy
+
+-- !sql_test_10 --
+\N
+\N
+
+-- !sql_test_11 --
+9 \N Ivan
+7 \N Grace
+5 \N \N
+1 100 Alice
+2 200 \N
+
+-- !sql_test_12 --
+1 100
+2 200
+3 300
+4 400
+6 600
+
+-- !sql_test_13 --
+\N Ivan
+\N Grace
+\N \N
+1000 Judy
+800 \N
+
+-- !sql_test_14 --
+10
+8
+6
+4
+3
+
+-- !sql_test_15 --
+\N
+\N
+\N
+Alice
+Charlie
+
+-- !sql_test_16 --
+1 Alice
+3 Charlie
+4 David
+6 Frank
+7 Grace
+
+-- !sql_test_17 --
+200
+\N
+800
+
+-- !sql_test_18 --
+10 1000 Judy
+9 \N Ivan
+7 \N Grace
+
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_topn_rf_null.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_topn_rf_null.groovy
new file mode 100644
index 00000000000..a191a086cbd
--- /dev/null
+++
b/regression-test/suites/external_table_p0/hive/test_hive_topn_rf_null.groovy
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_topn_rf_null",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test.")
+ return;
+ }
+ def runTopnRfNullTest = {
+ for (String table_name in ["test_topn_rf_null_orc",
"test_topn_rf_null_parquet"]) {
+
+ qt_sql_test_1 """SELECT id, value, name FROM ${table_name} ORDER
BY id ASC LIMIT 5;"""
+ qt_sql_test_2 """SELECT id, value FROM ${table_name} ORDER BY id
DESC LIMIT 3;"""
+ qt_sql_test_3 """SELECT value, name FROM ${table_name} ORDER BY
value ASC LIMIT 5;"""
+ qt_sql_test_4 """SELECT id FROM ${table_name} ORDER BY value DESC
LIMIT 5;"""
+ qt_sql_test_5 """SELECT name FROM ${table_name} ORDER BY name ASC
LIMIT 5;"""
+ qt_sql_test_6 """SELECT id, name FROM ${table_name} ORDER BY name
DESC LIMIT 5;"""
+ qt_sql_test_7 """SELECT value FROM ${table_name} ORDER BY value
DESC LIMIT 3;"""
+ qt_sql_test_8 """SELECT id, name FROM ${table_name} ORDER BY id
ASC LIMIT 7;"""
+ qt_sql_test_9 """SELECT * FROM ${table_name} ORDER BY value ASC
LIMIT 10;"""
+ qt_sql_test_10 """SELECT name FROM ${table_name} ORDER BY name ASC
LIMIT 2;"""
+ qt_sql_test_11 """SELECT id, value, name FROM ${table_name} ORDER
BY value ASC NULLS FIRST LIMIT 5;"""
+ qt_sql_test_12 """SELECT id, value FROM ${table_name} ORDER BY
value ASC NULLS LAST LIMIT 5;"""
+ qt_sql_test_13 """SELECT value, name FROM ${table_name} ORDER BY
value DESC NULLS FIRST LIMIT 5;"""
+ qt_sql_test_14 """SELECT id FROM ${table_name} ORDER BY value DESC
NULLS LAST LIMIT 5;"""
+ qt_sql_test_15 """SELECT name FROM ${table_name} ORDER BY name ASC
NULLS FIRST LIMIT 5;"""
+ qt_sql_test_16 """SELECT id, name FROM ${table_name} ORDER BY name
ASC NULLS LAST LIMIT 5;"""
+ qt_sql_test_17 """SELECT value FROM ${table_name} ORDER BY name
DESC NULLS FIRST LIMIT 3;"""
+ qt_sql_test_18 """SELECT * FROM ${table_name} ORDER BY name DESC
NULLS LAST LIMIT 3;"""
+ }
+ }
+
+
+
+
+ for (String hivePrefix : ["hive3"]) {
+ String hms_port = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog = "test_hive_topn_rf_null_${hivePrefix}"
+
+ sql """drop catalog if exists ${catalog}"""
+ sql """create catalog if not exists ${catalog} properties (
+ 'type'='hms',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ logger.info("catalog " + catalog + " created")
+ sql """switch ${catalog};"""
+
+ sql """ use `default`; """
+ sql """ set num_scanner_threads = 1 """
+
+ sql """ set topn_filter_ratio=1"""
+ runTopnRfNullTest();
+
+
+
+ sql """ set topn_filter_ratio=0 """
+ runTopnRfNullTest();
+
+
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
