This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new e5ef0aa6d4 [refactor](mysql result format) use new serde framework to tuple convert (#25006) e5ef0aa6d4 is described below commit e5ef0aa6d439c3f9b1f1fe5bc89c9ea6a71d4019 Author: zhiqiang <seuhezhiqi...@163.com> AuthorDate: Sat Oct 14 06:46:42 2023 -0500 [refactor](mysql result format) use new serde framework to tuple convert (#25006) --- be/cmake/thirdparty.cmake | 2 + be/src/gutil/CMakeLists.txt | 2 + be/src/gutil/strings/numbers.cc | 13 ++++- be/src/gutil/strings/numbers.h | 4 +- be/src/runtime/runtime_state.h | 4 ++ be/src/util/mysql_row_buffer.cpp | 10 ++-- be/src/util/mysql_row_buffer.h | 4 ++ be/src/vec/sink/vmysql_result_writer.cpp | 3 ++ be/src/vec/sink/vmysql_result_writer.h | 2 + be/test/util/mysql_row_buffer_test.cpp | 58 ++++++++++++++++++++++ .../main/java/org/apache/doris/qe/Coordinator.java | 2 + .../java/org/apache/doris/qe/SessionVariable.java | 12 +++++ gensrc/thrift/PaloInternalService.thrift | 4 +- .../mysql_tupleconvert_p0/mysql_tupleconvert.out | 10 ++++ .../mysql_tupleconvert.groovy | 40 +++++++++++++++ 15 files changed, 160 insertions(+), 10 deletions(-) diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 60d9c65055..5adf108ab6 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -164,6 +164,8 @@ add_thirdparty(krb5) add_thirdparty(com_err) add_thirdparty(k5crypto) add_thirdparty(gssapi_krb5) +add_thirdparty(dragonbox_to_chars LIB64) +target_include_directories(dragonbox_to_chars INTERFACE "${THIRDPARTY_DIR}/include/dragonbox-1.1.3") if (OS_MACOSX) add_thirdparty(bfd) diff --git a/be/src/gutil/CMakeLists.txt b/be/src/gutil/CMakeLists.txt index bac9aec303..12c2330884 100644 --- a/be/src/gutil/CMakeLists.txt +++ b/be/src/gutil/CMakeLists.txt @@ -57,6 +57,8 @@ add_library(Gutil STATIC ${SOURCE_FILES}) set_target_properties(Gutil PROPERTIES COMPILE_FLAGS "-funsigned-char -Wno-deprecated") target_compile_options(Gutil PRIVATE -Wno-char-subscripts -Wno-implicit-fallthrough) +target_link_libraries(Gutil PRIVATE dragonbox_to_chars) + # target_link_libraries(Gutil glog protobuf rt) #set(GUTIL_LIBS diff --git a/be/src/gutil/strings/numbers.cc b/be/src/gutil/strings/numbers.cc index 5550757d40..9aea10155f 100644 --- a/be/src/gutil/strings/numbers.cc +++ b/be/src/gutil/strings/numbers.cc @@ -27,6 +27,7 @@ using std::string; #include "common/logging.h" #include <fmt/format.h> +#include "dragonbox/dragonbox_to_chars.h" #include "gutil/gscoped_ptr.h" #include "gutil/int128.h" #include "gutil/integral_types.h" @@ -1274,7 +1275,11 @@ int FloatToBuffer(float value, int width, char* buffer) { return snprintf_result; } -int FastDoubleToBuffer(double value, char* buffer) { +int FastDoubleToBuffer(double value, char* buffer, bool faster_float_convert) { + if (faster_float_convert) { + return jkj::dragonbox::to_chars_n(value, buffer) - buffer; + } + auto end = fmt::format_to(buffer, "{:.15g}", value); *end = '\0'; if (strtod(buffer, nullptr) != value) { @@ -1283,7 +1288,11 @@ int FastDoubleToBuffer(double value, char* buffer) { return end - buffer; } -int FastFloatToBuffer(float value, char* buffer) { +int FastFloatToBuffer(float value, char* buffer, bool faster_float_convert) { + if (faster_float_convert) { + return jkj::dragonbox::to_chars_n(value, buffer) - buffer; + } + auto end = fmt::format_to(buffer, "{:.6g}", value); *end = '\0'; #ifdef _MSC_VER // has no strtof() diff --git a/be/src/gutil/strings/numbers.h b/be/src/gutil/strings/numbers.h index 8e2478794a..e7c6f93826 100644 --- a/be/src/gutil/strings/numbers.h +++ b/be/src/gutil/strings/numbers.h @@ -443,8 +443,8 @@ int FloatToBuffer(float i, int width, char* buffer); char* DoubleToBuffer(double i, char* buffer); char* FloatToBuffer(float i, char* buffer); -int FastDoubleToBuffer(double i, char* buffer); -int FastFloatToBuffer(float i, char* buffer); +int FastDoubleToBuffer(double i, char* buffer, bool faster_float_convert = false); +int FastFloatToBuffer(float i, char* buffer, bool faster_float_convert = false); // In practice, doubles should never need more than 24 bytes and floats // should never need more than 14 (including null terminators), but we // overestimate to be safe. diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 9396c41d96..06ceb699e7 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -151,6 +151,10 @@ public: _query_options.enable_common_expr_pushdown; } + bool enable_faster_float_convert() const { + return _query_options.__isset.faster_float_convert && _query_options.faster_float_convert; + } + Status query_status(); // Appends error to the _error_log if there is space diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index 00193481b7..e671ba2d01 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -174,12 +174,12 @@ static char* add_largeint(int128_t data, char* pos, bool dynamic_mode) { } template <typename T> -char* add_float(T data, char* pos, bool dynamic_mode) { +char* add_float(T data, char* pos, bool dynamic_mode, bool faster_float_convert = false) { int length = 0; if constexpr (std::is_same_v<T, float>) { - length = FastFloatToBuffer(data, pos + !dynamic_mode); + length = FastFloatToBuffer(data, pos + !dynamic_mode, faster_float_convert); } else if constexpr (std::is_same_v<T, double>) { - length = FastDoubleToBuffer(data, pos + !dynamic_mode); + length = FastDoubleToBuffer(data, pos + !dynamic_mode, faster_float_convert); } if (!dynamic_mode) { int1store(pos++, length); @@ -385,7 +385,7 @@ int MysqlRowBuffer<is_binary_format>::push_float(float data) { return ret; } - _pos = add_float(data, _pos, _dynamic_mode); + _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert); return 0; } @@ -405,7 +405,7 @@ int MysqlRowBuffer<is_binary_format>::push_double(double data) { return ret; } - _pos = add_float(data, _pos, _dynamic_mode); + _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert); return 0; } diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h index 6a6c2b0ad0..eec21feead 100644 --- a/be/src/util/mysql_row_buffer.h +++ b/be/src/util/mysql_row_buffer.h @@ -124,6 +124,8 @@ public: */ void close_dynamic_mode(); + void set_faster_float_convert(bool faster) { _faster_float_convert = faster; } + private: int reserve(int64_t size); @@ -142,6 +144,8 @@ private: uint64_t _len_pos; uint32_t _field_pos = 0; uint32_t _field_count = 0; + + bool _faster_float_convert = false; }; } // namespace doris diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp index bd372ffcbf..4452014b48 100644 --- a/be/src/vec/sink/vmysql_result_writer.cpp +++ b/be/src/vec/sink/vmysql_result_writer.cpp @@ -90,6 +90,8 @@ Status VMysqlResultWriter<is_binary_format>::init(RuntimeState* state) { } set_output_object_data(state->return_object_data_as_binary()); _is_dry_run = state->query_options().dry_run_query; + _enable_faster_float_convert = state->enable_faster_float_convert(); + return Status::OK(); } @@ -126,6 +128,7 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) { { SCOPED_TIMER(_convert_tuple_timer); MysqlRowBuffer<is_binary_format> row_buffer; + row_buffer.set_faster_float_convert(_enable_faster_float_convert); if constexpr (is_binary_format) { row_buffer.start_binary_row(_output_vexpr_ctxs.size()); } diff --git a/be/src/vec/sink/vmysql_result_writer.h b/be/src/vec/sink/vmysql_result_writer.h index f86b62b7e2..2e1ae53fff 100644 --- a/be/src/vec/sink/vmysql_result_writer.h +++ b/be/src/vec/sink/vmysql_result_writer.h @@ -89,6 +89,8 @@ private: bool _is_dry_run = false; uint64_t _bytes_sent = 0; + + bool _enable_faster_float_convert = false; }; } // namespace vectorized } // namespace doris diff --git a/be/test/util/mysql_row_buffer_test.cpp b/be/test/util/mysql_row_buffer_test.cpp index cfe012fdcd..3e2380b6c1 100644 --- a/be/test/util/mysql_row_buffer_test.cpp +++ b/be/test/util/mysql_row_buffer_test.cpp @@ -118,4 +118,62 @@ TEST(MysqlRowBufferTest, dynamic_mode) { EXPECT_EQ(0, strncmp(buf + 43, "test", 4)); } +TEST(MysqlRowBufferTest /*unused*/, faster_float_convert /*unused*/) { + MysqlRowBuffer mrb; + mrb.set_faster_float_convert(true); + + mrb.push_float(0); + mrb.push_float(1.0); + mrb.push_float(-1.0); + mrb.push_float(56.45); + mrb.push_double(10.12); + + const char* buf = mrb.buf(); + + // mem: size-data-size-data + // 3-'0E0'-3-'1E0'-4-'-1E0'-7-'5.645E1'-7-'1.012E1' + // 1b-3b---1b-3b---1b-4b----1b-7b-------1b-7b------ + // 0 1 4 5 8 9 13 14 21 22 + + EXPECT_EQ(29, mrb.length()); + + EXPECT_EQ(3, *((int8_t*)(buf + 0))); + EXPECT_EQ(0, strncmp(buf + 1, "0E0", 3)); + + EXPECT_EQ(3, *((int8_t*)(buf + 4))); + EXPECT_EQ(0, strncmp(buf + 5, "1E0", 3)); + + EXPECT_EQ(4, *((int8_t*)(buf + 8))); + EXPECT_EQ(0, strncmp(buf + 9, "-1E0", 4)); + + EXPECT_EQ(7, *((int8_t*)(buf + 13))); + EXPECT_EQ(0, strncmp(buf + 14, "5.645E1", 7)); + + EXPECT_EQ(7, *((int8_t*)(buf + 21))); + EXPECT_EQ(0, strncmp(buf + 22, "1.012E1", 7)); +} + +TEST(MysqlRowBufferTest /*unused*/, faster_float_convert_dynamic /*unused*/) { + MysqlRowBuffer mrb; + mrb.set_faster_float_convert(true); + mrb.open_dynamic_mode(); + + mrb.push_float(0); + mrb.push_float(1.0); + mrb.push_float(-1.0); + mrb.push_float(56.45); + mrb.push_double(10.12); + + const char* buf = mrb.buf(); + + EXPECT_EQ(33, mrb.length()); + EXPECT_EQ(254, *((uint8_t*)(buf))); + + EXPECT_EQ(0, strncmp(buf + 9, "0E0", 3)); + EXPECT_EQ(0, strncmp(buf + 12, "1E0", 3)); + EXPECT_EQ(0, strncmp(buf + 15, "-1E0", 4)); + EXPECT_EQ(0, strncmp(buf + 19, "5.645E1", 7)); + EXPECT_EQ(0, strncmp(buf + 26, "1.012E1", 7)); +} + } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 5247973f57..c84fb90f2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -253,6 +253,7 @@ public class Coordinator implements CoordInterface { private boolean enablePipelineEngine = false; private boolean enablePipelineXEngine = false; + private boolean fasterFloatConvert = false; // Runtime filter merge instance address and ID public TNetworkAddress runtimeFilterMergeAddr; @@ -324,6 +325,7 @@ public class Coordinator implements CoordInterface { && (fragments.size() > 0); this.enablePipelineXEngine = context.getSessionVariable().getEnablePipelineXEngine() && (fragments.size() > 0); + this.fasterFloatConvert = context.getSessionVariable().fasterFloatConvert(); initQueryOptions(context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a236918e18..da7e5a10d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -416,6 +416,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_FULL_AUTO_ANALYZE = "enable_full_auto_analyze"; + public static final String FASTER_FLOAT_CONVERT = "faster_float_convert"; + public static final List<String> DEBUG_VARIABLES = ImmutableList.of( SKIP_DELETE_PREDICATE, SKIP_DELETE_BITMAP, @@ -1224,6 +1226,10 @@ public class SessionVariable implements Serializable, Writable { flag = VariableMgr.GLOBAL) public boolean enableFullAutoAnalyze = true; + @VariableMgr.VarAttr(name = FASTER_FLOAT_CONVERT, + description = {"是否启用更快的浮点数转换算法,注意会影响输出格式", "Set true to enable faster float pointer number convert"}) + public boolean fasterFloatConvert = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. public void initFuzzyModeVariables() { @@ -2345,6 +2351,8 @@ public class SessionVariable implements Serializable, Writable { tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold); + tResult.setFasterFloatConvert(fasterFloatConvert); + return tResult; } @@ -2679,4 +2687,8 @@ public class SessionVariable implements Serializable, Writable { public int getProfileLevel() { return this.profileLevel; } + + public boolean fasterFloatConvert() { + return this.fasterFloatConvert; + } } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index cc7102e408..a56e4f98a4 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -246,7 +246,9 @@ struct TQueryOptions { // use is_report_success any more 84: optional bool enable_profile = false; 85: optional bool enable_page_cache = false; - 86: optional i32 analyze_timeout = 43200 + 86: optional i32 analyze_timeout = 43200; + + 87: optional bool faster_float_convert = false; } diff --git a/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out new file mode 100644 index 0000000000..c27a9d00b5 --- /dev/null +++ b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878] + +-- !select -- +1 0.33 3.1415925 [3.3E-1, 6.7E-1] [3.1415926E0, 8.78787878E-1] + +-- !select -- +1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878] +2 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878] diff --git a/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy new file mode 100644 index 0000000000..70278a9765 --- /dev/null +++ b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("mysql_tuple_convert_test") { + + testTable = "doule_faster_convert_test" + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + `col_0` INT, + `col_1` DOUBLE, + `col_2` FLOAT, + `col_3` ARRAY<FLOAT>, + `col_4` ARRAY<DOUBLE>) + ENGINE=OLAP DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + """ + sql """INSERT INTO ${testTable} VALUES (1, 0.33, 3.1415926, [0.33, 0.67], [3.1415926, 0.878787878]);""" + qt_select """SELECT /*+SET_VAR(faster_float_convert=false)*/ * FROM ${testTable};""" + qt_select """SELECT /*+SET_VAR(faster_float_convert=true)*/ * FROM ${testTable};""" + // make sure we can convert number from string to value + sql """INSERT INTO ${testTable} SELECT 2, col_1, col_2, col_3, col_4 from ${testTable} where col_0 = 1;""" + qt_select """SELECT * FROM ${testTable};""" + sql """SET faster_float_convert=false;""" + sql "DROP TABLE IF EXISTS ${testTable}" +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org