This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new e5ef0aa6d4 [refactor](mysql result format) use new serde framework to 
tuple convert (#25006)
e5ef0aa6d4 is described below

commit e5ef0aa6d439c3f9b1f1fe5bc89c9ea6a71d4019
Author: zhiqiang <seuhezhiqi...@163.com>
AuthorDate: Sat Oct 14 06:46:42 2023 -0500

    [refactor](mysql result format) use new serde framework to tuple convert 
(#25006)
---
 be/cmake/thirdparty.cmake                          |  2 +
 be/src/gutil/CMakeLists.txt                        |  2 +
 be/src/gutil/strings/numbers.cc                    | 13 ++++-
 be/src/gutil/strings/numbers.h                     |  4 +-
 be/src/runtime/runtime_state.h                     |  4 ++
 be/src/util/mysql_row_buffer.cpp                   | 10 ++--
 be/src/util/mysql_row_buffer.h                     |  4 ++
 be/src/vec/sink/vmysql_result_writer.cpp           |  3 ++
 be/src/vec/sink/vmysql_result_writer.h             |  2 +
 be/test/util/mysql_row_buffer_test.cpp             | 58 ++++++++++++++++++++++
 .../main/java/org/apache/doris/qe/Coordinator.java |  2 +
 .../java/org/apache/doris/qe/SessionVariable.java  | 12 +++++
 gensrc/thrift/PaloInternalService.thrift           |  4 +-
 .../mysql_tupleconvert_p0/mysql_tupleconvert.out   | 10 ++++
 .../mysql_tupleconvert.groovy                      | 40 +++++++++++++++
 15 files changed, 160 insertions(+), 10 deletions(-)

diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 60d9c65055..5adf108ab6 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -164,6 +164,8 @@ add_thirdparty(krb5)
 add_thirdparty(com_err)
 add_thirdparty(k5crypto)
 add_thirdparty(gssapi_krb5)
+add_thirdparty(dragonbox_to_chars LIB64)
+target_include_directories(dragonbox_to_chars INTERFACE 
"${THIRDPARTY_DIR}/include/dragonbox-1.1.3")
 
 if (OS_MACOSX)
     add_thirdparty(bfd)
diff --git a/be/src/gutil/CMakeLists.txt b/be/src/gutil/CMakeLists.txt
index bac9aec303..12c2330884 100644
--- a/be/src/gutil/CMakeLists.txt
+++ b/be/src/gutil/CMakeLists.txt
@@ -57,6 +57,8 @@ add_library(Gutil STATIC ${SOURCE_FILES})
 
 set_target_properties(Gutil PROPERTIES COMPILE_FLAGS "-funsigned-char 
-Wno-deprecated")
 target_compile_options(Gutil PRIVATE -Wno-char-subscripts 
-Wno-implicit-fallthrough)
+target_link_libraries(Gutil PRIVATE dragonbox_to_chars)
+
 # target_link_libraries(Gutil glog protobuf rt)
 
 #set(GUTIL_LIBS
diff --git a/be/src/gutil/strings/numbers.cc b/be/src/gutil/strings/numbers.cc
index 5550757d40..9aea10155f 100644
--- a/be/src/gutil/strings/numbers.cc
+++ b/be/src/gutil/strings/numbers.cc
@@ -27,6 +27,7 @@ using std::string;
 #include "common/logging.h"
 #include <fmt/format.h>
 
+#include "dragonbox/dragonbox_to_chars.h"
 #include "gutil/gscoped_ptr.h"
 #include "gutil/int128.h"
 #include "gutil/integral_types.h"
@@ -1274,7 +1275,11 @@ int FloatToBuffer(float value, int width, char* buffer) {
     return snprintf_result;
 }
 
-int FastDoubleToBuffer(double value, char* buffer) {
+int FastDoubleToBuffer(double value, char* buffer, bool faster_float_convert) {
+    if (faster_float_convert) {
+        return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
+    }
+ 
     auto end = fmt::format_to(buffer, "{:.15g}", value);
     *end = '\0';
     if (strtod(buffer, nullptr) != value) {
@@ -1283,7 +1288,11 @@ int FastDoubleToBuffer(double value, char* buffer) {
     return end - buffer;
 }
 
-int FastFloatToBuffer(float value, char* buffer) {
+int FastFloatToBuffer(float value, char* buffer, bool faster_float_convert) {
+    if (faster_float_convert) {
+        return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
+    }
+
     auto end = fmt::format_to(buffer, "{:.6g}", value);
     *end = '\0';
 #ifdef _MSC_VER // has no strtof()
diff --git a/be/src/gutil/strings/numbers.h b/be/src/gutil/strings/numbers.h
index 8e2478794a..e7c6f93826 100644
--- a/be/src/gutil/strings/numbers.h
+++ b/be/src/gutil/strings/numbers.h
@@ -443,8 +443,8 @@ int FloatToBuffer(float i, int width, char* buffer);
 char* DoubleToBuffer(double i, char* buffer);
 char* FloatToBuffer(float i, char* buffer);
 
-int FastDoubleToBuffer(double i, char* buffer);
-int FastFloatToBuffer(float i, char* buffer);
+int FastDoubleToBuffer(double i, char* buffer, bool faster_float_convert = 
false);
+int FastFloatToBuffer(float i, char* buffer, bool faster_float_convert = 
false);
 // In practice, doubles should never need more than 24 bytes and floats
 // should never need more than 14 (including null terminators), but we
 // overestimate to be safe.
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 9396c41d96..06ceb699e7 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -151,6 +151,10 @@ public:
                _query_options.enable_common_expr_pushdown;
     }
 
+    bool enable_faster_float_convert() const {
+        return _query_options.__isset.faster_float_convert && 
_query_options.faster_float_convert;
+    }
+
     Status query_status();
 
     // Appends error to the _error_log if there is space
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index 00193481b7..e671ba2d01 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -174,12 +174,12 @@ static char* add_largeint(int128_t data, char* pos, bool 
dynamic_mode) {
 }
 
 template <typename T>
-char* add_float(T data, char* pos, bool dynamic_mode) {
+char* add_float(T data, char* pos, bool dynamic_mode, bool 
faster_float_convert = false) {
     int length = 0;
     if constexpr (std::is_same_v<T, float>) {
-        length = FastFloatToBuffer(data, pos + !dynamic_mode);
+        length = FastFloatToBuffer(data, pos + !dynamic_mode, 
faster_float_convert);
     } else if constexpr (std::is_same_v<T, double>) {
-        length = FastDoubleToBuffer(data, pos + !dynamic_mode);
+        length = FastDoubleToBuffer(data, pos + !dynamic_mode, 
faster_float_convert);
     }
     if (!dynamic_mode) {
         int1store(pos++, length);
@@ -385,7 +385,7 @@ int MysqlRowBuffer<is_binary_format>::push_float(float 
data) {
         return ret;
     }
 
-    _pos = add_float(data, _pos, _dynamic_mode);
+    _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
     return 0;
 }
 
@@ -405,7 +405,7 @@ int MysqlRowBuffer<is_binary_format>::push_double(double 
data) {
         return ret;
     }
 
-    _pos = add_float(data, _pos, _dynamic_mode);
+    _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
     return 0;
 }
 
diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h
index 6a6c2b0ad0..eec21feead 100644
--- a/be/src/util/mysql_row_buffer.h
+++ b/be/src/util/mysql_row_buffer.h
@@ -124,6 +124,8 @@ public:
      */
     void close_dynamic_mode();
 
+    void set_faster_float_convert(bool faster) { _faster_float_convert = 
faster; }
+
 private:
     int reserve(int64_t size);
 
@@ -142,6 +144,8 @@ private:
     uint64_t _len_pos;
     uint32_t _field_pos = 0;
     uint32_t _field_count = 0;
+
+    bool _faster_float_convert = false;
 };
 
 } // namespace doris
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp 
b/be/src/vec/sink/vmysql_result_writer.cpp
index bd372ffcbf..4452014b48 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -90,6 +90,8 @@ Status 
VMysqlResultWriter<is_binary_format>::init(RuntimeState* state) {
     }
     set_output_object_data(state->return_object_data_as_binary());
     _is_dry_run = state->query_options().dry_run_query;
+    _enable_faster_float_convert = state->enable_faster_float_convert();
+
     return Status::OK();
 }
 
@@ -126,6 +128,7 @@ Status 
VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
     {
         SCOPED_TIMER(_convert_tuple_timer);
         MysqlRowBuffer<is_binary_format> row_buffer;
+        row_buffer.set_faster_float_convert(_enable_faster_float_convert);
         if constexpr (is_binary_format) {
             row_buffer.start_binary_row(_output_vexpr_ctxs.size());
         }
diff --git a/be/src/vec/sink/vmysql_result_writer.h 
b/be/src/vec/sink/vmysql_result_writer.h
index f86b62b7e2..2e1ae53fff 100644
--- a/be/src/vec/sink/vmysql_result_writer.h
+++ b/be/src/vec/sink/vmysql_result_writer.h
@@ -89,6 +89,8 @@ private:
     bool _is_dry_run = false;
 
     uint64_t _bytes_sent = 0;
+
+    bool _enable_faster_float_convert = false;
 };
 } // namespace vectorized
 } // namespace doris
diff --git a/be/test/util/mysql_row_buffer_test.cpp 
b/be/test/util/mysql_row_buffer_test.cpp
index cfe012fdcd..3e2380b6c1 100644
--- a/be/test/util/mysql_row_buffer_test.cpp
+++ b/be/test/util/mysql_row_buffer_test.cpp
@@ -118,4 +118,62 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
     EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
 }
 
+TEST(MysqlRowBufferTest /*unused*/, faster_float_convert /*unused*/) {
+    MysqlRowBuffer mrb;
+    mrb.set_faster_float_convert(true);
+
+    mrb.push_float(0);
+    mrb.push_float(1.0);
+    mrb.push_float(-1.0);
+    mrb.push_float(56.45);
+    mrb.push_double(10.12);
+
+    const char* buf = mrb.buf();
+
+    // mem: size-data-size-data
+    // 3-'0E0'-3-'1E0'-4-'-1E0'-7-'5.645E1'-7-'1.012E1'
+    // 1b-3b---1b-3b---1b-4b----1b-7b-------1b-7b------
+    // 0  1    4  5    8  9     13 14       21 22
+
+    EXPECT_EQ(29, mrb.length());
+
+    EXPECT_EQ(3, *((int8_t*)(buf + 0)));
+    EXPECT_EQ(0, strncmp(buf + 1, "0E0", 3));
+
+    EXPECT_EQ(3, *((int8_t*)(buf + 4)));
+    EXPECT_EQ(0, strncmp(buf + 5, "1E0", 3));
+
+    EXPECT_EQ(4, *((int8_t*)(buf + 8)));
+    EXPECT_EQ(0, strncmp(buf + 9, "-1E0", 4));
+
+    EXPECT_EQ(7, *((int8_t*)(buf + 13)));
+    EXPECT_EQ(0, strncmp(buf + 14, "5.645E1", 7));
+
+    EXPECT_EQ(7, *((int8_t*)(buf + 21)));
+    EXPECT_EQ(0, strncmp(buf + 22, "1.012E1", 7));
+}
+
+TEST(MysqlRowBufferTest /*unused*/, faster_float_convert_dynamic /*unused*/) {
+    MysqlRowBuffer mrb;
+    mrb.set_faster_float_convert(true);
+    mrb.open_dynamic_mode();
+
+    mrb.push_float(0);
+    mrb.push_float(1.0);
+    mrb.push_float(-1.0);
+    mrb.push_float(56.45);
+    mrb.push_double(10.12);
+
+    const char* buf = mrb.buf();
+
+    EXPECT_EQ(33, mrb.length());
+    EXPECT_EQ(254, *((uint8_t*)(buf)));
+
+    EXPECT_EQ(0, strncmp(buf + 9, "0E0", 3));
+    EXPECT_EQ(0, strncmp(buf + 12, "1E0", 3));
+    EXPECT_EQ(0, strncmp(buf + 15, "-1E0", 4));
+    EXPECT_EQ(0, strncmp(buf + 19, "5.645E1", 7));
+    EXPECT_EQ(0, strncmp(buf + 26, "1.012E1", 7));
+}
+
 } // namespace doris
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 5247973f57..c84fb90f2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -253,6 +253,7 @@ public class Coordinator implements CoordInterface {
 
     private boolean enablePipelineEngine = false;
     private boolean enablePipelineXEngine = false;
+    private boolean fasterFloatConvert = false;
 
     // Runtime filter merge instance address and ID
     public TNetworkAddress runtimeFilterMergeAddr;
@@ -324,6 +325,7 @@ public class Coordinator implements CoordInterface {
                 && (fragments.size() > 0);
         this.enablePipelineXEngine = 
context.getSessionVariable().getEnablePipelineXEngine()
                 && (fragments.size() > 0);
+        this.fasterFloatConvert = 
context.getSessionVariable().fasterFloatConvert();
 
         initQueryOptions(context);
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a236918e18..da7e5a10d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -416,6 +416,8 @@ public class SessionVariable implements Serializable, 
Writable {
 
     public static final String ENABLE_FULL_AUTO_ANALYZE = 
"enable_full_auto_analyze";
 
+    public static final String FASTER_FLOAT_CONVERT = "faster_float_convert";
+
     public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
             SKIP_DELETE_PREDICATE,
             SKIP_DELETE_BITMAP,
@@ -1224,6 +1226,10 @@ public class SessionVariable implements Serializable, 
Writable {
             flag = VariableMgr.GLOBAL)
     public boolean enableFullAutoAnalyze = true;
 
+    @VariableMgr.VarAttr(name = FASTER_FLOAT_CONVERT,
+            description = {"是否启用更快的浮点数转换算法,注意会影响输出格式", "Set true to enable 
faster float pointer number convert"})
+    public boolean fasterFloatConvert = false;
+
     // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to 
generate some variables,
     // not the default value set in the code.
     public void initFuzzyModeVariables() {
@@ -2345,6 +2351,8 @@ public class SessionVariable implements Serializable, 
Writable {
 
         
tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold);
 
+        tResult.setFasterFloatConvert(fasterFloatConvert);
+
         return tResult;
     }
 
@@ -2679,4 +2687,8 @@ public class SessionVariable implements Serializable, 
Writable {
     public int getProfileLevel() {
         return this.profileLevel;
     }
+
+    public boolean fasterFloatConvert() {
+        return this.fasterFloatConvert;
+    }
 }
diff --git a/gensrc/thrift/PaloInternalService.thrift 
b/gensrc/thrift/PaloInternalService.thrift
index cc7102e408..a56e4f98a4 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -246,7 +246,9 @@ struct TQueryOptions {
   // use is_report_success any more
   84: optional bool enable_profile = false;
   85: optional bool enable_page_cache = false;
-  86: optional i32 analyze_timeout = 43200
+  86: optional i32 analyze_timeout = 43200;
+
+  87: optional bool faster_float_convert = false;
 }
 
 
diff --git a/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out 
b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
new file mode 100644
index 0000000000..c27a9d00b5
--- /dev/null
+++ b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you 
want to edit this
+-- !select --
+1      0.33    3.1415925       [0.33, 0.67]    [3.1415926, 0.878787878]
+
+-- !select --
+1      0.33    3.1415925       [3.3E-1, 6.7E-1]        [3.1415926E0, 
8.78787878E-1]
+
+-- !select --
+1      0.33    3.1415925       [0.33, 0.67]    [3.1415926, 0.878787878]
+2      0.33    3.1415925       [0.33, 0.67]    [3.1415926, 0.878787878]
diff --git 
a/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy 
b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
new file mode 100644
index 0000000000..70278a9765
--- /dev/null
+++ b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("mysql_tuple_convert_test") {
+    
+    testTable = "doule_faster_convert_test"
+    sql "DROP TABLE IF EXISTS ${testTable}"
+    sql """
+        CREATE TABLE IF NOT EXISTS ${testTable} (
+            `col_0` INT,
+            `col_1` DOUBLE,
+            `col_2` FLOAT,
+            `col_3` ARRAY<FLOAT>,
+            `col_4` ARRAY<DOUBLE>)
+        ENGINE=OLAP DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) 
BUCKETS 1
+        PROPERTIES("replication_num" = "1");
+    """
+    sql """INSERT INTO ${testTable} VALUES (1, 0.33, 3.1415926, [0.33, 0.67], 
[3.1415926, 0.878787878]);"""
+    qt_select """SELECT /*+SET_VAR(faster_float_convert=false)*/ * FROM 
${testTable};"""
+    qt_select """SELECT /*+SET_VAR(faster_float_convert=true)*/ * FROM 
${testTable};"""
+    // make sure we can convert number from string to value
+    sql """INSERT INTO ${testTable} SELECT 2, col_1, col_2, col_3, col_4 from 
${testTable} where col_0 = 1;"""
+    qt_select """SELECT * FROM ${testTable};"""
+    sql """SET faster_float_convert=false;"""
+    sql "DROP TABLE IF EXISTS ${testTable}"
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Reply via email to