This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new c6d80452bac [refactor](mysql result format) use new serde framework to
tuple conv… (#25526)
c6d80452bac is described below
commit c6d80452bac9acdd7913c1a308dcde658376428c
Author: zhiqiang <seuhezhiqi...@163.com>
AuthorDate: Wed Oct 18 00:10:50 2023 -0500
[refactor](mysql result format) use new serde framework to tuple conv…
(#25526)
* [refactor](mysql result format) use new serde framework to tuple convert
(#25006)
* fix compile
* Update mysql_tupleconvert.groovy
---------
Co-authored-by: yiguolei <676222...@qq.com>
---
be/cmake/thirdparty.cmake | 4 ++
be/src/gutil/CMakeLists.txt | 2 +
be/src/gutil/strings/numbers.cc | 13 ++++-
be/src/gutil/strings/numbers.h | 4 +-
be/src/runtime/runtime_state.h | 4 ++
be/src/util/mysql_row_buffer.cpp | 10 ++--
be/src/util/mysql_row_buffer.h | 4 ++
be/src/vec/sink/vmysql_result_writer.cpp | 3 ++
be/src/vec/sink/vmysql_result_writer.h | 2 +
be/test/util/mysql_row_buffer_test.cpp | 58 ++++++++++++++++++++++
.../main/java/org/apache/doris/qe/Coordinator.java | 5 ++
.../java/org/apache/doris/qe/SessionVariable.java | 11 ++++
gensrc/thrift/PaloInternalService.thrift | 4 +-
.../mysql_tupleconvert_p0/mysql_tupleconvert.out | 10 ++++
.../mysql_tupleconvert.groovy | 40 +++++++++++++++
15 files changed, 164 insertions(+), 10 deletions(-)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index bcb04cdf290..e8ea8c33308 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -283,6 +283,10 @@ set_target_properties(k5crypto PROPERTIES
IMPORTED_LOCATION ${THIRDPARTY_DIR}/li
add_library(gssapi_krb5 STATIC IMPORTED)
set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib/libgssapi_krb5.a)
+add_library(dragonbox_to_chars STATIC IMPORTED)
+set_target_properties(dragonbox_to_chars PROPERTIES IMPORTED_LOCATION
${THIRDPARTY_DIR}/lib/libdragonbox_to_chars.a)
+target_include_directories(dragonbox_to_chars INTERFACE
${THIRDPARTY_DIR}/include/dragonbox-1.1.3)
+
find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)
if (OS_MACOSX)
diff --git a/be/src/gutil/CMakeLists.txt b/be/src/gutil/CMakeLists.txt
index bac9aec3037..12c2330884f 100644
--- a/be/src/gutil/CMakeLists.txt
+++ b/be/src/gutil/CMakeLists.txt
@@ -57,6 +57,8 @@ add_library(Gutil STATIC ${SOURCE_FILES})
set_target_properties(Gutil PROPERTIES COMPILE_FLAGS "-funsigned-char
-Wno-deprecated")
target_compile_options(Gutil PRIVATE -Wno-char-subscripts
-Wno-implicit-fallthrough)
+target_link_libraries(Gutil PRIVATE dragonbox_to_chars)
+
# target_link_libraries(Gutil glog protobuf rt)
#set(GUTIL_LIBS
diff --git a/be/src/gutil/strings/numbers.cc b/be/src/gutil/strings/numbers.cc
index 5550757d403..9aea10155f3 100644
--- a/be/src/gutil/strings/numbers.cc
+++ b/be/src/gutil/strings/numbers.cc
@@ -27,6 +27,7 @@ using std::string;
#include "common/logging.h"
#include <fmt/format.h>
+#include "dragonbox/dragonbox_to_chars.h"
#include "gutil/gscoped_ptr.h"
#include "gutil/int128.h"
#include "gutil/integral_types.h"
@@ -1274,7 +1275,11 @@ int FloatToBuffer(float value, int width, char* buffer) {
return snprintf_result;
}
-int FastDoubleToBuffer(double value, char* buffer) {
+int FastDoubleToBuffer(double value, char* buffer, bool faster_float_convert) {
+ if (faster_float_convert) {
+ return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
+ }
+
auto end = fmt::format_to(buffer, "{:.15g}", value);
*end = '\0';
if (strtod(buffer, nullptr) != value) {
@@ -1283,7 +1288,11 @@ int FastDoubleToBuffer(double value, char* buffer) {
return end - buffer;
}
-int FastFloatToBuffer(float value, char* buffer) {
+int FastFloatToBuffer(float value, char* buffer, bool faster_float_convert) {
+ if (faster_float_convert) {
+ return jkj::dragonbox::to_chars_n(value, buffer) - buffer;
+ }
+
auto end = fmt::format_to(buffer, "{:.6g}", value);
*end = '\0';
#ifdef _MSC_VER // has no strtof()
diff --git a/be/src/gutil/strings/numbers.h b/be/src/gutil/strings/numbers.h
index 8e2478794a0..e7c6f938267 100644
--- a/be/src/gutil/strings/numbers.h
+++ b/be/src/gutil/strings/numbers.h
@@ -443,8 +443,8 @@ int FloatToBuffer(float i, int width, char* buffer);
char* DoubleToBuffer(double i, char* buffer);
char* FloatToBuffer(float i, char* buffer);
-int FastDoubleToBuffer(double i, char* buffer);
-int FastFloatToBuffer(float i, char* buffer);
+int FastDoubleToBuffer(double i, char* buffer, bool faster_float_convert =
false);
+int FastFloatToBuffer(float i, char* buffer, bool faster_float_convert =
false);
// In practice, doubles should never need more than 24 bytes and floats
// should never need more than 14 (including null terminators), but we
// overestimate to be safe.
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 8c2d32b05a4..3c4ebb9feed 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -146,6 +146,10 @@ public:
return _process_status;
}
+ bool enable_faster_float_convert() const {
+ return _query_options.__isset.faster_float_convert &&
_query_options.faster_float_convert;
+ }
+
// Appends error to the _error_log if there is space
bool log_error(const std::string& error);
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index 00193481b71..e671ba2d012 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -174,12 +174,12 @@ static char* add_largeint(int128_t data, char* pos, bool
dynamic_mode) {
}
template <typename T>
-char* add_float(T data, char* pos, bool dynamic_mode) {
+char* add_float(T data, char* pos, bool dynamic_mode, bool
faster_float_convert = false) {
int length = 0;
if constexpr (std::is_same_v<T, float>) {
- length = FastFloatToBuffer(data, pos + !dynamic_mode);
+ length = FastFloatToBuffer(data, pos + !dynamic_mode,
faster_float_convert);
} else if constexpr (std::is_same_v<T, double>) {
- length = FastDoubleToBuffer(data, pos + !dynamic_mode);
+ length = FastDoubleToBuffer(data, pos + !dynamic_mode,
faster_float_convert);
}
if (!dynamic_mode) {
int1store(pos++, length);
@@ -385,7 +385,7 @@ int MysqlRowBuffer<is_binary_format>::push_float(float
data) {
return ret;
}
- _pos = add_float(data, _pos, _dynamic_mode);
+ _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
return 0;
}
@@ -405,7 +405,7 @@ int MysqlRowBuffer<is_binary_format>::push_double(double
data) {
return ret;
}
- _pos = add_float(data, _pos, _dynamic_mode);
+ _pos = add_float(data, _pos, _dynamic_mode, _faster_float_convert);
return 0;
}
diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h
index 6a6c2b0ad0c..eec21feeadb 100644
--- a/be/src/util/mysql_row_buffer.h
+++ b/be/src/util/mysql_row_buffer.h
@@ -124,6 +124,8 @@ public:
*/
void close_dynamic_mode();
+ void set_faster_float_convert(bool faster) { _faster_float_convert =
faster; }
+
private:
int reserve(int64_t size);
@@ -142,6 +144,8 @@ private:
uint64_t _len_pos;
uint32_t _field_pos = 0;
uint32_t _field_count = 0;
+
+ bool _faster_float_convert = false;
};
} // namespace doris
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp
b/be/src/vec/sink/vmysql_result_writer.cpp
index f1094673b0f..75d8606a3e8 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -90,6 +90,8 @@ Status
VMysqlResultWriter<is_binary_format>::init(RuntimeState* state) {
}
set_output_object_data(state->return_object_data_as_binary());
_is_dry_run = state->query_options().dry_run_query;
+ _enable_faster_float_convert = state->enable_faster_float_convert();
+
return Status::OK();
}
@@ -619,6 +621,7 @@ Status
VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
{
SCOPED_TIMER(_convert_tuple_timer);
MysqlRowBuffer<is_binary_format> row_buffer;
+ row_buffer.set_faster_float_convert(_enable_faster_float_convert);
if constexpr (is_binary_format) {
row_buffer.start_binary_row(_output_vexpr_ctxs.size());
}
diff --git a/be/src/vec/sink/vmysql_result_writer.h
b/be/src/vec/sink/vmysql_result_writer.h
index b1954f2b019..1761a1a3bef 100644
--- a/be/src/vec/sink/vmysql_result_writer.h
+++ b/be/src/vec/sink/vmysql_result_writer.h
@@ -89,6 +89,8 @@ private:
bool _is_dry_run = false;
uint64_t _bytes_sent = 0;
+
+ bool _enable_faster_float_convert = false;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/test/util/mysql_row_buffer_test.cpp
b/be/test/util/mysql_row_buffer_test.cpp
index cfe012fdcd5..3e2380b6c11 100644
--- a/be/test/util/mysql_row_buffer_test.cpp
+++ b/be/test/util/mysql_row_buffer_test.cpp
@@ -118,4 +118,62 @@ TEST(MysqlRowBufferTest, dynamic_mode) {
EXPECT_EQ(0, strncmp(buf + 43, "test", 4));
}
+TEST(MysqlRowBufferTest /*unused*/, faster_float_convert /*unused*/) {
+ MysqlRowBuffer mrb;
+ mrb.set_faster_float_convert(true);
+
+ mrb.push_float(0);
+ mrb.push_float(1.0);
+ mrb.push_float(-1.0);
+ mrb.push_float(56.45);
+ mrb.push_double(10.12);
+
+ const char* buf = mrb.buf();
+
+ // mem: size-data-size-data
+ // 3-'0E0'-3-'1E0'-4-'-1E0'-7-'5.645E1'-7-'1.012E1'
+ // 1b-3b---1b-3b---1b-4b----1b-7b-------1b-7b------
+ // 0 1 4 5 8 9 13 14 21 22
+
+ EXPECT_EQ(29, mrb.length());
+
+ EXPECT_EQ(3, *((int8_t*)(buf + 0)));
+ EXPECT_EQ(0, strncmp(buf + 1, "0E0", 3));
+
+ EXPECT_EQ(3, *((int8_t*)(buf + 4)));
+ EXPECT_EQ(0, strncmp(buf + 5, "1E0", 3));
+
+ EXPECT_EQ(4, *((int8_t*)(buf + 8)));
+ EXPECT_EQ(0, strncmp(buf + 9, "-1E0", 4));
+
+ EXPECT_EQ(7, *((int8_t*)(buf + 13)));
+ EXPECT_EQ(0, strncmp(buf + 14, "5.645E1", 7));
+
+ EXPECT_EQ(7, *((int8_t*)(buf + 21)));
+ EXPECT_EQ(0, strncmp(buf + 22, "1.012E1", 7));
+}
+
+TEST(MysqlRowBufferTest /*unused*/, faster_float_convert_dynamic /*unused*/) {
+ MysqlRowBuffer mrb;
+ mrb.set_faster_float_convert(true);
+ mrb.open_dynamic_mode();
+
+ mrb.push_float(0);
+ mrb.push_float(1.0);
+ mrb.push_float(-1.0);
+ mrb.push_float(56.45);
+ mrb.push_double(10.12);
+
+ const char* buf = mrb.buf();
+
+ EXPECT_EQ(33, mrb.length());
+ EXPECT_EQ(254, *((uint8_t*)(buf)));
+
+ EXPECT_EQ(0, strncmp(buf + 9, "0E0", 3));
+ EXPECT_EQ(0, strncmp(buf + 12, "1E0", 3));
+ EXPECT_EQ(0, strncmp(buf + 15, "-1E0", 4));
+ EXPECT_EQ(0, strncmp(buf + 19, "5.645E1", 7));
+ EXPECT_EQ(0, strncmp(buf + 26, "1.012E1", 7));
+}
+
} // namespace doris
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 49afb6e6851..4ad725ae663 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -241,6 +241,8 @@ public class Coordinator implements CoordInterface {
private boolean enablePipelineEngine = false;
+ private boolean fasterFloatConvert = false;
+
// Runtime filter merge instance address and ID
public TNetworkAddress runtimeFilterMergeAddr;
public TUniqueId runtimeFilterMergeInstanceId;
@@ -291,6 +293,9 @@ public class Coordinator implements CoordInterface {
// Only enable pipeline query engine in query, not load
this.enablePipelineEngine =
context.getSessionVariable().getEnablePipelineEngine()
&& (fragments.size() > 0 && fragments.get(0).getSink()
instanceof ResultSink);
+
+ this.fasterFloatConvert =
context.getSessionVariable().fasterFloatConvert();
+
initQueryOptions(context);
setFromUserProperty(context);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index f01e67a5107..65fbbf46026 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -408,6 +408,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_FULL_AUTO_ANALYZE =
"enable_full_auto_analyze";
+ public static final String FASTER_FLOAT_CONVERT = "faster_float_convert";
+
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@@ -1168,6 +1170,9 @@ public class SessionVariable implements Serializable,
Writable {
flag = VariableMgr.GLOBAL)
public String fullAutoAnalyzeEndTime = "02:00:00";
+ @VariableMgr.VarAttr(name = FASTER_FLOAT_CONVERT,
+ description = {"是否启用更快的浮点数转换算法，注意会影响输出格式", "Set true to enable
faster float pointer number convert"})
+ public boolean fasterFloatConvert = false;
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to
generate some variables,
// not the default value set in the code.
@@ -2272,6 +2277,8 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setInvertedIndexConjunctionOptThreshold(invertedIndexConjunctionOptThreshold);
+ tResult.setFasterFloatConvert(fasterFloatConvert);
+
return tResult;
}
@@ -2580,6 +2587,10 @@ public class SessionVariable implements Serializable,
Writable {
return connectContext.getSessionVariable().enableAggState;
}
+ public boolean fasterFloatConvert() {
+ return this.fasterFloatConvert;
+ }
+
public void checkAnalyzeTimeFormat(String time) {
try {
DateTimeFormatter timeFormatter =
DateTimeFormatter.ofPattern("HH:mm:ss");
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 818bc538b20..62eb5a0827b 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -246,7 +246,9 @@ struct TQueryOptions {
// use is_report_success any more
84: optional bool enable_profile = false;
85: optional bool enable_page_cache = false;
- 86: optional i32 analyze_timeout = 43200
+ 86: optional i32 analyze_timeout = 43200;
+
+ 87: optional bool faster_float_convert = false;
}
diff --git a/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
new file mode 100644
index 00000000000..c27a9d00b53
--- /dev/null
+++ b/regression-test/data/mysql_tupleconvert_p0/mysql_tupleconvert.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
+
+-- !select --
+1 0.33 3.1415925 [3.3E-1, 6.7E-1] [3.1415926E0,
8.78787878E-1]
+
+-- !select --
+1 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
+2 0.33 3.1415925 [0.33, 0.67] [3.1415926, 0.878787878]
diff --git
a/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
new file mode 100644
index 00000000000..68dcd74f9d7
--- /dev/null
+++ b/regression-test/suites/mysql_tupleconvert_p0/mysql_tupleconvert.groovy
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("mysql_tuple_convert_test") {
+
+ testTable = "doule_faster_convert_test"
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${testTable} (
+ `col_0` INT,
+ `col_1` DOUBLE,
+ `col_2` FLOAT,
+ `col_3` ARRAY<FLOAT>,
+ `col_4` ARRAY<DOUBLE>)
+ ENGINE=OLAP DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`)
BUCKETS 1
+ PROPERTIES("replication_num" = "1");
+ """
+ sql """INSERT INTO ${testTable} VALUES (1, 0.33, 3.1415926, [0.33, 0.67],
[3.1415926, 0.878787878]);"""
+ qt_select """SELECT /*+SET_VAR(faster_float_convert=false)*/ * FROM
${testTable};"""
+ qt_select """SELECT /*+SET_VAR(faster_float_convert=true)*/ * FROM
${testTable};"""
+ // make sure we can convert number from string to value
+ sql """INSERT INTO ${testTable} SELECT 2, col_1, col_2, col_3, col_4 from
${testTable} where col_0 = 1;"""
+ qt_select """SELECT * FROM ${testTable} ORDER BY col_0;"""
+ sql """SET faster_float_convert=false;"""
+ sql "DROP TABLE IF EXISTS ${testTable}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
