This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a1a37c8 [Feature] Support calc constant expr by BE (#6233)
a1a37c8 is described below
commit a1a37c8cba6654f7e2e789f71435492ebba36f59
Author: qiye <[email protected]>
AuthorDate: Mon Jul 19 10:25:53 2021 +0800
[Feature] Support calc constant expr by BE (#6233)
At present, some constant expression calculations are implemented on the FE
side,
but they are incomplete, and some expressions cannot be completely
consistent with
the value calculated by BE (such as part of the time function)
Therefore, we provide a way to pass all the constants in SQL to BE for
calculation,
and then begin to analyze and plan SQL. This method can also solve the
problem that some
complex constant calculations issued by BI cannot be processed on the FE
side.
Here through a session variable enable_fold_constant_by_be to control this
function,
which is disabled by default.
---
be/src/runtime/CMakeLists.txt | 1 +
be/src/runtime/exec_env.h | 3 +
be/src/runtime/exec_env_init.cpp | 3 +
be/src/runtime/fold_constant_mgr.cpp | 213 +++++++++++++++
be/src/runtime/fold_constant_mgr.h | 57 ++++
be/src/service/internal_service.cpp | 39 +++
be/src/service/internal_service.h | 8 +
docs/en/administrator-guide/variables.md | 5 +
docs/zh-CN/administrator-guide/variables.md | 5 +
.../java/org/apache/doris/analysis/Analyzer.java | 8 +-
.../main/java/org/apache/doris/analysis/Expr.java | 2 +-
.../org/apache/doris/analysis/GroupByClause.java | 8 +
.../apache/doris/analysis/InformationFunction.java | 32 +++
.../java/org/apache/doris/analysis/InsertStmt.java | 5 +
.../java/org/apache/doris/analysis/QueryStmt.java | 58 ++++
.../java/org/apache/doris/analysis/SelectStmt.java | 188 +++++++++++++
.../apache/doris/analysis/SetOperationStmt.java | 35 +++
.../org/apache/doris/analysis/StatementBase.java | 10 +
.../org/apache/doris/analysis/SysVariableDesc.java | 17 ++
.../org/apache/doris/catalog/PrimitiveType.java | 10 +
.../java/org/apache/doris/qe/ConnectProcessor.java | 3 +
.../java/org/apache/doris/qe/SessionVariable.java | 8 +
.../java/org/apache/doris/qe/StmtExecutor.java | 9 +-
.../org/apache/doris/rewrite/ExprRewriter.java | 24 ++
.../apache/doris/rewrite/FoldConstantsRule.java | 303 +++++++++++++++++++++
.../org/apache/doris/rpc/BackendServiceClient.java | 4 +
.../org/apache/doris/rpc/BackendServiceProxy.java | 15 +
.../org/apache/doris/analysis/QueryStmtTest.java | 280 +++++++++++++++++++
gensrc/proto/internal_service.proto | 20 ++
gensrc/proto/palo_internal_service.proto | 1 +
gensrc/thrift/FrontendService.thrift | 1 +
gensrc/thrift/PaloInternalService.thrift | 9 +
32 files changed, 1380 insertions(+), 4 deletions(-)
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index 0fffb5a..26a4f2d 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -107,6 +107,7 @@ set(RUNTIME_FILES
mysql_result_writer.cpp
memory/system_allocator.cpp
memory/chunk_allocator.cpp
+ fold_constant_mgr.cpp
cache/result_node.cpp
cache/result_cache.cpp
odbc_table_sink.cpp
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 2fcf901..91e8e4c 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -55,6 +55,7 @@ class RoutineLoadTaskExecutor;
class SmallFileMgr;
class FileBlockManager;
class PluginMgr;
+class FoldConstantMgr;
class BackendServiceClient;
class FrontendServiceClient;
@@ -128,6 +129,7 @@ public:
LoadChannelMgr* load_channel_mgr() { return _load_channel_mgr; }
LoadStreamMgr* load_stream_mgr() { return _load_stream_mgr; }
SmallFileMgr* small_file_mgr() { return _small_file_mgr; }
+ FoldConstantMgr* fold_constant_mgr() { return _fold_constant_mgr; }
const std::vector<StorePath>& store_paths() const { return _store_paths; }
void set_store_paths(const std::vector<StorePath>& paths) { _store_paths =
paths; }
@@ -179,6 +181,7 @@ private:
LoadPathMgr* _load_path_mgr = nullptr;
DiskIoMgr* _disk_io_mgr = nullptr;
TmpFileMgr* _tmp_file_mgr = nullptr;
+ FoldConstantMgr* _fold_constant_mgr = nullptr;
BfdParser* _bfd_parser = nullptr;
BrokerMgr* _broker_mgr = nullptr;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 63095d5..04e3ee1 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -38,6 +38,7 @@
#include "runtime/etl_job_mgr.h"
#include "runtime/exec_env.h"
#include "runtime/external_scan_context_mgr.h"
+#include "runtime/fold_constant_mgr.h"
#include "runtime/fragment_mgr.h"
#include "runtime/heartbeat_flags.h"
#include "runtime/load_channel_mgr.h"
@@ -98,6 +99,7 @@ Status ExecEnv::_init(const std::vector<StorePath>&
store_paths) {
_fragment_mgr = new FragmentMgr(this);
_result_cache = new ResultCache(config::query_cache_max_size_mb,
config::query_cache_elasticity_size_mb);
+ _fold_constant_mgr = new FoldConstantMgr(this);
_master_info = new TMasterInfo();
_etl_job_mgr = new EtlJobMgr(this);
_load_path_mgr = new LoadPathMgr(this);
@@ -250,6 +252,7 @@ void ExecEnv::_destroy() {
SAFE_DELETE(_etl_job_mgr);
SAFE_DELETE(_master_info);
SAFE_DELETE(_fragment_mgr);
+ SAFE_DELETE(_fold_constant_mgr);
SAFE_DELETE(_cgroups_mgr);
SAFE_DELETE(_etl_thread_pool);
SAFE_DELETE(_thread_pool);
diff --git a/be/src/runtime/fold_constant_mgr.cpp
b/be/src/runtime/fold_constant_mgr.cpp
new file mode 100644
index 0000000..4d94f1d
--- /dev/null
+++ b/be/src/runtime/fold_constant_mgr.cpp
@@ -0,0 +1,213 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <map>
+#include <string>
+
+#include "runtime/fold_constant_mgr.h"
+#include "runtime/tuple_row.h"
+#include "runtime/exec_env.h"
+#include "runtime/runtime_state.h"
+#include "runtime/mem_tracker.h"
+#include "exprs/expr_context.h"
+#include "exprs/expr.h"
+#include "common/object_pool.h"
+#include "common/status.h"
+
+#include "gen_cpp/internal_service.pb.h"
+#include "gen_cpp/PaloInternalService_types.h"
+
+using std::string;
+using std::map;
+
+namespace doris {
+
+TUniqueId FoldConstantMgr::_dummy_id;
+
+FoldConstantMgr::FoldConstantMgr(ExecEnv* exec_env)
+ : _exec_env(exec_env), _pool(){
+
+}
+
+Status FoldConstantMgr::fold_constant_expr(
+ const TFoldConstantParams& params, PConstantExprResult* response) {
+ auto expr_map = params.expr_map;
+ auto expr_result_map = response->mutable_expr_result_map();
+
+ TQueryGlobals query_globals = params.query_globals;
+
+ // init
+ Status status = init(query_globals);
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to init mem trackers, msg: " <<
status.get_error_msg();
+ return status;
+ }
+
+ for (auto m : expr_map) {
+ PExprResultMap pexpr_result_map;
+ for (auto n : m.second) {
+ ExprContext* ctx = nullptr;
+ TExpr& texpr = n.second;
+ // create expr tree from TExpr
+ RETURN_IF_ERROR(Expr::create_expr_tree(&_pool, texpr, &ctx));
+ // prepare and open context
+ status = prepare_and_open(ctx);
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to init mem trackers, msg: " <<
status.get_error_msg();
+ return status;
+ }
+
+ TupleRow* row = nullptr;
+ // calc expr
+ void* src = ctx->get_value(row);
+ PrimitiveType root_type = ctx->root()->type().type;
+ // covert to thrift type
+ TPrimitiveType::type t_type = doris::to_thrift(root_type);
+
+ // collect result
+ PExprResult expr_result;
+ string result;
+ if (src == nullptr) {
+ expr_result.set_success(false);
+ } else {
+ expr_result.set_success(true);
+ result = get_result(src, ctx->root()->type().type);
+ }
+
+ expr_result.set_content(result);
+ expr_result.mutable_type()->set_type(t_type);
+
+ pexpr_result_map.mutable_map()->insert({n.first, expr_result});
+
+ // close context expr
+ ctx->close(_runtime_state.get());
+ }
+
+ expr_result_map->insert({m.first, pexpr_result_map});
+ }
+
+ return Status::OK();
+
+}
+
+Status FoldConstantMgr::init(TQueryGlobals query_globals) {
+ // init runtime state, runtime profile
+ TPlanFragmentExecParams params;
+ params.fragment_instance_id = FoldConstantMgr::_dummy_id;
+ params.query_id = FoldConstantMgr::_dummy_id;
+ TExecPlanFragmentParams fragment_params;
+ fragment_params.params = params;
+ fragment_params.protocol_version = PaloInternalServiceVersion::V1;
+ TQueryOptions query_options;
+ _runtime_state.reset(new RuntimeState(fragment_params.params,
query_options, query_globals,
+ ExecEnv::GetInstance()));
+ DescriptorTbl* desc_tbl = NULL;
+ TDescriptorTable* t_desc_tbl = new TDescriptorTable();
+ Status status = DescriptorTbl::create(_runtime_state->obj_pool(),
*t_desc_tbl, &desc_tbl);
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to create descriptor table, msg: " <<
status.get_error_msg();
+ return Status::Uninitialized(status.get_error_msg());
+ }
+ _runtime_state->set_desc_tbl(desc_tbl);
+ status = _runtime_state->init_mem_trackers(FoldConstantMgr::_dummy_id);
+ if (UNLIKELY(!status.ok())) {
+ LOG(WARNING) << "Failed to init mem trackers, msg: " <<
status.get_error_msg();
+ return Status::Uninitialized(status.get_error_msg());
+ }
+
+ _runtime_profile = _runtime_state->runtime_profile();
+ _runtime_profile->set_name("FoldConstantExpr");
+ _mem_tracker = MemTracker::CreateTracker(-1, "FoldConstantExpr",
_runtime_state->instance_mem_tracker());
+ _mem_pool.reset(new MemPool(_mem_tracker.get()));
+
+ return Status::OK();
+}
+
+Status FoldConstantMgr::prepare_and_open(ExprContext* ctx) {
+ RowDescriptor* desc = new RowDescriptor();
+ ctx -> prepare(_runtime_state.get(), *desc, _mem_tracker);
+ return ctx -> open(_runtime_state.get());
+}
+
+string FoldConstantMgr::get_result(void* src, PrimitiveType slot_type){
+ switch (slot_type) {
+ case TYPE_NULL: {
+ return NULL;
+ }
+ case TYPE_BOOLEAN: {
+ bool val = *reinterpret_cast<const bool*>(src);
+ return val ? "true" : "false";
+ }
+ case TYPE_TINYINT: {
+ int8_t val = *reinterpret_cast<const int8_t*>(src);
+ string s;
+ s.push_back(val);
+ return s;
+ }
+ case TYPE_SMALLINT: {
+ int16_t val = *reinterpret_cast<const int16_t*>(src);
+ return std::to_string(val);
+ }
+ case TYPE_INT: {
+ int32_t val = *reinterpret_cast<const int32_t*>(src);
+ return std::to_string(val);
+ }
+ case TYPE_BIGINT: {
+ int64_t val = *reinterpret_cast<const int64_t*>(src);
+ return std::to_string(val);
+ }
+ case TYPE_LARGEINT: {
+ char buf[48];
+ int len = 48;
+ char* v = LargeIntValue::to_string(*reinterpret_cast<__int128*>(src),
buf, &len);
+ return std::string(v, len);
+ }
+ case TYPE_FLOAT: {
+ float val = *reinterpret_cast<const float*>(src);
+ return std::to_string(val);
+ }
+ case TYPE_TIME:
+ case TYPE_DOUBLE: {
+ double val = *reinterpret_cast<double*>(src);
+ return std::to_string(val);
+ }
+ case TYPE_CHAR:
+ case TYPE_VARCHAR:
+ case TYPE_HLL:
+ case TYPE_OBJECT: {
+ return (reinterpret_cast<StringValue*>(src))->debug_string();
+ }
+ case TYPE_DATE:
+ case TYPE_DATETIME: {
+ const DateTimeValue date_value =
*reinterpret_cast<DateTimeValue*>(src);
+ char str[MAX_DTVALUE_STR_LEN];
+ date_value.to_string(str);
+ return str;
+ }
+ case TYPE_DECIMALV2: {
+ return reinterpret_cast<DecimalV2Value*>(src)->to_string();
+ }
+ default:
+ DCHECK(false) << "Type not implemented: " << slot_type;
+ return NULL;
+ }
+ return NULL;
+}
+
+
+}
+
diff --git a/be/src/runtime/fold_constant_mgr.h
b/be/src/runtime/fold_constant_mgr.h
new file mode 100644
index 0000000..353dda9
--- /dev/null
+++ b/be/src/runtime/fold_constant_mgr.h
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/tuple_row.h"
+#include "util/runtime_profile.h"
+#include "exprs/expr_context.h"
+#include "exprs/expr.h"
+#include "common/object_pool.h"
+#include "common/status.h"
+#include "runtime/exec_env.h"
+#include "gen_cpp/internal_service.pb.h"
+#include "gen_cpp/PaloInternalService_types.h"
+
+namespace doris {
+
+class TFoldConstantParams;
+class TExpr;
+class TQueryGlobals;
+
+// This class used to fold constant expr from fe
+class FoldConstantMgr {
+public:
+ FoldConstantMgr(ExecEnv* exec_env);
+ // fold constant expr
+ Status fold_constant_expr(const TFoldConstantParams& params,
PConstantExprResult* response);
+ // init runtime_state and mem_tracker
+ Status init(TQueryGlobals query_globals);
+ // prepare expr
+ Status prepare_and_open(ExprContext* ctx);
+
+ std::string get_result(void* src, PrimitiveType slot_type);
+
+private:
+ std::unique_ptr<RuntimeState> _runtime_state;
+ std::shared_ptr<MemTracker> _mem_tracker;
+ RuntimeProfile* _runtime_profile;
+ std::unique_ptr<MemPool> _mem_pool;
+ ExecEnv* _exec_env;
+ ObjectPool _pool;
+ static TUniqueId _dummy_id;
+};
+}
+
diff --git a/be/src/service/internal_service.cpp
b/be/src/service/internal_service.cpp
index 1141d7d..a24f356 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -23,10 +23,12 @@
#include "runtime/buffer_control_block.h"
#include "runtime/data_stream_mgr.h"
#include "runtime/exec_env.h"
+#include "runtime/fold_constant_mgr.h"
#include "runtime/fragment_mgr.h"
#include "runtime/load_channel_mgr.h"
#include "runtime/result_buffer_mgr.h"
#include "runtime/routine_load/routine_load_task_executor.h"
+#include "runtime/runtime_state.h"
#include "service/brpc.h"
#include "util/thrift_util.h"
#include "util/uid_util.h"
@@ -285,6 +287,43 @@ void
PInternalServiceImpl<T>::apply_filter(::google::protobuf::RpcController* co
}
st.to_protobuf(response->mutable_status());
}
+
+template<typename T>
+void PInternalServiceImpl<T>::fold_constant_expr(
+ google::protobuf::RpcController* cntl_base,
+ const PConstantExprRequest* request,
+ PConstantExprResult* response,
+ google::protobuf::Closure* done) {
+
+ brpc::ClosureGuard closure_guard(done);
+ brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_base);
+
+ Status st = Status::OK();
+ if (request->has_request()) {
+ st = _fold_constant_expr(request->request(), response);
+ } else {
+ // TODO(yangzhengguo) this is just for compatible with old version,
this should be removed in the release 0.15
+ st = _fold_constant_expr(cntl->request_attachment().to_string(),
response);
+ }
+ if (!st.ok()) {
+ LOG(WARNING) << "exec fold constant expr failed, errmsg=" <<
st.get_error_msg();
+ }
+ st.to_protobuf(response->mutable_status());
+}
+
+template<typename T>
+Status PInternalServiceImpl<T>::_fold_constant_expr(const std::string&
ser_request,
+ PConstantExprResult*
response) {
+
+ TFoldConstantParams t_request;
+ {
+ const uint8_t* buf = (const uint8_t*)ser_request.data();
+ uint32_t len = ser_request.size();
+ RETURN_IF_ERROR(deserialize_thrift_msg(buf, &len, false, &t_request));
+ }
+ FoldConstantMgr mgr(_exec_env);
+ return mgr.fold_constant_expr(t_request, response);
+}
template class PInternalServiceImpl<PBackendService>;
template class PInternalServiceImpl<palo::PInternalService>;
diff --git a/be/src/service/internal_service.h
b/be/src/service/internal_service.h
index 247f985..b51680d 100644
--- a/be/src/service/internal_service.h
+++ b/be/src/service/internal_service.h
@@ -87,14 +87,22 @@ public:
const ::doris::PMergeFilterRequest* request,
::doris::PMergeFilterResponse* response,
::google::protobuf::Closure* done) override;
+
void apply_filter(::google::protobuf::RpcController* controller,
const ::doris::PPublishFilterRequest* request,
::doris::PPublishFilterResponse* response,
::google::protobuf::Closure* done) override;
+ void fold_constant_expr(google::protobuf::RpcController* controller,
+ const PConstantExprRequest* request,
+ PConstantExprResult* response,
+ google::protobuf::Closure* done) override;
+
private:
Status _exec_plan_fragment(const std::string& s_request);
+ Status _fold_constant_expr(const std::string& ser_request,
PConstantExprResult* response);
+
private:
ExecEnv* _exec_env;
PriorityThreadPool _tablet_worker_pool;
diff --git a/docs/en/administrator-guide/variables.md
b/docs/en/administrator-guide/variables.md
index 1379a2b..38b6030 100644
--- a/docs/en/administrator-guide/variables.md
+++ b/docs/en/administrator-guide/variables.md
@@ -75,6 +75,7 @@ Variables that support both session-level and global-level
setting include:
* `parallel_exchange_instance_num`
* `allow_partition_column_nullable`
* `insert_visible_timeout_ms`
+* `enable_fold_constant_by_be`
Variables that support only global-level setting include:
@@ -392,3 +393,7 @@ Note that the comment must start with /*+ and can only
follow the SELECT.
* `extract_wide_range_expr`
Used to control whether turn on the 'Wide Common Factors' rule. The value
has two: true or false. On by default.
+
+* `enable_fold_constant_by_be`
+
+ Used to control the calculation method of constant folding. The default is
`false`, that is, calculation is performed in `FE`; if it is set to `true`, it
will be calculated by `BE` through `RPC` request.
diff --git a/docs/zh-CN/administrator-guide/variables.md
b/docs/zh-CN/administrator-guide/variables.md
index 441cb6f..a4d05de 100644
--- a/docs/zh-CN/administrator-guide/variables.md
+++ b/docs/zh-CN/administrator-guide/variables.md
@@ -73,6 +73,7 @@ SET GLOBAL exec_mem_limit = 137438953472
* `batch_size`
* `allow_partition_column_nullable`
* `insert_visible_timeout_ms`
+* `enable_fold_constant_by_be`
只支持全局生效的变量包括：
@@ -387,3 +388,7 @@ SELECT /*+ SET_VAR(query_timeout = 1,
enable_partition_cache=true) */ sleep(3);
* `extract_wide_range_expr`
用于控制是否开启 「宽泛公因式提取」的优化。取值有两种：true 和 false 。默认情况下开启。
+
+* `enable_fold_constant_by_be`
+
+ 用于控制常量折叠的计算方式。默认是 `false`，即在 `FE` 进行计算；若设置为 `true`，则通过 `RPC` 请求经 `BE` 计算。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index 9030377..4230c18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -853,7 +853,6 @@ public class Analyzer {
/**
* register expr id
- *
* @param expr
*/
void registerExprId(Expr expr) {
@@ -1628,6 +1627,13 @@ public class Analyzer {
}
return
globalState.context.getSessionVariable().isEnableJoinReorderBasedCost();
}
+
+ public boolean safeIsEnableFoldConstantByBe() {
+ if (globalState.context == null) {
+ return false;
+ }
+ return
globalState.context.getSessionVariable().isEnableFoldConstantByBe();
+ }
/**
* Returns true if predicate 'e' can be correctly evaluated by a tree
materializing
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 17a7405..2823d98 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -283,7 +283,7 @@ abstract public class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
return id;
}
- protected void setId(ExprId id) {
+ public void setId(ExprId id) {
this.id = id;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java
index 9ce4744..14d1804 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/GroupByClause.java
@@ -118,6 +118,10 @@ public class GroupByClause implements ParseNode {
return oriGroupingExprs;
}
+ public void setOriGroupingExprs(ArrayList<Expr> list) {
+ oriGroupingExprs = list;
+ }
+
public ArrayList<Expr> getGroupingExprs() {
if (!exprGenerated) {
try {
@@ -130,6 +134,10 @@ public class GroupByClause implements ParseNode {
return groupingExprs;
}
+ public void setGroupingExpr(ArrayList<Expr> list) {
+ groupingExprs = list;
+ }
+
// generate grouping exprs from group by, grouping sets, cube, rollup
clause
public void genGroupingExprs() throws AnalysisException {
if (exprGenerated) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java
index d1f2fc7..a9fd4a3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java
@@ -46,6 +46,12 @@ public class InformationFunction extends Expr {
return strValue;
}
+ public String getIntValue() {
+ return String.valueOf(intValue);
+ }
+
+ public String getFuncType() {return funcType; }
+
@Override
public Expr clone() {
return new InformationFunction(this);
@@ -79,4 +85,30 @@ public class InformationFunction extends Expr {
public String toSqlImpl() {
return funcType + "()";
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (!(obj instanceof InformationFunction)) {
+ return false;
+ }
+
+ if (!funcType.equals(((InformationFunction) obj).getFuncType())) {
+ return false;
+ }
+
+ if (type.equals(Type.VARCHAR)) {
+ if (!strValue.equals(((InformationFunction) obj).getStrValue())) {
+ return false;
+ }
+ } else if (type.equals(Type.BIGINT)) {
+ if (intValue != Integer.parseInt(((InformationFunction)
obj).getIntValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
index 8ded93d..8c6e0d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
@@ -222,6 +222,11 @@ public class InsertStmt extends DdlStmt {
this.queryStmt = queryStmt;
}
+ @Override
+ public void foldConstant(ExprRewriter rewriter) throws AnalysisException {
+ Preconditions.checkState(isAnalyzed());
+ queryStmt.foldConstant(rewriter);
+ }
@Override
public void rewriteExprs(ExprRewriter rewriter) throws AnalysisException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
index 15bc170..7645340 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
@@ -29,6 +29,7 @@ import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
@@ -435,6 +436,63 @@ public abstract class QueryStmt extends StatementBase {
}
}
+ /**
+ * collect all exprs of a QueryStmt to a map
+ * @param exprMap
+ */
+ public void collectExprs(Map<String, Expr> exprMap) {}
+
+ /**
+ * put all rewritten exprs back to the ori QueryStmt
+ * @param rewrittenExprMap
+ */
+ public void putBackExprs(Map<String, Expr> rewrittenExprMap) {}
+
+
+ @Override
+ public void foldConstant(ExprRewriter rewriter) throws AnalysisException {
+ Preconditions.checkState(isAnalyzed());
+ Map<String, Expr> exprMap = new HashMap<>();
+ collectExprs(exprMap);
+ rewriter.rewriteConstant(exprMap, analyzer);
+ if (rewriter.changed()) {
+ putBackExprs(exprMap);
+ }
+
+ }
+
+
+ /**
+ * register expr_id of expr and its children, if not set
+ * @param expr
+ */
+ public void registerExprId(Expr expr) {
+ if (expr.getId() == null) {
+ analyzer.registerExprId(expr);
+ }
+ for (Expr child : expr.getChildren()) {
+ registerExprId(child);
+ }
+ }
+
+ /**
+ * check whether expr and it's children contain alias
+ * @param expr expr to be checked
+ * @return true if contains, otherwise false
+ */
+ public boolean containAlias(Expr expr) {
+ for (Expr child : expr.getChildren()) {
+ if (containAlias(child)) {
+ return true;
+ }
+ }
+
+ if (null != aliasSMap.get(expr)) {
+ return true;
+ }
+ return false;
+ }
+
// get tables used by this query.
// Set<String> parentViewNameSet contain parent stmt view name
// to make sure query like "with tmp as (select * from db1.table1) " +
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 05569a7..6a60dde 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -1325,6 +1325,194 @@ public class SelectStmt extends QueryStmt {
}
}
+ @Override
+ public void collectExprs(Map<String, Expr> exprMap) {
+ // subquery
+ List<Subquery> subqueryExprs = Lists.newArrayList();
+
+ // select clause
+ for (SelectListItem item : selectList.getItems()) {
+ if (item.isStar()) {
+ continue;
+ }
+ // register expr id
+ registerExprId(item.getExpr());
+
+ exprMap.put(item.getExpr().getId().toString(), item.getExpr());
+
+ // equal subquery in select list
+ if
(item.getExpr().contains(Predicates.instanceOf(Subquery.class))) {
+ item.getExpr().collect(Subquery.class, subqueryExprs);
+ }
+ }
+
+ // from clause
+ for (TableRef ref : fromClause_) {
+ Preconditions.checkState(ref.isAnalyzed);
+ if (ref.onClause != null) {
+ registerExprId(ref.onClause);
+ exprMap.put(ref.onClause.getId().toString(), ref.onClause);
+ }
+ }
+
+ if (whereClause != null) {
+ registerExprId(whereClause);
+ exprMap.put(whereClause.getId().toString(), whereClause);
+ whereClause.collect(Subquery.class, subqueryExprs);
+
+ }
+ if (havingClause != null) {
+ registerExprId(havingClause);
+ exprMap.put(havingClause.getId().toString(), havingClause);
+ havingClauseAfterAnaylzed.collect(Subquery.class, subqueryExprs);
+ }
+ for (Subquery subquery : subqueryExprs) {
+ registerExprId(subquery);
+ subquery.getStatement().collectExprs(exprMap);
+ }
+ if (groupByClause != null) {
+ ArrayList<Expr> groupingExprs = groupByClause.getGroupingExprs();
+ if (groupingExprs != null) {
+ for (Expr expr : groupingExprs) {
+ if (containAlias(expr)) {
+ continue;
+ }
+ registerExprId(expr);
+ exprMap.put(expr.getId().toString(), expr);
+ }
+ }
+ List<Expr> oriGroupingExprs = groupByClause.getOriGroupingExprs();
+ if (oriGroupingExprs != null) {
+ for (Expr expr : oriGroupingExprs) {
+ /*
+ * Suppose there is a query statement:
+ *
+ * ```
+ * select
+ * i_item_sk as b
+ * from item
+ * group by b
+ * order by b desc
+ * ```
+ *
+ * where `b` is an alias for `i_item_sk`.
+ *
+ * When analyze is done, it becomes
+ *
+ * ```
+ * SELECT
+ * `i_item_sk`
+ * FROM `item`
+ * GROUP BY `b`
+ * ORDER BY `b` DESC
+ * ```
+ * Aliases information of groupBy and orderBy clauses is
recorded in `QueryStmt.aliasSMap`.
+ * The select clause has it's own alias info in
`SelectListItem.alias`.
+ *
+ * Aliases expr in the `group by` and `order by` clauses
are not analyzed, i.e. `Expr.isAnalyzed=false`
+ * Subsequent constant folding will analyze the unanalyzed
Expr before collecting the constant
+ * expressions, preventing the `INVALID_TYPE` expr from
being sent to BE.
+ *
+ * But when analyzing the alias, the meta information
corresponding to the slot cannot be found
+ * in the catalog, an error will be reported.
+ *
+ * So the alias needs to be removed here.
+ *
+ */
+ if (containAlias(expr)) {
+ continue;
+ }
+ registerExprId(expr);
+ exprMap.put(expr.getId().toString(), expr);
+ }
+ }
+ }
+ if (orderByElements != null) {
+ for (OrderByElement orderByElem : orderByElements) {
+ // same as above
+ if (containAlias(orderByElem.getExpr())) {
+ continue;
+ }
+ registerExprId(orderByElem.getExpr());
+ exprMap.put(orderByElem.getExpr().getId().toString(),
orderByElem.getExpr());
+ }
+ }
+ }
+
+ @Override
+ public void putBackExprs(Map<String, Expr> rewrittenExprMap) {
+ // subquery
+ List<Subquery> subqueryExprs = Lists.newArrayList();
+ for (SelectListItem item : selectList.getItems()) {
+ if (item.isStar()) {
+ continue;
+ }
+
item.setExpr(rewrittenExprMap.get(item.getExpr().getId().toString()));
+ // equal subquery in select list
+ if
(item.getExpr().contains(Predicates.instanceOf(Subquery.class))) {
+ item.getExpr().collect(Subquery.class, subqueryExprs);
+ }
+ }
+
+ // from clause
+ for (TableRef ref : fromClause_) {
+ if (ref.onClause != null) {
+
ref.setOnClause(rewrittenExprMap.get(ref.onClause.getId().toString()));
+ }
+ }
+
+ if (whereClause != null) {
+
setWhereClause(rewrittenExprMap.get(whereClause.getId().toString()));
+ whereClause.collect(Subquery.class, subqueryExprs);
+ }
+ if (havingClause != null) {
+ havingClause =
rewrittenExprMap.get(havingClause.getId().toString());
+ havingClauseAfterAnaylzed.collect(Subquery.class, subqueryExprs);
+ }
+
+ for (Subquery subquery : subqueryExprs) {
+ subquery.getStatement().putBackExprs(rewrittenExprMap);
+ }
+
+ if (groupByClause != null) {
+ ArrayList<Expr> groupingExprs = groupByClause.getGroupingExprs();
+ if (groupingExprs != null) {
+ ArrayList<Expr> newGroupingExpr = new ArrayList<>();
+ for (Expr expr : groupingExprs) {
+ if (expr.getId() == null) {
+ newGroupingExpr.add(expr);
+ } else {
+
newGroupingExpr.add(rewrittenExprMap.get(expr.getId().toString()));
+ }
+ }
+ groupByClause.setGroupingExpr(newGroupingExpr);
+
+ }
+ List<Expr> oriGroupingExprs = groupByClause.getOriGroupingExprs();
+ if (oriGroupingExprs != null) {
+ ArrayList<Expr> newOriGroupingExprs = new ArrayList<>();
+ for (Expr expr : oriGroupingExprs) {
+ if (expr.getId() == null) {
+ newOriGroupingExprs.add(expr);
+ } else {
+
newOriGroupingExprs.add(rewrittenExprMap.get(expr.getId().toString()));
+ }
+ }
+ groupByClause.setOriGroupingExprs(newOriGroupingExprs);
+ }
+ }
+ if (orderByElements != null) {
+ for (OrderByElement orderByElem : orderByElements) {
+ Expr expr = orderByElem.getExpr();
+ if (expr.getId() == null) {
+ orderByElem.setExpr(expr);
+ } else {
+
orderByElem.setExpr(rewrittenExprMap.get(expr.getId().toString()));
+ }
+ }
+ }
+ }
+
private void rewriteSelectList(ExprRewriter rewriter) throws
AnalysisException {
for (SelectListItem item : selectList.getItems()) {
if (item.getExpr() instanceof CaseExpr &&
item.getExpr().contains(Predicates.instanceOf(Subquery.class))) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
index a0908fb..875b2d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
@@ -526,6 +526,41 @@ public class SetOperationStmt extends QueryStmt {
}
@Override
+ public void collectExprs(Map<String, Expr> exprMap) {
+ for (SetOperand op : operands) {
+ op.getQueryStmt().collectExprs(exprMap);
+ }
+ if (orderByElements != null) {
+ for (OrderByElement orderByElement : orderByElements) {
+ Expr expr = orderByElement.getExpr();
+ // see SelectStmt.collectExprs comments
+ if (containAlias(expr)) {
+ continue;
+ }
+ registerExprId(expr);
+ exprMap.put(expr.getId().toString(), expr);
+ }
+ }
+ }
+
+ @Override
+ public void putBackExprs(Map<String, Expr> rewrittenExprMap) {
+ for (SetOperand op : operands) {
+ op.getQueryStmt().putBackExprs(rewrittenExprMap);
+ }
+ if (orderByElements != null) {
+ for (OrderByElement orderByElement : orderByElements) {
+ Expr expr = orderByElement.getExpr();
+ if (expr.getId() == null) {
+ orderByElement.setExpr(expr);
+ } else {
+
orderByElement.setExpr(rewrittenExprMap.get(expr.getId().toString()));
+ }
+ }
+ }
+ }
+
+ @Override
public void rewriteExprs(ExprRewriter rewriter) throws AnalysisException {
for (SetOperand op: operands) op.getQueryStmt().rewriteExprs(rewriter);
if (orderByElements != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
index 79047ac..da78614 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java
@@ -171,6 +171,16 @@ public abstract class StatementBase implements ParseNode {
"rewriteExprs() not implemented for this stmt: " +
getClass().getSimpleName());
}
+ /**
+ * fold constant exprs in statement
+ * @throws AnalysisException
+ * @param rewriter
+ */
+ public void foldConstant(ExprRewriter rewriter) throws AnalysisException {
+ throw new IllegalStateException(
+ "foldConstant() not implemented for this stmt: " +
getClass().getSimpleName());
+ }
+
public String getClusterName() {
return clusterName;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
index c000baf..dd49dcc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SysVariableDesc.java
@@ -175,4 +175,21 @@ public class SysVariableDesc extends Expr {
public String toString() {
return toSql();
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof SysVariableDesc)) {
+ return false;
+ }
+ if (!name.equals(((SysVariableDesc) obj).getName())) {
+ return false;
+ }
+ if (!setType.equals(((SysVariableDesc) obj).getSetType())) {
+ return false;
+ }
+ return literalExpr.equals(((SysVariableDesc) obj).getLiteralExpr());
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index 15082c4..989f7a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -538,6 +538,16 @@ public enum PrimitiveType {
return FLOAT;
case DOUBLE:
return DOUBLE;
+ case DATE:
+ return DATE;
+ case DATETIME:
+ return DATETIME;
+ case BINARY:
+ return BINARY;
+ case DECIMALV2:
+ return DECIMALV2;
+ case TIME:
+ return TIME;
case VARCHAR:
return VARCHAR;
case CHAR:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 6aae692..0f567ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -421,6 +421,9 @@ public class ConnectProcessor {
UserIdentity currentUserIdentity =
UserIdentity.fromThrift(request.getCurrentUserIdent());
ctx.setCurrentUserIdentity(currentUserIdentity);
}
+ if (request.isFoldConstantByBe()) {
+
ctx.getSessionVariable().setEnableFoldConstantByBe(request.foldConstantByBe);
+ }
if (request.isSetSessionVariables()) {
ctx.getSessionVariable().setForwardedSessionVariables(request.getSessionVariables());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 19bedaa..9818ab1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -80,6 +80,7 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_EXCHANGE_NODE_PARALLEL_MERGE =
"enable_exchange_node_parallel_merge";
public static final String PREFER_JOIN_METHOD = "prefer_join_method";
+ public static final String ENABLE_FOLD_CONSTANT_BY_BE =
"enable_fold_constant_by_be";
public static final String ENABLE_ODBC_TRANSCATION =
"enable_odbc_transcation";
public static final String ENABLE_SQL_CACHE = "enable_sql_cache";
public static final String ENABLE_PARTITION_CACHE =
"enable_partition_cache";
@@ -271,6 +272,9 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = PREFER_JOIN_METHOD)
public String preferJoinMethod = "broadcast";
+ @VariableMgr.VarAttr(name = ENABLE_FOLD_CONSTANT_BY_BE)
+ private boolean enableFoldConstantByBe = false;
+
/*
* the parallel exec instance num for one Fragment in one BE
* 1 means disable this feature
@@ -517,6 +521,10 @@ public class SessionVariable implements Serializable,
Writable {
this.preferJoinMethod = preferJoinMethod;
}
+ public boolean isEnableFoldConstantByBe() { return enableFoldConstantByBe;
}
+
+ public void setEnableFoldConstantByBe(boolean foldConstantByBe)
{this.enableFoldConstantByBe = foldConstantByBe; }
+
public int getParallelExecInstanceNum() {
return parallelExecInstanceNum;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 7ec6222..f9b0ca9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -544,12 +544,17 @@ public class StmtExecutor implements ProfileWriter {
private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions)
throws UserException {
parsedStmt.analyze(analyzer);
if (parsedStmt instanceof QueryStmt || parsedStmt instanceof
InsertStmt) {
+ ExprRewriter rewriter = analyzer.getExprRewriter();
+ rewriter.reset();
+ if (context.getSessionVariable().isEnableFoldConstantByBe()) {
+ // fold constant expr
+ parsedStmt.foldConstant(rewriter);
+
+ }
// Apply expr and subquery rewrites.
ExplainOptions explainOptions = parsedStmt.getExplainOptions();
boolean reAnalyze = false;
- ExprRewriter rewriter = analyzer.getExprRewriter();
- rewriter.reset();
parsedStmt.rewriteExprs(rewriter);
reAnalyze = rewriter.changed();
if (analyzer.containSubquery()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java
index 1217f41..269f12a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/ExprRewriter.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.AnalysisException;
import com.google.common.collect.Lists;
import java.util.List;
+import java.util.Map;
/**
* Helper class that drives the transformation of Exprs according to a given
list of
@@ -68,6 +69,10 @@ public class ExprRewriter {
do {
oldNumChanges = numChanges_;
for (ExprRewriteRule rule: rules_) {
+ // when foldConstantByBe is on, fold all constant expr by BE
instead of applying FoldConstantsRule in FE.
+ if (rule instanceof FoldConstantsRule &&
analyzer.safeIsEnableFoldConstantByBe()) {
+ continue;
+ }
rewrittenExpr = applyRuleRepeatedly(rewrittenExpr, rule,
analyzer);
}
} while (oldNumChanges != numChanges_);
@@ -87,6 +92,25 @@ public class ExprRewriter {
}
/**
+ * FoldConstantsRule rewrite
+ */
+ public void rewriteConstant(Map<String, Expr> exprMap, Analyzer analyzer)
throws AnalysisException {
+ if (exprMap.isEmpty()) {
+ return;
+ }
+ boolean changed = false;
+ // rewrite constant expr
+ for (ExprRewriteRule rule : rules_) {
+ if (rule instanceof FoldConstantsRule) {
+ changed = ((FoldConstantsRule) rule).apply(exprMap, analyzer,
changed);
+ }
+ }
+ if (changed) {
+ ++numChanges_;
+ }
+ }
+
+ /**
* Applies 'rule' on the Expr tree rooted at 'expr' until there are no
more changes.
* Returns the transformed Expr or 'expr' if there were no changes.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
index f697150..6042ad8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FoldConstantsRule.java
@@ -22,11 +22,43 @@ import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.CaseExpr;
import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.InformationFunction;
+import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.NullLiteral;
+import org.apache.doris.analysis.SysVariableDesc;
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.LoadException;
+import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.proto.InternalService;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.VariableMgr;
+import org.apache.doris.rpc.BackendServiceProxy;
+import org.apache.doris.system.Backend;
+import org.apache.doris.thrift.TExpr;
+import org.apache.doris.thrift.TFoldConstantParams;
+import org.apache.doris.thrift.TNetworkAddress;
+import org.apache.doris.thrift.TPrimitiveType;
+import org.apache.doris.thrift.TQueryGlobals;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
/**
* This rule replaces a constant Expr with its equivalent LiteralExpr by
evaluating the
* Expr in the BE. Exprs that are already LiteralExprs are not changed.
@@ -44,6 +76,7 @@ import org.apache.logging.log4j.Logger;
*/
public class FoldConstantsRule implements ExprRewriteRule {
private static final Logger LOG =
LogManager.getLogger(FoldConstantsRule.class);
+ private static final DateFormat DATE_FORMAT = new
SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
public static ExprRewriteRule INSTANCE = new FoldConstantsRule();
@@ -86,5 +119,275 @@ public class FoldConstantsRule implements ExprRewriteRule {
}
return expr.getResultValue();
}
+
+ /**
+ * fold constant expr by BE
+ * SysVariableDesc and InformationFunction need handling specially
+ * @param exprMap
+ * @param analyzer
+ * @return
+ * @throws AnalysisException
+ */
+ public boolean apply(Map<String, Expr> exprMap, Analyzer analyzer, boolean
changed)
+ throws AnalysisException {
+ // root_expr_id_string:
+ // child_expr_id_string : texpr
+ // child_expr_id_string : texpr
+ Map<String, Map<String, TExpr>> paramMap = new HashMap<>();
+ Map<String, Expr> allConstMap = new HashMap<>();
+ // map to collect SysVariableDesc
+ Map<String, Map<String, Expr>> sysVarsMap = new HashMap<>();
+ // map to collect InformationFunction
+ Map<String, Map<String, Expr>> infoFnsMap = new HashMap<>();
+ for (Map.Entry<String, Expr> entry : exprMap.entrySet()){
+ Map<String, TExpr> constMap = new HashMap<>();
+ Map<String, Expr> oriConstMap = new HashMap<>();
+ Map<String, Expr> sysVarMap = new HashMap<>();
+ Map<String, Expr> infoFnMap = new HashMap<>();
+ getConstExpr(entry.getValue(), constMap, oriConstMap, analyzer,
sysVarMap, infoFnMap);
+
+ if (!constMap.isEmpty()) {
+ paramMap.put(entry.getKey(), constMap);
+ allConstMap.putAll(oriConstMap);
+ }
+ if (!sysVarMap.isEmpty()) {
+ sysVarsMap.put(entry.getKey(), sysVarMap);
+ }
+ if (!infoFnMap.isEmpty()) {
+ infoFnsMap.put(entry.getKey(), infoFnMap);
+ }
+ }
+
+ if (!sysVarsMap.isEmpty()) {
+ putBackConstExpr(exprMap, sysVarsMap);
+ changed = true;
+ }
+
+ if (!infoFnsMap.isEmpty()) {
+ putBackConstExpr(exprMap, infoFnsMap);
+ changed = true;
+ }
+
+ if (!paramMap.isEmpty()) {
+ Map<String, Map<String, Expr>> resultMap = calcConstExpr(paramMap,
allConstMap, analyzer.getContext());
+
+ if (!resultMap.isEmpty()) {
+ putBackConstExpr(exprMap, resultMap);
+ changed = true;
+
+ }
+
+ }
+ return changed;
+ }
+
+ /**
+ * get all constant children expr from a expr
+ * @param expr
+ * @param constExprMap
+ * @param analyzer
+ * @throws AnalysisException
+ */
+ private void getConstExpr(Expr expr, Map<String,TExpr> constExprMap,
Map<String, Expr> oriConstMap,
+ Analyzer analyzer, Map<String, Expr> sysVarMap,
Map<String, Expr> infoFnMap)
+ throws AnalysisException {
+ // Analyze constant exprs, if necessary. Note that the 'expr' may
become non-constant
+ // after analysis (e.g., aggregate functions).
+ if (!expr.isAnalyzed()) {
+ expr.analyze(analyzer);
+ }
+ if (expr.isConstant()) {
+ // Do not constant fold cast(null as dataType) because we cannot
preserve the
+ // cast-to-types and that can lead to query failures, e.g., CTAS
+ if (expr instanceof CastExpr) {
+ CastExpr castExpr = (CastExpr) expr;
+ if (castExpr.getChild(0) instanceof NullLiteral) {
+ return;
+ }
+ }
+ // skip literal expr
+ if (expr instanceof LiteralExpr) {
+ return;
+ }
+ // collect sysVariableDesc expr
+ if (expr.contains(Predicates.instanceOf(SysVariableDesc.class))) {
+ getSysVarDescExpr(expr, sysVarMap);
+ return;
+ }
+ // collect InformationFunction
+ if
(expr.contains(Predicates.instanceOf(InformationFunction.class))) {
+ getInfoFnExpr(expr, infoFnMap);
+ return;
+ }
+ constExprMap.put(expr.getId().toString(),expr.treeToThrift());
+ oriConstMap.put(expr.getId().toString(), expr);
+ } else {
+ recursiveGetChildrenConstExpr(expr, constExprMap, oriConstMap,
analyzer, sysVarMap, infoFnMap);
+
+ }
+ }
+
+ private void recursiveGetChildrenConstExpr(Expr expr, Map<String,TExpr>
constExprMap, Map<String, Expr> oriConstMap,
+ Analyzer analyzer, Map<String,
Expr> sysVarMap,
+ Map<String, Expr>
infoFnMap)throws AnalysisException {
+ for (int i = 0; i < expr.getChildren().size(); i++) {
+ final Expr child = expr.getChildren().get(i);
+ getConstExpr(child, constExprMap, oriConstMap, analyzer,
sysVarMap, infoFnMap);
+ }
+
+ }
+
+ private void getSysVarDescExpr(Expr expr, Map<String, Expr> sysVarMap) {
+ if (expr instanceof SysVariableDesc) {
+ Expr literalExpr = ((SysVariableDesc) expr).getLiteralExpr();
+ if (literalExpr == null) {
+ try {
+
VariableMgr.fillValue(ConnectContext.get().getSessionVariable(),
(SysVariableDesc) expr);
+ literalExpr = ((SysVariableDesc) expr).getLiteralExpr();
+ } catch (AnalysisException e) {
+ LOG.warn("failed to get session variable value: " +
((SysVariableDesc) expr).getName());
+ }
+ }
+ sysVarMap.put(expr.getId().toString(), literalExpr);
+ } else {
+ for (Expr child : expr.getChildren()) {
+ getSysVarDescExpr(child, sysVarMap);
+ }
+ }
+ }
+
+ private void getInfoFnExpr(Expr expr, Map<String, Expr> infoFnMap) {
+ if (expr instanceof InformationFunction) {
+ Type type = expr.getType();
+ LiteralExpr literalExpr = null;
+ try {
+ String str = null;
+ if (type.equals(Type.VARCHAR)) {
+ str = ((InformationFunction) expr).getStrValue();
+ } else if (type.equals(Type.BIGINT)) {
+ str = ((InformationFunction) expr).getIntValue();
+ }
+ Preconditions.checkNotNull(str);
+ literalExpr = LiteralExpr.create(str, type);
+ infoFnMap.put(expr.getId().toString(), literalExpr);
+ } catch (AnalysisException e) {
+ LOG.warn("failed to get const expr value from
InformationFunction: {}", e.getMessage());
+ }
+
+ } else {
+ for (Expr child : expr.getChildren()) {
+ getInfoFnExpr(child, infoFnMap);
+ }
+ }
+ }
+
+ /**
+ * put all rewritten expr back to ori expr map
+ * @param exprMap
+ * @param resultMap
+ */
+ private void putBackConstExpr(Map<String, Expr> exprMap, Map<String,
Map<String, Expr>> resultMap) {
+ for (Map.Entry<String, Map<String, Expr>> entry :
resultMap.entrySet()) {
+ Expr rewrittenExpr = putBackConstExpr(exprMap.get(entry.getKey()),
entry.getValue());
+ exprMap.put(entry.getKey(), rewrittenExpr);
+ }
+ }
+
+ private Expr putBackConstExpr(Expr expr, Map<String, Expr> resultMap) {
+ for (Map.Entry<String, Expr> entry : resultMap.entrySet()) {
+ if (entry.getValue() instanceof LiteralExpr) {
+ expr = replaceExpr(expr, entry.getKey(), (LiteralExpr)
entry.getValue());
+
+ }
+ }
+ return expr;
+ }
+
+ /**
+ * find and replace constant child expr of a expr by literal expr
+ * @param expr
+ * @param key
+ * @param literalExpr
+ * @return
+ */
+ private Expr replaceExpr(Expr expr, String key, LiteralExpr literalExpr) {
+ if (expr.getId().toString().equals(key)) {
+ return literalExpr;
+ }
+ // ATTN: make sure the child order of expr keep unchanged
+ for (int i = 0; i < expr.getChildren().size(); i++) {
+ Expr child = expr.getChild(i);
+ if (literalExpr.equals(replaceExpr(child, key, literalExpr))) {
+ literalExpr.setId(child.getId());
+ expr.setChild(i, literalExpr);
+ break;
+ }
+ }
+ return expr;
+ }
+
+ /**
+ * calc all constant exprs by BE
+ * @param map
+ * @param context
+ * @return
+ */
+ private Map<String, Map<String, Expr>> calcConstExpr(Map<String,
Map<String, TExpr>> map,
+ Map<String, Expr>
allConstMap,
+ ConnectContext
context) {
+ TNetworkAddress brpcAddress = null;
+ Map<String, Map<String, Expr>> resultMap = new HashMap<>();
+ try {
+ List<Long> backendIds =
Catalog.getCurrentSystemInfo().getBackendIds(true);
+ if (backendIds.isEmpty()) {
+ throw new LoadException("Failed to get all partitions. No
alive backends");
+ }
+ Collections.shuffle(backendIds);
+ Backend be =
Catalog.getCurrentSystemInfo().getBackend(backendIds.get(0));
+ brpcAddress = new TNetworkAddress(be.getHost(), be.getBrpcPort());
+
+ TQueryGlobals queryGlobals = new TQueryGlobals();
+ queryGlobals.setNowString(DATE_FORMAT.format(new Date()));
+ queryGlobals.setTimestampMs(new Date().getTime());
+ queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
+ if (context.getSessionVariable().getTimeZone().equals("CST")) {
+ queryGlobals.setTimeZone(TimeUtils.DEFAULT_TIME_ZONE);
+ } else {
+
queryGlobals.setTimeZone(context.getSessionVariable().getTimeZone());
+ }
+
+ TFoldConstantParams tParams = new TFoldConstantParams(map,
queryGlobals);
+
+ Future<InternalService.PConstantExprResult> future =
BackendServiceProxy.getInstance().foldConstantExpr(brpcAddress, tParams);
+ InternalService.PConstantExprResult result = future.get(5,
TimeUnit.SECONDS);
+
+ if (result.getStatus().getStatusCode() == 0) {
+ for (Map.Entry<String, InternalService.PExprResultMap> entry :
result.getExprResultMapMap().entrySet()) {
+ Map<String, Expr> tmp = new HashMap<>();
+ for (Map.Entry<String, InternalService.PExprResult> entry1
: entry.getValue().getMapMap().entrySet()) {
+ TPrimitiveType type =
TPrimitiveType.findByValue(entry1.getValue().getType().getType());
+ Expr retExpr = null;
+ if (entry1.getValue().getSuccess()) {
+ retExpr =
LiteralExpr.create(entry1.getValue().getContent(),
+
Type.fromPrimitiveType(PrimitiveType.fromThrift(type)));
+ } else {
+ retExpr = allConstMap.get(entry1.getKey());
+ }
+ tmp.put(entry1.getKey(), retExpr);
+ }
+ if (!tmp.isEmpty()) {
+ resultMap.put(entry.getKey(), tmp);
+ }
+ }
+
+ } else {
+ LOG.warn("failed to get const expr value from be: {}",
result.getStatus().getErrorMsgsList());
+ }
+ } catch (Exception e) {
+ LOG.warn("failed to get const expr value from be: {}",
e.getMessage());
+ }
+ return resultMap;
+
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
index d9871c2..ffd7e25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
@@ -76,4 +76,8 @@ public class BackendServiceClient {
public Future<InternalService.PProxyResult>
getInfo(InternalService.PProxyRequest request) {
return stub.getInfo(request);
}
+
+ public Future<InternalService.PConstantExprResult>
foldConstantExpr(InternalService.PConstantExprRequest request) {
+ return stub.foldConstantExpr(request);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
index b488dfd..40f2897 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
@@ -20,6 +20,7 @@ package org.apache.doris.rpc;
import org.apache.doris.proto.InternalService;
import org.apache.doris.proto.Types;
import org.apache.doris.thrift.TExecPlanFragmentParams;
+import org.apache.doris.thrift.TFoldConstantParams;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TUniqueId;
@@ -167,4 +168,18 @@ public class BackendServiceProxy {
throw new RpcException(address.hostname, e.getMessage());
}
}
+
+ public Future<InternalService.PConstantExprResult> foldConstantExpr(
+ TNetworkAddress address, TFoldConstantParams tParams) throws
RpcException, TException {
+ final InternalService.PConstantExprRequest pRequest =
InternalService.PConstantExprRequest.newBuilder()
+ .setRequest(ByteString.copyFrom(new
TSerializer().serialize(tParams))).build();
+
+ try {
+ final BackendServiceClient client = getProxy(address);
+ return client.foldConstantExpr(pRequest);
+ } catch (Throwable e) {
+ LOG.warn("failed to fold constant expr, address={}:{}",
address.getHostname(), address.getPort(), e);
+ throw new RpcException(address.hostname, e.getMessage());
+ }
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
new file mode 100644
index 0000000..e76f24e
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/QueryStmtTest.java
@@ -0,0 +1,280 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import com.google.common.collect.Lists;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.UserException;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.utframe.DorisAssert;
+import org.apache.doris.utframe.UtFrameUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+public class QueryStmtTest {
+ private static String runningDir = "fe/mocked/DemoTest/" +
UUID.randomUUID().toString() + "/";
+ private static DorisAssert dorisAssert;
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ UtFrameUtils.cleanDorisFeDir(runningDir);
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Config.enable_batch_delete_by_default = true;
+ UtFrameUtils.createMinDorisCluster(runningDir);
+ String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2
varchar(32), k3 varchar(32), k4 int) "
+ + "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets
3 properties('replication_num' = '1');";
+ String createBaseAllStmtStr = "create table db1.baseall(k1 int, k2
varchar(32)) distributed by hash(k1) "
+ + "buckets 3 properties('replication_num' = '1');";
+ String tbl1 = "CREATE TABLE db1.table1 (\n" +
+ " `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" +
+ " `citycode` smallint(6) NULL COMMENT \"\",\n" +
+ " `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" +
+ " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" +
+ ") ENGINE=OLAP\n" +
+ "UNIQUE KEY(`siteid`, `citycode`, `username`)\n" +
+ "COMMENT \"OLAP\"\n" +
+ "DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" +
+ "PROPERTIES (\n" +
+ "\"replication_num\" = \"1\",\n" +
+ "\"in_memory\" = \"false\",\n" +
+ "\"storage_format\" = \"V2\"\n" +
+ ")";
+ dorisAssert = new DorisAssert();
+ dorisAssert.withDatabase("db1").useDatabase("db1");
+ dorisAssert.withTable(createTblStmtStr)
+ .withTable(createBaseAllStmtStr)
+ .withTable(tbl1);
+ }
+
+ @Test
+ public void testCollectExprs() throws Exception {
+ ConnectContext ctx = UtFrameUtils.createDefaultCtx();
+ String sql = "SELECT CASE\n" +
+ " WHEN (\n" +
+ " SELECT COUNT(*) / 2\n" +
+ " FROM db1.tbl1\n" +
+ " ) > k4 THEN (\n" +
+ " SELECT AVG(k4)\n" +
+ " FROM db1.tbl1\n" +
+ " )\n" +
+ " ELSE (\n" +
+ " SELECT SUM(k4)\n" +
+ " FROM db1.tbl1\n" +
+ " )\n" +
+ " END AS kk4\n" +
+ "FROM db1.tbl1;";
+ QueryStmt stmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(sql,
ctx);
+ Map<String, Expr> exprsMap = new HashMap<>();
+ stmt.collectExprs(exprsMap);
+ Assert.assertEquals(4, exprsMap.size());
+
+ sql = "SELECT username\n" +
+ "FROM db1.table1\n" +
+ "WHERE siteid in\n" +
+ " (SELECT abs(5+abs(0))+1)\n" +
+ "UNION\n" +
+ "SELECT CASE\n" +
+ " WHEN\n" +
+ " (SELECT count(*)+abs(8)\n" +
+ " FROM db1.table1\n" +
+ " WHERE username='helen')>1 THEN 888\n" +
+ " ELSE 999\n" +
+ " END AS ccc\n" +
+ "FROM\n" +
+ " (SELECT curdate()) a;";
+ stmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
+ exprsMap.clear();
+ stmt.collectExprs(exprsMap);
+ Assert.assertEquals(6, exprsMap.size());
+
+ sql = "select\n" +
+ " avg(t1.k4)\n" +
+ "from\n" +
+ " db1.tbl1 t1,\n" +
+ " db1.tbl1 t2,\n" +
+ " db1.tbl1 t3,\n" +
+ " db1.tbl1 t4,\n" +
+ " db1.tbl1 t5,\n" +
+ " db1.tbl1 t6\n" +
+ "where\n" +
+ " t2.k1 = t1.k1\n" +
+ " and t1.k2 = t6.k2\n" +
+ " and t6.k4 = 2001\n" +
+ " and(\n" +
+ " (\n" +
+ " t1.k2 = t4.k2\n" +
+ " and t3.k3 = t1.k3\n" +
+ " and t3.k1 = 'D'\n" +
+ " and t4.k3 = '2 yr Degree'\n" +
+ " and t1.k4 between 100.00\n" +
+ " and 150.00\n" +
+ " and t4.k4 = 3\n" +
+ " )\n" +
+ " or (\n" +
+ " t1.k2 = t4.k2\n" +
+ " and t3.k3 = t1.k3\n" +
+ " and t3.k1 = 'S'\n" +
+ " and t4.k3 = 'Secondary'\n" +
+ " and t1.k4 between 50.00\n" +
+ " and 100.00\n" +
+ " and t4.k4 = 1\n" +
+ " )\n" +
+ " or (\n" +
+ " t1.k2 = t4.k2\n" +
+ " and t3.k3 = t1.k3\n" +
+ " and t3.k1 = 'W'\n" +
+ " and t4.k3 = 'Advanced Degree'\n" +
+ " and t1.k4 between 150.00\n" +
+ " and 200.00\n" +
+ " and t4.k4 = 1\n" +
+ " )\n" +
+ " )\n" +
+ " and(\n" +
+ " (\n" +
+ " t1.k1 = t5.k1\n" +
+ " and t5.k2 = 'United States'\n" +
+ " and t5.k3 in ('CO', 'IL', 'MN')\n" +
+ " and t1.k4 between 100\n" +
+ " and 200\n" +
+ " )\n" +
+ " or (\n" +
+ " t1.k1 = t5.k1\n" +
+ " and t5.k2 = 'United States'\n" +
+ " and t5.k3 in ('OH', 'MT', 'NM')\n" +
+ " and t1.k4 between 150\n" +
+ " and 300\n" +
+ " )\n" +
+ " or (\n" +
+ " t1.k1 = t5.k1\n" +
+ " and t5.k2 = 'United States'\n" +
+ " and t5.k3 in ('TX', 'MO', 'MI')\n" +
+ " and t1.k4 between 50 and 250\n" +
+ " )\n" +
+ " );";
+ stmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
+ exprsMap.clear();
+ stmt.collectExprs(exprsMap);
+ Assert.assertEquals(2, exprsMap.size());
+
+ sql = "SELECT k1 FROM db1.baseall GROUP BY k1 HAVING EXISTS(SELECT k4
FROM db1.tbl1 GROUP BY k4 " +
+ "HAVING SUM(k4) = k4);";
+ stmt = (QueryStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
+ exprsMap.clear();
+ stmt.collectExprs(exprsMap);
+ Assert.assertEquals(4, exprsMap.size());
+ }
+
+ @Test
+ public void testPutBackExprs() throws Exception {
+ ConnectContext ctx = UtFrameUtils.createDefaultCtx();
+ String sql = "SELECT username, @@license, @@time_zone\n" +
+ "FROM db1.table1\n" +
+ "WHERE siteid in\n" +
+ " (SELECT abs(5+abs(0))+1)\n" +
+ "UNION\n" +
+ "SELECT CASE\n" +
+ " WHEN\n" +
+ " (SELECT count(*)+abs(8)\n" +
+ " FROM db1.table1\n" +
+ " WHERE username='helen')>1 THEN 888\n" +
+ " ELSE 999\n" +
+ " END AS ccc, @@language, @@storage_engine\n" +
+ "FROM\n" +
+ " (SELECT curdate()) a;";
+ StatementBase stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
+ stmt.foldConstant(new Analyzer(ctx.getCatalog(),
ctx).getExprRewriter());
+
+ // reAnalyze
+ reAnalyze(stmt, ctx);
+ Assert.assertTrue(stmt.toSql().contains("Apache License, Version
2.0"));
+ Assert.assertTrue(stmt.toSql().contains("/palo/share/english/"));
+
+ // test sysVariableDescs
+ sql = "SELECT\n" +
+ " avg(t1.k4)\n" +
+ "FROM\n" +
+ " db1.tbl1 t1,\n" +
+ " db1.tbl1 t2\n" +
+ "WHERE\n" +
+ "(\n" +
+ " t2.k2 = 'United States'\n" +
+ " AND t2.k3 in (@@license, @@version)\n" +
+ ")\n" +
+ "OR (\n" +
+ " t2.k2 = @@language\n" +
+ ")";
+ stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
+ stmt.foldConstant(new Analyzer(ctx.getCatalog(),
ctx).getExprRewriter());
+ // reAnalyze
+ reAnalyze(stmt, ctx);
+ Assert.assertTrue(stmt.toSql().contains("Apache License, Version
2.0"));
+ Assert.assertTrue(stmt.toSql().contains("/palo/share/english/"));
+
+ // test informationFunctions
+ sql = "SELECT\n" +
+ " avg(t1.k4)\n" +
+ "FROM\n" +
+ " db1.tbl1 t1,\n" +
+ " db1.tbl1 t2\n" +
+ "WHERE\n" +
+ "(\n" +
+ " t2.k2 = 'United States'\n" +
+ " AND t2.k1 in (USER(), CURRENT_USER(), SCHEMA())\n" +
+ ")\n" +
+ "OR (\n" +
+ " t2.k2 = CONNECTION_ID()\n" +
+ ")";
+ stmt = UtFrameUtils.parseAndAnalyzeStmt(sql, ctx);
+ stmt.foldConstant(new Analyzer(ctx.getCatalog(),
ctx).getExprRewriter());
+ // reAnalyze
+ reAnalyze(stmt, ctx);
+ Assert.assertTrue(stmt.toSql().contains("root''@''%"));
+ Assert.assertTrue(stmt.toSql().contains("root''@''127.0.0.1"));
+
+ }
+
+ private void reAnalyze(StatementBase stmt, ConnectContext ctx) throws
UserException {
+ // reAnalyze
+ List<Type> origResultTypes = Lists.newArrayList();
+ for (Expr e: stmt.getResultExprs()) {
+ origResultTypes.add(e.getType());
+ }
+ List<String> origColLabels =
+ Lists.newArrayList(stmt.getColLabels());
+
+ // query re-analyze
+ stmt.reset();
+ // Re-analyze the stmt with a new analyzer.
+ stmt.analyze(new Analyzer(ctx.getCatalog(), ctx));
+
+ // Restore the original result types and column labels.
+ stmt.castResultExprs(origResultTypes);
+ stmt.setColLabels(origColLabels);
+ }
+}
diff --git a/gensrc/proto/internal_service.proto
b/gensrc/proto/internal_service.proto
index 95b1be4..d900073 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -325,6 +325,25 @@ message PPublishFilterResponse {
required PStatus status = 1;
};
+message PExprResult {
+ required PScalarType type = 1;
+ required string content = 2;
+ required bool success = 3;
+};
+
+message PExprResultMap {
+ map<string, PExprResult> map = 1;
+};
+
+message PConstantExprRequest {
+ optional bytes request = 1;
+};
+
+message PConstantExprResult {
+ required PStatus status = 1;
+ map<string, PExprResultMap> expr_result_map = 2;
+};
+
// NOTE(zc): If you want to add new method here,
// you MUST add same method to palo_internal_service.proto
service PBackendService {
@@ -341,5 +360,6 @@ service PBackendService {
rpc clear_cache(PClearCacheRequest) returns (PCacheResponse);
rpc merge_filter(PMergeFilterRequest) returns (PMergeFilterResponse);
rpc apply_filter(PPublishFilterRequest) returns (PPublishFilterResponse);
+ rpc fold_constant_expr(PConstantExprRequest) returns (PConstantExprResult);
};
diff --git a/gensrc/proto/palo_internal_service.proto
b/gensrc/proto/palo_internal_service.proto
index 592fe58..8439f4a 100644
--- a/gensrc/proto/palo_internal_service.proto
+++ b/gensrc/proto/palo_internal_service.proto
@@ -40,4 +40,5 @@ service PInternalService {
rpc clear_cache(doris.PClearCacheRequest) returns (doris.PCacheResponse);
rpc merge_filter(doris.PMergeFilterRequest) returns
(doris.PMergeFilterResponse);
rpc apply_filter(doris.PPublishFilterRequest) returns
(doris.PPublishFilterResponse);
+ rpc fold_constant_expr(doris.PConstantExprRequest) returns
(doris.PConstantExprResult);
};
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index df87a5e..0f8b420 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -451,6 +451,7 @@ struct TMasterOpRequest {
17: optional Types.TUniqueId query_id // when this is a query, we
translate this query id to master
18: optional i64 insert_visible_timeout_ms // deprecated, move into
session_variables
19: optional map<string, string> session_variables
+ 20: optional bool foldConstantByBe
}
struct TColumnDefinition {
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index b2f0c89..ae64ef5 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -319,6 +319,15 @@ struct TCancelPlanFragmentResult {
1: optional Status.TStatus status
}
+// fold constant expr
+struct TExprMap {
+ 1: required map<string, Exprs.TExpr> expr_map
+}
+
+struct TFoldConstantParams {
+ 1: required map<string, map<string, Exprs.TExpr>> expr_map
+ 2: required TQueryGlobals query_globals
+}
// TransmitData
struct TTransmitDataParams {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
