This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/spill_and_reserve by this push:
new d8f61b57a72 pretty print the size info (#41516)
d8f61b57a72 is described below
commit d8f61b57a72c8e83777c46d7773b6570b7e43475
Author: yiguolei <[email protected]>
AuthorDate: Tue Oct 1 23:01:36 2024 +0800
pretty print the size info (#41516)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
Co-authored-by: yiguolei <[email protected]>
---
be/src/pipeline/pipeline_task.cpp | 9 +++++----
be/src/runtime/memory/thread_mem_tracker_mgr.h | 12 ++++++++----
be/src/runtime/workload_group/workload_group_manager.cpp | 6 ++++--
3 files changed, 17 insertions(+), 10 deletions(-)
diff --git a/be/src/pipeline/pipeline_task.cpp
b/be/src/pipeline/pipeline_task.cpp
index 37b44c6e6f1..4d7fcc4b53b 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -392,10 +392,11 @@ Status PipelineTask::execute(bool* eos) {
COUNTER_UPDATE(_memory_reserve_times, 1);
if (!st.ok()) {
COUNTER_UPDATE(_memory_reserve_failed_times, 1);
- LOG(INFO) << "query: " << print_id(query_id)
- << ", try to reserve: " << reserve_size <<
"(sink reserve size:("
- << sink_reserve_size << ")"
- << ", sink name: " << _sink->get_name()
+ LOG(INFO) << "query: " << print_id(query_id) << ", try to
reserve: "
+ << PrettyPrinter::print(reserve_size,
TUnit::BYTES)
+ << "(sink reserve size:("
+ << PrettyPrinter::print(sink_reserve_size,
TUnit::BYTES)
+ << "), sink name: " << _sink->get_name()
<< ", node id: " << _sink->node_id() << "
failed: " << st.to_string()
<< ", debug info: " <<
GlobalMemoryArbitrator::process_mem_log_str();
diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h
b/be/src/runtime/memory/thread_mem_tracker_mgr.h
index 165153fc75f..1d7fa8bba07 100644
--- a/be/src/runtime/memory/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h
@@ -293,7 +293,8 @@ inline doris::Status
ThreadMemTrackerMgr::try_reserve(int64_t size) {
// Only do a check here, do not real reserve. If we could reserve it,
it is better, but the logic is too complicated.
if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) {
return doris::Status::Error<ErrorCode::PROCESS_MEMORY_EXCEEDED>(
- "reserve memory failed, size: {}, because {}", size,
+ "reserve memory failed, size: {}, because {}",
+ PrettyPrinter::print(size, TUnit::BYTES),
GlobalMemoryArbitrator::process_mem_log_str());
} else {
doris::GlobalMemoryArbitrator::release_process_reserved_memory(size);
@@ -304,14 +305,16 @@ inline doris::Status
ThreadMemTrackerMgr::try_reserve(int64_t size) {
auto err_msg = fmt::format(
"reserve memory failed, size: {}, because query memory
exceeded, memory tracker "
"consumption: {}, limit: {}",
- size, _limiter_tracker->consumption(),
_limiter_tracker->limit());
+ PrettyPrinter::print(size, TUnit::BYTES),
+ PrettyPrinter::print(_limiter_tracker->consumption(),
TUnit::BYTES),
+ PrettyPrinter::print(_limiter_tracker->limit(), TUnit::BYTES));
return doris::Status::Error<ErrorCode::QUERY_MEMORY_EXCEEDED>(err_msg);
}
if (wg_ptr) {
if (!wg_ptr->add_wg_refresh_interval_memory_growth(size)) {
auto err_msg = fmt::format(
"reserve memory failed, size: {}, because wg memory
exceeded, wg info: {}",
- size, wg_ptr->memory_debug_string());
+ PrettyPrinter::print(size, TUnit::BYTES),
wg_ptr->memory_debug_string());
_limiter_tracker->release(size); // rollback
_limiter_tracker->release_reserved(size); // rollback
return
doris::Status::Error<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>(err_msg);
@@ -320,7 +323,8 @@ inline doris::Status
ThreadMemTrackerMgr::try_reserve(int64_t size) {
if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) {
auto err_msg =
fmt::format("reserve memory failed, size: {}, because proccess
memory exceeded, {}",
- size,
GlobalMemoryArbitrator::process_mem_log_str());
+ PrettyPrinter::print(size, TUnit::BYTES),
+ GlobalMemoryArbitrator::process_mem_log_str());
_limiter_tracker->release(size); // rollback
_limiter_tracker->release_reserved(size); // rollback
if (wg_ptr) {
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp
b/be/src/runtime/workload_group/workload_group_manager.cpp
index 2e9f4b0b0ad..bc87f342ac0 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -639,6 +639,8 @@ void
WorkloadGroupMgr::update_queries_limit(WorkloadGroupPtr wg, bool enable_har
int64_t expected_query_weighted_mem_limit = 0;
// If the query enable hard limit, then it should not use the soft
limit
if (query_ctx->enable_query_slot_hard_limit()) {
+ LOG(INFO) << "query info " << wg_high_water_mark_except_load << ","
+ << query_ctx->get_slot_count() << "," <<
total_slot_count;
if (total_slot_count < 1) {
LOG(WARNING)
<< "query " << print_id(query_ctx->query_id())
@@ -673,8 +675,8 @@ void
WorkloadGroupMgr::update_queries_limit(WorkloadGroupPtr wg, bool enable_har
query_ctx->set_expected_mem_limit(expected_query_weighted_mem_limit);
}
}
- //LOG(INFO) << debug_msg;
- LOG_EVERY_T(INFO, 60) << debug_msg;
+ LOG(INFO) << debug_msg;
+ //LOG_EVERY_T(INFO, 60) << debug_msg;
}
void WorkloadGroupMgr::stop() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
