This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 64c4dae48dd branch-4.2: [fix](be) Wait before cancelling paused
queries under process memory pressure (#70432)
64c4dae48dd is described below
commit 64c4dae48dd418a5f69088a1d9d60e0c7cd52126
Author: Jerry Hu <[email protected]>
AuthorDate: Fri Nov 20 11:26:45 2026 +0800
branch-4.2: [fix](be) Wait before cancelling paused queries under process
memory pressure (#70432)
## Summary
Backport #67845 to `branch-4.2`.
- Allow queries paused by process memory pressure to wait for a bounded
interval while soft pressure persists. Resume them when pressure clears;
cancel them on timeout or hard-limit pressure.
- Preserve the branch-4.2 inline paused-query dispatch while porting
recovery, spill, and cross-workload-group memory reclamation behavior.
- Measure the wait from the first failed reservation and reset it after
a successful reservation. Add focused BE coverage for these transitions.
## Testing
- `./run-be-ut.sh --run
--filter='WorkloadGroupManagerTest.*:PipelineTaskTest.TEST_RESERVE_MEMORY_FAIL_SPILLABLE'
-j 24` — ASAN build and 53/53 tests passed.
- `./build-support/check-format.sh` — passed.
- `clang-format --dry-run --Werror` on the eight changed files — passed.
- `git diff --check origin/branch-4.2...HEAD` — passed.
SQL regression, multi-BE, and mixed-version tests were not run locally.
---
be/src/exec/pipeline/pipeline_task.cpp | 4 +
.../workload_group/workload_group_manager.cpp | 433 ++++++++---
.../workload_group/workload_group_manager.h | 31 +-
.../runtime/workload_management/task_controller.h | 15 +
be/src/util/mem_info.h | 7 +
be/test/exec/pipeline/pipeline_task_test.cpp | 22 +
.../workload_group/workload_group_manager_test.cpp | 844 ++++++++++++++++++++-
be/test/testutil/mock/mock_query_task_controller.h | 55 ++
8 files changed, 1279 insertions(+), 132 deletions(-)
diff --git a/be/src/exec/pipeline/pipeline_task.cpp
b/be/src/exec/pipeline/pipeline_task.cpp
index f71d3eac22c..e9498a387e9 100644
--- a/be/src/exec/pipeline/pipeline_task.cpp
+++ b/be/src/exec/pipeline/pipeline_task.cpp
@@ -862,6 +862,10 @@ bool PipelineTask::_try_to_reserve_memory(const size_t
reserve_size, OperatorBas
_spilling = true;
return false;
}
+ // The query made progress: a later reservation that fails for process
memory starts a new
+ // bounded wait instead of continuing the one that this reservation may
have been retried
+ // from.
+
_state->get_query_ctx()->resource_ctx()->task_controller()->end_process_memory_wait();
return true;
}
diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp
b/be/src/runtime/workload_group/workload_group_manager.cpp
index 9c50c3c5fb6..e9bb3e79b31 100644
--- a/be/src/runtime/workload_group/workload_group_manager.cpp
+++ b/be/src/runtime/workload_group/workload_group_manager.cpp
@@ -23,6 +23,7 @@
#include <memory>
#include <mutex>
#include <unordered_map>
+#include <vector>
#include "common/config.h"
#include "common/status.h"
@@ -47,13 +48,12 @@ const static std::string INTERNAL_NORMAL_WG_NAME = "normal";
const static uint64_t INTERNAL_NORMAL_WG_ID = 1;
PausedQuery::PausedQuery(std::shared_ptr<ResourceContext> resource_ctx, double
cache_ratio,
- int64_t reserve_size)
+ int64_t reserve_size, int64_t enqueue_at_ms)
: resource_ctx_(resource_ctx),
+ enqueue_at(enqueue_at_ms),
cache_ratio_(cache_ratio),
reserve_size_(reserve_size),
- query_id_(print_id(resource_ctx->task_controller()->task_id())) {
- enqueue_at = std::chrono::system_clock::now();
-}
+ query_id_(print_id(resource_ctx->task_controller()->task_id())) {}
WorkloadGroupMgr::~WorkloadGroupMgr() = default;
@@ -289,18 +289,51 @@ void WorkloadGroupMgr::add_paused_query(const
std::shared_ptr<ResourceContext>&
resource_ctx->task_controller()->update_paused_reason(status);
resource_ctx->task_controller()->set_low_memory_mode(true);
resource_ctx->task_controller()->set_memory_sufficient(false);
+ // A query paused for process memory is resumed once its recorded
reservation fits and every
+ // blocked task then retries its own request. A retry that fails again
(another task took the
+ // memory first, or a sibling that fit on its own still does not fit next
to the others)
+ // pauses the query again: it continues the wait it was resumed from, so
that the wait stays
+ // bounded by spill_in_paused_queue_timeout_ms. The wait ends with the
first reservation that
+ // succeeds, see PipelineTask::_try_to_reserve_memory().
+ const int64_t now_ms = MonotonicMillis();
+ const int64_t enqueue_at =
+ status.is<ErrorCode::PROCESS_MEMORY_EXCEEDED>()
+ ?
resource_ctx->task_controller()->start_process_memory_wait(now_ms)
+ : now_ms;
std::lock_guard<std::mutex> lock(_paused_queries_lock);
auto wg = resource_ctx->workload_group();
- auto&& [it, inserted] = _paused_queries_list[wg].emplace(
+ auto& queries_list = _paused_queries_list[wg];
+ auto&& [it, inserted] = queries_list.emplace(
resource_ctx,
doris::GlobalMemoryArbitrator::last_affected_cache_capacity_adjust_weighted,
- reserve_size);
+ reserve_size, enqueue_at);
// Check if this is an invalid reserve, for example, if the reserve size
is too large, larger than the query limit
// if hard limit is enabled, then not need enable other queries hard limit.
if (inserted) {
LOG(INFO) << "Insert one new paused query: "
<< resource_ctx->task_controller()->debug_string()
- << ", workload group: " << wg->debug_string();
+ << ", workload group: " << wg->debug_string() << ", waited "
<< it->elapsed_time()
+ << " ms";
+ } else if (reserve_size > it->reserve_size_) {
+ // Another task of the same query already failed a smaller
reservation. The query is
+ // resumed as a whole and every blocked task then retries its own
request, so record the
+ // largest pending one: the query is woken up once that one fits,
which is when the
+ // tasks can reserve one after another (a task releases its
reservation after each
+ // block, so the pending requests do not need to fit at the same
time). Waking it up
+ // for a smaller one would fail the largest again right away.
+ LOG(INFO) << "Query: " <<
print_id(resource_ctx->task_controller()->task_id())
+ << " is already paused, raise the recorded reservation from "
+ << PrettyPrinter::print_bytes(it->reserve_size_) << " to "
+ << PrettyPrinter::print_bytes(reserve_size);
+ auto node = queries_list.extract(it);
+ node.value().reserve_size_ = reserve_size;
+ queries_list.insert(std::move(node));
+ } else {
+ LOG(INFO) << "Query: " <<
print_id(resource_ctx->task_controller()->task_id())
+ << " is already paused, keep the recorded reservation "
+ << PrettyPrinter::print_bytes(it->reserve_size_)
+ << " which is not smaller than the pending "
+ << PrettyPrinter::print_bytes(reserve_size);
}
}
@@ -579,48 +612,10 @@ void WorkloadGroupMgr::handle_paused_queries() {
}
}
} else {
- // PROCESS_MEMORY_EXCEEDED: a recently cancelled query
anywhere may be releasing
- // process-level memory, so wait globally before spilling or
revoking.
- if (has_recently_cancelled_query) {
- ++query_it;
- continue;
- }
- // If workload group's memory usage > min memory, then it
means the workload group use too much memory
- // in memory contention state. Should just spill
- if (wg->total_mem_used() > wg->min_memory_limit()) {
- bool spill_res = handle_single_query_(
- resource_ctx, query_it->reserve_size_,
query_it->elapsed_time(),
- resource_ctx->task_controller()->paused_reason());
- if (!spill_res) {
- ++query_it;
- continue;
- } else {
- VLOG_DEBUG
- << "Query: " <<
print_id(resource_ctx->task_controller()->task_id())
- << " remove from paused list";
- query_it = queries_list.erase(query_it);
- // The query is cancelled, just break. And wait for
the query to release the memory. Other query maybe not need spill.
- if (resource_ctx->task_controller()->is_cancelled()) {
- revoking_memory_from_other_query_ = true;
- }
- // If any query is cancelled or spilled to disk, we
need to stop and not revoke memory from other queries.
- return;
- }
- }
-
- // Other workload groups many use a lot of memory, should
revoke memory from other workload groups
- // by cancelling their queries.
- int64_t revoked_size = revoke_memory_from_other_groups_();
- if (revoked_size > 0) {
- // Revoke memory from other workload groups will cancel
some queries, wait them cancel finished
- // and then check it again.
- revoking_memory_from_other_query_ = true;
+ if (handle_process_memory_exceeded_(wg, queries_list,
query_it, resource_ctx,
+
has_recently_cancelled_query)) {
return;
}
-
- // TODO revoke from memtable
-
- ++query_it;
}
}
@@ -636,19 +631,121 @@ void WorkloadGroupMgr::handle_paused_queries() {
}
}
-// Find the workload group that could revoke lot of memory:
-// 1. workload group = max(total used memory - min memory that should reserved
for it)
+bool WorkloadGroupMgr::handle_process_memory_exceeded_(
+ const WorkloadGroupPtr& wg, PausedQuerySet& queries_list,
PausedQueryIterator& query_it,
+ const std::shared_ptr<ResourceContext>& resource_ctx, bool
has_recently_cancelled_query) {
+ // PROCESS_MEMORY_EXCEEDED: a recently cancelled query anywhere may be
releasing
+ // process-level memory, so wait globally before spilling or revoking.
+ if (has_recently_cancelled_query) {
+ ++query_it;
+ return false;
+ }
+
+ // The process memory pressure may have been relieved by cache reclamation
or by other
+ // queries that finished. Check it before routing the query below,
otherwise a query in a
+ // workload group that uses less than its min memory limit has to wait for
the timeout.
+ // Test the recorded reservation itself: this is the predicate the failed
reservation used,
+ // so the query is resumed exactly when its request fits now.
+ if
(!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(query_it->reserve_size_)) {
+ resume_process_paused_query_(wg, queries_list, query_it, resource_ctx);
+ return false;
+ }
+
+ // If workload group's memory usage > min memory, then it means the
workload group use too much memory
+ // in memory contention state. Should just spill
+ if (wg->total_mem_used() > wg->min_memory_limit()) {
+ return release_query_memory_(queries_list, query_it, resource_ctx,
true);
+ }
+
+ // Other workload groups many use a lot of memory, should revoke memory
from other workload groups
+ // by cancelling their queries. Only the memory actually revoked counts: a
workload group that
+ // exceeds its min memory but has nothing cancellable releases nothing
(the next exceeding
+ // workload group is tried instead), and waiting for it would resume this
query without any
+ // memory being freed.
+ int64_t revoked_size =
revoke_memory_from_other_groups_(query_it->reserve_size_);
+ if (revoked_size > 0) {
+ // Revoke memory from other workload groups will cancel some queries,
wait them cancel finished
+ // and then check it again.
+ revoking_memory_from_other_query_ = true;
+ return true;
+ }
+
+ // While the other workload groups were scanned, other queries or the
cache may have released
+ // enough memory for the recorded reservation (the walk stops as soon as
it fits). Resume the
+ // query instead of cancelling it below from the stale observation.
+ if
(!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(query_it->reserve_size_)) {
+ resume_process_paused_query_(wg, queries_list, query_it, resource_ctx);
+ return false;
+ }
+
+ // TODO revoke from memtable
+
+ // Fallback: if the process reaches the hard limit or we have waited too
long and cannot
+ // revoke from anywhere, let this query spill or cancel it to protect the
process.
+ if (GlobalMemoryArbitrator::is_exceed_hard_mem_limit() ||
+ query_it->elapsed_time() > config::spill_in_paused_queue_timeout_ms) {
+ // No other workload group can release memory, and process memory is
still exceeded.
+ return release_query_memory_(queries_list, query_it, resource_ctx,
true);
+ }
+ ++query_it;
+ return false;
+}
+
+void WorkloadGroupMgr::resume_process_paused_query_(
+ const WorkloadGroupPtr& wg, PausedQuerySet& queries_list,
PausedQueryIterator& query_it,
+ const std::shared_ptr<ResourceContext>& resource_ctx) {
+ LOG(INFO) << "Query: " <<
print_id(resource_ctx->task_controller()->task_id())
+ << ", process limit not exceeded now, resume this query"
+ << ", process memory info: "
+ << GlobalMemoryArbitrator::process_memory_used_details_str()
+ << ", wg info: " << wg->debug_string();
+ resource_ctx->task_controller()->set_memory_sufficient(true);
+ query_it = queries_list.erase(query_it);
+}
+
+bool WorkloadGroupMgr::release_query_memory_(PausedQuerySet& queries_list,
+ PausedQueryIterator& query_it,
+ const
std::shared_ptr<ResourceContext>& resource_ctx,
+ bool stop_after_release) {
+ bool spill_res =
+ handle_single_query_(resource_ctx, query_it->reserve_size_,
query_it->elapsed_time(),
+
resource_ctx->task_controller()->paused_reason());
+ if (!spill_res) {
+ ++query_it;
+ return false;
+ }
+
+ VLOG_DEBUG << "Query: " <<
print_id(resource_ctx->task_controller()->task_id())
+ << " remove from paused list";
+ query_it = queries_list.erase(query_it);
+ if (resource_ctx->task_controller()->is_cancelled()) {
+ // The query is cancelled, just break. And wait for the query to
release the memory.
+ // Other query maybe not need spill.
+ revoking_memory_from_other_query_ = true;
+ return true;
+ }
+ return stop_after_release;
+}
+
+// Find the workload groups that could revoke lot of memory:
+// 1. order the workload groups by (total used memory - min memory that should
reserved for it),
+// starting with the one that exceeds most.
// 2. revoke 10% memory of the workload group that exceeded. For example, if
the workload group exceed 10g,
-// then revoke 1g memory.
+// then revoke 1g memory. A workload group may release nothing although it
exceeds its min
+// memory (for example, every query is too small to be cancelled), then try
the next one.
// 3. After revoke memory, go to the loop and wait for the query to be
cancelled and check again.
-int64_t WorkloadGroupMgr::revoke_memory_from_other_groups_() {
+// `reserve_size` is the reservation of the paused query this walk serves: the
walk stops once it
+// fits, nothing needs to be revoked for it any more.
+// Returns the memory actually revoked, which is 0 when no query of any of
these workload groups
+// could be cancelled.
+int64_t WorkloadGroupMgr::revoke_memory_from_other_groups_(int64_t
reserve_size) {
MonotonicStopWatch watch;
watch.start();
std::unique_ptr<RuntimeProfile> profile =
std::make_unique<RuntimeProfile>("RevokeMemoryFromOtherGroups");
- WorkloadGroupPtr max_wg = nullptr;
- int64_t max_exceeded_memory = 0;
+ // (exceeded memory, workload group), the most exceeded first.
+ std::vector<std::pair<int64_t, WorkloadGroupPtr>> exceeded_wgs;
{
std::shared_lock<std::shared_mutex> r_lock(_group_mutex);
for (auto& workload_group : _workload_groups) {
@@ -659,32 +756,76 @@ int64_t
WorkloadGroupMgr::revoke_memory_from_other_groups_() {
// then not revoke memory from it.
continue;
}
- if (total_used_memory - min_memory_limit > max_exceeded_memory) {
- max_wg = workload_group.second;
- max_exceeded_memory = total_used_memory - min_memory_limit;
- }
+ exceeded_wgs.emplace_back(total_used_memory - min_memory_limit,
workload_group.second);
}
}
- if (max_wg == nullptr) {
- return 0;
- }
- if (max_exceeded_memory < 1 << 27) {
- LOG(INFO) << "The workload group that exceed most memory is :"
- << max_wg->memory_debug_string() << ", max_exceeded_memory: "
- << PrettyPrinter::print(max_exceeded_memory, TUnit::BYTES)
- << " less than 128MB, no need to revoke memory";
- return 0;
- }
- int64_t freed_mem = static_cast<int64_t>((double)max_exceeded_memory *
0.1);
- // Revoke 10% of memory from the workload group that exceed most memory
- max_wg->revoke_memory(freed_mem, "exceed_memory", profile.get());
- std::stringstream ss;
- profile->pretty_print(&ss);
- LOG(INFO) << fmt::format(
- "[MemoryGC] process memory not enough, revoke memory from
workload_group: {}, "
- "free memory {}. cost(us): {}, details: {}",
- max_wg->memory_debug_string(),
PrettyPrinter::print_bytes(freed_mem),
- watch.elapsed_time() / 1000, ss.str());
+ std::sort(exceeded_wgs.begin(), exceeded_wgs.end(),
+ [](const auto& lhs, const auto& rhs) { return lhs.first >
rhs.first; });
+
+ int64_t freed_mem = 0;
+ size_t tried_wgs = 0;
+ for (const auto& [snapshot_exceeded_memory, wg] : exceeded_wgs) {
+ if (snapshot_exceeded_memory < 1 << 27) {
+ // The remaining workload groups exceed even less.
+ LOG(INFO) << "The workload group that exceed most memory among the
untried ones is :"
+ << wg->memory_debug_string() << ", exceeded_memory: "
+ << PrettyPrinter::print(snapshot_exceeded_memory,
TUnit::BYTES)
+ << " less than 128MB, no need to revoke memory";
+ break;
+ }
+ // Scanning the earlier workload groups took time (a cancellation in
flight is waited for
+ // up to revoke_memory_max_tolerance_ms), during which other queries
or the cache may have
+ // released memory. The caller checked the reservation before the walk
only: stop as soon
+ // as it fits, so that no further query is cancelled for pressure that
is gone.
+ if (tried_wgs > 0 &&
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(reserve_size)) {
+ LOG(INFO) << "Process memory pressure is relieved after trying "
<< tried_wgs
+ << " workload group(s), the reservation "
+ << PrettyPrinter::print_bytes(reserve_size)
+ << " fits now, stop revoking memory from other workload
groups";
+ break;
+ }
+ // The snapshot above was taken before the earlier workload groups
were scanned, during
+ // which this one may have released memory (its queries finished or
were cancelled).
+ // total_mem_used() is the usage cached by the last maintenance
refresh, so refresh it and
+ // decide from the current usage: the min memory stays reserved for a
workload group that
+ // no longer exceeds it, and the amount to revoke is what it exceeds
by now.
+ const int64_t exceeded_memory = wg->refresh_memory_usage() -
wg->min_memory_limit();
+ if (exceeded_memory < 1 << 27) {
+ LOG(INFO) << "The workload group " << wg->memory_debug_string()
+ << " exceeded its min memory by "
+ << PrettyPrinter::print(snapshot_exceeded_memory,
TUnit::BYTES)
+ << " when the walk started, now by "
+ << PrettyPrinter::print(exceeded_memory, TUnit::BYTES)
+ << " less than 128MB, skip it";
+ continue;
+ }
+ auto need_free_mem = static_cast<int64_t>((double)exceeded_memory *
0.1);
+ // Revoke 10% of memory from the workload group that exceed most memory
+ freed_mem = wg->revoke_memory(need_free_mem, "exceed_memory",
profile.get());
+ ++tried_wgs;
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] process memory not enough, revoke memory from
workload_group: {}, "
+ "need free memory {}, freed memory {}. cost(us): {}",
+ wg->memory_debug_string(),
PrettyPrinter::print_bytes(need_free_mem),
+ PrettyPrinter::print_bytes(freed_mem), watch.elapsed_time() /
1000);
+ if (freed_mem > 0) {
+ break;
+ }
+ // Nothing could be cancelled in this workload group, try the next one
that exceeds
+ // its min memory.
+ }
+ if (tried_wgs > 0) {
+ // Every tried workload group appended its own child to the profile,
so print it once
+ // after the walk instead of once per group: this runs under
_paused_queries_lock on
+ // every maintenance round while the pressure lasts.
+ std::stringstream ss;
+ profile->pretty_print(&ss);
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] process memory not enough, tried to revoke memory
from {} workload "
+ "group(s), freed memory {}. cost(us): {}, details: {}",
+ tried_wgs, PrettyPrinter::print_bytes(freed_mem),
watch.elapsed_time() / 1000,
+ ss.str());
+ }
return freed_mem;
}
@@ -707,20 +848,23 @@ bool WorkloadGroupMgr::handle_single_query_(const
std::shared_ptr<ResourceContex
const auto query_id = print_id(requestor->task_controller()->task_id());
requestor->task_controller()->get_revocable_info(&revocable_size,
&memory_usage,
&has_running_task);
+ if (!paused_reason.is<ErrorCode::QUERY_MEMORY_EXCEEDED>() &&
+ !paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>()) {
+ // PROCESS_MEMORY_EXCEEDED is resolved separately: the process hard
limit and the
+ // timeout must be enforced even when a running task prevents
spilling, which the
+ // early exits below skip.
+ return resolve_process_memory_exceeded_query_(requestor,
size_to_reserve, time_in_queue,
+ memory_usage,
has_running_task);
+ }
if (has_running_task) {
- LOG(INFO) << "Query: " <<
print_id(requestor->task_controller()->task_id())
- << " is paused, but still has running task, skip it.";
+ LOG(INFO) << "Query: " << query_id << " is paused, but still has
running task, skip it.";
return false;
}
const auto wg = requestor->workload_group();
auto revocable_tasks = requestor->task_controller()->get_revocable_tasks();
if (!revocable_tasks.empty()) {
- SCOPED_ATTACH_TASK(requestor);
- auto status = requestor->task_controller()->revoke_memory();
- if (!status.ok()) {
- requestor->task_controller()->cancel(status);
- }
+ spill_query_(requestor);
return true;
}
const auto limit = requestor->memory_context()->mem_limit();
@@ -762,7 +906,10 @@ bool WorkloadGroupMgr::handle_single_query_(const
std::shared_ptr<ResourceContex
requestor->task_controller()->set_memory_sufficient(true);
return true;
}
- } else if (paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>()) {
+ }
+
+ DCHECK(paused_reason.is<ErrorCode::WORKLOAD_GROUP_MEMORY_EXCEEDED>());
+ {
bool exceed_low_watermark = false;
bool exceed_high_watermark = false;
wg->check_mem_used(size_to_reserve, &exceed_low_watermark,
&exceed_high_watermark);
@@ -795,42 +942,92 @@ bool WorkloadGroupMgr::handle_single_query_(const
std::shared_ptr<ResourceContex
requestor->task_controller()->cancel(error_status);
return true;
}
- } else {
- // Should not consider about process memory. For example, the query's
limit is 100g, workload
- // group's memlimit is 10g, process memory is 20g. The query reserve
will always failed in wg
- // limit, and process is always have memory, so that it will resume
and failed reserve again.
- const size_t test_memory_size = std::max<size_t>(size_to_reserve, 32L
* 1024 * 1024);
- if
(!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(test_memory_size)) {
- LOG(INFO) << "Query: " << query_id
- << ", process limit not exceeded now, resume this query"
- << ", process memory info: "
- <<
GlobalMemoryArbitrator::process_memory_used_details_str()
- << ", wg info: " << wg->debug_string();
- requestor->task_controller()->set_memory_sufficient(true);
- return true;
- } else {
- // if cannot find any memory to release, then let the query
continue to run as far as possible
- // or cancelled by gc if memory is really not enough.
- Status error_status = Status::MemoryLimitExceeded(
- "Query {} process memory is exceeded"
- ", and there is no cache now. And could not find task to
spill, disable "
- "reserve memory and resume it. "
- "Query memory usage: {}, limit: {}, reserved "
- "size: {}, try to reserve: {}, wg info: {}."
- " Maybe you should set the workload group's limit to a
lower value. {}",
- query_id, PrettyPrinter::print_bytes(memory_usage),
- PrettyPrinter::print_bytes(limit),
PrettyPrinter::print_bytes(reserved_size),
- PrettyPrinter::print_bytes(size_to_reserve),
wg->memory_debug_string(),
- doris::ProcessProfile::instance()
- ->memory_profile()
- ->process_memory_detail_str());
- LOG_LONG_STRING(INFO, error_status.to_string());
- requestor->task_controller()->cancel(error_status);
- return true;
- }
}
}
+void WorkloadGroupMgr::spill_query_(const std::shared_ptr<ResourceContext>&
requestor) {
+ SCOPED_ATTACH_TASK(requestor);
+ auto status = requestor->task_controller()->revoke_memory();
+ if (!status.ok()) {
+ requestor->task_controller()->cancel(status);
+ }
+}
+
+bool WorkloadGroupMgr::resolve_process_memory_exceeded_query_(
+ const std::shared_ptr<ResourceContext>& requestor, size_t
size_to_reserve,
+ int64_t time_in_queue, size_t memory_usage, bool has_running_task) {
+ const auto query_id = print_id(requestor->task_controller()->task_id());
+ const auto wg = requestor->workload_group();
+
+ // Spilling needs every task of the query to be idle, so the revocable
tasks are only
+ // collected while none is running. A running task does not postpone the
cancellation
+ // below: cancelling is safe while the task runs.
+ std::vector<PipelineTask*> revocable_tasks;
+ if (!has_running_task) {
+ revocable_tasks = requestor->task_controller()->get_revocable_tasks();
+ }
+
+ // The caller observed the pressure before it inspected the pipeline tasks
and tried to
+ // revoke memory from other workload groups, during which other queries or
the cache may have
+ // released memory. Re-check the recorded reservation after the last
inspection and right
+ // before acting, so that the query is never spilled or cancelled from a
stale observation.
+ if (!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(size_to_reserve)) {
+ LOG(INFO) << "Query: " << query_id << ", process limit not exceeded
now, resume this query"
+ << ", process memory info: "
+ << GlobalMemoryArbitrator::process_memory_used_details_str()
+ << ", wg info: " << wg->debug_string();
+ requestor->task_controller()->set_memory_sufficient(true);
+ return true;
+ }
+
+ if (!revocable_tasks.empty()) {
+ // The query is resumed by the spill callback and reserves memory
again. If that
+ // still fails, it comes back here without revocable memory.
+ spill_query_(requestor);
+ return true;
+ }
+
+ const bool exceed_hard_mem_limit =
GlobalMemoryArbitrator::is_exceed_hard_mem_limit();
+ if (!exceed_hard_mem_limit && time_in_queue <
config::spill_in_paused_queue_timeout_ms) {
+ // The query has no revocable memory (or a running task prevents
spilling), cancelling
+ // it will not release much memory. Keep it paused so that it can be
resumed once other
+ // queries release memory, and cancel it only after it has waited for
+ // `spill_in_paused_queue_timeout_ms`. Stop waiting once the process
reaches the hard
+ // memory limit, so that the protection does not depend on memory gc,
which may be
+ // disabled.
+ LOG_EVERY_T(INFO, 1) << "Query: " << query_id
+ << " process memory is exceeded, and could not
find task to "
+ "spill, keep it paused. Waited "
+ << time_in_queue
+ << " ms, timeout: " <<
config::spill_in_paused_queue_timeout_ms
+ << " ms, has running task: " << has_running_task
+ << ", process memory info: "
+ <<
GlobalMemoryArbitrator::process_memory_used_details_str()
+ << ", wg info: " << wg->debug_string();
+ return false;
+ }
+
+ // Waited long enough or the process reached the hard memory limit, and
still could not
+ // find any memory to release, cancel the query to protect the process.
+ Status error_status = Status::MemoryLimitExceeded(
+ "Query {} process memory is exceeded"
+ ", and there is no cache now. And could not find task to spill
after "
+ "waiting {} ms in paused queue (timeout: {} ms, exceed hard limit:
{}, has running "
+ "task: {}), try to cancel query. "
+ "Query memory usage: {}, limit: {}, reserved "
+ "size: {}, try to reserve: {}, wg info: {}."
+ " Maybe you should set the workload group's limit to a lower
value. {}",
+ query_id, time_in_queue, config::spill_in_paused_queue_timeout_ms,
+ exceed_hard_mem_limit, has_running_task,
PrettyPrinter::print_bytes(memory_usage),
+
PrettyPrinter::print_bytes(requestor->memory_context()->mem_limit()),
+
PrettyPrinter::print_bytes(requestor->memory_context()->reserved_consumption()),
+ PrettyPrinter::print_bytes(size_to_reserve),
wg->memory_debug_string(),
+
doris::ProcessProfile::instance()->memory_profile()->process_memory_detail_str());
+ LOG_LONG_STRING(INFO, error_status.to_string());
+ requestor->task_controller()->cancel(error_status);
+ return true;
+}
+
void WorkloadGroupMgr::update_queries_limit_(WorkloadGroupPtr wg, bool
enable_hard_limit) {
auto wg_mem_limit = wg->memory_limit();
auto all_resource_ctxs = wg->resource_ctxs();
diff --git a/be/src/runtime/workload_group/workload_group_manager.h
b/be/src/runtime/workload_group/workload_group_manager.h
index 5b2b1160d60..9e087594b7e 100644
--- a/be/src/runtime/workload_group/workload_group_manager.h
+++ b/be/src/runtime/workload_group/workload_group_manager.h
@@ -23,6 +23,7 @@
#include "common/be_mock_util.h"
#include "runtime/workload_group/workload_group.h"
+#include "util/time.h"
namespace doris {
@@ -39,18 +40,17 @@ public:
// Use weak ptr to save resource ctx, to make sure if the query is
cancelled
// the resource will be released
std::weak_ptr<ResourceContext> resource_ctx_;
- std::chrono::system_clock::time_point enqueue_at;
+ // Monotonic ms. For a query paused for process memory this is the start
of its bounded wait,
+ // which may predate this entry: see
TaskController::start_process_memory_wait().
+ int64_t enqueue_at {0};
size_t last_mem_usage {0};
double cache_ratio_ {0.0};
int64_t reserve_size_ {0};
PausedQuery(std::shared_ptr<ResourceContext> resource_ctx_, double
cache_ratio,
- int64_t reserve_size);
+ int64_t reserve_size, int64_t enqueue_at_ms);
- int64_t elapsed_time() const {
- auto now = std::chrono::system_clock::now();
- return std::chrono::duration_cast<std::chrono::milliseconds>(now -
enqueue_at).count();
- }
+ int64_t elapsed_time() const { return MonotonicMillis() - enqueue_at; }
std::string query_id() const { return query_id_; }
@@ -102,9 +102,26 @@ private:
WorkloadGroupPtr get_or_create_workload_group(const WorkloadGroupInfo&
workload_group_info);
+ using PausedQuerySet = std::set<PausedQuery>;
+ using PausedQueryIterator = PausedQuerySet::iterator;
+
+ bool handle_process_memory_exceeded_(const WorkloadGroupPtr& wg,
PausedQuerySet& queries_list,
+ PausedQueryIterator& query_it,
+ const
std::shared_ptr<ResourceContext>& resource_ctx,
+ bool has_recently_cancelled_query);
+ void resume_process_paused_query_(const WorkloadGroupPtr& wg,
PausedQuerySet& queries_list,
+ PausedQueryIterator& query_it,
+ const std::shared_ptr<ResourceContext>&
resource_ctx);
+ bool release_query_memory_(PausedQuerySet& queries_list,
PausedQueryIterator& query_it,
+ const std::shared_ptr<ResourceContext>&
resource_ctx,
+ bool stop_after_release);
bool handle_single_query_(const std::shared_ptr<ResourceContext>&
requestor,
size_t size_to_reserve, int64_t time_in_queue,
Status paused_reason);
- int64_t revoke_memory_from_other_groups_();
+ bool resolve_process_memory_exceeded_query_(const
std::shared_ptr<ResourceContext>& requestor,
+ size_t size_to_reserve,
int64_t time_in_queue,
+ size_t memory_usage, bool
has_running_task);
+ void spill_query_(const std::shared_ptr<ResourceContext>& requestor);
+ int64_t revoke_memory_from_other_groups_(int64_t reserve_size);
void update_queries_limit_(WorkloadGroupPtr wg, bool enable_hard_limit);
std::shared_mutex _group_mutex;
diff --git a/be/src/runtime/workload_management/task_controller.h
b/be/src/runtime/workload_management/task_controller.h
index 37f2c6ab5a6..97399bb7094 100644
--- a/be/src/runtime/workload_management/task_controller.h
+++ b/be/src/runtime/workload_management/task_controller.h
@@ -98,6 +98,20 @@ public:
void reset_paused_reason() { paused_reason_.reset(); }
Status paused_reason() { return paused_reason_.status(); }
void add_paused_count() { paused_count_.fetch_add(1); }
+ // The wait of a task paused for process memory is bounded from the first
reservation that
+ // failed since its last successful one, not from its latest pause. The
task is resumed to
+ // retry its pending reservations, and a retry that fails again before any
of them succeeded
+ // continues the same wait instead of starting a new one. Returns the
start of the wait
+ // (monotonic ms), starting it at `now_ms` when none is pending.
+ int64_t start_process_memory_wait(int64_t now_ms) {
+ int64_t expected = 0;
+ return process_memory_wait_start_ms_.compare_exchange_strong(expected,
now_ms) ? now_ms
+
: expected;
+ }
+ // A reservation succeeded: the task made progress, the next failed
reservation starts a new
+ // wait.
+ void end_process_memory_wait() { process_memory_wait_start_ms_ = 0; }
+ int64_t process_memory_wait_start_ms() const { return
process_memory_wait_start_ms_; }
/* memory status action
*/
@@ -147,6 +161,7 @@ protected:
*/
AtomicStatus paused_reason_;
std::atomic<int64_t> paused_count_ = 0;
+ std::atomic<int64_t> process_memory_wait_start_ms_ = 0;
/* memory status property
*/
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 113b3352c25..3a3ef3cecc0 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -87,6 +87,13 @@ public:
static void set_mem_limit_for_test(int64_t mem_limit) {
_s_mem_limit.store(mem_limit, std::memory_order_relaxed);
}
+ static void set_soft_mem_limit_for_test(int64_t soft_mem_limit) {
+ _s_soft_mem_limit.store(soft_mem_limit, std::memory_order_relaxed);
+ }
+ // Returns the previous value so that the test can restore it.
+ static int64_t set_sys_mem_available_for_test(int64_t sys_mem_available) {
+ return _s_sys_mem_available.exchange(sys_mem_available,
std::memory_order_relaxed);
+ }
#endif
static inline std::string mem_limit_str() {
DCHECK(_s_initialized);
diff --git a/be/test/exec/pipeline/pipeline_task_test.cpp
b/be/test/exec/pipeline/pipeline_task_test.cpp
index 9e2b5e0892d..28f01a00487 100644
--- a/be/test/exec/pipeline/pipeline_task_test.cpp
+++ b/be/test/exec/pipeline/pipeline_task_test.cpp
@@ -1371,6 +1371,28 @@ TEST_F(PipelineTaskTest,
TEST_RESERVE_MEMORY_FAIL_SPILLABLE) {
EXPECT_TRUE(
((MockWorkloadGroupMgr*)ExecEnv::GetInstance()->_workload_group_manager)->_paused);
}
+ {
+ // The reservations succeed now. The query made progress: the wait
that a reservation
+ // failed for process memory may have started is ended, and the task
is not paused.
+
((MockThreadMemTrackerMgr*)thread_context()->thread_mem_tracker_mgr.get())
+ ->_test_low_memory = false;
+
_query_ctx->resource_ctx()->task_controller()->start_process_memory_wait(1);
+
EXPECT_EQ(_query_ctx->resource_ctx()->task_controller()->process_memory_wait_start_ms(),
1);
+ task->_spilling = false;
+
((MockWorkloadGroupMgr*)ExecEnv::GetInstance()->_workload_group_manager)->_paused
= false;
+ EXPECT_EQ(task->_exec_state, PipelineTask::State::RUNNABLE);
+ bool done = false;
+ EXPECT_TRUE(task->execute(&done).ok());
+ EXPECT_FALSE(task->_eos);
+ EXPECT_FALSE(done);
+ EXPECT_FALSE(task->_spilling);
+
EXPECT_TRUE(_query_ctx->resource_ctx()->task_controller()->is_enable_reserve_memory());
+ EXPECT_FALSE(
+
((MockWorkloadGroupMgr*)ExecEnv::GetInstance()->_workload_group_manager)->_paused);
+
EXPECT_EQ(_query_ctx->resource_ctx()->task_controller()->process_memory_wait_start_ms(),
0);
+
((MockThreadMemTrackerMgr*)thread_context()->thread_mem_tracker_mgr.get())
+ ->_test_low_memory = true;
+ }
{
// Reserve failed and paused.
task->_operators.front()->cast<DummyOperator>()._disable_reserve_mem =
true;
diff --git a/be/test/runtime/workload_group/workload_group_manager_test.cpp
b/be/test/runtime/workload_group/workload_group_manager_test.cpp
index 2a61c1319c5..455226c0bf2 100644
--- a/be/test/runtime/workload_group/workload_group_manager_test.cpp
+++ b/be/test/runtime/workload_group/workload_group_manager_test.cpp
@@ -17,6 +17,7 @@
#include "runtime/workload_group/workload_group_manager.h"
+#include <fmt/format.h>
#include <gen_cpp/BackendService_types.h>
#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/Types_types.h>
@@ -27,9 +28,14 @@
#include <chrono>
#include <cmath>
#include <filesystem>
+#include <functional>
+#include <limits>
#include <memory>
#include <sstream>
+#include <string>
#include <tuple>
+#include <utility>
+#include <vector>
#include "common/config.h"
#include "common/status.h"
@@ -39,6 +45,7 @@
#include "exec/scan/scanner_scheduler.h"
#include "exec/spill/spill_file_manager.h"
#include "runtime/exec_env.h"
+#include "runtime/memory/global_memory_arbitrator.h"
#include "runtime/query_context.h"
#include "runtime/runtime_query_statistics_mgr.h"
#include "runtime/thread_context.h"
@@ -53,10 +60,27 @@
namespace doris {
+// Process memory limit for the PROCESS_MEMORY_EXCEEDED cases. The workload
group memory limits
+// are derived from the process memory limit whenever their usage is
refreshed, so it is set before
+// the workload group is created and kept; process memory pressure is
simulated by the soft memory
+// limit, the memory growth since the last refresh and the system available
memory instead.
+static constexpr int64_t kProcessMemLimitForWg = 1024L * 1024 * 1000;
+// System available memory far above the warning water mark, so that only the
process memory
+// limits decide the process memory predicates unless a case lowers it on
purpose.
+static constexpr int64_t kLargeSysMemAvailable =
std::numeric_limits<int64_t>::max() / 2;
+static constexpr int64_t kProcessPausedReserveSize = 1024L;
+
class WorkloadGroupManagerTest : public testing::Test {
public:
protected:
void SetUp() override {
+ _original_mem_limit = MemInfo::mem_limit();
+ _original_soft_mem_limit = MemInfo::soft_mem_limit();
+ // The process memory predicates also depend on the system available
memory, which is
+ // read from the runner otherwise: a runner below the water marks
reports pressure before
+ // a case sets any process memory limit.
+ _original_sys_mem_available =
+ MemInfo::set_sys_mem_available_for_test(kLargeSysMemAvailable);
_wg_manager = std::make_unique<WorkloadGroupMgr>();
// generate a unique test directory to avoid conflicts between
parallel runs
std::ostringstream _oss;
@@ -98,6 +122,14 @@ protected:
doris::ExecEnv::GetInstance()->set_memtable_memory_limiter(new
MemTableMemoryLimiter());
}
void TearDown() override {
+ for (auto& [query, memory] : _consumed_memory) {
+ query->query_mem_tracker()->consume(-memory);
+ }
+ _consumed_memory.clear();
+ MemInfo::set_mem_limit_for_test(_original_mem_limit);
+ MemInfo::set_soft_mem_limit_for_test(_original_soft_mem_limit);
+ MemInfo::set_sys_mem_available_for_test(_original_sys_mem_available);
+ GlobalMemoryArbitrator::reset_refresh_interval_memory_growth();
_wg_manager.reset();
ExecEnv::GetInstance()->spill_file_mgr()->stop();
SAFE_DELETE(ExecEnv::GetInstance()->_spill_file_mgr);
@@ -156,9 +188,151 @@ private:
}
}
+ // Helpers for the PROCESS_MEMORY_EXCEEDED cases.
+
+ // A workload group whose min memory limit is 100 MiB, so that a query
consuming more than
+ // that routes into handle_single_query_ directly, and a smaller one is
routed through the
+ // other workload groups first.
+ std::shared_ptr<WorkloadGroup> _create_wg_with_min_memory(uint64_t id) {
+ MemInfo::set_mem_limit_for_test(kProcessMemLimitForWg);
+ WorkloadGroupInfo wg_info {.id = id,
+ .memory_limit = kProcessMemLimitForWg,
+ .min_memory_percent = 10,
+ .max_memory_percent = 100};
+ auto wg = _wg_manager->get_or_create_workload_group(wg_info);
+ EXPECT_EQ(wg->min_memory_limit(), 1024L * 1024 * 100);
+ return wg;
+ }
+
+ // A query on `wg` that consumes `memory` bytes until TearDown.
+ std::shared_ptr<QueryContext>
_create_query_with_memory(std::shared_ptr<WorkloadGroup>& wg,
+ int64_t memory) {
+ auto query = _generate_on_query(wg);
+ query->query_mem_tracker()->consume(memory);
+ _consumed_memory.emplace_back(query, memory);
+ wg->refresh_memory_usage();
+ return query;
+ }
+
+ // Release `memory` bytes of `query` without refreshing the usage of its
workload group:
+ // total_mem_used() keeps the value cached by the last refresh, as it does
in production
+ // until the next maintenance round or revocation refreshes it.
+ void _release_query_memory(const std::shared_ptr<QueryContext>& query,
int64_t memory) {
+ query->query_mem_tracker()->consume(-memory);
+ _consumed_memory.emplace_back(query, -memory);
+ }
+
+ // The process memory predicates are controlled from both sides in the
following helpers:
+ // the process memory usage is the vm_rss of PerfCounters (never refreshed
in a unit test)
+ // plus the memory growth since the last refresh, which the helpers set;
the system available
+ // memory is set in SetUp far above the water marks. The process memory
limit is kept at
+ // kProcessMemLimitForWg, which refresh_memory_usage() derives the
workload group limits from.
+
+ // Process soft memory limit is exceeded, hard memory limit is not.
+ static void _exceed_process_soft_mem_limit() {
+ GlobalMemoryArbitrator::reset_refresh_interval_memory_growth();
+ MemInfo::set_mem_limit_for_test(kProcessMemLimitForWg);
+ MemInfo::set_soft_mem_limit_for_test(1);
+
ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(kProcessPausedReserveSize));
+ ASSERT_FALSE(GlobalMemoryArbitrator::is_exceed_hard_mem_limit());
+ }
+
+ // Both the soft and the hard process memory limits are exceeded: the
process has grown by
+ // its whole memory limit since the last refresh.
+ static void _exceed_process_hard_mem_limit() {
+ MemInfo::set_mem_limit_for_test(kProcessMemLimitForWg);
+ MemInfo::set_soft_mem_limit_for_test(kProcessMemLimitForWg);
+ GlobalMemoryArbitrator::refresh_interval_memory_growth =
kProcessMemLimitForWg;
+ ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_hard_mem_limit());
+ }
+
+ // Neither process memory limit is exceeded: the process memory limit has
its whole headroom
+ // and the system available memory is far above the warning water mark.
+ static void _relieve_process_mem_limit() {
+ GlobalMemoryArbitrator::reset_refresh_interval_memory_growth();
+ MemInfo::set_mem_limit_for_test(kProcessMemLimitForWg);
+ MemInfo::set_soft_mem_limit_for_test(kProcessMemLimitForWg);
+ MemInfo::set_sys_mem_available_for_test(kLargeSysMemAvailable);
+
ASSERT_FALSE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(kProcessPausedReserveSize));
+ }
+
+ void _pause_for_process_memory(const std::shared_ptr<QueryContext>& query)
{
+ _wg_manager->add_paused_query(query->resource_ctx(),
kProcessPausedReserveSize,
+
Status::Error(ErrorCode::PROCESS_MEMORY_EXCEEDED, "test"));
+ }
+
+ size_t _paused_query_count(const std::shared_ptr<WorkloadGroup>& wg) {
+ std::unique_lock<std::mutex> lock(_wg_manager->_paused_queries_lock);
+ if (!_wg_manager->_paused_queries_list.contains(wg)) {
+ return 0;
+ }
+ return _wg_manager->_paused_queries_list[wg].size();
+ }
+
+ // The reservation recorded for the single paused query of `wg`.
+ int64_t _recorded_reserve_size(const std::shared_ptr<WorkloadGroup>& wg) {
+ std::unique_lock<std::mutex> lock(_wg_manager->_paused_queries_lock);
+ auto& queries = _wg_manager->_paused_queries_list[wg];
+ EXPECT_EQ(queries.size(), 1);
+ return queries.begin()->reserve_size_;
+ }
+
+ // Backdate the wait of the single paused query of `wg` by `ms` instead of
sleeping, the timer
+ // is monotonic: both its entry and the wait start that its task
controller carries across a
+ // resume are moved back.
+ void _backdate_paused_query(const std::shared_ptr<WorkloadGroup>& wg,
int64_t ms) {
+ std::unique_lock<std::mutex> lock(_wg_manager->_paused_queries_lock);
+ auto& queries = _wg_manager->_paused_queries_list[wg];
+ ASSERT_EQ(queries.size(), 1);
+ auto node = queries.extract(queries.begin());
+ node.value().enqueue_at -= ms;
+ auto resource_ctx = node.value().resource_ctx_.lock();
+ ASSERT_TRUE(resource_ctx != nullptr);
+ resource_ctx->task_controller()->end_process_memory_wait();
+ ASSERT_EQ(
+
resource_ctx->task_controller()->start_process_memory_wait(node.value().enqueue_at),
+ node.value().enqueue_at);
+ queries.insert(std::move(node));
+ }
+
+ // How long the single paused query of `wg` has waited.
+ int64_t _paused_elapsed_time(const std::shared_ptr<WorkloadGroup>& wg) {
+ std::unique_lock<std::mutex> lock(_wg_manager->_paused_queries_lock);
+ auto& queries = _wg_manager->_paused_queries_list[wg];
+ EXPECT_EQ(queries.size(), 1);
+ return queries.begin()->elapsed_time();
+ }
+
+ static void _assert_still_paused(const std::shared_ptr<QueryContext>&
query) {
+ ASSERT_FALSE(query->is_cancelled()) <<
query->exec_status().to_string();
+ ASSERT_TRUE(query->resource_ctx()
+ ->task_controller()
+ ->paused_reason()
+ .is<ErrorCode::PROCESS_MEMORY_EXCEEDED>());
+ }
+
+ static void _assert_resumed(const std::shared_ptr<QueryContext>& query) {
+ ASSERT_FALSE(query->is_cancelled()) <<
query->exec_status().to_string();
+
ASSERT_TRUE(query->resource_ctx()->task_controller()->paused_reason().ok());
+ }
+
+ static void _assert_cancelled_by_process_memory(const
std::shared_ptr<QueryContext>& query,
+ bool exceed_hard_limit) {
+ ASSERT_TRUE(query->is_cancelled());
+ const auto status = query->exec_status().to_string();
+ ASSERT_TRUE(query->exec_status().is<ErrorCode::MEM_LIMIT_EXCEEDED>())
<< status;
+ ASSERT_NE(status.find(fmt::format("exceed hard limit: {}",
exceed_hard_limit)),
+ std::string::npos)
+ << status;
+ }
+
std::unique_ptr<WorkloadGroupMgr> _wg_manager;
std::string _test_dir;
const int64_t _spill_in_paused_queue_timeout_ms =
config::spill_in_paused_queue_timeout_ms;
+ int64_t _original_mem_limit {0};
+ int64_t _original_soft_mem_limit {0};
+ int64_t _original_sys_mem_available {0};
+ std::vector<std::pair<std::shared_ptr<QueryContext>, int64_t>>
_consumed_memory;
};
TEST_F(WorkloadGroupManagerTest, get_or_create_workload_group) {
@@ -634,7 +808,7 @@ TEST_F(WorkloadGroupManagerTest, ProcessMemoryNotEnough) {
wg3->refresh_memory_usage();
// There is no query in workload groups, so that revoke memory will return 0
- EXPECT_EQ(0, _wg_manager->revoke_memory_from_other_groups_());
+ EXPECT_EQ(0,
_wg_manager->revoke_memory_from_other_groups_(kProcessPausedReserveSize));
// If exceed memory less than 128MB, then not revoke
auto query_context21 = _generate_on_query(wg2);
@@ -644,7 +818,7 @@ TEST_F(WorkloadGroupManagerTest, ProcessMemoryNotEnough) {
EXPECT_EQ(wg2->total_mem_used(), 1024 * 1024 * 50);
EXPECT_EQ(wg2->min_memory_limit(), 1024 * 1024 * 100);
// There is not workload group's memory usage > it's min memory limit.
- EXPECT_EQ(0, _wg_manager->revoke_memory_from_other_groups_());
+ EXPECT_EQ(0,
_wg_manager->revoke_memory_from_other_groups_(kProcessPausedReserveSize));
ASSERT_FALSE(query_context21->is_cancelled());
// Add another query that use a lot of memory
@@ -655,7 +829,7 @@ TEST_F(WorkloadGroupManagerTest, ProcessMemoryNotEnough) {
EXPECT_EQ(wg2->total_mem_used(), 1024 * 1024 * 110);
EXPECT_EQ(wg2->min_memory_limit(), 1024 * 1024 * 100);
// Could not revoke larger than 128MB, not revoke.
- EXPECT_EQ(0, _wg_manager->revoke_memory_from_other_groups_());
+ EXPECT_EQ(0,
_wg_manager->revoke_memory_from_other_groups_(kProcessPausedReserveSize));
ASSERT_FALSE(query_context21->is_cancelled());
ASSERT_FALSE(query_context22->is_cancelled());
@@ -666,7 +840,10 @@ TEST_F(WorkloadGroupManagerTest, ProcessMemoryNotEnough) {
wg2->refresh_memory_usage();
EXPECT_EQ(wg2->total_mem_used(), 1024 * 1024 * 410);
EXPECT_EQ(wg2->min_memory_limit(), 1024 * 1024 * 100);
- EXPECT_EQ(31 * 1024 * 1024,
_wg_manager->revoke_memory_from_other_groups_());
+ // WG2 exceeds its min memory by 310MB, so 31MB should be revoked. The
largest query (300MB)
+ // is cancelled and its whole memory is reported as revoked.
+ EXPECT_EQ(300L * 1024 * 1024,
+
_wg_manager->revoke_memory_from_other_groups_(kProcessPausedReserveSize));
ASSERT_FALSE(query_context21->is_cancelled());
ASSERT_FALSE(query_context22->is_cancelled());
ASSERT_TRUE(query_context23->is_cancelled());
@@ -692,15 +869,19 @@ TEST_F(WorkloadGroupManagerTest, ProcessMemoryNotEnough) {
wg3->refresh_memory_usage();
EXPECT_EQ(wg3->total_mem_used(), 1024 * 1024 * 500); // WG3 exceed 400MB
- EXPECT_EQ(40 * 1024 * 1024,
_wg_manager->revoke_memory_from_other_groups_());
+ // WG3 exceeds most, 40MB should be revoked, and the cancelled query31
frees 500MB.
+ EXPECT_EQ(500L * 1024 * 1024,
+
_wg_manager->revoke_memory_from_other_groups_(kProcessPausedReserveSize));
wg1->refresh_memory_usage();
wg2->refresh_memory_usage();
wg3->refresh_memory_usage();
ASSERT_TRUE(query_context31->is_cancelled());
- // query31 is still in wg3, so that it is not cancel again.
- EXPECT_EQ(40 * 1024 * 1024,
_wg_manager->revoke_memory_from_other_groups_());
+ // query31 is still in wg3, so that it is not cancel again. It was
cancelled just now and is
+ // still releasing memory, so its memory is counted as revoked again.
+ EXPECT_EQ(500L * 1024 * 1024,
+
_wg_manager->revoke_memory_from_other_groups_(kProcessPausedReserveSize));
ASSERT_FALSE(query_context11->is_cancelled());
ASSERT_FALSE(query_context21->is_cancelled());
ASSERT_FALSE(query_context22->is_cancelled());
@@ -1417,4 +1598,653 @@ TEST_F(WorkloadGroupManagerTest,
DefaultScanSchedulersUseThreadPool) {
EXPECT_NE(dynamic_cast<ThreadPoolSimplifiedScanScheduler*>(remote_scan_sched),
nullptr);
}
+// When the process memory is exceeded and the paused query has no revocable
memory, the query
+// should be kept paused instead of being cancelled immediately, so that it
can be resumed once
+// other queries release memory.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_keeps_paused_and_resumes) {
+ auto wg = _create_wg_with_min_memory(1);
+ // Let the workload group use more than its min memory limit, so that the
paused query is
+ // handled by handle_single_query_ directly instead of waiting for other
workload groups.
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ for (int i = 0; i < 3; ++i) {
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+ }
+
+ // Process memory pressure is relieved, the query should be resumed.
+ _relieve_process_mem_limit();
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
+// When the process memory is still exceeded after the paused query has waited
for
+// `spill_in_paused_queue_timeout_ms`, the query should be cancelled.
+TEST_F(WorkloadGroupManagerTest, process_mem_exceeded_cancels_after_timeout) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, false);
+}
+
+// The recovery check must test the recorded reservation, not a larger probe:
after other
+// queries release some memory, a small reservation fits under the process
soft limit while a
+// 32 MiB probe does not, and the query must be resumed instead of waiting for
the timeout.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_resumes_when_reservation_fits_soft_limit) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+
+ // Leave 16 MiB headroom below the soft limit: more than the recorded
reservation, less
+ // than a 32 MiB probe.
+
MemInfo::set_soft_mem_limit_for_test(GlobalMemoryArbitrator::process_memory_usage()
+
+ 1024L * 1024 * 16);
+
ASSERT_FALSE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(kProcessPausedReserveSize));
+ ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(1024L * 1024
* 32));
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
+// Same as above for the system available memory boundary: the recorded
reservation keeps the
+// system available memory above the warning water mark while a 32 MiB probe
does not.
+TEST_F(WorkloadGroupManagerTest,
+ process_mem_exceeded_resumes_when_reservation_fits_sys_mem_available) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+
+ // The process limits have their whole headroom, only the system available
memory is close
+ // to the warning water mark: 16 MiB above it.
+ _relieve_process_mem_limit();
+ ASSERT_GT(MemInfo::sys_mem_available_warning_water_mark(), 0);
+
MemInfo::set_sys_mem_available_for_test(MemInfo::sys_mem_available_warning_water_mark()
+
+ 1024L * 1024 * 16);
+
ASSERT_FALSE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(kProcessPausedReserveSize));
+ ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(1024L * 1024
* 32));
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
+// A paused query whose workload group uses no more than its min memory limit
is not handled by
+// handle_single_query_. It should still be resumed as soon as the process
memory pressure is
+// relieved, instead of waiting for the timeout.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_resumes) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+
+ _relieve_process_mem_limit();
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
+// A paused query whose workload group uses no more than its min memory limit
should still be
+// cancelled once it has waited for `spill_in_paused_queue_timeout_ms` under
process memory
+// pressure.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_cancels_after_timeout) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, false);
+}
+
+// A paused query whose workload group uses no more than its min memory limit
should also be
+// cancelled immediately at the process hard limit when no other workload
group can release
+// memory. This path must not rely on the memory GC daemon, which may be
disabled.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_cancels_at_hard_limit) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, true);
+}
+
+// Another workload group exceeds its min memory limit by more than 128 MiB,
but every query in
+// it is too small to be cancelled, so revoking memory from it frees nothing.
The paused query
+// must not be treated as if memory had been revoked (which would resume it
without any memory
+// being freed), it keeps waiting below the hard limit and is cancelled at the
hard limit.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_with_non_reclaimable_peer) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ auto peer_wg = _create_wg_with_min_memory(2);
+ std::vector<std::shared_ptr<QueryContext>> peer_queries;
+ for (int i = 0; i < 10; ++i) {
+ // Not larger than SMALL_MEMORY_TASK (32 MiB), so memory reclamation
skips it.
+ peer_queries.push_back(_create_query_with_memory(peer_wg, 1024L * 1024
* 30));
+ }
+ ASSERT_GT(peer_wg->total_mem_used(), peer_wg->min_memory_limit() + (1 <<
27));
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ for (int i = 0; i < 3; ++i) {
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+ ASSERT_FALSE(_wg_manager->revoking_memory_from_other_query_);
+ }
+
+ _exceed_process_hard_mem_limit();
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, true);
+ for (const auto& peer_query : peer_queries) {
+ ASSERT_FALSE(peer_query->is_cancelled());
+ }
+}
+
+// Two workload groups exceed their min memory limit by more than 128 MiB. The
one that exceeds
+// most only holds queries too small to be cancelled, so revoking memory from
it frees nothing;
+// the other one holds a single cancellable query. The paused query must not
fall through to the
+// hard-limit fallback while the second workload group can still release
memory: its query is
+// cancelled and the paused query waits for the release.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_tries_next_peer) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ auto small_peer_wg = _create_wg_with_min_memory(2);
+ std::vector<std::shared_ptr<QueryContext>> small_peer_queries;
+ for (int i = 0; i < 10; ++i) {
+ // Not larger than SMALL_MEMORY_TASK (32 MiB), so memory reclamation
skips it.
+ small_peer_queries.push_back(_create_query_with_memory(small_peer_wg,
1024L * 1024 * 30));
+ }
+ auto large_peer_wg = _create_wg_with_min_memory(3);
+ auto large_peer_query = _create_query_with_memory(large_peer_wg, 1024L *
1024 * 250);
+ ASSERT_GT(large_peer_wg->total_mem_used(),
large_peer_wg->min_memory_limit() + (1 << 27));
+ // The non-reclaimable workload group exceeds its min memory most, so it
is tried first.
+ ASSERT_GT(small_peer_wg->total_mem_used() -
small_peer_wg->min_memory_limit(),
+ large_peer_wg->total_mem_used() -
large_peer_wg->min_memory_limit());
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ ASSERT_TRUE(large_peer_query->is_cancelled());
+ for (const auto& small_peer_query : small_peer_queries) {
+ ASSERT_FALSE(small_peer_query->is_cancelled());
+ }
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+ ASSERT_TRUE(_wg_manager->revoking_memory_from_other_query_);
+}
+
+// Revoking memory from another workload group cancels one of its queries,
which holds its
+// memory until the cancellation completes. While that cancellation is in
flight (within
+// `revoke_memory_max_tolerance_ms`), memory reclamation keeps reporting its
memory as revoked,
+// so the paused query waits for the release instead of being cancelled, also
at the hard limit.
+// Once the cancellation has taken longer than the tolerance, the peer no
longer counts,
+// nothing is revoked and the paused query falls through to the hard-limit
fallback.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_waits_for_cancelling_peer) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ auto peer_wg = _create_wg_with_min_memory(2);
+ auto peer_query = _create_query_with_memory(peer_wg, 1024L * 1024 * 300);
+ auto* peer_controller = _install_mock_query_task_controller(peer_query);
+ ASSERT_GT(peer_wg->total_mem_used(), peer_wg->min_memory_limit() + (1 <<
27));
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ // The peer query is cancelled, the paused query waits for its memory to
be released.
+ _wg_manager->handle_paused_queries();
+ ASSERT_TRUE(peer_query->is_cancelled());
+ _assert_still_paused(query);
+ ASSERT_TRUE(_wg_manager->revoking_memory_from_other_query_);
+
+ // The cancelled peer is not in the paused list, so the next round resumes
the paused
+ // query, whose reservation fails again and pauses it again.
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_FALSE(_wg_manager->revoking_memory_from_other_query_);
+ _pause_for_process_memory(query);
+
+ // The peer still holds its memory and its cancellation is in flight: it
is reported as
+ // revoked memory again, and the paused query keeps waiting for it.
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_TRUE(_wg_manager->revoking_memory_from_other_query_);
+
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ _pause_for_process_memory(query);
+
+ // The cancellation has taken longer than the tolerance: the peer no
longer counts, nothing
+ // is revoked and the paused query is cancelled at the hard limit.
+ peer_controller->set_cancelled_time(peer_controller->cancelled_time() -
+ config::revoke_memory_max_tolerance_ms
- 1);
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, true);
+}
+
+// When the process reaches the hard memory limit, the paused query should be
cancelled without
+// waiting for the timeout, so that the protection does not depend on memory
gc.
+TEST_F(WorkloadGroupManagerTest, process_mem_exceeded_cancels_at_hard_limit) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, true);
+}
+
+// A query with a running task cannot be spilled, so it keeps waiting for the
task to yield.
+// The wait is still bounded: at the timeout the query is cancelled,
cancelling is safe while
+// the task runs.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_running_task_cancels_after_timeout) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+ auto* mock_controller = _install_mock_query_task_controller(query);
+ mock_controller->has_running_task_ = true;
+ // Revocable memory is not spilled while a task runs.
+ mock_controller->has_revocable_task_ = true;
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ for (int i = 0; i < 3; ++i) {
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+ ASSERT_EQ(mock_controller->revoke_memory_calls_, 0);
+ }
+
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+ _wg_manager->handle_paused_queries();
+ ASSERT_EQ(mock_controller->revoke_memory_calls_, 0);
+ _assert_cancelled_by_process_memory(query, false);
+ const auto status = query->exec_status().to_string();
+ ASSERT_NE(status.find("has running task: true"), std::string::npos) <<
status;
+}
+
+// At the hard limit a running task must not postpone the protection: spilling
is unsafe while
+// the task runs, so the query is cancelled at once.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_running_task_cancels_at_hard_limit) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+ auto* mock_controller = _install_mock_query_task_controller(query);
+ mock_controller->has_running_task_ = true;
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, true);
+ const auto status = query->exec_status().to_string();
+ ASSERT_NE(status.find("has running task: true"), std::string::npos) <<
status;
+}
+
+// A query with revocable memory is spilled rather than cancelled, also at the
hard limit. If
+// the spill resumes the query without freeing memory and it is paused again
without revocable
+// memory, the next round cancels it at the hard limit.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_spills_revocable_tasks_at_hard_limit) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+ auto* mock_controller = _install_mock_query_task_controller(query);
+ mock_controller->has_revocable_task_ = true;
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+ ASSERT_FALSE(query->get_memory_sufficient_dependency()->ready());
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ ASSERT_EQ(mock_controller->revoke_memory_calls_, 1);
+ // The spill completed and resumed the query: its memory dependency is
ready again, the
+ // paused reason is cleared and the entry is gone, so its tasks retry
their reservations.
+ _assert_resumed(query);
+ ASSERT_TRUE(query->get_memory_sufficient_dependency()->ready());
+ ASSERT_EQ(_paused_query_count(wg), 0);
+
+ // The spill freed nothing and the process is still at its hard limit, so
the retried
+ // reservation fails and the query is paused again, now without revocable
memory.
+ _pause_for_process_memory(query);
+ ASSERT_FALSE(query->get_memory_sufficient_dependency()->ready());
+ ASSERT_FALSE(mock_controller->has_revocable_task_);
+ _wg_manager->handle_paused_queries();
+ ASSERT_EQ(mock_controller->revoke_memory_calls_, 1);
+ _assert_cancelled_by_process_memory(query, true);
+}
+
+// The process memory pressure observed at the beginning of the round may be
relieved while the
+// manager inspects the pipeline tasks, up to the last scan for revocable
tasks. The timeout
+// decision must re-check the pressure after that scan and resume the query
instead of
+// cancelling it from the stale observation.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_recovery_before_timeout_decision_resumes) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+ auto* mock_controller = _install_mock_query_task_controller(query);
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ // The pressure is relieved after the round has observed it, routed the
query and
+ // inspected its pipeline tasks for the last time.
+ mock_controller->on_get_revocable_tasks_ = []() {
_relieve_process_mem_limit(); };
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
+// Two tasks of one query fail process reservations of different sizes before
the next
+// maintenance round. The query is resumed as a whole, so the recorded
reservation must be the
+// largest pending one: once the smaller one fits, the query stays paused
(keeping its timer)
+// until the larger one fits too. The pending requests do not have to fit at
the same time: a
+// task releases its reservation after each block, so the tasks can reserve
one after another
+// once the largest request fits.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_keeps_largest_pending_reservation) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+ ASSERT_EQ(_recorded_reserve_size(wg), kProcessPausedReserveSize);
+ // A sibling task fails a larger reservation: the entry is shared and
keeps the larger size.
+ _wg_manager->add_paused_query(query->resource_ctx(), 1024L * 1024 * 64,
+
Status::Error(ErrorCode::PROCESS_MEMORY_EXCEEDED, "test"));
+ ASSERT_EQ(_recorded_reserve_size(wg), 1024L * 1024 * 64);
+ // A later smaller failure does not lower it.
+ _wg_manager->add_paused_query(query->resource_ctx(), 1024L * 1024 * 4,
+
Status::Error(ErrorCode::PROCESS_MEMORY_EXCEEDED, "test"));
+ ASSERT_EQ(_recorded_reserve_size(wg), 1024L * 1024 * 64);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+
+ // 16 MiB headroom: the 1 KiB and 4 MiB reservations fit, the 64 MiB one
does not. The
+ // query must not be woken up only to fail the 64 MiB reservation again.
+
MemInfo::set_soft_mem_limit_for_test(GlobalMemoryArbitrator::process_memory_usage()
+
+ 1024L * 1024 * 16);
+ ASSERT_FALSE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(1024L * 1024
* 4));
+ ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(1024L * 1024
* 64));
+ for (int i = 0; i < 3; ++i) {
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+ }
+
+ // 65 MiB headroom: the largest pending reservation fits, although not all
of them do at the
+ // same time. The query is resumed; the tasks reserve one after another.
+
MemInfo::set_soft_mem_limit_for_test(GlobalMemoryArbitrator::process_memory_usage()
+
+ 1024L * 1024 * 65);
+ ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(1024L * 1024
* 68));
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
+// Same two pending reservations near the timeout: the query keeps the entry
it was paused with
+// instead of being woken up by the smaller reservation and starting a fresh
wait, so the bounded
+// wait still ends with the cancellation.
+TEST_F(WorkloadGroupManagerTest,
+ process_mem_exceeded_largest_pending_reservation_cancels_after_timeout)
{
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+ _wg_manager->add_paused_query(query->resource_ctx(), 1024L * 1024 * 64,
+
Status::Error(ErrorCode::PROCESS_MEMORY_EXCEEDED, "test"));
+ ASSERT_EQ(_recorded_reserve_size(wg), 1024L * 1024 * 64);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+
+
MemInfo::set_soft_mem_limit_for_test(GlobalMemoryArbitrator::process_memory_usage()
+
+ 1024L * 1024 * 16);
+
ASSERT_FALSE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(kProcessPausedReserveSize));
+ ASSERT_TRUE(GlobalMemoryArbitrator::is_exceed_soft_mem_limit(1024L * 1024
* 64));
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, false);
+}
+
+// A query resumed because its recorded reservation fits is paused again when
the retry fails
+// (another query took the memory first, or a smaller sibling does not fit
next to the request
+// that was resumed for). The new entry continues the wait the query was
resumed from instead of
+// starting a fresh one, so the bounded wait still ends with the cancellation.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_failed_retry_continues_the_wait) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ // The pressure is relieved and the query is resumed to retry its
reservation.
+ _relieve_process_mem_limit();
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+
+ // The retry fails before any reservation of the query succeeded: the
query has been waiting
+ // since its first failure and is cancelled at the timeout instead of
waiting another one.
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+ ASSERT_GT(_paused_elapsed_time(wg),
config::spill_in_paused_queue_timeout_ms);
+ _wg_manager->handle_paused_queries();
+ _assert_cancelled_by_process_memory(query, false);
+}
+
+// A successful reservation ends the wait: the query made progress, so a later
failure starts a
+// new bounded wait.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_successful_reservation_starts_new_wait) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 1024 * 128);
+ ASSERT_GT(wg->total_mem_used(), wg->min_memory_limit());
+
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ _backdate_paused_query(wg, config::spill_in_paused_queue_timeout_ms + 1);
+
+ _relieve_process_mem_limit();
+ _wg_manager->handle_paused_queries();
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+
+ // The retried reservation succeeds
(PipelineTask::_try_to_reserve_memory), a later one fails.
+ query->resource_ctx()->task_controller()->end_process_memory_wait();
+ _exceed_process_soft_mem_limit();
+ _pause_for_process_memory(query);
+ ASSERT_LT(_paused_elapsed_time(wg),
config::spill_in_paused_queue_timeout_ms);
+ for (int i = 0; i < 3; ++i) {
+ _wg_manager->handle_paused_queries();
+ _assert_still_paused(query);
+ ASSERT_EQ(_paused_query_count(wg), 1);
+ }
+}
+
+// Two workload groups exceed their min memory limit by more than 128 MiB when
the walk starts.
+// The first one holds only a query whose cancellation has already taken
longer than
+// `revoke_memory_max_tolerance_ms`, so it releases nothing; while it is
scanned, the second one
+// drops back within its min memory. The second one must then be skipped from
its current usage
+// instead of having its remaining query cancelled for the excess it had when
the walk started.
+TEST_F(WorkloadGroupManagerTest,
process_mem_exceeded_below_min_memory_skips_peer_within_min) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ auto first_peer_wg = _create_wg_with_min_memory(2);
+ auto first_peer_query = _create_query_with_memory(first_peer_wg, 1024L *
1024 * 300);
+ auto* first_peer_controller =
_install_mock_query_task_controller(first_peer_query);
+ first_peer_controller->cancel(Status::InternalError("memory gc cancel"));
+
first_peer_controller->set_cancelled_time(first_peer_controller->cancelled_time()
-
+
config::revoke_memory_max_tolerance_ms - 1);
+
+ auto second_peer_wg = _create_wg_with_min_memory(3);
+ auto second_peer_query = _create_query_with_memory(second_peer_wg, 1024L *
1024 * 250);
+ ASSERT_GT(second_peer_wg->total_mem_used(),
second_peer_wg->min_memory_limit() + (1 << 27));
+ // The first peer exceeds its min memory most, so it is scanned first.
+ ASSERT_GT(first_peer_wg->total_mem_used() -
first_peer_wg->min_memory_limit(),
+ second_peer_wg->total_mem_used() -
second_peer_wg->min_memory_limit());
+
+ // While the first peer is scanned, the second peer's query releases 180
MiB and the
+ // workload group drops to 70 MiB, within its 100 MiB min memory; its
query is still larger
+ // than SMALL_MEMORY_TASK, so it could be cancelled. Nothing refreshes the
workload group
+ // usage meanwhile, so its cached usage still shows the 250 MiB of the
snapshot.
+ bool released = false;
+ first_peer_controller->on_is_cancelled_ = [&]() {
+ if (released) {
+ return;
+ }
+ released = true;
+ _release_query_memory(second_peer_query, 1024L * 1024 * 180);
+ ASSERT_GT(second_peer_wg->total_mem_used(),
second_peer_wg->min_memory_limit());
+ };
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ ASSERT_TRUE(released);
+ ASSERT_FALSE(second_peer_query->is_cancelled());
+ // The walk refreshed the second peer's usage before deciding on it.
+ ASSERT_EQ(second_peer_wg->total_mem_used(), 1024L * 1024 * 70);
+ ASSERT_EQ(second_peer_wg->min_memory_limit(), 1024L * 1024 * 100);
+ // Nothing could be revoked, the paused query falls through to the
hard-limit fallback.
+ _assert_cancelled_by_process_memory(query, true);
+}
+
+// Two workload groups exceed their min memory limit by more than 128 MiB. The
first one holds
+// only a query whose cancellation has already taken longer than
`revoke_memory_max_tolerance_ms`,
+// so it releases nothing; while it is scanned, the process memory pressure is
relieved (other
+// queries or the cache released memory). The second one must not have its
query cancelled for
+// pressure that is gone, and the paused query must be resumed instead of
being cancelled at the
+// hard limit it was observed under before the walk.
+TEST_F(WorkloadGroupManagerTest,
+
process_mem_exceeded_below_min_memory_resumes_when_relieved_during_peer_scan) {
+ auto wg = _create_wg_with_min_memory(1);
+ auto query = _create_query_with_memory(wg, 1024L * 4);
+ ASSERT_LE(wg->total_mem_used(), wg->min_memory_limit());
+
+ auto first_peer_wg = _create_wg_with_min_memory(2);
+ auto first_peer_query = _create_query_with_memory(first_peer_wg, 1024L *
1024 * 300);
+ auto* first_peer_controller =
_install_mock_query_task_controller(first_peer_query);
+ first_peer_controller->cancel(Status::InternalError("memory gc cancel"));
+
first_peer_controller->set_cancelled_time(first_peer_controller->cancelled_time()
-
+
config::revoke_memory_max_tolerance_ms - 1);
+
+ auto second_peer_wg = _create_wg_with_min_memory(3);
+ auto second_peer_query = _create_query_with_memory(second_peer_wg, 1024L *
1024 * 250);
+ ASSERT_GT(second_peer_wg->total_mem_used(),
second_peer_wg->min_memory_limit() + (1 << 27));
+ // The first peer exceeds its min memory most, so it is scanned first.
+ ASSERT_GT(first_peer_wg->total_mem_used() -
first_peer_wg->min_memory_limit(),
+ second_peer_wg->total_mem_used() -
second_peer_wg->min_memory_limit());
+
+ bool relieved = false;
+ first_peer_controller->on_is_cancelled_ = [&]() {
+ if (relieved) {
+ return;
+ }
+ relieved = true;
+ _relieve_process_mem_limit();
+ };
+
+ _exceed_process_hard_mem_limit();
+ _pause_for_process_memory(query);
+
+ config::spill_in_paused_queue_timeout_ms = 60 * 1000;
+ _wg_manager->handle_paused_queries();
+ ASSERT_TRUE(relieved);
+ ASSERT_FALSE(second_peer_query->is_cancelled());
+ ASSERT_FALSE(_wg_manager->revoking_memory_from_other_query_);
+ _assert_resumed(query);
+ ASSERT_EQ(_paused_query_count(wg), 0);
+}
+
} // namespace doris
diff --git a/be/test/testutil/mock/mock_query_task_controller.h
b/be/test/testutil/mock/mock_query_task_controller.h
index a9c6886aada..96c4a2d1fae 100644
--- a/be/test/testutil/mock/mock_query_task_controller.h
+++ b/be/test/testutil/mock/mock_query_task_controller.h
@@ -16,6 +16,9 @@
// under the License.
#pragma once
+#include <functional>
+#include <vector>
+
#include "runtime/workload_management/query_task_controller.h"
namespace doris {
@@ -35,6 +38,58 @@ struct MockQueryTaskController : public QueryTaskController {
}
void set_cancelled_time(int64_t ctime) { cancelled_time_ = ctime; }
+
+ // Memory reclamation asks every candidate query whether it is cancelled
while it scans a
+ // workload group, so this is where a test can let another query release
memory during that
+ // scan.
+ bool is_cancelled() const override {
+ if (on_is_cancelled_) {
+ on_is_cancelled_();
+ }
+ return QueryTaskController::is_cancelled();
+ }
+
+ // Pipeline state seen by WorkloadGroupMgr::handle_single_query_. The
query context of a
+ // unit test has no fragments, so the real implementation never reports a
running or a
+ // revocable task; these knobs simulate them.
+ // NOLINTNEXTLINE(readability-make-member-function-const): overrides a
non-const virtual.
+ void get_revocable_info(size_t* revocable_size, size_t* memory_usage,
+ bool* has_running_task) override {
+ QueryTaskController::get_revocable_info(revocable_size, memory_usage,
has_running_task);
+ *has_running_task = has_running_task_;
+ }
+
+ // The manager only checks whether the list is empty before it calls
revoke_memory(), which
+ // is mocked below, so a placeholder entry is enough to stand for a
revocable task.
+ std::vector<PipelineTask*> get_revocable_tasks() override {
+ if (on_get_revocable_tasks_) {
+ on_get_revocable_tasks_();
+ }
+ return has_revocable_task_ ? std::vector<PipelineTask*> {nullptr}
+ : std::vector<PipelineTask*> {};
+ }
+
+ // The spill of a unit test completes at once: like the spill callback of
the real
+ // implementation, resume the query so that its blocked tasks retry their
reservations. The
+ // spilled tasks have nothing left to revoke.
+ Status revoke_memory() override {
+ ++revoke_memory_calls_;
+ RETURN_IF_ERROR(revoke_memory_status_);
+ has_revocable_task_ = false;
+ set_memory_sufficient(true);
+ return Status::OK();
+ }
+
+ bool has_running_task_ {false};
+ bool has_revocable_task_ {false};
+ Status revoke_memory_status_ {Status::OK()};
+ int revoke_memory_calls_ {0};
+ // Runs inside get_revocable_tasks(), the last inspection of the pipeline
tasks after the
+ // manager has observed the memory pressure and before it decides what to
do with the query.
+ std::function<void()> on_get_revocable_tasks_;
+ // Runs inside is_cancelled(), while memory reclamation scans the workload
group of this
+ // query.
+ std::function<void()> on_is_cancelled_;
};
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]