This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a45946dec0a [Improvement](memory) reduce some memtracker overhead
(#50063)
a45946dec0a is described below
commit a45946dec0a502e0bd67743577c8590125719693
Author: Pxl <[email protected]>
AuthorDate: Wed Apr 16 09:56:01 2025 +0800
[Improvement](memory) reduce some memtracker overhead (#50063)
### What problem does this PR solve?
1. some subcolumn will shared by multiple tasks, and do
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER at ctor/dtor, but some
subcolumn from ColumnObject no need to do those things.
2. Allocator::realloc_impl has some overhead coz
`doris::thread_context()
->thread_mem_tracker_mgr->limiter_mem_tracker()`, add a shorcut to avoid
this.
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [x] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/olap/rowset/segment_v2/stream_reader.h | 4 +--
be/src/vec/columns/column_object.h | 2 +-
be/src/vec/columns/subcolumn_tree.h | 35 ++++++++++++++++++---------
be/src/vec/common/allocator.cpp | 6 +++++
4 files changed, 32 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/stream_reader.h
b/be/src/olap/rowset/segment_v2/stream_reader.h
index 9aac3c0f232..6cf7366fba2 100644
--- a/be/src/olap/rowset/segment_v2/stream_reader.h
+++ b/be/src/olap/rowset/segment_v2/stream_reader.h
@@ -41,13 +41,13 @@ struct SubstreamIterator {
};
// path -> StreamReader
-using SubstreamReaderTree = vectorized::SubcolumnsTree<SubstreamIterator>;
+using SubstreamReaderTree = vectorized::SubcolumnsTree<SubstreamIterator,
false>;
// Reader for the storage layer, the file_column_type indicates the read type
of the column in segment file
struct SubcolumnReader {
std::unique_ptr<ColumnReader> reader;
std::shared_ptr<const vectorized::IDataType> file_column_type;
};
-using SubcolumnColumnReaders = vectorized::SubcolumnsTree<SubcolumnReader>;
+using SubcolumnColumnReaders = vectorized::SubcolumnsTree<SubcolumnReader,
true>;
} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/vec/columns/column_object.h
b/be/src/vec/columns/column_object.h
index 04a9cb4a7c1..76400075186 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -235,7 +235,7 @@ public:
// the root Node should be JSONB type when finalize
bool is_root = false;
};
- using Subcolumns = SubcolumnsTree<Subcolumn>;
+ using Subcolumns = SubcolumnsTree<Subcolumn, false>;
private:
/// If true then all subcolumns are nullable.
diff --git a/be/src/vec/columns/subcolumn_tree.h
b/be/src/vec/columns/subcolumn_tree.h
index 8b53d1912f3..f1290737003 100644
--- a/be/src/vec/columns/subcolumn_tree.h
+++ b/be/src/vec/columns/subcolumn_tree.h
@@ -26,13 +26,11 @@
#include "vec/columns/column.h"
#include "vec/common/arena.h"
#include "vec/common/string_ref.h"
-#include "vec/data_types/data_type.h"
#include "vec/json/path_in_data.h"
namespace doris::vectorized {
-/// Tree that represents paths in document
-/// with additional data in nodes.
-
-template <typename NodeData>
+// Tree that represents paths in document with additional data in nodes.
+// IsShared mean this object shared above multiple tasks, need swtich to
subcolumns_tree_tracker
+template <typename NodeData, bool IsShared>
class SubcolumnsTree {
public:
struct Node {
@@ -75,10 +73,12 @@ public:
void add_child(std::string_view key, std::shared_ptr<Node> next_node,
Arena& strings_pool) {
next_node->parent = this;
StringRef key_ref;
- {
+ if constexpr (IsShared) {
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
ExecEnv::GetInstance()->subcolumns_tree_tracker());
key_ref = {strings_pool.insert(key.data(), key.length()),
key.length()};
+ } else {
+ key_ref = {strings_pool.insert(key.data(), key.length()),
key.length()};
}
children[key_ref] = std::move(next_node);
}
@@ -256,7 +256,7 @@ public:
/// for the last rows.
/// If there are no leaves, skip current node and find
/// the next node up to the current.
- leaf = SubcolumnsTree<NodeData>::find_leaf(node_nested, pred);
+ leaf = SubcolumnsTree<NodeData, IsShared>::find_leaf(node_nested,
pred);
if (leaf) {
break;
@@ -308,14 +308,25 @@ public:
const_iterator end() const { return leaves.end(); }
~SubcolumnsTree() {
-
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->subcolumns_tree_tracker());
- strings_pool.reset();
+ if constexpr (IsShared) {
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
+ ExecEnv::GetInstance()->subcolumns_tree_tracker());
+ strings_pool.reset();
+ } else {
+ strings_pool.reset();
+ }
}
SubcolumnsTree() {
-
SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->subcolumns_tree_tracker());
- SCOPED_SKIP_MEMORY_CHECK();
- strings_pool = std::make_shared<Arena>();
+ if constexpr (IsShared) {
+ SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(
+ ExecEnv::GetInstance()->subcolumns_tree_tracker());
+ SCOPED_SKIP_MEMORY_CHECK();
+ strings_pool = std::make_shared<Arena>();
+ } else {
+ SCOPED_SKIP_MEMORY_CHECK();
+ strings_pool = std::make_shared<Arena>();
+ }
}
private:
diff --git a/be/src/vec/common/allocator.cpp b/be/src/vec/common/allocator.cpp
index 57394f65c1b..22bc5ae3660 100644
--- a/be/src/vec/common/allocator.cpp
+++ b/be/src/vec/common/allocator.cpp
@@ -289,6 +289,9 @@ void Allocator<clear_memory_, mmap_populate, use_mmap,
MemoryAllocator>::add_add
return;
}
#endif
+ if (!doris::config::crash_in_memory_tracker_inaccurate) {
+ return;
+ }
doris::thread_context()->thread_mem_tracker_mgr->limiter_mem_tracker()->add_address_sanitizers(
buf, size);
}
@@ -301,6 +304,9 @@ void Allocator<clear_memory_, mmap_populate, use_mmap,
MemoryAllocator>::remove_
return;
}
#endif
+ if (!doris::config::crash_in_memory_tracker_inaccurate) {
+ return;
+ }
doris::thread_context()
->thread_mem_tracker_mgr->limiter_mem_tracker()
->remove_address_sanitizers(buf, size);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
