yiguolei commented on code in PR #41193: URL: https://github.com/apache/doris/pull/41193#discussion_r1774372420
########## be/src/vec/common/custom_allocator.h: ########## @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "vec/common/allocator.h" +#include "vec/common/allocator_fwd.h" + +template <class T, typename MemoryAllocator = Allocator<true>> +class CustomStdAllocator; + +template <typename T> +using DorisVector = std::vector<T, CustomStdAllocator<T>>; + +template <typename K, typename T> +using DorisMap = std::map<K, T, std::less<K>, CustomStdAllocator<std::pair<const K, T>>>; Review Comment: https://en.cppreference.com/w/cpp/container/map template< class Key, class T, class Compare = [std::less](http://en.cppreference.com/w/cpp/utility/functional/less)<Key>, class Allocator = [std::allocator](http://en.cppreference.com/w/cpp/memory/allocator)<[std::pair](http://en.cppreference.com/w/cpp/utility/pair)<const Key, T>> > class map; ########## be/src/olap/rowid_conversion.h: ########## @@ -33,17 +34,36 @@ namespace doris { class RowIdConversion { public: RowIdConversion() = default; - ~RowIdConversion() = default; + ~RowIdConversion() { + if (_row_id_thread_ctx_memtracker_ptr) { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_row_id_thread_ctx_memtracker_ptr); Review Comment: 这里没必要switch，直接release 就可以 ########## be/src/olap/rowid_conversion.h: ########## @@ -33,17 +34,36 @@ namespace doris { class RowIdConversion { public: RowIdConversion() = default; - ~RowIdConversion() = default; + ~RowIdConversion() { + if (_row_id_thread_ctx_memtracker_ptr) { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_row_id_thread_ctx_memtracker_ptr); + RELEASE_THREAD_MEM_TRACKER(_seg_rowid_map_mem_used); + } + } // resize segment rowid map to its rows num void init_segment_map(const RowsetId& src_rowset_id, const std::vector<uint32_t>& num_rows) { + size_t mem_used = 0; for (size_t i = 0; i < num_rows.size(); i++) { uint32_t id = _segments_rowid_map.size(); _segment_to_id_map.emplace(std::pair<RowsetId, uint32_t> {src_rowset_id, i}, id); _id_to_segment_map.emplace_back(src_rowset_id, i); _segments_rowid_map.emplace_back(std::vector<std::pair<uint32_t, uint32_t>>( num_rows[i], std::pair<uint32_t, uint32_t>(UINT32_MAX, UINT32_MAX))); + mem_used += sizeof(std::pair<uint32_t, uint32_t>) * _segments_rowid_map[i].capacity(); + } + //NOTE: manually count _segments_rowid_map's memory here, because _segments_rowid_map could be used by indexCompaction. + // indexCompaction is a thridparty code, it's too complex to modify it. + // refer compact_column. + mem_used += + sizeof(std::vector<std::pair<uint32_t, uint32_t>>) * _segments_rowid_map.capacity(); + _seg_rowid_map_mem_used = mem_used; + auto* t_ctx = doris::thread_context(true); + if (t_ctx) { Review Comment: 这里也不用保存，直接consume 就可以 ########## be/src/olap/rowid_conversion.h: ########## @@ -115,6 +135,8 @@ class RowIdConversion { // value indicates row id of destination segment. // <UINT32_MAX, UINT32_MAX> indicates current row not exist. std::vector<std::vector<std::pair<uint32_t, uint32_t>>> _segments_rowid_map; + size_t _seg_rowid_map_mem_used; + std::shared_ptr<MemTrackerLimiter> _row_id_thread_ctx_memtracker_ptr = nullptr; Review Comment: 不用保存这个 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
