morningman commented on a change in pull request #1954: Limit the memory usage
of Loading process
URL: https://github.com/apache/incubator-doris/pull/1954#discussion_r334475376
##########
File path: be/src/runtime/load_channel_mgr.h
##########
@@ -30,56 +28,58 @@
#include "gen_cpp/PaloInternalService_types.h"
#include "gen_cpp/internal_service.pb.h"
#include "runtime/tablets_channel.h"
-#include "util/hash_util.hpp"
#include "util/uid_util.h"
-#include "service/brpc.h"
-
namespace doris {
-class ExecEnv;
-
class Cache;
+class LoadChannel;
-// Mgr -> load -> tablet
+// LoadChannelMgr -> LoadChannel -> TabletsChannel -> DeltaWrtier
// All dispached load data for this backend is routed from this class
-class TabletWriterMgr {
+class LoadChannelMgr {
public:
- TabletWriterMgr(ExecEnv* exec_env);
- ~TabletWriterMgr();
+ LoadChannelMgr();
+ ~LoadChannelMgr();
+
+ Status init(int64_t process_mem_limit);
- // open a new backend
+ // open a new load channel if not exist
Status open(const PTabletWriterOpenRequest& request);
- // this batch must belong to a index in one transaction
- // when batch.
Status add_batch(const PTabletWriterAddBatchRequest& request,
google::protobuf::RepeatedPtrField<PTabletInfo>*
tablet_vec,
int64_t* wait_lock_time_ns);
// cancel all tablet stream for 'load_id' load
- // id: stream load's id
Status cancel(const PTabletWriterCancelRequest& request);
Status start_bg_worker();
private:
- ExecEnv* _exec_env;
- // lock protect the channel map
- std::mutex _lock;
+ // calculate the totol memory limit of all load processes on this Backend
+ int64_t _calc_total_mem_limit(int64_t process_mem_limit);
+ // calculate the memory limit for a single load process.
+ int64_t _calc_load_mem_limit(int64_t mem_limit);
- // A map from load_id|index_id to load channel
- butil::FlatMap<
- TabletsChannelKey,
- std::shared_ptr<TabletsChannel>,
- TabletsChannelKeyHasher> _tablets_channels;
+ // check if the total load mem consumption exceeds limit.
+ // If yes, it will pick a load channel to try to reduce memory consumption.
+ void _handle_mem_exceed_limit();
+private:
+ bool _init = false;
Review comment:
If you while true, how does this thread exit?
I will change it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
