This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit a879bece767336da220f5493e47be1913848f0b0 Author: Riza Suminto <[email protected]> AuthorDate: Thu Mar 23 19:25:31 2023 -0700 IMPALA-12025: Hide misleading TotalTime and InactiveTotalTime RuntimeProfileBase class in runtime-profile.cc always include TotalTime and InactiveTotalTime counter to query profile. These are being used to measure fragment instance activities, but meaningless and potentially misleading in some other parts of query profile. THis patch hide TotalTime and InactiveTotalTime in the following runtime profile nodes: - Right under "Query" (first line) - All under "Summary:" and "ImpalaServer:" - "Execution Profile" - All under "Per Node Profiles:" except for "CodeGen:" - "Fragment Instance Lifecycle Timings:" - "Buffer pool:" - "Dequeue:" and "Enqueue:" - "Hash Table:" - "Filter x:" - "GroupingAggregator x:" Testing: - Pass core tests. Change-Id: I32e68d36549f9730c08232fbc270e518f2acaffa Reviewed-on: http://gerrit.cloudera.org:8080/19652 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/exec/aggregator.cc | 2 +- be/src/exec/grouping-aggregator.cc | 5 +- be/src/exec/hash-table.cc | 4 +- be/src/exec/scan-node.cc | 7 +- be/src/runtime/bufferpool/buffer-pool.cc | 3 +- be/src/runtime/coordinator-backend-state.cc | 9 +-- be/src/runtime/coordinator.cc | 6 +- be/src/runtime/fragment-instance-state.cc | 4 +- be/src/runtime/fragment-state.cc | 2 +- be/src/runtime/krpc-data-stream-recvr.cc | 9 ++- be/src/runtime/query-state.cc | 4 +- be/src/service/client-request-state.cc | 6 +- be/src/util/runtime-profile.cc | 101 ++++++++++++++++++---------- be/src/util/runtime-profile.h | 34 +++++++--- 14 files changed, 122 insertions(+), 74 deletions(-) diff --git a/be/src/exec/aggregator.cc b/be/src/exec/aggregator.cc index cf3e208f3..1ffcd81f4 100644 --- a/be/src/exec/aggregator.cc +++ b/be/src/exec/aggregator.cc @@ -98,7 +98,7 @@ Aggregator::Aggregator(ExecNode* exec_node, ObjectPool* pool, needs_finalize_(config.needs_finalize_), agg_fns_(config.aggregate_functions_), conjuncts_(config.conjuncts_), - runtime_profile_(RuntimeProfile::Create(pool_, name)) {} + runtime_profile_(RuntimeProfile::Create(pool_, name, false)) {} Aggregator::~Aggregator() {} diff --git a/be/src/exec/grouping-aggregator.cc b/be/src/exec/grouping-aggregator.cc index 75425d599..cd6162c83 100644 --- a/be/src/exec/grouping-aggregator.cc +++ b/be/src/exec/grouping-aggregator.cc @@ -39,6 +39,7 @@ #include "runtime/tuple-row.h" #include "runtime/tuple.h" #include "util/runtime-profile-counters.h" +#include "util/runtime-profile.h" #include "util/string-parser.h" #include "gen-cpp/PlanNodes_types.h" @@ -134,8 +135,8 @@ static const int STREAMING_HT_MIN_REDUCTION_SIZE = GroupingAggregator::GroupingAggregator(ExecNode* exec_node, ObjectPool* pool, const GroupingAggregatorConfig& config, int64_t estimated_input_cardinality, bool needUnsetLimit) - : Aggregator( - exec_node, pool, config, Substitute("GroupingAggregator $0", config.agg_idx_)), + : Aggregator(exec_node, pool, config, + Substitute("$0$1", RuntimeProfile::PREFIX_GROUPING_AGGREGATOR, config.agg_idx_)), hash_table_config_(*config.hash_table_config_), intermediate_row_desc_(config.intermediate_row_desc_), is_streaming_preagg_(config.is_streaming_preagg_), diff --git a/be/src/exec/hash-table.cc b/be/src/exec/hash-table.cc index 47a1d4e77..c134ff7d7 100644 --- a/be/src/exec/hash-table.cc +++ b/be/src/exec/hash-table.cc @@ -501,8 +501,8 @@ Status HashTable::Init(bool* got_memory) { unique_ptr<HashTableStatsProfile> HashTable::AddHashTableCounters( RuntimeProfile* parent_profile) { unique_ptr<HashTableStatsProfile> stats_profile(new HashTableStatsProfile()); - RuntimeProfile *hashtable_profile = stats_profile->hashtable_profile = - parent_profile->CreateChild("Hash Table", true, true); + RuntimeProfile* hashtable_profile = stats_profile->hashtable_profile = + parent_profile->CreateChild(RuntimeProfile::HASH_TABLE, true, true, false); stats_profile->num_hash_probes_ = ADD_COUNTER(hashtable_profile, "Probes", TUnit::UNIT); stats_profile->num_hash_travels_ = diff --git a/be/src/exec/scan-node.cc b/be/src/exec/scan-node.cc index fb9042334..afe6ef6fd 100644 --- a/be/src/exec/scan-node.cc +++ b/be/src/exec/scan-node.cc @@ -155,10 +155,11 @@ Status ScanNode::Prepare(RuntimeState* state) { filter_ctxs_.emplace_back(); FilterContext& filter_ctx = filter_ctxs_.back(); filter_ctx.filter = state->filter_bank()->RegisterConsumer(filter_desc); - string filter_profile_title = Substitute("Filter $0 ($1)", filter_desc.filter_id, - PrettyPrinter::Print(filter_ctx.filter->filter_size(), TUnit::BYTES)); + string filter_profile_title = + Substitute("$0$1 ($2)", RuntimeProfile::PREFIX_FILTER, filter_desc.filter_id, + PrettyPrinter::Print(filter_ctx.filter->filter_size(), TUnit::BYTES)); RuntimeProfile* profile = - RuntimeProfile::Create(state->obj_pool(), filter_profile_title); + RuntimeProfile::Create(state->obj_pool(), filter_profile_title, false); runtime_profile_->AddChild(profile); filter_ctx.stats = state->obj_pool()->Add(new FilterStats(profile)); } diff --git a/be/src/runtime/bufferpool/buffer-pool.cc b/be/src/runtime/bufferpool/buffer-pool.cc index 1b2c19679..a3b3c02a0 100644 --- a/be/src/runtime/bufferpool/buffer-pool.cc +++ b/be/src/runtime/bufferpool/buffer-pool.cc @@ -427,7 +427,8 @@ BufferPool::Client::Client(BufferPool* pool, TmpFileGroup* file_group, num_pages_(0), buffers_allocated_bytes_(0) { // Set up a child profile with buffer pool info. - RuntimeProfile* child_profile = profile->CreateChild("Buffer pool", true, true); + RuntimeProfile* child_profile = + profile->CreateChild(RuntimeProfile::BUFFER_POOL, true, true, false); reservation_.InitChildTracker( child_profile, parent_reservation, mem_tracker, reservation_limit, mem_limit_mode); counters_.alloc_time = ADD_TIMER(child_profile, "AllocTime"); diff --git a/be/src/runtime/coordinator-backend-state.cc b/be/src/runtime/coordinator-backend-state.cc index 2df0626cc..9ee2d6e24 100644 --- a/be/src/runtime/coordinator-backend-state.cc +++ b/be/src/runtime/coordinator-backend-state.cc @@ -86,7 +86,8 @@ Coordinator::BackendState::BackendState(const QueryExecParams& exec_params, int void Coordinator::BackendState::Init(const vector<FragmentStats*>& fragment_stats, RuntimeProfile* host_profile_parent, ObjectPool* obj_pool) { - host_profile_ = RuntimeProfile::Create(obj_pool, NetworkAddressPBToString(host_)); + host_profile_ = + RuntimeProfile::Create(obj_pool, NetworkAddressPBToString(host_), false); host_profile_parent->AddChild(host_profile_); RuntimeProfile::Counter* admission_slots = ADD_COUNTER(host_profile_, "AdmissionSlots", TUnit::UNIT); @@ -568,7 +569,7 @@ void Coordinator::BackendState::UpdateHostProfile( const TRuntimeProfileTree& thrift_profile) { // We do not take 'lock_' here because RuntimeProfile::Update() is thread-safe. DCHECK(!IsEmptyBackend()); - host_profile_->Update(thrift_profile); + host_profile_->Update(thrift_profile, false); } void Coordinator::BackendState::UpdateExecStats( @@ -893,8 +894,8 @@ void Coordinator::BackendState::InstanceStats::ToJson(Value* value, Document* do Coordinator::FragmentStats::FragmentStats(const string& agg_profile_name, const string& root_profile_name, int num_instances, ObjectPool* obj_pool) : agg_profile_( - AggregatedRuntimeProfile::Create(obj_pool, agg_profile_name, num_instances)), - root_profile_(RuntimeProfile::Create(obj_pool, root_profile_name)), + AggregatedRuntimeProfile::Create(obj_pool, agg_profile_name, num_instances)), + root_profile_(RuntimeProfile::Create(obj_pool, root_profile_name, false)), num_instances_(num_instances) {} void Coordinator::FragmentStats::AddSplitStats() { diff --git a/be/src/runtime/coordinator.cc b/be/src/runtime/coordinator.cc index 3d2fae021..aa9e00106 100644 --- a/be/src/runtime/coordinator.cc +++ b/be/src/runtime/coordinator.cc @@ -152,12 +152,12 @@ Status Coordinator::Exec() { << " stmt=" << request.query_ctx.client_request.stmt; stmt_type_ = request.stmt_type; - query_profile_ = - RuntimeProfile::Create(obj_pool(), "Execution Profile " + PrintId(query_id())); + query_profile_ = RuntimeProfile::Create( + obj_pool(), "Execution Profile " + PrintId(query_id()), false); finalization_timer_ = PROFILE_FinalizationTimer.Instantiate(query_profile_); filter_updates_received_ = PROFILE_FiltersReceived.Instantiate(query_profile_); - host_profiles_ = RuntimeProfile::Create(obj_pool(), "Per Node Profiles"); + host_profiles_ = RuntimeProfile::Create(obj_pool(), "Per Node Profiles", false); query_profile_->AddChild(host_profiles_); SCOPED_TIMER(query_profile_->total_time_counter()); diff --git a/be/src/runtime/fragment-instance-state.cc b/be/src/runtime/fragment-instance-state.cc index 876383cac..8d6c19f77 100644 --- a/be/src/runtime/fragment-instance-state.cc +++ b/be/src/runtime/fragment-instance-state.cc @@ -158,8 +158,8 @@ Status FragmentInstanceState::Prepare() { // total_time_counter() is in the runtime_state_ so start it up now. SCOPED_TIMER(profile()->total_time_counter()); - timings_profile_ = - RuntimeProfile::Create(obj_pool(), "Fragment Instance Lifecycle Timings"); + timings_profile_ = RuntimeProfile::Create( + obj_pool(), RuntimeProfile::FRAGMENT_INSTANCE_LIFECYCLE_TIMINGS, false); profile()->AddChild(timings_profile_); SCOPED_TIMER(ADD_TIMER(timings_profile_, PREPARE_TIMER_NAME)); diff --git a/be/src/runtime/fragment-state.cc b/be/src/runtime/fragment-state.cc index a9e8d23b5..787be20f7 100644 --- a/be/src/runtime/fragment-state.cc +++ b/be/src/runtime/fragment-state.cc @@ -140,7 +140,7 @@ FragmentState::FragmentState(QueryState* query_state, const TPlanFragment& fragm const PlanFragmentCtxPB& fragment_ctx) : query_state_(query_state), fragment_(fragment), fragment_ctx_(fragment_ctx) { runtime_profile_ = RuntimeProfile::Create( - query_state->obj_pool(), Substitute("Fragment $0", fragment_.display_name)); + query_state->obj_pool(), Substitute("Fragment $0", fragment_.display_name), false); query_state_->host_profile()->AddChild(runtime_profile_); } diff --git a/be/src/runtime/krpc-data-stream-recvr.cc b/be/src/runtime/krpc-data-stream-recvr.cc index eab53eaba..a3b804e7f 100644 --- a/be/src/runtime/krpc-data-stream-recvr.cc +++ b/be/src/runtime/krpc-data-stream-recvr.cc @@ -673,9 +673,8 @@ void KrpcDataStreamRecvr::TransferAllResources(RowBatch* transfer_batch) { KrpcDataStreamRecvr::KrpcDataStreamRecvr(KrpcDataStreamMgr* stream_mgr, MemTracker* parent_tracker, const RowDescriptor* row_desc, const RuntimeState& runtime_state, const TUniqueId& fragment_instance_id, - PlanNodeId dest_node_id, int num_senders, bool is_merging, - int64_t total_buffer_limit, RuntimeProfile* profile, - BufferPool::ClientHandle* client) + PlanNodeId dest_node_id, int num_senders, bool is_merging, int64_t total_buffer_limit, + RuntimeProfile* profile, BufferPool::ClientHandle* client) : mgr_(stream_mgr), runtime_state_(runtime_state), fragment_instance_id_(fragment_instance_id), @@ -689,8 +688,8 @@ KrpcDataStreamRecvr::KrpcDataStreamRecvr(KrpcDataStreamMgr* stream_mgr, parent_tracker_(parent_tracker), buffer_pool_client_(client), profile_(profile), - dequeue_profile_(RuntimeProfile::Create(&pool_, "Dequeue")), - enqueue_profile_(RuntimeProfile::Create(&pool_, "Enqueue")) { + dequeue_profile_(RuntimeProfile::Create(&pool_, RuntimeProfile::DEQUEUE, false)), + enqueue_profile_(RuntimeProfile::Create(&pool_, RuntimeProfile::ENQUEUE, false)) { // Create one queue per sender if is_merging is true. int num_queues = is_merging ? num_senders : 1; sender_queues_.reserve(num_queues); diff --git a/be/src/runtime/query-state.cc b/be/src/runtime/query-state.cc index 1aa5cb7ec..6f7522939 100644 --- a/be/src/runtime/query-state.cc +++ b/be/src/runtime/query-state.cc @@ -111,7 +111,7 @@ QueryState::QueryState( refcnt_(0), is_cancelled_(0), query_spilled_(0), - host_profile_(RuntimeProfile::Create(obj_pool(), "<track resource usage>")) { + host_profile_(RuntimeProfile::Create(obj_pool(), "<track resource usage>", false)) { if (query_ctx_.request_pool.empty()) { // fix up pool name for tests DCHECK(!request_pool.empty()); @@ -176,7 +176,7 @@ Status QueryState::Init(const ExecQueryFInstancesRequestPB* exec_rpc_params, ExecEnv* exec_env = ExecEnv::GetInstance(); - RuntimeProfile* jvm_host_profile = RuntimeProfile::Create(&obj_pool_, "JVM"); + RuntimeProfile* jvm_host_profile = RuntimeProfile::Create(&obj_pool_, "JVM", false); host_profile_->AddChild(jvm_host_profile); int64_t gc_count = JvmMemoryCounterMetric::GC_COUNT->GetValue(); diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc index 5ef85b7fa..f863d86a7 100644 --- a/be/src/service/client-request-state.cc +++ b/be/src/service/client-request-state.cc @@ -108,10 +108,10 @@ ClientRequestState::ClientRequestState(const TQueryCtx& query_ctx, Frontend* fro session_(session), coord_exec_called_(false), // Profile is assigned name w/ id after planning - profile_(RuntimeProfile::Create(&profile_pool_, "Query")), + profile_(RuntimeProfile::Create(&profile_pool_, "Query", false)), frontend_profile_(RuntimeProfile::Create(&profile_pool_, "Frontend", false)), - server_profile_(RuntimeProfile::Create(&profile_pool_, "ImpalaServer")), - summary_profile_(RuntimeProfile::Create(&profile_pool_, "Summary")), + server_profile_(RuntimeProfile::Create(&profile_pool_, "ImpalaServer", false)), + summary_profile_(RuntimeProfile::Create(&profile_pool_, "Summary", false)), exec_request_(exec_request), frontend_(frontend), parent_server_(server), diff --git a/be/src/util/runtime-profile.cc b/be/src/util/runtime-profile.cc index 44dc8df11..660c1067f 100644 --- a/be/src/util/runtime-profile.cc +++ b/be/src/util/runtime-profile.cc @@ -82,6 +82,15 @@ const string RuntimeProfileBase::INACTIVE_TIME_COUNTER_NAME = "InactiveTotalTime const string RuntimeProfileBase::SKEW_SUMMARY = "skew(s) found at"; const string RuntimeProfileBase::SKEW_DETAILS = "Skew details"; +const string RuntimeProfile::FRAGMENT_INSTANCE_LIFECYCLE_TIMINGS = + "Fragment Instance Lifecycle Timings"; +const string RuntimeProfile::BUFFER_POOL = "Buffer pool"; +const string RuntimeProfile::DEQUEUE = "Dequeue"; +const string RuntimeProfile::ENQUEUE = "Enqueue"; +const string RuntimeProfile::HASH_TABLE = "Hash Table"; +const string RuntimeProfile::PREFIX_FILTER = "Filter "; +const string RuntimeProfile::PREFIX_GROUPING_AGGREGATOR = "GroupingAggregator "; + constexpr ProfileEntryPrototype::Significance ProfileEntryPrototype::ALLSIGNIFICANCE[]; /// Helper to interpret the bit pattern of 'val' as T, which can either be an int64_t or @@ -111,6 +120,16 @@ static int32_t GetProfileVersion(const TRuntimeProfileTree& tree) { return tree.__isset.profile_version ? tree.profile_version : 1; } +static bool UntimedProfileNode(const string& name) { + return name.compare(RuntimeProfile::FRAGMENT_INSTANCE_LIFECYCLE_TIMINGS) == 0 + || name.compare(RuntimeProfile::BUFFER_POOL) == 0 + || name.compare(RuntimeProfile::DEQUEUE) == 0 + || name.compare(RuntimeProfile::ENQUEUE) == 0 + || name.compare(RuntimeProfile::HASH_TABLE) == 0 + || name.rfind(RuntimeProfile::PREFIX_FILTER, 0) == 0 + || name.rfind(RuntimeProfile::PREFIX_GROUPING_AGGREGATOR, 0) == 0; +} + static RuntimeProfile::Verbosity DefaultVerbosity() { return FLAGS_gen_experimental_profile ? RuntimeProfile::Verbosity::DEFAULT : RuntimeProfile::Verbosity::LEGACY; @@ -354,7 +373,7 @@ void RuntimeProfile::InitFromThrift(const TRuntimeProfileNode& node, ObjectPool* } void AggregatedRuntimeProfile::UpdateAggregatedFromInstance( - RuntimeProfile* other, int idx) { + RuntimeProfile* other, int idx, bool add_default_counters) { { lock_guard<SpinLock> l(input_profile_name_lock_); DCHECK(!input_profile_names_.empty()) @@ -362,14 +381,14 @@ void AggregatedRuntimeProfile::UpdateAggregatedFromInstance( input_profile_names_[idx] = other->name(); } - UpdateAggregatedFromInstanceRecursive(other, idx); + UpdateAggregatedFromInstanceRecursive(other, idx, add_default_counters); // Recursively compute times on the whole tree. ComputeTimeInProfile(); } void AggregatedRuntimeProfile::UpdateAggregatedFromInstanceRecursive( - RuntimeProfile* other, int idx) { + RuntimeProfile* other, int idx, bool add_default_counters) { DCHECK(other != NULL); DCHECK_GE(idx, 0); DCHECK_LT(idx, num_input_profiles_); @@ -395,15 +414,19 @@ void AggregatedRuntimeProfile::UpdateAggregatedFromInstanceRecursive( DCHECK(other_child != nullptr) << other->children_[i].first->name() << " must be a RuntimeProfile"; bool indent_other_child = other->children_[i].second; - AggregatedRuntimeProfile* child = dynamic_cast<AggregatedRuntimeProfile*>( - AddOrCreateChild(other_child->name(), &insert_pos, [this, other_child] () { - AggregatedRuntimeProfile* child2 = Create( - pool_, other_child->name(), num_input_profiles_, /*is_root=*/false); - child2->metadata_ = other_child->metadata(); - return child2; - }, indent_other_child)); + bool is_timed = add_default_counters && !UntimedProfileNode(other_child->name()); + AggregatedRuntimeProfile* child = + dynamic_cast<AggregatedRuntimeProfile*>(AddOrCreateChild( + other_child->name(), &insert_pos, + [this, other_child, is_timed]() { + AggregatedRuntimeProfile* child2 = Create(pool_, other_child->name(), + num_input_profiles_, /*is_root=*/false, is_timed); + child2->metadata_ = other_child->metadata(); + return child2; + }, + indent_other_child)); DCHECK(child != nullptr); - child->UpdateAggregatedFromInstanceRecursive(other_child, idx); + child->UpdateAggregatedFromInstanceRecursive(other_child, idx, is_timed); } } } @@ -531,7 +554,7 @@ void AggregatedRuntimeProfile::UpdateEventSequencesFromInstance( // TODO: do we need any special handling of the computed timers? Like local/total time. void AggregatedRuntimeProfile::UpdateAggregatedFromInstances( - const TRuntimeProfileTree& src, int start_idx) { + const TRuntimeProfileTree& src, int start_idx, bool add_default_counters) { DCHECK(FLAGS_gen_experimental_profile) << "Aggregated profiles should only be used on the coordinator when enabled by flag"; if (UNLIKELY(src.nodes.size()) == 0) return; @@ -551,14 +574,15 @@ void AggregatedRuntimeProfile::UpdateAggregatedFromInstances( } int node_idx = 0; - UpdateAggregatedFromInstancesRecursive(src, &node_idx, start_idx); + UpdateAggregatedFromInstancesRecursive(src, &node_idx, start_idx, add_default_counters); // Recursively compute times on the whole tree. ComputeTimeInProfile(); } void AggregatedRuntimeProfile::UpdateAggregatedFromInstancesRecursive( - const TRuntimeProfileTree& src, int* node_idx, int start_idx) { + const TRuntimeProfileTree& src, int* node_idx, int start_idx, + bool add_default_counters) { DCHECK_LT(*node_idx, src.nodes.size()); const TRuntimeProfileNode& node = src.nodes[*node_idx]; DCHECK(node.__isset.aggregated); @@ -572,15 +596,19 @@ void AggregatedRuntimeProfile::UpdateAggregatedFromInstancesRecursive( // Update children with matching names; create new ones if they don't match. for (int i = 0; i < node.num_children; ++i) { const TRuntimeProfileNode& tchild = src.nodes[*node_idx]; - AggregatedRuntimeProfile* child = dynamic_cast<AggregatedRuntimeProfile*>( - AddOrCreateChild(tchild.name, &insert_pos, [this, tchild] () { - AggregatedRuntimeProfile* child2 = - Create(pool_, tchild.name, num_input_profiles_, /*is_root=*/false); - child2->metadata_ = tchild.node_metadata; - return child2; - }, tchild.indent)); + bool is_timed = add_default_counters && !UntimedProfileNode(tchild.name); + AggregatedRuntimeProfile* child = + dynamic_cast<AggregatedRuntimeProfile*>(AddOrCreateChild( + tchild.name, &insert_pos, + [this, tchild, is_timed]() { + AggregatedRuntimeProfile* child2 = Create( + pool_, tchild.name, num_input_profiles_, /*is_root=*/false, is_timed); + child2->metadata_ = tchild.node_metadata; + return child2; + }, + tchild.indent)); DCHECK(child != nullptr); - child->UpdateAggregatedFromInstancesRecursive(src, node_idx, start_idx); + child->UpdateAggregatedFromInstancesRecursive(src, node_idx, start_idx, is_timed); } } } @@ -859,16 +887,17 @@ void RuntimeProfile::Update( // Update children with matching names; create new ones if they don't match. for (int i = 0; i < node.num_children; ++i) { const TRuntimeProfileNode& tchild = nodes[*idx]; + bool is_timed = add_default_counters && !UntimedProfileNode(tchild.name); RuntimeProfile* child = dynamic_cast<RuntimeProfile*>(AddOrCreateChild( tchild.name, &insert_pos, - [this, tchild, add_default_counters]() { - RuntimeProfile* child2 = Create(pool_, tchild.name, add_default_counters); + [this, tchild, is_timed]() { + RuntimeProfile* child2 = Create(pool_, tchild.name, is_timed); child2->metadata_ = tchild.node_metadata; return child2; }, tchild.indent)); DCHECK(child != nullptr); - child->Update(nodes, idx, add_default_counters); + child->Update(nodes, idx, is_timed); } } } @@ -952,11 +981,11 @@ void RuntimeProfile::PrependChild(RuntimeProfileBase* child, bool indent) { AddChildLocked(child, indent, children_.begin()); } -RuntimeProfile* RuntimeProfile::CreateChild(const string& name, bool indent, - bool prepend) { +RuntimeProfile* RuntimeProfile::CreateChild( + const string& name, bool indent, bool prepend, bool add_default_counters) { lock_guard<SpinLock> l(children_lock_); DCHECK(child_map_.find(name) == child_map_.end()); - RuntimeProfile* child = Create(pool_, name); + RuntimeProfile* child = Create(pool_, name, add_default_counters); AddChildLocked(child, indent, prepend ? children_.begin() : children_.end()); return child; } @@ -2592,19 +2621,21 @@ void RuntimeProfile::EventSequence::ToJson( *value = event_sequence_json; } -AggregatedRuntimeProfile::AggregatedRuntimeProfile( - ObjectPool* pool, const string& name, int num_input_profiles, bool is_root) +AggregatedRuntimeProfile::AggregatedRuntimeProfile(ObjectPool* pool, const string& name, + int num_input_profiles, bool is_root, bool add_default_counters) : RuntimeProfileBase(pool, name, - pool->Add(new AveragedCounter(TUnit::TIME_NS, num_input_profiles)), - pool->Add(new AveragedCounter(TUnit::TIME_NS, num_input_profiles))), + pool->Add(new AveragedCounter(TUnit::TIME_NS, num_input_profiles)), + pool->Add(new AveragedCounter(TUnit::TIME_NS, num_input_profiles)), + add_default_counters), num_input_profiles_(num_input_profiles) { DCHECK_GE(num_input_profiles, 0); if (is_root) input_profile_names_.resize(num_input_profiles); } -AggregatedRuntimeProfile* AggregatedRuntimeProfile::Create( - ObjectPool* pool, const string& name, int num_input_profiles, bool is_root) { - return pool->Add(new AggregatedRuntimeProfile(pool, name, num_input_profiles, is_root)); +AggregatedRuntimeProfile* AggregatedRuntimeProfile::Create(ObjectPool* pool, + const string& name, int num_input_profiles, bool is_root, bool add_default_counters) { + return pool->Add(new AggregatedRuntimeProfile( + pool, name, num_input_profiles, is_root, add_default_counters)); } void AggregatedRuntimeProfile::InitFromThrift( diff --git a/be/src/util/runtime-profile.h b/be/src/util/runtime-profile.h index 2b6a2cde6..8e1a1fb2e 100644 --- a/be/src/util/runtime-profile.h +++ b/be/src/util/runtime-profile.h @@ -433,6 +433,15 @@ class RuntimeProfile : public RuntimeProfileBase { class SamplingTimeSeriesCounter; class ChunkedTimeSeriesCounter; + /// String constants for instruction count names. + static const string FRAGMENT_INSTANCE_LIFECYCLE_TIMINGS; + static const string BUFFER_POOL; + static const string DEQUEUE; + static const string ENQUEUE; + static const string HASH_TABLE; + static const string PREFIX_FILTER; + static const string PREFIX_GROUPING_AGGREGATOR; + /// Create a runtime profile object with 'name'. The profile, counters and any other /// structures owned by the profile are allocated from 'pool'. /// If add_default_counters is false, TotalTime and InactiveTotalTime will be @@ -464,8 +473,10 @@ class RuntimeProfile : public RuntimeProfileBase { /// Creates a new child profile with the given 'name'. A child profile with that name /// must not already exist. If 'prepend' is true, prepended before other child profiles, /// otherwise appended after other child profiles. - RuntimeProfile* CreateChild( - const std::string& name, bool indent = true, bool prepend = false); + /// If 'add_default_counters' is false, TotalTime and InactiveTotalTime will be + /// hidden in any newly created RuntimeProfile node. + RuntimeProfile* CreateChild(const std::string& name, bool indent = true, + bool prepend = false, bool add_default_counters = false); /// Sorts all children according to descending total time. Does not /// invalidate pointers to profiles. @@ -829,7 +840,7 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase { /// will have UpdateAggregated*() called on it. The descendants of the root are /// created and updated by that call. static AggregatedRuntimeProfile* Create(ObjectPool* pool, const std::string& name, - int num_input_profiles, bool is_root = true); + int num_input_profiles, bool is_root = true, bool add_default_counters = true); /// Updates the AveragedCounter counters in this profile with the counters from the /// 'src' profile, which is the input instance that was assigned index 'idx'. If a @@ -839,7 +850,8 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase { /// /// Note that 'src' must be all instances of RuntimeProfile - no /// AggregatedRuntimeProfiles can be part of the input. - void UpdateAggregatedFromInstance(RuntimeProfile* src, int idx); + void UpdateAggregatedFromInstance( + RuntimeProfile* src, int idx, bool add_default_counters = true); /// Updates the AveragedCounter counters in this profile with the counters from the /// 'src' profile, which must be a serialized AggregatedProfile. The instances in @@ -847,7 +859,8 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase { /// this profile. If a counter is present in 'src' but missing in this profile, a /// new AveragedCounter is created with the same name. Obtains locks on the counter /// maps and child counter maps in this profile. - void UpdateAggregatedFromInstances(const TRuntimeProfileTree& src, int start_idx); + void UpdateAggregatedFromInstances( + const TRuntimeProfileTree& src, int start_idx, bool add_default_counters = true); protected: virtual int GetNumInputProfiles() const override { return num_input_profiles_; } @@ -922,8 +935,8 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase { /// Time series counters. Protected by 'counter_map_lock_'. std::map<std::string, TAggTimeSeriesCounter> time_series_counter_map_; - AggregatedRuntimeProfile( - ObjectPool* pool, const std::string& name, int num_input_profiles, bool is_root); + AggregatedRuntimeProfile(ObjectPool* pool, const std::string& name, + int num_input_profiles, bool is_root, bool add_default_counters = true); /// Group the values in 'info_string_values' by value, with a vector of indices where /// that value appears. 'info_string_values' must be a value from 'info_strings_'. @@ -931,7 +944,8 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase { const std::vector<std::string>& info_string_values) const; /// Helper for UpdateAggregatedFromInstance() that are invoked recursively on 'src'. - void UpdateAggregatedFromInstanceRecursive(RuntimeProfile* src, int idx); + void UpdateAggregatedFromInstanceRecursive( + RuntimeProfile* src, int idx, bool add_default_counter = true); /// Helpers for UpdateAggregatedFromInstanceRecursive() that update particular parts /// of this profile node from 'src'. @@ -941,8 +955,8 @@ class AggregatedRuntimeProfile : public RuntimeProfileBase { void UpdateEventSequencesFromInstance(RuntimeProfile* src, int idx); /// Helper for UpdateAggregatedFromInstances() that are invoked recursively on 'src'. - void UpdateAggregatedFromInstancesRecursive( - const TRuntimeProfileTree& src, int* node_idx, int start_idx); + void UpdateAggregatedFromInstancesRecursive(const TRuntimeProfileTree& src, + int* node_idx, int start_idx, bool add_default_counter = true); /// Helpers for UpdateAggregatedFromInstancesRecursive() that update particular parts /// of this profile node from 'src'. 'src' must have an aggregated profile set.
