This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 93d4f2236 IMPALA-12617: Fix DCHECK failure for Statestore
93d4f2236 is described below
commit 93d4f2236ca5afb33e3770ba3814f36fe159288c
Author: wzhou-code <[email protected]>
AuthorDate: Mon Dec 11 22:15:06 2023 -0800
IMPALA-12617: Fix DCHECK failure for Statestore
Statestore uses thread pools to periodically send catalog topic update
and cluster membership. It adds sending tasks to the queues of thread
pools when receiving registration requests from subscribers so the
thread pools have to be ready before the Thrift server of Statestore
is started to accept registration request.
Current code call ThreadPool::Init() after the Thrift server is started.
This could cause Statestore to hit DCHECK failure when calling
ThreadPool::Offer(). It's more likely to happen when Statestore HA is
enabled since Statestore takes more time for initialization.
This patch changes the order to call ThreadPool::Init() before starting
Thrift server of the Statestore server.
Testing:
- Repeatedly ran custom_cluster/test_statestored_ha.py on local machine
and Jenkins over night without failure.
- Passed core tests.
Change-Id: I91423f3de2d64cb617a06ea7adbe5ee2937bde66
Reviewed-on: http://gerrit.cloudera.org:8080/20775
Reviewed-by: Riza Suminto <[email protected]>
Reviewed-by: Michael Smith <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
be/src/statestore/statestore.cc | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/be/src/statestore/statestore.cc b/be/src/statestore/statestore.cc
index ef1f5f539..0054da633 100644
--- a/be/src/statestore/statestore.cc
+++ b/be/src/statestore/statestore.cc
@@ -775,14 +775,15 @@ Status Statestore::Init(int32_t state_store_port) {
.ssl_version(ssl_version)
.cipher_list(FLAGS_ssl_cipher_list);
}
+ RETURN_IF_ERROR(subscriber_topic_update_threadpool_.Init());
+ RETURN_IF_ERROR(subscriber_priority_topic_update_threadpool_.Init());
+ RETURN_IF_ERROR(subscriber_heartbeat_threadpool_.Init());
+
ThriftServer* server;
RETURN_IF_ERROR(builder.metrics(metrics_).Build(&server));
thrift_server_.reset(server);
RETURN_IF_ERROR(thrift_server_->Start());
- RETURN_IF_ERROR(subscriber_topic_update_threadpool_.Init());
- RETURN_IF_ERROR(subscriber_priority_topic_update_threadpool_.Init());
- RETURN_IF_ERROR(subscriber_heartbeat_threadpool_.Init());
RETURN_IF_ERROR(Thread::Create("statestore-heartbeat",
"heartbeat-monitoring-thread",
&Statestore::MonitorSubscriberHeartbeat, this,
&heartbeat_monitoring_thread_));
RETURN_IF_ERROR(Thread::Create("statestore-update-catalogd",
"update-catalogd-thread",
