[
https://issues.apache.org/jira/browse/KUDU-2165?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16260315#comment-16260315
]
Todd Lipcon commented on KUDU-2165:
-----------------------------------
I think this is just because the cache is a Singleton, and each
internal-minicluster daemon ends up calling StartInstrumentation. So, when
multiple daemons start, they all race to set metrics, and end up overwriting
the metrics set by other daemons. Additionally, if you restart a server while
another one is actively working, you'll also end up destructing the
CacheMetrics underneath a concurrent-operating cache, causing the race.
> alter_table-test has mysterious TSAN warnings
> ---------------------------------------------
>
> Key: KUDU-2165
> URL: https://issues.apache.org/jira/browse/KUDU-2165
> Project: Kudu
> Issue Type: Bug
> Affects Versions: 1.6.0
> Reporter: Adar Dembo
> Attachments: alter_table-test.txt
>
>
> I say "mysterious" because the warnings don't show one half of the stack
> trace, so it's not immediately clear where the race is.
> Here are the warnings. I'm attaching the full log too.
> {noformat}
> ==================
> WARNING: ThreadSanitizer: data race (pid=1945)
> Write of size 8 at 0x7b0c000913b0 by main thread:
> #0 operator delete(void*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73
> (alter_table-test+0x4db471)
> #1 kudu::AtomicGauge<unsigned long>::~AtomicGauge()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/metrics.h:768:7
> (libtablet.so+0x15a561)
> #2 kudu::RefCountedThreadSafe<kudu::Metric,
> kudu::DefaultRefCountedThreadSafeTraits<kudu::Metric>
> >::DeleteInternal(kudu::Metric const*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:153:44
> (libmaster.so+0xc1787)
> #3
> kudu::DefaultRefCountedThreadSafeTraits<kudu::Metric>::Destruct(kudu::Metric
> const*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:116:5
> (libmaster.so+0xc1749)
> #4 kudu::RefCountedThreadSafe<kudu::Metric,
> kudu::DefaultRefCountedThreadSafeTraits<kudu::Metric> >::Release() const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:144:7
> (libmaster.so+0xc1709)
> #5 scoped_refptr<kudu::AtomicGauge<unsigned long> >::~scoped_refptr()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:266:13
> (libtablet.so+0x1596aa)
> #6 kudu::CacheMetrics::~CacheMetrics()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache_metrics.h:27:8
> (libkudu_util.so+0xe8cda)
> #7
> kudu::DefaultDeleter<kudu::CacheMetrics>::operator()(kudu::CacheMetrics*)
> const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:145:5
> (libkudu_util.so+0xe8c9e)
> #8 kudu::internal::gscoped_ptr_impl<kudu::CacheMetrics,
> kudu::DefaultDeleter<kudu::CacheMetrics> >::reset(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:254:7
> (libkudu_util.so+0xe9404)
> #9 gscoped_ptr<kudu::CacheMetrics,
> kudu::DefaultDeleter<kudu::CacheMetrics> >::reset(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:375:46
> (libkudu_util.so+0xe9370)
> #10 kudu::(anonymous
> namespace)::ShardedLRUCache::SetMetrics(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:464:14
> (libkudu_util.so+0xe6840)
> #11
> kudu::cfile::BlockCache::StartInstrumentation(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:96:11
> (libcfile.so+0x72af4)
> #12 kudu::master::Master::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:118:38
> (libmaster.so+0xe5ff6)
> #13 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:92:3
> (libmaster.so+0x10070e)
> #14 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #15 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #16 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #17 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #18 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #19 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #20 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #21 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #22 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #23 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #24 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #25 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #26 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #27 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Previous read of size 8 at 0x7b0c000913b0 by thread T170 (mutexes: write
> M627261677020195488):
> As if synchronized via sleep:
> #0 nanosleep
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357
> (alter_table-test+0x4768f7)
> #1 boost::detail::yield(unsigned int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/smart_ptr/detail/yield_k.hpp:151:9
> (libmaster.so+0xd2b16)
> #2 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:555:7
> (libkudu_util.so+0x1b2cb1)
> #3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #4 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #5 kudu::ThreadPool::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:350:21
> (libkudu_util.so+0x1b8f2a)
> #6 kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool,
> kudu::DefaultDeleter<kudu::ThreadPool> >*) const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:127:3
> (libkudu_util.so+0x1b8e37)
> #7
> kudu::MaintenanceManager::MaintenanceManager(kudu::MaintenanceManager::Options
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/maintenance_manager.cc:154:3
> (libkudu_util.so+0x159fd5)
> #8 kudu::master::Master::Master(kudu::master::MasterOptions const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:101:30
> (libmaster.so+0xe5939)
> #9 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:91:33
> (libmaster.so+0x1006f9)
> #10 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #11 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #12 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #13 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #14 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #15 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #16 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #17 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #18 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #19 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #20 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #21 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #22 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #23 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Mutex M627261677020195488 is already destroyed.
> Thread T170 (tid=5115, finished) created by thread T154 at:
> #0 pthread_create
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897
> (alter_table-test+0x47afe3)
> #1 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:524:15
> (libkudu_util.so+0x1b2ac7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #3 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #4 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>,
> kudu::ThreadPoolToken*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:493:21
> (libkudu_util.so+0x1b9b01)
> #5 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:454:10
> (libkudu_util.so+0x1bb95f)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:446:10
> (libkudu_util.so+0x1bb899)
> #7 kudu::tablet::TransactionDriver::ApplyAsync()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:482:23
> (libtablet.so+0x1548a9)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:436:5
> (libtablet.so+0x152188)
> #9
> _ZNSt3__18__invokeIRMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEERPS3_JS6_EvEEDTcldsdeclsr3std3__1E7forwardIT0_Efp0_Efp_spclsr3std3__1E7forwardIT1_Efp1_EEEOT_OSC_DpOSD_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4242:1
> (libtablet.so+0x14e2c7)
> #10
> _ZNSt3__115__apply_functorIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEENS_5tupleIJPS3_NS_12placeholders4__phILi1EEEEEEJLm0ELm1EENS9_IJS6_EEEEENS_13__bind_returnIT_T0_T2_Xsr22__is_valid_bind_returnISH_SI_SJ_EE5valueEE4typeERSH_RSI_NS_15__tuple_indicesIJXspT1_EEEEOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2217
> (libtablet.so+0x14e2c7)
> #11
> _ZNSt3__16__bindIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEEJPS3_RKNS_12placeholders4__phILi1EEEEEclIJS6_EEENS_13__bind_returnIS8_NS_5tupleIJS9_SC_EEENSI_IJDpOT_EEEXsr22__is_valid_bind_returnIS8_SJ_SN_EE5valueEE4typeESM_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2250
> (libtablet.so+0x14e2c7)
> #12
> _ZNSt3__18__invokeIRNS_6__bindIMN4kudu6tablet17TransactionDriverEFvRKNS2_6StatusEEJPS4_RKNS_12placeholders4__phILi1EEEEEEJS7_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSI_DpOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4301
> (libtablet.so+0x14e2c7)
> #13 void
> std::__1::__invoke_void_return_wrapper<void>::__call<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&>(std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/__functional_base:359
> (libtablet.so+0x14e2c7)
> #14 std::__1::__function::__func<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>,
> std::__1::allocator<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&> >,
> void (kudu::Status const&)>::operator()(kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1552:12
> (libtablet.so+0x14e1a4)
> #15 std::__1::function<void (kudu::Status
> const&)>::operator()(kudu::Status const&) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1914:12
> (libconsensus.so+0xc45ab)
> #16
> kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:2751:3
> (libconsensus.so+0xbfd94)
> #17 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/pending_rounds.cc:185:12
> (libconsensus.so+0xa180a)
> #18 kudu::consensus::RaftConsensus::NotifyCommitIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:700:13
> (libconsensus.so+0xb27c5)
> #19
> kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChangeTask(long)
>
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/consensus_queue.cc:1022:15
> (libconsensus.so+0x83c6d)
> #20 kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>::Run(kudu::consensus::PeerMessageQueue*,
> long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:192:12
> (libconsensus.so+0x8f310)
> #21 kudu::internal::InvokeHelper<false, void,
> kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long
> const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>,
> kudu::consensus::PeerMessageQueue*, long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:889:14
> (libconsensus.so+0x8f248)
> #22 kudu::internal::Invoker<2,
> kudu::internal::BindState<kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long), void
> (kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>,
> long)>, void (kudu::consensus::PeerMessageQueue*,
> long)>::Run(kudu::internal::BindStateBase*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:1147:12
> (libconsensus.so+0x8f183)
> #23 kudu::Callback<void ()>::Run() const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/callback.h:396:12
> (libconsensus.so+0x9371d)
> #24 kudu::ClosureRunnable::Run()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:74:9
> (libkudu_util.so+0x1beced)
> #25 kudu::ThreadPool::DispatchThread(bool)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:631:22
> (libkudu_util.so+0x1bc021)
> #26 boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>::operator()(kudu::ThreadPool*, bool) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:165:29
> (libkudu_util.so+0x1c48de)
> #27 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void,
> kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:319:9
> (libkudu_util.so+0x1c481d)
> #28 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >::operator()()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libkudu_util.so+0x1c4783)
> #29
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>,
> boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libkudu_util.so+0x1c4521)
> #30 boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb6eb1)
> #31 kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:601:3
> (libkudu_util.so+0x1b32ce)
> SUMMARY: ThreadSanitizer: data race
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73
> in operator delete(void*)
> ==================
> ==================
> WARNING: ThreadSanitizer: data race (pid=1945)
> Write of size 8 at 0x7b4000010c80 by main thread:
> #0 operator delete(void*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73
> (alter_table-test+0x4db471)
> #1 kudu::Counter::~Counter()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/metrics.h:943:7
> (libkudu_util.so+0x175b61)
> #2 kudu::RefCountedThreadSafe<kudu::Metric,
> kudu::DefaultRefCountedThreadSafeTraits<kudu::Metric>
> >::DeleteInternal(kudu::Metric const*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:153:44
> (libmaster.so+0xc1787)
> #3
> kudu::DefaultRefCountedThreadSafeTraits<kudu::Metric>::Destruct(kudu::Metric
> const*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:116:5
> (libmaster.so+0xc1749)
> #4 kudu::RefCountedThreadSafe<kudu::Metric,
> kudu::DefaultRefCountedThreadSafeTraits<kudu::Metric> >::Release() const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:144:7
> (libmaster.so+0xc1709)
> #5 scoped_refptr<kudu::Counter>::~scoped_refptr()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/ref_counted.h:266:13
> (libtserver.so+0xaa30a)
> #6 kudu::CacheMetrics::~CacheMetrics()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache_metrics.h:27:8
> (libkudu_util.so+0xe8d07)
> #7
> kudu::DefaultDeleter<kudu::CacheMetrics>::operator()(kudu::CacheMetrics*)
> const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:145:5
> (libkudu_util.so+0xe8c9e)
> #8 kudu::internal::gscoped_ptr_impl<kudu::CacheMetrics,
> kudu::DefaultDeleter<kudu::CacheMetrics> >::reset(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:254:7
> (libkudu_util.so+0xe9404)
> #9 gscoped_ptr<kudu::CacheMetrics,
> kudu::DefaultDeleter<kudu::CacheMetrics> >::reset(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:375:46
> (libkudu_util.so+0xe9370)
> #10 kudu::(anonymous
> namespace)::ShardedLRUCache::SetMetrics(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:464:14
> (libkudu_util.so+0xe6840)
> #11
> kudu::cfile::BlockCache::StartInstrumentation(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:96:11
> (libcfile.so+0x72af4)
> #12 kudu::master::Master::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:118:38
> (libmaster.so+0xe5ff6)
> #13 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:92:3
> (libmaster.so+0x10070e)
> #14 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #15 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #16 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #17 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #18 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #19 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #20 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #21 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #22 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #23 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #24 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #25 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #26 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #27 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Previous atomic write of size 8 at 0x7b4000010c80 by thread T170 (mutexes:
> write M627261677020195488):
> As if synchronized via sleep:
> #0 nanosleep
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357
> (alter_table-test+0x4768f7)
> #1 boost::detail::yield(unsigned int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/smart_ptr/detail/yield_k.hpp:151:9
> (libmaster.so+0xd2b16)
> #2 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:555:7
> (libkudu_util.so+0x1b2cb1)
> #3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #4 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #5 kudu::ThreadPool::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:350:21
> (libkudu_util.so+0x1b8f2a)
> #6 kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool,
> kudu::DefaultDeleter<kudu::ThreadPool> >*) const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:127:3
> (libkudu_util.so+0x1b8e37)
> #7
> kudu::MaintenanceManager::MaintenanceManager(kudu::MaintenanceManager::Options
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/maintenance_manager.cc:154:3
> (libkudu_util.so+0x159fd5)
> #8 kudu::master::Master::Master(kudu::master::MasterOptions const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:101:30
> (libmaster.so+0xe5939)
> #9 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:91:33
> (libmaster.so+0x1006f9)
> #10 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #11 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #12 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #13 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #14 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #15 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #16 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #17 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #18 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #19 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #20 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #21 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #22 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #23 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Mutex M627261677020195488 is already destroyed.
> Thread T170 (tid=5115, finished) created by thread T154 at:
> #0 pthread_create
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897
> (alter_table-test+0x47afe3)
> #1 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:524:15
> (libkudu_util.so+0x1b2ac7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #3 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #4 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>,
> kudu::ThreadPoolToken*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:493:21
> (libkudu_util.so+0x1b9b01)
> #5 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:454:10
> (libkudu_util.so+0x1bb95f)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:446:10
> (libkudu_util.so+0x1bb899)
> #7 kudu::tablet::TransactionDriver::ApplyAsync()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:482:23
> (libtablet.so+0x1548a9)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:436:5
> (libtablet.so+0x152188)
> #9
> _ZNSt3__18__invokeIRMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEERPS3_JS6_EvEEDTcldsdeclsr3std3__1E7forwardIT0_Efp0_Efp_spclsr3std3__1E7forwardIT1_Efp1_EEEOT_OSC_DpOSD_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4242:1
> (libtablet.so+0x14e2c7)
> #10
> _ZNSt3__115__apply_functorIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEENS_5tupleIJPS3_NS_12placeholders4__phILi1EEEEEEJLm0ELm1EENS9_IJS6_EEEEENS_13__bind_returnIT_T0_T2_Xsr22__is_valid_bind_returnISH_SI_SJ_EE5valueEE4typeERSH_RSI_NS_15__tuple_indicesIJXspT1_EEEEOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2217
> (libtablet.so+0x14e2c7)
> #11
> _ZNSt3__16__bindIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEEJPS3_RKNS_12placeholders4__phILi1EEEEEclIJS6_EEENS_13__bind_returnIS8_NS_5tupleIJS9_SC_EEENSI_IJDpOT_EEEXsr22__is_valid_bind_returnIS8_SJ_SN_EE5valueEE4typeESM_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2250
> (libtablet.so+0x14e2c7)
> #12
> _ZNSt3__18__invokeIRNS_6__bindIMN4kudu6tablet17TransactionDriverEFvRKNS2_6StatusEEJPS4_RKNS_12placeholders4__phILi1EEEEEEJS7_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSI_DpOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4301
> (libtablet.so+0x14e2c7)
> #13 void
> std::__1::__invoke_void_return_wrapper<void>::__call<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&>(std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/__functional_base:359
> (libtablet.so+0x14e2c7)
> #14 std::__1::__function::__func<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>,
> std::__1::allocator<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&> >,
> void (kudu::Status const&)>::operator()(kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1552:12
> (libtablet.so+0x14e1a4)
> #15 std::__1::function<void (kudu::Status
> const&)>::operator()(kudu::Status const&) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1914:12
> (libconsensus.so+0xc45ab)
> #16
> kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:2751:3
> (libconsensus.so+0xbfd94)
> #17 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/pending_rounds.cc:185:12
> (libconsensus.so+0xa180a)
> #18 kudu::consensus::RaftConsensus::NotifyCommitIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:700:13
> (libconsensus.so+0xb27c5)
> #19
> kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChangeTask(long)
>
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/consensus_queue.cc:1022:15
> (libconsensus.so+0x83c6d)
> #20 kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>::Run(kudu::consensus::PeerMessageQueue*,
> long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:192:12
> (libconsensus.so+0x8f310)
> #21 kudu::internal::InvokeHelper<false, void,
> kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long
> const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>,
> kudu::consensus::PeerMessageQueue*, long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:889:14
> (libconsensus.so+0x8f248)
> #22 kudu::internal::Invoker<2,
> kudu::internal::BindState<kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long), void
> (kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>,
> long)>, void (kudu::consensus::PeerMessageQueue*,
> long)>::Run(kudu::internal::BindStateBase*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:1147:12
> (libconsensus.so+0x8f183)
> #23 kudu::Callback<void ()>::Run() const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/callback.h:396:12
> (libconsensus.so+0x9371d)
> #24 kudu::ClosureRunnable::Run()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:74:9
> (libkudu_util.so+0x1beced)
> #25 kudu::ThreadPool::DispatchThread(bool)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:631:22
> (libkudu_util.so+0x1bc021)
> #26 boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>::operator()(kudu::ThreadPool*, bool) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:165:29
> (libkudu_util.so+0x1c48de)
> #27 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void,
> kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:319:9
> (libkudu_util.so+0x1c481d)
> #28 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >::operator()()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libkudu_util.so+0x1c4783)
> #29
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>,
> boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libkudu_util.so+0x1c4521)
> #30 boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb6eb1)
> #31 kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:601:3
> (libkudu_util.so+0x1b32ce)
> SUMMARY: ThreadSanitizer: data race
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73
> in operator delete(void*)
> ==================
> ==================
> WARNING: ThreadSanitizer: data race (pid=1945)
> Write of size 8 at 0x7b1000063190 by main thread:
> #0 operator delete(void*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73
> (alter_table-test+0x4db471)
> #1
> kudu::DefaultDeleter<kudu::CacheMetrics>::operator()(kudu::CacheMetrics*)
> const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:145:5
> (libkudu_util.so+0xe8ca6)
> #2 kudu::internal::gscoped_ptr_impl<kudu::CacheMetrics,
> kudu::DefaultDeleter<kudu::CacheMetrics> >::reset(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:254:7
> (libkudu_util.so+0xe9404)
> #3 gscoped_ptr<kudu::CacheMetrics,
> kudu::DefaultDeleter<kudu::CacheMetrics> >::reset(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/gscoped_ptr.h:375:46
> (libkudu_util.so+0xe9370)
> #4 kudu::(anonymous
> namespace)::ShardedLRUCache::SetMetrics(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:464:14
> (libkudu_util.so+0xe6840)
> #5
> kudu::cfile::BlockCache::StartInstrumentation(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:96:11
> (libcfile.so+0x72af4)
> #6 kudu::master::Master::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:118:38
> (libmaster.so+0xe5ff6)
> #7 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:92:3
> (libmaster.so+0x10070e)
> #8 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #9 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #10 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #11 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #12 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #13 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #14 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #15 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #16 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #17 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #18 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #19 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #20 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #21 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Previous read of size 8 at 0x7b1000063190 by thread T170 (mutexes: write
> M627261677020195488):
> As if synchronized via sleep:
> #0 nanosleep
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357
> (alter_table-test+0x4768f7)
> #1 boost::detail::yield(unsigned int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/smart_ptr/detail/yield_k.hpp:151:9
> (libmaster.so+0xd2b16)
> #2 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:555:7
> (libkudu_util.so+0x1b2cb1)
> #3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #4 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #5 kudu::ThreadPool::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:350:21
> (libkudu_util.so+0x1b8f2a)
> #6 kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool,
> kudu::DefaultDeleter<kudu::ThreadPool> >*) const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:127:3
> (libkudu_util.so+0x1b8e37)
> #7
> kudu::MaintenanceManager::MaintenanceManager(kudu::MaintenanceManager::Options
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/maintenance_manager.cc:154:3
> (libkudu_util.so+0x159fd5)
> #8 kudu::master::Master::Master(kudu::master::MasterOptions const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:101:30
> (libmaster.so+0xe5939)
> #9 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:91:33
> (libmaster.so+0x1006f9)
> #10 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #11 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #12 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #13 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #14 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #15 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #16 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #17 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #18 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #19 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #20 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #21 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #22 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #23 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Mutex M627261677020195488 is already destroyed.
> Thread T170 (tid=5115, finished) created by thread T154 at:
> #0 pthread_create
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897
> (alter_table-test+0x47afe3)
> #1 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:524:15
> (libkudu_util.so+0x1b2ac7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #3 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #4 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>,
> kudu::ThreadPoolToken*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:493:21
> (libkudu_util.so+0x1b9b01)
> #5 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:454:10
> (libkudu_util.so+0x1bb95f)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:446:10
> (libkudu_util.so+0x1bb899)
> #7 kudu::tablet::TransactionDriver::ApplyAsync()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:482:23
> (libtablet.so+0x1548a9)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:436:5
> (libtablet.so+0x152188)
> #9
> _ZNSt3__18__invokeIRMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEERPS3_JS6_EvEEDTcldsdeclsr3std3__1E7forwardIT0_Efp0_Efp_spclsr3std3__1E7forwardIT1_Efp1_EEEOT_OSC_DpOSD_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4242:1
> (libtablet.so+0x14e2c7)
> #10
> _ZNSt3__115__apply_functorIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEENS_5tupleIJPS3_NS_12placeholders4__phILi1EEEEEEJLm0ELm1EENS9_IJS6_EEEEENS_13__bind_returnIT_T0_T2_Xsr22__is_valid_bind_returnISH_SI_SJ_EE5valueEE4typeERSH_RSI_NS_15__tuple_indicesIJXspT1_EEEEOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2217
> (libtablet.so+0x14e2c7)
> #11
> _ZNSt3__16__bindIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEEJPS3_RKNS_12placeholders4__phILi1EEEEEclIJS6_EEENS_13__bind_returnIS8_NS_5tupleIJS9_SC_EEENSI_IJDpOT_EEEXsr22__is_valid_bind_returnIS8_SJ_SN_EE5valueEE4typeESM_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2250
> (libtablet.so+0x14e2c7)
> #12
> _ZNSt3__18__invokeIRNS_6__bindIMN4kudu6tablet17TransactionDriverEFvRKNS2_6StatusEEJPS4_RKNS_12placeholders4__phILi1EEEEEEJS7_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSI_DpOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4301
> (libtablet.so+0x14e2c7)
> #13 void
> std::__1::__invoke_void_return_wrapper<void>::__call<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&>(std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/__functional_base:359
> (libtablet.so+0x14e2c7)
> #14 std::__1::__function::__func<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>,
> std::__1::allocator<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&> >,
> void (kudu::Status const&)>::operator()(kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1552:12
> (libtablet.so+0x14e1a4)
> #15 std::__1::function<void (kudu::Status
> const&)>::operator()(kudu::Status const&) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1914:12
> (libconsensus.so+0xc45ab)
> #16
> kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:2751:3
> (libconsensus.so+0xbfd94)
> #17 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/pending_rounds.cc:185:12
> (libconsensus.so+0xa180a)
> #18 kudu::consensus::RaftConsensus::NotifyCommitIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:700:13
> (libconsensus.so+0xb27c5)
> #19
> kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChangeTask(long)
>
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/consensus_queue.cc:1022:15
> (libconsensus.so+0x83c6d)
> #20 kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>::Run(kudu::consensus::PeerMessageQueue*,
> long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:192:12
> (libconsensus.so+0x8f310)
> #21 kudu::internal::InvokeHelper<false, void,
> kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long
> const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>,
> kudu::consensus::PeerMessageQueue*, long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:889:14
> (libconsensus.so+0x8f248)
> #22 kudu::internal::Invoker<2,
> kudu::internal::BindState<kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long), void
> (kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>,
> long)>, void (kudu::consensus::PeerMessageQueue*,
> long)>::Run(kudu::internal::BindStateBase*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:1147:12
> (libconsensus.so+0x8f183)
> #23 kudu::Callback<void ()>::Run() const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/callback.h:396:12
> (libconsensus.so+0x9371d)
> #24 kudu::ClosureRunnable::Run()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:74:9
> (libkudu_util.so+0x1beced)
> #25 kudu::ThreadPool::DispatchThread(bool)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:631:22
> (libkudu_util.so+0x1bc021)
> #26 boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>::operator()(kudu::ThreadPool*, bool) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:165:29
> (libkudu_util.so+0x1c48de)
> #27 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void,
> kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:319:9
> (libkudu_util.so+0x1c481d)
> #28 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >::operator()()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libkudu_util.so+0x1c4783)
> #29
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>,
> boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libkudu_util.so+0x1c4521)
> #30 boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb6eb1)
> #31 kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:601:3
> (libkudu_util.so+0x1b32ce)
> SUMMARY: ThreadSanitizer: data race
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73
> in operator delete(void*)
> ==================
> ==================
> WARNING: ThreadSanitizer: data race (pid=1945)
> Write of size 8 at 0x7b2000001370 by main thread:
> #0 kudu::(anonymous namespace)::LRUCache::SetMetrics(kudu::CacheMetrics*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:187:53
> (libkudu_util.so+0xe7fee)
> #1 kudu::(anonymous
> namespace)::ShardedLRUCache::SetMetrics(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:466:14
> (libkudu_util.so+0xe687d)
> #2
> kudu::cfile::BlockCache::StartInstrumentation(scoped_refptr<kudu::MetricEntity>
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:96:11
> (libcfile.so+0x72af4)
> #3 kudu::master::Master::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:118:38
> (libmaster.so+0xe5ff6)
> #4 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:92:3
> (libmaster.so+0x10070e)
> #5 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #6 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #7 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #8 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #9 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #10 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #11 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #12 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #13 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #14 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #15 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #16 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #17 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #18 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Previous read of size 8 at 0x7b2000001370 by thread T170 (mutexes: write
> M627261677020195488):
> As if synchronized via sleep:
> #0 nanosleep
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357
> (alter_table-test+0x4768f7)
> #1 boost::detail::yield(unsigned int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/smart_ptr/detail/yield_k.hpp:151:9
> (libmaster.so+0xd2b16)
> #2 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:555:7
> (libkudu_util.so+0x1b2cb1)
> #3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #4 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #5 kudu::ThreadPool::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:350:21
> (libkudu_util.so+0x1b8f2a)
> #6 kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool,
> kudu::DefaultDeleter<kudu::ThreadPool> >*) const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:127:3
> (libkudu_util.so+0x1b8e37)
> #7
> kudu::MaintenanceManager::MaintenanceManager(kudu::MaintenanceManager::Options
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/maintenance_manager.cc:154:3
> (libkudu_util.so+0x159fd5)
> #8 kudu::master::Master::Master(kudu::master::MasterOptions const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:101:30
> (libmaster.so+0xe5939)
> #9 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:91:33
> (libmaster.so+0x1006f9)
> #10 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #11 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #12 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #13 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #14 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #15 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #16 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #17 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #18 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #19 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #20 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #21 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #22 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #23 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Location is heap block of size 120 at 0x7b2000001300 allocated by main
> thread:
> #0 operator new(unsigned long)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:41
> (alter_table-test+0x4db1d3)
> #1 kudu::(anonymous namespace)::ShardedLRUCache::ShardedLRUCache(unsigned
> long, std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:428:35
> (libkudu_util.so+0xe608c)
> #2 kudu::NewLRUCache(kudu::CacheType, unsigned long,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:505:18
> (libkudu_util.so+0xe5dc6)
> #3 kudu::cfile::(anonymous namespace)::CreateCache(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:60:10
> (libcfile.so+0x727f1)
> #4 kudu::cfile::BlockCache::BlockCache(unsigned long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:70:12
> (libcfile.so+0x7260d)
> #5 kudu::cfile::BlockCache::BlockCache()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.cc:66:5
> (libcfile.so+0x725d0)
> #6 Singleton<kudu::cfile::BlockCache>::CreateInstance()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/singleton.h:124:18
> (libmaster.so+0xe9c14)
> #7 Singleton<kudu::cfile::BlockCache>::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/singleton.h:117:17
> (libmaster.so+0xe9b74)
> #8 GoogleOnceInternalInit(int*, void (*)(), void (*)(void*), void*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/once.cc:43:7
> (libgutil.so+0x2d2c7)
> #9 GoogleOnceInit(GoogleOnceType*, void (*)())
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/once.h:54:5
> (libmaster.so+0xe9b43)
> #10 Singleton<kudu::cfile::BlockCache>::get()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/singleton.h:79:5
> (libmaster.so+0xe9ab1)
> #11 kudu::cfile::BlockCache::GetSingleton()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/cfile/block_cache.h:117:12
> (libmaster.so+0xe81b3)
> #12 kudu::master::Master::Init()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/master.cc:118:3
> (libmaster.so+0xe5fe0)
> #13 kudu::master::MiniMaster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/master/mini_master.cc:92:3
> (libmaster.so+0x10070e)
> #14 kudu::cluster::InternalMiniCluster::StartSingleMaster()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:173:3
> (libmini-cluster.so+0x3f0cf)
> #15 kudu::cluster::InternalMiniCluster::Start()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/mini-cluster/internal_mini_cluster.cc:102:5
> (libmini-cluster.so+0x3e03b)
> #16 kudu::AlterTableTest::SetUp()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/integration-tests/alter_table-test.cc:137:5
> (alter_table-test+0x512f9f)
> #17 void
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x52b39)
> #18 void
> testing::internal::HandleExceptionsInMethodIfSupported<testing::Test,
> void>(testing::Test*, void (testing::Test::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x52b39)
> #19 testing::Test::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2470:3
> (libgmock.so+0x32b26)
> #20 testing::TestInfo::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11
> (libgmock.so+0x34036)
> #21 testing::TestCase::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28
> (libgmock.so+0x34da6)
> #22 testing::internal::UnitTestImpl::RunAllTests()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43
> (libgmock.so+0x40896)
> #23 bool
> testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10
> (libgmock.so+0x53a19)
> #24 bool
> testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl,
> bool>(testing::internal::UnitTestImpl*, bool
> (testing::internal::UnitTestImpl::*)(), char const*)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438
> (libgmock.so+0x53a19)
> #25 testing::UnitTest::Run()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10
> (libgmock.so+0x402e1)
> #26 RUN_ALL_TESTS()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46
> (libkudu_test_main.so+0x33db)
> #27 main
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/test_main.cc:106:13
> (libkudu_test_main.so+0x2b96)
> Mutex M627261677020195488 is already destroyed.
> Thread T170 (tid=5115, finished) created by thread T154 at:
> #0 pthread_create
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897
> (alter_table-test+0x47afe3)
> #1 kudu::Thread::StartThread(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned
> long, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:524:15
> (libkudu_util.so+0x1b2ac7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool),
> kudu::ThreadPool*, bool>(std::__1::basic_string<char,
> std::__1::char_traits<char>, std::__1::allocator<char> > const&,
> std::__1::basic_string<char, std::__1::char_traits<char>,
> std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool),
> kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.h:171:12
> (libkudu_util.so+0x1bdc66)
> #3 kudu::ThreadPool::CreateThreadUnlocked()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:696:14
> (libkudu_util.so+0x1bb641)
> #4 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>,
> kudu::ThreadPoolToken*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:493:21
> (libkudu_util.so+0x1b9b01)
> #5 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:454:10
> (libkudu_util.so+0x1bb95f)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()>)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:446:10
> (libkudu_util.so+0x1bb899)
> #7 kudu::tablet::TransactionDriver::ApplyAsync()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:482:23
> (libtablet.so+0x1548a9)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/tablet/transactions/transaction_driver.cc:436:5
> (libtablet.so+0x152188)
> #9
> _ZNSt3__18__invokeIRMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEERPS3_JS6_EvEEDTcldsdeclsr3std3__1E7forwardIT0_Efp0_Efp_spclsr3std3__1E7forwardIT1_Efp1_EEEOT_OSC_DpOSD_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4242:1
> (libtablet.so+0x14e2c7)
> #10
> _ZNSt3__115__apply_functorIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEENS_5tupleIJPS3_NS_12placeholders4__phILi1EEEEEEJLm0ELm1EENS9_IJS6_EEEEENS_13__bind_returnIT_T0_T2_Xsr22__is_valid_bind_returnISH_SI_SJ_EE5valueEE4typeERSH_RSI_NS_15__tuple_indicesIJXspT1_EEEEOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2217
> (libtablet.so+0x14e2c7)
> #11
> _ZNSt3__16__bindIMN4kudu6tablet17TransactionDriverEFvRKNS1_6StatusEEJPS3_RKNS_12placeholders4__phILi1EEEEEclIJS6_EEENS_13__bind_returnIS8_NS_5tupleIJS9_SC_EEENSI_IJDpOT_EEEXsr22__is_valid_bind_returnIS8_SJ_SN_EE5valueEE4typeESM_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:2250
> (libtablet.so+0x14e2c7)
> #12
> _ZNSt3__18__invokeIRNS_6__bindIMN4kudu6tablet17TransactionDriverEFvRKNS2_6StatusEEJPS4_RKNS_12placeholders4__phILi1EEEEEEJS7_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSI_DpOSJ_
>
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/type_traits:4301
> (libtablet.so+0x14e2c7)
> #13 void
> std::__1::__invoke_void_return_wrapper<void>::__call<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&>(std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>&,
> kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/__functional_base:359
> (libtablet.so+0x14e2c7)
> #14 std::__1::__function::__func<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&>,
> std::__1::allocator<std::__1::__bind<void
> (kudu::tablet::TransactionDriver::*)(kudu::Status const&),
> kudu::tablet::TransactionDriver*, std::__1::placeholders::__ph<1> const&> >,
> void (kudu::Status const&)>::operator()(kudu::Status const&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1552:12
> (libtablet.so+0x14e1a4)
> #15 std::__1::function<void (kudu::Status
> const&)>::operator()(kudu::Status const&) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/c++/v1/functional:1914:12
> (libconsensus.so+0xc45ab)
> #16
> kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status
> const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:2751:3
> (libconsensus.so+0xbfd94)
> #17 kudu::consensus::PendingRounds::AdvanceCommittedIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/pending_rounds.cc:185:12
> (libconsensus.so+0xa180a)
> #18 kudu::consensus::RaftConsensus::NotifyCommitIndex(long)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/raft_consensus.cc:700:13
> (libconsensus.so+0xb27c5)
> #19
> kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChangeTask(long)
>
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/consensus/consensus_queue.cc:1022:15
> (libconsensus.so+0x83c6d)
> #20 kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>::Run(kudu::consensus::PeerMessageQueue*,
> long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:192:12
> (libconsensus.so+0x8f310)
> #21 kudu::internal::InvokeHelper<false, void,
> kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long
> const&)>::MakeItSo(kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>,
> kudu::consensus::PeerMessageQueue*, long const&)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:889:14
> (libconsensus.so+0x8f248)
> #22 kudu::internal::Invoker<2,
> kudu::internal::BindState<kudu::internal::RunnableAdapter<void
> (kudu::consensus::PeerMessageQueue::*)(long)>, void
> (kudu::consensus::PeerMessageQueue*, long), void
> (kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>,
> long)>, void (kudu::consensus::PeerMessageQueue*,
> long)>::Run(kudu::internal::BindStateBase*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/bind_internal.h:1147:12
> (libconsensus.so+0x8f183)
> #23 kudu::Callback<void ()>::Run() const
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/gutil/callback.h:396:12
> (libconsensus.so+0x9371d)
> #24 kudu::ClosureRunnable::Run()
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:74:9
> (libkudu_util.so+0x1beced)
> #25 kudu::ThreadPool::DispatchThread(bool)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/threadpool.cc:631:22
> (libkudu_util.so+0x1bc021)
> #26 boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>::operator()(kudu::ThreadPool*, bool) const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:165:29
> (libkudu_util.so+0x1c48de)
> #27 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void,
> kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:319:9
> (libkudu_util.so+0x1c481d)
> #28 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool,
> bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >::operator()()
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16
> (libkudu_util.so+0x1c4783)
> #29
> boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void,
> boost::_mfi::mf1<void, kudu::ThreadPool, bool>,
> boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>,
> boost::_bi::value<bool> > >,
> void>::invoke(boost::detail::function::function_buffer&)
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11
> (libkudu_util.so+0x1c4521)
> #30 boost::function0<void>::operator()() const
> /home/jenkins-slave/workspace/kudu-master/2/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14
> (libkrpc.so+0xb6eb1)
> #31 kudu::Thread::SuperviseThread(void*)
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/thread.cc:601:3
> (libkudu_util.so+0x1b32ce)
> SUMMARY: ThreadSanitizer: data race
> /home/jenkins-slave/workspace/kudu-master/2/src/kudu/util/cache.cc:187:53 in
> kudu::(anonymous namespace)::LRUCache::SetMetrics(kudu::CacheMetrics*)
> ==================
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
