Details
-
Bug
-
Resolution: Duplicate
-
Critical
-
None
-
6.5.0
-
None
-
Triaged
-
No
Description
Summary
TSan reports the below error on a 4-node cluster run (2 replicas) when issuing SyncWrites in a loop.
Steps to Reproduce
- Build with ThreadSanitizer:
make -j8 EXTRA_CMAKE_OPTIONS="-D CB_THREADSANITIZER=1"
- Start cluster with 4 nodes, 2 replicas with TSan set to log:
TSAN_OPTIONS="log_path=sanitizers.log second_deadlock_stack=1" ./cluster_run --nodes 4
./cluster_connect -n4 -r2
- Direct SyncWrites in a loop at vBucket:0
kv_engine/engines/ep/management$ ./sync_repl.py 127.0.0.1:12000 Administrator asdasd default loop_setD key value 1000000 1
Expected Results
Empty tsan.log files (no TSan issues reported).
Actual Results
Deadlock reported:
WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=5841)
|
Cycle in lock order graph: M33101 (0x7b3800010028) => M738725681898217512 (0x000000000000) => M730000232498396792 (0x000000000000) => M33101
|
|
Mutex M738725681898217512 acquired here while holding mutex M33101 in thread T10:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x000000131230)
|
#3 cb::RWLock::lock_shared() platform/include/platform/rwlock.h:67 (ep.so+0x000000131230)
|
#4 std::shared_lock<cb::RWLock>::shared_lock(cb::RWLock&) /usr/local/include/c++/7.3.0/shared_mutex:553 (ep.so+0x000000131230)
|
#5 StreamContainer<std::shared_ptr<Stream> >::ReadLockedHandle::ReadLockedHandle(StreamContainer<std::shared_ptr<Stream> > const&) kv_engine/engines/ep/src/dcp/stream_container.h:213 (ep.so+0x000000131230)
|
#6 StreamContainer<std::shared_ptr<Stream> >::rlock() const kv_engine/engines/ep/src/dcp/stream_container.h:273 (ep.so+0x000000121c1b)
|
#7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1397 (ep.so+0x000000121c1b)
|
#8 DcpConnMap::notifyVBConnections(Vbid, unsigned long) kv_engine/engines/ep/src/dcp/dcpconnmap.cc:428 (ep.so+0x0000000f9eb1)
|
#9 KVBucket::notifyReplication(Vbid, long) kv_engine/engines/ep/src/kv_bucket.cc:2588 (ep.so+0x000000213a0d)
|
#10 EPBucket::notifyNewSeqno(Vbid, VBNotifyCtx const&) kv_engine/engines/ep/src/ep_bucket.cc:1327 (ep.so+0x000000162ebb)
|
#11 NotifyNewSeqnoCB::callback(Vbid const&, VBNotifyCtx const&) kv_engine/engines/ep/src/kv_bucket.h:838 (ep.so+0x000000229ab7)
|
#12 VBucket::notifyNewSeqno(VBNotifyCtx const&) kv_engine/engines/ep/src/vbucket.cc:3627 (ep.so+0x00000027179d)
|
#13 VBucket::set(Item&, void const*, EventuallyPersistentEngine&, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>, Collections::VB::Manifest::CachingReadHandle const&) kv_engine/engines/ep/src/vbucket.cc:1579 (ep.so+0x0000002790cd)
|
#14 KVBucket::set(Item&, void const*, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>) kv_engine/engines/ep/src/kv_bucket.cc:692 (ep.so+0x000000223b70)
|
#15 EventuallyPersistentEngine::storeIfInner(void const*, Item&, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)> const&) kv_engine/engines/ep/src/ep_engine.cc:2402 (ep.so+0x000000182ab1)
|
#16 EventuallyPersistentEngine::store_if(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)> const&, boost::optional<cb::durability::Requirements> const&, DocumentState) kv_engine/engines/ep/src/ep_engine.cc:376 (ep.so+0x0000001830a0)
|
#17 bucket_store_if(Cookie&, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>, boost::optional<cb::durability::Requirements>, DocumentState) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:139 (memcached+0x00000048d0ae)
|
#18 MutationCommandContext::storeItem() kv_engine/daemon/protocol/mcbp/mutation_context.cc:296 (memcached+0x00000053e417)
|
#19 MutationCommandContext::step() kv_engine/daemon/protocol/mcbp/mutation_context.cc:63 (memcached+0x00000053e69a)
|
#20 SteppableCommandContext::drive() kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:37 (memcached+0x00000054d8d8)
|
#21 add_set_replace_executor kv_engine/daemon/mcbp_executors.cc:170 (memcached+0x00000051a8eb)
|
#22 set_executor kv_engine/daemon/mcbp_executors.cc:178 (memcached+0x00000051ab4e)
|
#23 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#24 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#25 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#26 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#27 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#28 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#29 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
#30 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#31 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#32 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#33 event_handler(int, short, void*) kv_engine/daemon/memcached.cc:848 (memcached+0x00000043d8a8)
|
#34 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#35 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#36 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#37 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#38 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
Mutex M33101 previously acquired by the same thread here:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x00000016dd2b)
|
#3 cb::ReaderLock::lock() platform/include/platform/rwlock.h:131 (ep.so+0x00000016dd2b)
|
#4 std::unique_lock<cb::ReaderLock>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (ep.so+0x00000016dd2b)
|
#5 std::unique_lock<cb::ReaderLock>::unique_lock(cb::ReaderLock&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (ep.so+0x00000016dd2b)
|
#6 Collections::VB::Manifest::ReadHandle::ReadHandle(Collections::VB::Manifest const*, cb::RWLock&) kv_engine/engines/ep/src/collections/vbucket_manifest.h:99 (ep.so+0x00000016dd2b)
|
#7 Collections::VB::Manifest::CachingReadHandle::CachingReadHandle(Collections::VB::Manifest const*, cb::RWLock&, DocKey, bool) kv_engine/engines/ep/src/collections/vbucket_manifest.h:297 (ep.so+0x00000016dd2b)
|
#8 Collections::VB::Manifest::lock(DocKey, bool) const kv_engine/engines/ep/src/collections/vbucket_manifest.h:683 (ep.so+0x0000002238cf)
|
#9 VBucket::lockCollections(DocKey const&, bool) const kv_engine/engines/ep/src/vbucket.h:786 (ep.so+0x0000002238cf)
|
#10 KVBucket::set(Item&, void const*, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>) kv_engine/engines/ep/src/kv_bucket.cc:680 (ep.so+0x0000002238cf)
|
#11 EventuallyPersistentEngine::storeIfInner(void const*, Item&, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)> const&) kv_engine/engines/ep/src/ep_engine.cc:2402 (ep.so+0x000000182ab1)
|
#12 EventuallyPersistentEngine::store_if(gsl::not_null<void const*>, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)> const&, boost::optional<cb::durability::Requirements> const&, DocumentState) kv_engine/engines/ep/src/ep_engine.cc:376 (ep.so+0x0000001830a0)
|
#13 bucket_store_if(Cookie&, gsl::not_null<void*>, unsigned long, ENGINE_STORE_OPERATION, std::function<cb::StoreIfStatus (boost::optional<item_info> const&, cb::vbucket_info)>, boost::optional<cb::durability::Requirements>, DocumentState) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:139 (memcached+0x00000048d0ae)
|
#14 MutationCommandContext::storeItem() kv_engine/daemon/protocol/mcbp/mutation_context.cc:296 (memcached+0x00000053e417)
|
#15 MutationCommandContext::step() kv_engine/daemon/protocol/mcbp/mutation_context.cc:63 (memcached+0x00000053e69a)
|
#16 SteppableCommandContext::drive() kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:37 (memcached+0x00000054d8d8)
|
#17 add_set_replace_executor kv_engine/daemon/mcbp_executors.cc:170 (memcached+0x00000051a8eb)
|
#18 set_executor kv_engine/daemon/mcbp_executors.cc:178 (memcached+0x00000051ab4e)
|
#19 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#20 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#21 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#22 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#23 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#24 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#25 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
#26 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#27 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#28 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#29 event_handler(int, short, void*) kv_engine/daemon/memcached.cc:848 (memcached+0x00000043d8a8)
|
#30 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#31 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#32 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#33 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#34 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
Mutex M730000232498396792 acquired here while holding mutex M738725681898217512 in thread T7:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (ep.so+0x0000000bc8ae)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (ep.so+0x0000000bc8ae)
|
#3 std::lock_guard<std::mutex>::lock_guard(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 (ep.so+0x0000000bc8ae)
|
#4 ActiveStream::next() kv_engine/engines/ep/src/dcp/active_stream.cc:146 (ep.so+0x0000000bc8ae)
|
#5 DcpProducer::getNextItem() kv_engine/engines/ep/src/dcp/producer.cc:1545 (ep.so+0x000000124981)
|
#6 DcpProducer::step(dcp_message_producers*) kv_engine/engines/ep/src/dcp/producer.cc:599 (ep.so+0x00000012b057)
|
#7 EventuallyPersistentEngine::step(gsl::not_null<void const*>, gsl::not_null<dcp_message_producers*>) kv_engine/engines/ep/src/ep_engine.cc:1194 (ep.so+0x00000018f258)
|
#8 StateMachine::conn_ship_log() kv_engine/daemon/statemachine.cc:196 (memcached+0x000000554767)
|
#9 StateMachine::execute() kv_engine/daemon/statemachine.cc:141 (memcached+0x000000554f08)
|
#10 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#11 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#12 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#13 thread_libevent_process kv_engine/daemon/thread.cc:340 (memcached+0x0000004bb9ac)
|
#14 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#15 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#16 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#17 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#18 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
Mutex M738725681898217512 previously acquired by the same thread here:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x0000001245d3)
|
#3 cb::RWLock::lock_shared() platform/include/platform/rwlock.h:67 (ep.so+0x0000001245d3)
|
#4 std::shared_lock<cb::RWLock>::shared_lock(cb::RWLock&) /usr/local/include/c++/7.3.0/shared_mutex:553 (ep.so+0x0000001245d3)
|
#5 StreamContainer<std::shared_ptr<Stream> >::ResumableIterationHandle::ResumableIterationHandle(StreamContainer<std::shared_ptr<Stream> >&) kv_engine/engines/ep/src/dcp/stream_container.h:119 (ep.so+0x0000001245d3)
|
#6 StreamContainer<std::shared_ptr<Stream> >::startResumable() kv_engine/engines/ep/src/dcp/stream_container.h:269 (ep.so+0x0000001245d3)
|
#7 DcpProducer::getNextItem() kv_engine/engines/ep/src/dcp/producer.cc:1539 (ep.so+0x0000001245d3)
|
#8 DcpProducer::step(dcp_message_producers*) kv_engine/engines/ep/src/dcp/producer.cc:599 (ep.so+0x00000012b057)
|
#9 EventuallyPersistentEngine::step(gsl::not_null<void const*>, gsl::not_null<dcp_message_producers*>) kv_engine/engines/ep/src/ep_engine.cc:1194 (ep.so+0x00000018f258)
|
#10 StateMachine::conn_ship_log() kv_engine/daemon/statemachine.cc:196 (memcached+0x000000554767)
|
#11 StateMachine::execute() kv_engine/daemon/statemachine.cc:141 (memcached+0x000000554f08)
|
#12 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#13 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#14 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#15 thread_libevent_process kv_engine/daemon/thread.cc:340 (memcached+0x0000004bb9ac)
|
#16 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#17 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#18 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#19 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#20 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
Mutex M33101 acquired here while holding mutex M730000232498396792 in thread T7:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x00000014bcbd)
|
#3 cb::ReaderLock::lock() platform/include/platform/rwlock.h:131 (ep.so+0x00000014bcbd)
|
#4 std::unique_lock<cb::ReaderLock>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (ep.so+0x00000014bcbd)
|
#5 std::unique_lock<cb::ReaderLock>::unique_lock(cb::ReaderLock&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (ep.so+0x00000014bcbd)
|
#6 Collections::VB::Manifest::ReadHandle::ReadHandle(Collections::VB::Manifest const*, cb::RWLock&) kv_engine/engines/ep/src/collections/vbucket_manifest.h:99 (ep.so+0x00000014bcbd)
|
#7 Collections::VB::Manifest::CachingReadHandle::CachingReadHandle(Collections::VB::Manifest const*, cb::RWLock&, DocKey, bool) kv_engine/engines/ep/src/collections/vbucket_manifest.h:297 (ep.so+0x00000014bcbd)
|
#8 Collections::VB::Manifest::lock(DocKey, bool) const kv_engine/engines/ep/src/collections/vbucket_manifest.h:683 (ep.so+0x00000014bcbd)
|
#9 VBucket::lockCollections(DocKey const&, bool) const kv_engine/engines/ep/src/vbucket.h:786 (ep.so+0x00000014bcbd)
|
#10 ActiveDurabilityMonitor::commit(DurabilityMonitor::SyncWrite const&) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:983 (ep.so+0x00000014bcbd)
|
#11 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() kv_engine/engines/ep/src/durability/active_durability_monitor.cc:653 (ep.so+0x00000014d496)
|
#12 ActiveDurabilityMonitor::seqnoAckReceived(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:540 (ep.so+0x00000014e116)
|
#13 VBucket::seqnoAcknowledged(folly::SharedMutexImpl<false, void, std::atomic, false, true>::ReadHolder const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/vbucket.cc:3832 (ep.so+0x000000271b6e)
|
#14 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1774 (ep.so+0x0000000b69b5)
|
#15 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#16 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#17 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#18 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#19 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#20 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#21 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#22 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#23 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#24 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#25 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
#26 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#27 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#28 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#29 event_handler(int, short, void*) kv_engine/daemon/memcached.cc:848 (memcached+0x00000043d8a8)
|
#30 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#31 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#32 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#33 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#34 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
Mutex M730000232498396792 previously acquired by the same thread here:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (ep.so+0x0000000b6913)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (ep.so+0x0000000b6913)
|
#3 std::lock_guard<std::mutex>::lock_guard(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 (ep.so+0x0000000b6913)
|
#4 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1766 (ep.so+0x0000000b6913)
|
#5 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#6 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#7 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#8 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#9 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#10 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#11 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#12 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#13 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#14 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#15 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
#16 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#17 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#18 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#19 event_handler(int, short, void*) kv_engine/daemon/memcached.cc:848 (memcached+0x00000043d8a8)
|
#20 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#21 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#22 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#23 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#24 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
Thread T10 'mc:worker_4' (tid=5957, running) created by main thread at:
|
#0 pthread_create <null> (libtsan.so.0+0x00000002b6f0)
|
#1 cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) platform/src/cb_pthreads.cc:109 (libplatform_so.so.0.1.0+0x000000009fc5)
|
#2 create_worker kv_engine/daemon/thread.cc:111 (memcached+0x0000004b9be8)
|
#3 thread_init(unsigned long, event_base*, void (*)(int, short, void*)) kv_engine/daemon/thread.cc:460 (memcached+0x0000004bad7b)
|
#4 memcached_main kv_engine/daemon/memcached.cc:2479 (memcached+0x000000442b42)
|
#5 main kv_engine/daemon/main.cc:33 (memcached+0x000000433a75)
|
|
Thread T7 'mc:worker_1' (tid=5949, running) created by main thread at:
|
#0 pthread_create <null> (libtsan.so.0+0x00000002b6f0)
|
#1 cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) platform/src/cb_pthreads.cc:109 (libplatform_so.so.0.1.0+0x000000009fc5)
|
#2 create_worker kv_engine/daemon/thread.cc:111 (memcached+0x0000004b9be8)
|
#3 thread_init(unsigned long, event_base*, void (*)(int, short, void*)) kv_engine/daemon/thread.cc:460 (memcached+0x0000004bad7b)
|
#4 memcached_main kv_engine/daemon/memcached.cc:2479 (memcached+0x000000442b42)
|
#5 main kv_engine/daemon/main.cc:33 (memcached+0x000000433a75)
|
|
SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) (install/bin/../lib/libtsan.so.0+0x2c98b) in __interceptor_pthread_rwlock_rdlock
|
Attachments
Issue Links
- duplicates
-
MB-35458 lock-order-inversion when handling seqno-ack (between StreamContainer and streamMutex)
- Closed