Details
-
Bug
-
Resolution: Duplicate
-
Critical
-
None
-
6.5.0
-
None
-
Untriaged
-
No
Description
Summary
TSan reports the below error on a 4-node cluster run (2 replicas) when issuing SyncWrites in a loop.
Note this is similar to the deadlock in MB-35420 but this one seems more complex - involves four mutexes instead of three.
Steps to Reproduce
- Build with ThreadSanitizer:
make -j8 EXTRA_CMAKE_OPTIONS="-D CB_THREADSANITIZER=1" - Start cluster with 4 nodes, 2 replicas with TSan set to log:
TSAN_OPTIONS="log_path=sanitizers.log second_deadlock_stack=1" ./cluster_run --nodes 4./cluster_connect -n4 -r2 - Direct SyncWrites in a loop at vBucket:0
kv_engine/engines/ep/management$ ./sync_repl.py 127.0.0.1:12000 Administrator asdasd default loop_setD key value 1000000 1
Expected Results
Empty tsan.log files (no TSan issues reported).
Actual Results
Deadlock reported:
WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=5841)
|
Cycle in lock order graph: M54741866449338568 (0x000000000000) => M738725681898217512 (0x000000000000) => M730000232498396792 (0x000000000000) => M33101 (0x7b3800010028) => M54741866449338568
|
|
|
Mutex M738725681898217512 acquired here while holding mutex M54741866449338568 in thread T7:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x000000131230)
|
#3 cb::RWLock::lock_shared() platform/include/platform/rwlock.h:67 (ep.so+0x000000131230)
|
#4 std::shared_lock<cb::RWLock>::shared_lock(cb::RWLock&) /usr/local/include/c++/7.3.0/shared_mutex:553 (ep.so+0x000000131230)
|
#5 StreamContainer<std::shared_ptr<Stream> >::ReadLockedHandle::ReadLockedHandle(StreamContainer<std::shared_ptr<Stream> > const&) kv_engine/engines/ep/src/dcp/stream_container.h:213 (ep.so+0x000000131230)
|
#6 StreamContainer<std::shared_ptr<Stream> >::rlock() const kv_engine/engines/ep/src/dcp/stream_container.h:273 (ep.so+0x000000121c1b)
|
#7 DcpProducer::notifySeqnoAvailable(Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1397 (ep.so+0x000000121c1b)
|
#8 DcpConnMap::notifyVBConnections(Vbid, unsigned long) kv_engine/engines/ep/src/dcp/dcpconnmap.cc:428 (ep.so+0x0000000f9eb1)
|
#9 KVBucket::notifyReplication(Vbid, long) kv_engine/engines/ep/src/kv_bucket.cc:2588 (ep.so+0x000000213a0d)
|
#10 EPBucket::notifyNewSeqno(Vbid, VBNotifyCtx const&) kv_engine/engines/ep/src/ep_bucket.cc:1327 (ep.so+0x000000162ebb)
|
#11 NotifyNewSeqnoCB::callback(Vbid const&, VBNotifyCtx const&) kv_engine/engines/ep/src/kv_bucket.h:838 (ep.so+0x000000229ab7)
|
#12 VBucket::notifyNewSeqno(VBNotifyCtx const&) kv_engine/engines/ep/src/vbucket.cc:3627 (ep.so+0x00000027179d)
|
#13 VBucket::commit(DocKey const&, unsigned long, boost::optional<long>, Collections::VB::Manifest::CachingReadHandle const&, void const*) kv_engine/engines/ep/src/vbucket.cc:855 (ep.so+0x00000028b284)
|
#14 ActiveDurabilityMonitor::commit(DurabilityMonitor::SyncWrite const&) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:984 (ep.so+0x00000014bf3f)
|
#15 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() kv_engine/engines/ep/src/durability/active_durability_monitor.cc:653 (ep.so+0x00000014d496)
|
#16 ActiveDurabilityMonitor::seqnoAckReceived(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:540 (ep.so+0x00000014e116)
|
#17 VBucket::seqnoAcknowledged(folly::SharedMutexImpl<false, void, std::atomic, false, true>::ReadHolder const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/vbucket.cc:3832 (ep.so+0x000000271b6e)
|
#18 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1774 (ep.so+0x0000000b69b5)
|
#19 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#20 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#21 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#22 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#23 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#24 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#25 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#26 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#27 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#28 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
|
|
|
|
Mutex M54741866449338568 previously acquired by the same thread here:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (ep.so+0x0000001f1c09)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (ep.so+0x0000001f1c09)
|
#3 std::unique_lock<std::mutex>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (ep.so+0x0000001f1c09)
|
#4 std::unique_lock<std::mutex>::unique_lock(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (ep.so+0x0000001f1c09)
|
#5 HashTable::HashBucketLock::HashBucketLock(int, std::mutex&) kv_engine/engines/ep/src/hash_table.h:393 (ep.so+0x0000001f1c09)
|
#6 HashTable::getLockedBucketForHash(int) kv_engine/engines/ep/src/hash_table.h:1279 (ep.so+0x0000001f1c09)
|
#7 HashTable::getLockedBucket(DocKey const&) kv_engine/engines/ep/src/hash_table.h:1066 (ep.so+0x0000001f1c09)
|
#8 HashTable::findInner(DocKey const&) kv_engine/engines/ep/src/hash_table.cc:316 (ep.so+0x0000001f1c09)
|
#9 HashTable::findForCommit(DocKey const&) kv_engine/engines/ep/src/hash_table.cc:748 (ep.so+0x0000001f4bf7)
|
#10 VBucket::commit(DocKey const&, unsigned long, boost::optional<long>, Collections::VB::Manifest::CachingReadHandle const&, void const*) kv_engine/engines/ep/src/vbucket.cc:825 (ep.so+0x00000028ac28)
|
#11 ActiveDurabilityMonitor::commit(DurabilityMonitor::SyncWrite const&) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:984 (ep.so+0x00000014bf3f)
|
#12 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() kv_engine/engines/ep/src/durability/active_durability_monitor.cc:653 (ep.so+0x00000014d496)
|
#13 ActiveDurabilityMonitor::seqnoAckReceived(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:540 (ep.so+0x00000014e116)
|
#14 VBucket::seqnoAcknowledged(folly::SharedMutexImpl<false, void, std::atomic, false, true>::ReadHolder const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/vbucket.cc:3832 (ep.so+0x000000271b6e)
|
#15 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1774 (ep.so+0x0000000b69b5)
|
#16 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#17 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#18 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#19 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#20 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#21 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#22 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#23 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#24 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#25 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
|
|
|
|
Mutex M730000232498396792 acquired here while holding mutex M738725681898217512 in thread T7:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (ep.so+0x0000000bc8ae)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (ep.so+0x0000000bc8ae)
|
#3 std::lock_guard<std::mutex>::lock_guard(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 (ep.so+0x0000000bc8ae)
|
#4 ActiveStream::next() kv_engine/engines/ep/src/dcp/active_stream.cc:146 (ep.so+0x0000000bc8ae)
|
#5 DcpProducer::getNextItem() kv_engine/engines/ep/src/dcp/producer.cc:1545 (ep.so+0x000000124981)
|
#6 DcpProducer::step(dcp_message_producers*) kv_engine/engines/ep/src/dcp/producer.cc:599 (ep.so+0x00000012b057)
|
#7 EventuallyPersistentEngine::step(gsl::not_null<void const*>, gsl::not_null<dcp_message_producers*>) kv_engine/engines/ep/src/ep_engine.cc:1194 (ep.so+0x00000018f258)
|
#8 StateMachine::conn_ship_log() kv_engine/daemon/statemachine.cc:196 (memcached+0x000000554767)
|
#9 StateMachine::execute() kv_engine/daemon/statemachine.cc:141 (memcached+0x000000554f08)
|
#10 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#11 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
#12 run_event_loop(Connection*, short) kv_engine/daemon/connections.cc:148 (memcached+0x000000502340)
|
#13 thread_libevent_process kv_engine/daemon/thread.cc:340 (memcached+0x0000004bb9ac)
|
#14 event_persist_closure /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1580 (libevent_core.so.2.1.8+0x000000017856)
|
#15 event_process_active_single_queue /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639 (libevent_core.so.2.1.8+0x000000017856)
|
#16 CouchbaseThread::run() platform/src/cb_pthreads.cc:58 (libplatform_so.so.0.1.0+0x00000000a217)
|
#17 platform_thread_wrap platform/src/cb_pthreads.cc:71 (libplatform_so.so.0.1.0+0x00000000a217)
|
#18 <null> <null> (libtsan.so.0+0x00000002843b)
|
|
|
Mutex M738725681898217512 previously acquired by the same thread here:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x0000001245d3)
|
#3 cb::RWLock::lock_shared() platform/include/platform/rwlock.h:67 (ep.so+0x0000001245d3)
|
#4 std::shared_lock<cb::RWLock>::shared_lock(cb::RWLock&) /usr/local/include/c++/7.3.0/shared_mutex:553 (ep.so+0x0000001245d3)
|
#5 StreamContainer<std::shared_ptr<Stream> >::ResumableIterationHandle::ResumableIterationHandle(StreamContainer<std::shared_ptr<Stream> >&) kv_engine/engines/ep/src/dcp/stream_container.h:119 (ep.so+0x0000001245d3)
|
#6 StreamContainer<std::shared_ptr<Stream> >::startResumable() kv_engine/engines/ep/src/dcp/stream_container.h:269 (ep.so+0x0000001245d3)
|
#7 DcpProducer::getNextItem() kv_engine/engines/ep/src/dcp/producer.cc:1539 (ep.so+0x0000001245d3)
|
#8 DcpProducer::step(dcp_message_producers*) kv_engine/engines/ep/src/dcp/producer.cc:599 (ep.so+0x00000012b057)
|
#9 EventuallyPersistentEngine::step(gsl::not_null<void const*>, gsl::not_null<dcp_message_producers*>) kv_engine/engines/ep/src/ep_engine.cc:1194 (ep.so+0x00000018f258)
|
#10 StateMachine::conn_ship_log() kv_engine/daemon/statemachine.cc:196 (memcached+0x000000554767)
|
#11 StateMachine::execute() kv_engine/daemon/statemachine.cc:141 (memcached+0x000000554f08)
|
#12 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#13 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
|
|
|
|
Mutex M33101 acquired here while holding mutex M730000232498396792 in thread T7:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x00000014bcbd)
|
#3 cb::ReaderLock::lock() platform/include/platform/rwlock.h:131 (ep.so+0x00000014bcbd)
|
#4 std::unique_lock<cb::ReaderLock>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (ep.so+0x00000014bcbd)
|
#5 std::unique_lock<cb::ReaderLock>::unique_lock(cb::ReaderLock&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (ep.so+0x00000014bcbd)
|
#6 Collections::VB::Manifest::ReadHandle::ReadHandle(Collections::VB::Manifest const*, cb::RWLock&) kv_engine/engines/ep/src/collections/vbucket_manifest.h:99 (ep.so+0x00000014bcbd)
|
#7 Collections::VB::Manifest::CachingReadHandle::CachingReadHandle(Collections::VB::Manifest const*, cb::RWLock&, DocKey, bool) kv_engine/engines/ep/src/collections/vbucket_manifest.h:297 (ep.so+0x00000014bcbd)
|
#8 Collections::VB::Manifest::lock(DocKey, bool) const kv_engine/engines/ep/src/collections/vbucket_manifest.h:683 (ep.so+0x00000014bcbd)
|
#9 VBucket::lockCollections(DocKey const&, bool) const kv_engine/engines/ep/src/vbucket.h:786 (ep.so+0x00000014bcbd)
|
#10 ActiveDurabilityMonitor::commit(DurabilityMonitor::SyncWrite const&) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:983 (ep.so+0x00000014bcbd)
|
#11 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() kv_engine/engines/ep/src/durability/active_durability_monitor.cc:653 (ep.so+0x00000014d496)
|
#12 ActiveDurabilityMonitor::seqnoAckReceived(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:540 (ep.so+0x00000014e116)
|
#13 VBucket::seqnoAcknowledged(folly::SharedMutexImpl<false, void, std::atomic, false, true>::ReadHolder const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/vbucket.cc:3832 (ep.so+0x000000271b6e)
|
#14 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1774 (ep.so+0x0000000b69b5)
|
#15 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#16 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#17 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#18 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#19 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#20 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#21 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#22 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#23 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#24 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#25 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
#26 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
|
|
|
|
Mutex M730000232498396792 previously acquired by the same thread here:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (ep.so+0x0000000b6913)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (ep.so+0x0000000b6913)
|
#3 std::lock_guard<std::mutex>::lock_guard(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 (ep.so+0x0000000b6913)
|
#4 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1766 (ep.so+0x0000000b6913)
|
#5 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#6 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#7 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#8 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#9 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#10 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#11 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#12 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#13 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#14 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#15 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
#16 Connection::runStateMachinery() kv_engine/daemon/connection.cc:1340 (memcached+0x0000004ec21b)
|
#17 Connection::runEventLoop(short) kv_engine/daemon/connection.cc:1414 (memcached+0x0000004ec2aa)
|
|
|
|
|
Mutex M54741866449338568 acquired here while holding mutex M33101 in thread T7:
|
#0 pthread_mutex_lock <null> (libtsan.so.0+0x00000003bbbf)
|
#1 __gthread_mutex_lock /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 (ep.so+0x0000001f1c09)
|
#2 std::mutex::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 (ep.so+0x0000001f1c09)
|
#3 std::unique_lock<std::mutex>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (ep.so+0x0000001f1c09)
|
#4 std::unique_lock<std::mutex>::unique_lock(std::mutex&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (ep.so+0x0000001f1c09)
|
#5 HashTable::HashBucketLock::HashBucketLock(int, std::mutex&) kv_engine/engines/ep/src/hash_table.h:393 (ep.so+0x0000001f1c09)
|
#6 HashTable::getLockedBucketForHash(int) kv_engine/engines/ep/src/hash_table.h:1279 (ep.so+0x0000001f1c09)
|
#7 HashTable::getLockedBucket(DocKey const&) kv_engine/engines/ep/src/hash_table.h:1066 (ep.so+0x0000001f1c09)
|
#8 HashTable::findInner(DocKey const&) kv_engine/engines/ep/src/hash_table.cc:316 (ep.so+0x0000001f1c09)
|
#9 HashTable::findForCommit(DocKey const&) kv_engine/engines/ep/src/hash_table.cc:748 (ep.so+0x0000001f4bf7)
|
#10 VBucket::commit(DocKey const&, unsigned long, boost::optional<long>, Collections::VB::Manifest::CachingReadHandle const&, void const*) kv_engine/engines/ep/src/vbucket.cc:825 (ep.so+0x00000028ac28)
|
#11 ActiveDurabilityMonitor::commit(DurabilityMonitor::SyncWrite const&) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:984 (ep.so+0x00000014bf3f)
|
#12 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() kv_engine/engines/ep/src/durability/active_durability_monitor.cc:653 (ep.so+0x00000014d496)
|
#13 ActiveDurabilityMonitor::seqnoAckReceived(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:540 (ep.so+0x00000014e116)
|
#14 VBucket::seqnoAcknowledged(folly::SharedMutexImpl<false, void, std::atomic, false, true>::ReadHolder const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/vbucket.cc:3832 (ep.so+0x000000271b6e)
|
#15 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1774 (ep.so+0x0000000b69b5)
|
#16 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#17 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#18 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#19 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#20 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#21 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#22 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#23 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
#24 Cookie::execute() kv_engine/daemon/cookie.cc:124 (memcached+0x000000504438)
|
#25 StateMachine::conn_execute() kv_engine/daemon/statemachine.cc:410 (memcached+0x0000005530c5)
|
#26 StateMachine::execute() kv_engine/daemon/statemachine.cc:137 (memcached+0x000000554eea)
|
|
|
Mutex M33101 previously acquired by the same thread here:
|
#0 pthread_rwlock_rdlock <null> (libtsan.so.0+0x00000002c98b)
|
#1 cb_rw_reader_enter(pthread_rwlock_t*) platform/src/cb_pthreads.cc:195 (libplatform_so.so.0.1.0+0x00000000a43a)
|
#2 cb::RWLock::readerLock() platform/include/platform/rwlock.h:87 (ep.so+0x00000014bcbd)
|
#3 cb::ReaderLock::lock() platform/include/platform/rwlock.h:131 (ep.so+0x00000014bcbd)
|
#4 std::unique_lock<cb::ReaderLock>::lock() /usr/local/include/c++/7.3.0/bits/std_mutex.h:267 (ep.so+0x00000014bcbd)
|
#5 std::unique_lock<cb::ReaderLock>::unique_lock(cb::ReaderLock&) /usr/local/include/c++/7.3.0/bits/std_mutex.h:197 (ep.so+0x00000014bcbd)
|
#6 Collections::VB::Manifest::ReadHandle::ReadHandle(Collections::VB::Manifest const*, cb::RWLock&) kv_engine/engines/ep/src/collections/vbucket_manifest.h:99 (ep.so+0x00000014bcbd)
|
#7 Collections::VB::Manifest::CachingReadHandle::CachingReadHandle(Collections::VB::Manifest const*, cb::RWLock&, DocKey, bool) kv_engine/engines/ep/src/collections/vbucket_manifest.h:297 (ep.so+0x00000014bcbd)
|
#8 Collections::VB::Manifest::lock(DocKey, bool) const kv_engine/engines/ep/src/collections/vbucket_manifest.h:683 (ep.so+0x00000014bcbd)
|
#9 VBucket::lockCollections(DocKey const&, bool) const kv_engine/engines/ep/src/vbucket.h:786 (ep.so+0x00000014bcbd)
|
#10 ActiveDurabilityMonitor::commit(DurabilityMonitor::SyncWrite const&) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:983 (ep.so+0x00000014bcbd)
|
#11 ActiveDurabilityMonitor::processCompletedSyncWriteQueue() kv_engine/engines/ep/src/durability/active_durability_monitor.cc:653 (ep.so+0x00000014d496)
|
#12 ActiveDurabilityMonitor::seqnoAckReceived(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, long) kv_engine/engines/ep/src/durability/active_durability_monitor.cc:540 (ep.so+0x00000014e116)
|
#13 VBucket::seqnoAcknowledged(folly::SharedMutexImpl<false, void, std::atomic, false, true>::ReadHolder const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/vbucket.cc:3832 (ep.so+0x000000271b6e)
|
#14 ActiveStream::seqnoAck(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long) kv_engine/engines/ep/src/dcp/active_stream.cc:1774 (ep.so+0x0000000b69b5)
|
#15 DcpProducer::seqno_acknowledged(unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/dcp/producer.cc:1078 (ep.so+0x00000012d9d8)
|
#16 EventuallyPersistentEngine::seqno_acknowledged(gsl::not_null<void const*>, unsigned int, Vbid, unsigned long) kv_engine/engines/ep/src/ep_engine.cc:1583 (ep.so+0x00000018ed0a)
|
#17 dcpSeqnoAcknowledged(Cookie&, unsigned int, Vbid, unsigned long) kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:749 (memcached+0x00000048eddd)
|
#18 dcp_seqno_acknowledged_executor(Cookie&) kv_engine/daemon/protocol/mcbp/dcp_seqno_acknowledged_executor.cc:33 (memcached+0x000000533e02)
|
#19 std::_Function_handler<void (Cookie&), void (*)(Cookie&)>::_M_invoke(std::_Any_data const&, Cookie&) /usr/local/include/c++/7.3.0/bits/std_function.h:316 (memcached+0x00000051fffd)
|
#20 std::function<void (Cookie&)>::operator()(Cookie&) const /usr/local/include/c++/7.3.0/bits/std_function.h:706 (memcached+0x0000005203a3)
|
#21 execute_client_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:850 (memcached+0x00000051f8c2)
|
#22 execute_request_packet(Cookie&, cb::mcbp::Request const&) kv_engine/daemon/mcbp_executors.cc:874 (memcached+0x00000051f9ea)
|
|
|
|
|
Thread T7 'mc:worker_1' (tid=5949, running) created by main thread at:
|
#0 pthread_create <null> (libtsan.so.0+0x00000002b6f0)
|
#1 cb_create_named_thread(unsigned long*, void (*)(void*), void*, int, char const*) platform/src/cb_pthreads.cc:109 (libplatform_so.so.0.1.0+0x000000009fc5)
|
#2 create_worker kv_engine/daemon/thread.cc:111 (memcached+0x0000004b9be8)
|
#3 thread_init(unsigned long, event_base*, void (*)(int, short, void*)) kv_engine/daemon/thread.cc:460 (memcached+0x0000004bad7b)
|
#4 memcached_main kv_engine/daemon/memcached.cc:2479 (memcached+0x000000442b42)
|
#5 main kv_engine/daemon/main.cc:33 (memcached+0x000000433a75)
|
|
|
SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) (install/bin/../lib/libtsan.so.0+0x2c98b) in __interceptor_pthread_rwlock_rdlock
|
Attachments
Issue Links
- duplicates
-
-
- Closed
-