TSAN: Lock inversion between DCP consumer and set

07:52:31 ==================

07:52:31 WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=20124)

07:52:31   Cycle in lock order graph: M21516 (0x7d640005edf0) => M21519 (0x7d640005ef20) => M21516

07:52:31 

07:52:31   Mutex M21519 acquired here while holding mutex M21516 in main thread:

07:52:31     #0 pthread_mutex_lock <null> (engine_testapp+0x00000047e9e0)

07:52:31     #1 cb_mutex_enter <null> (libplatform.so.0.1.0+0x0000000039c0)

07:52:31     #2 Mutex::acquire() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e615e)

07:52:31     #3 LockHolder::lock() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/locks.h:71 (ep.so+0x000000080bc3)

07:52:31     #4 LockHolder::LockHolder(Mutex&, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/locks.h:48 (ep.so+0x000000080832)

07:52:31     #5 VBucket::getSnapshotLock() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/vbucket.h:212 (ep.so+0x000000104292)

07:52:31     #6 EventuallyPersistentStore::queueDirty(RCPtr<VBucket>&, StoredValue*, LockHolder*, bool, bool, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep.cc:2921 (ep.so+0x0000000d56a3)

07:52:31     #7 EventuallyPersistentStore::set(Item const&, void const*, bool, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep.cc:681 (ep.so+0x0000000d847d)

07:52:31     #8 EventuallyPersistentEngine::store(void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:2128 (ep.so+0x00000013dd58)

07:52:31     #9 EvpStore(engine_interface*, void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:229 (ep.so+0x0000001377ad)

07:52:31     #10 mock_store /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/memcached/programs/engine_testapp/engine_testapp.c (engine_testapp+0x0000004c7624)

07:52:31     #11 storeCasVb11(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, unsigned long, unsigned int, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/tests/ep_test_apis.cc:630 (ep_testsuite.so+0x0000000e96e7)

07:52:31     #12 store(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/tests/ep_test_apis.cc:602 (ep_testsuite.so+0x0000000e6f6a)

07:52:31     #13 test_mb19982(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/tests/ep_testsuite.cc:11973 (ep_testsuite.so+0x0000000b1379)

07:52:31     #14 execute_test /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/memcached/programs/engine_testapp/engine_testapp.c (engine_testapp+0x0000004c511f)

07:52:31     #15 main crtstuff.c (engine_testapp+0x0000004c2f08)

07:52:31 

07:52:31   Mutex M21516 previously acquired by the same thread here:

07:52:31     #0 pthread_rwlock_rdlock <null> (engine_testapp+0x000000462260)

07:52:31     #1 cb_rw_reader_enter <null> (libplatform.so.0.1.0+0x000000004800)

07:52:31     #2 RWLock::readerLock() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/rwlock.h:38 (ep.so+0x000000132e10)

07:52:31     #3 ReaderLockHolder::ReaderLockHolder(RWLock&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/locks.h:167 (ep.so+0x0000000f84f7)

07:52:31     #4 EventuallyPersistentStore::set(Item const&, void const*, bool, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep.cc:635 (ep.so+0x0000000d7d98)

07:52:31     #5 EventuallyPersistentEngine::store(void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:2128 (ep.so+0x00000013dd58)

07:52:31     #6 EvpStore(engine_interface*, void const*, void*, unsigned long*, ENGINE_STORE_OPERATION, unsigned short) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:229 (ep.so+0x0000001377ad)

07:52:31     #7 mock_store /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/memcached/programs/engine_testapp/engine_testapp.c (engine_testapp+0x0000004c7624)

07:52:31     #8 storeCasVb11(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, unsigned long, unsigned int, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/tests/ep_test_apis.cc:630 (ep_testsuite.so+0x0000000e96e7)

07:52:31     #9 store(engine_interface*, engine_interface_v1*, void const*, ENGINE_STORE_OPERATION, char const*, char const*, void**, unsigned long, unsigned short, unsigned int, unsigned char) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/tests/ep_test_apis.cc:602 (ep_testsuite.so+0x0000000e6f6a)

07:52:31     #10 test_mb19982(engine_interface*, engine_interface_v1*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/tests/ep_testsuite.cc:11973 (ep_testsuite.so+0x0000000b1379)

07:52:31     #11 execute_test /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/memcached/programs/engine_testapp/engine_testapp.c (engine_testapp+0x0000004c511f)

07:52:31     #12 main crtstuff.c (engine_testapp+0x0000004c2f08)

07:52:31 

07:52:31   Mutex M21516 acquired here while holding mutex M21519 in thread T18:

07:52:31     #0 pthread_rwlock_rdlock <null> (engine_testapp+0x000000462260)

07:52:31     #1 cb_rw_reader_enter <null> (libplatform.so.0.1.0+0x000000004800)

07:52:31     #2 RWLock::readerLock() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/rwlock.h:38 (ep.so+0x000000132e10)

07:52:31     #3 ReaderLockHolder::ReaderLockHolder(RWLock&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/locks.h:167 (ep.so+0x0000000f84f7)

07:52:31     #4 EventuallyPersistentStore::setWithMeta(Item const&, unsigned long, void const*, bool, bool, unsigned char, bool, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep.cc:1866 (ep.so+0x0000000e585e)

07:52:31     #5 PassiveStream::commitMutation(MutationResponse*, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-stream.cc:1392 (ep.so+0x0000002a3551)

07:52:31     #6 PassiveStream::processMutation(MutationResponse*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-stream.cc:1363 (ep.so+0x0000002a155b)

07:52:31     #7 PassiveStream::processBufferedMessages(unsigned int&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-stream.cc:1303 (ep.so+0x0000002a0b52)

07:52:31     #8 DcpConsumer::processBufferedItems() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-consumer.cc:608 (ep.so+0x000000265154)

07:52:31     #9 Processer::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-consumer.cc:48 (ep.so+0x000000264d4f)

07:52:31     #10 ExecutorThread::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorthread.cc:109 (ep.so+0x0000001e7631)

07:52:31     #11 launch_executor_thread(void*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e6c5a)

07:52:31     #12 platform_thread_wrap /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/platform/src/cb_pthreads.c (libplatform.so.0.1.0+0x00000000377c)

07:52:31 

07:52:31   Mutex M21519 previously acquired by the same thread here:

07:52:31     #0 pthread_mutex_lock <null> (engine_testapp+0x00000047e9e0)

07:52:31     #1 cb_mutex_enter <null> (libplatform.so.0.1.0+0x0000000039c0)

07:52:31     #2 Mutex::acquire() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e615e)

07:52:31     #3 LockHolder::lock() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/locks.h:71 (ep.so+0x000000080bc3)

07:52:31     #4 LockHolder::LockHolder(Mutex&, bool) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/locks.h:48 (ep.so+0x000000080832)

07:52:31     #5 VBucket::getSnapshotLock() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/vbucket.h:212 (ep.so+0x000000104292)

07:52:31     #6 PassiveStream::processMutation(MutationResponse*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-stream.cc:1362 (ep.so+0x0000002a14f2)

07:52:31     #7 PassiveStream::processBufferedMessages(unsigned int&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-stream.cc:1303 (ep.so+0x0000002a0b52)

07:52:31     #8 DcpConsumer::processBufferedItems() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-consumer.cc:608 (ep.so+0x000000265154)

07:52:31     #9 Processer::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/dcp-consumer.cc:48 (ep.so+0x000000264d4f)

07:52:31     #10 ExecutorThread::run() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorthread.cc:109 (ep.so+0x0000001e7631)

07:52:31     #11 launch_executor_thread(void*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e6c5a)

07:52:31     #12 platform_thread_wrap /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/platform/src/cb_pthreads.c (libplatform.so.0.1.0+0x00000000377c)

07:52:31 

07:52:31   Thread T18 (tid=20143, running) created by main thread at:

07:52:31     #0 pthread_create <null> (engine_testapp+0x00000045fb91)

07:52:31     #1 cb_create_thread <null> (libplatform.so.0.1.0+0x0000000035ee)

07:52:31     #2 ExecutorThread::start() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorthread.cc:47 (ep.so+0x0000001e68a8)

07:52:31     #3 ExecutorPool::_startWorkers() /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorpool.cc:488 (ep.so+0x0000001b3814)

07:52:31     #4 ExecutorPool::_registerBucket(EventuallyPersistentEngine*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorpool.cc:438 (ep.so+0x0000001b2776)

07:52:31     #5 ExecutorPool::registerBucket(EventuallyPersistentEngine*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/executorpool.cc:443 (ep.so+0x0000001b3c83)

07:52:31     #6 EventuallyPersistentStore::EventuallyPersistentStore(EventuallyPersistentEngine&) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep.cc:220 (ep.so+0x0000000cf39d)

07:52:31     #7 EventuallyPersistentEngine::initialize(char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:2018 (ep.so+0x00000013cd1f)

07:52:31     #8 EvpInitialize(engine_interface*, char const*) /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/ep-engine/src/ep_engine.cc:135 (ep.so+0x000000136c65)

07:52:31     #9 init_engine <null> (libmcd_util.so.1.0.0+0x000000003e3b)

07:52:31     #10 start_your_engines /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/memcached/programs/engine_testapp/engine_testapp.c (engine_testapp+0x0000004c42df)

07:52:31     #11 execute_test /home/couchbase/jenkins/workspace/ep-engine-threadsanitizer-3.0.x/memcached/programs/engine_testapp/engine_testapp.c (engine_testapp+0x0000004c4fc2)

07:52:31     #12 main crtstuff.c (engine_testapp+0x0000004c2f08)