WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=47479) [10/9598] Cycle in lock order graph: M43515 (0x7d480000b2e8) => M36787 (0x7d480000b708) => M36848 (0x7d4400007e88) => M43515 Mutex M36787 acquired here while holding mutex M43515 in main thread: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 TaskQueue::_schedule(SingleThreadedRCPtr&) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:222 (ep.so+0x00000025a9cd) #6 TaskQueue::schedule(SingleThreadedRCPtr&) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:240 (ep.so+0x00000025ad23) #7 ExecutorPool::_schedule(SingleThreadedRCPtr, task_type_t) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:390 (ep.so+0x0000001b17e7) #8 ExecutorPool::schedule(SingleThreadedRCPtr, task_type_t) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:397 (ep.so+0x0000001b19c0) #9 ActiveStream::scheduleBackfill() /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.cc:894 (ep.so+0x00000029b488) #10 ActiveStream::transitionState(stream_state_t) /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.cc:950 (ep.so+0x0000002975e1) #11 ActiveStream::setActive() /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.h:187 (ep.so+0x0000002868e2) #12 DcpProducer::streamRequest(unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/daver/repos/couchbase/server/ep-engine/src/dcp-producer.cc:263 (ep.so+0x00000027ee6e) #13 EvpDcpStreamReq(engine_interface*, void const*, unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/daver/repos/couchbase/server/ep-engine/src/ep_engine.cc:1471 (ep.so+0x0000 0013c613) #14 mock_dcp_stream_req /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:549 (engine_testapp+0x0000004c09e1) #15 dcp_stream_req(engine_interface*, engine_interface_v1*, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, ENGINE_ERROR_CODE) /home/daver/repos/couchbase/server/ep-engine/tests/ep_testsuite.cc:3645 (ep_testsuite.so+0x0000000b66a9) #16 test_failover_log_dcp(engine_interface*, engine_interface_v1*) /home/daver/repos/couchbase/server/ep-engine/tests/ep_testsuite.cc:11593 (ep_testsuite.so+0x0000000a3124) #17 execute_test /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1042 (engine_testapp+0x0000004ba8ff) #18 main /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1296 (engine_testapp+0x0000004b8861) Mutex M43515 previously acquired by the same thread here: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 ActiveStream::setActive() /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.h:185 (ep.so+0x0000002868b7) #6 DcpProducer::streamRequest(unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/daver/repos/couchbase/server/ep-engine/src/dcp-producer.cc:263 (ep.so+0x00000027ee6e) #7 EvpDcpStreamReq(engine_interface*, void const*, unsigned int, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, ENGINE_ERROR_CODE (*)(vbucket_failover_t*, unsigned long, void const*)) /home/daver/repos/couchbase/server/ep-engine/src/ep_engine.cc:1471 (ep.so+0x00000 013c613) #8 mock_dcp_stream_req /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:549 (engine_testapp+0x0000004c09e1) #9 dcp_stream_req(engine_interface*, engine_interface_v1*, unsigned int, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, ENGINE_ERROR_CODE) /home/daver/repos/couchbase/server/ep-engine/tests/ep_testsuite.cc:3645 (ep_testsuite.so+0x0000000b66a9) #10 test_failover_log_dcp(engine_interface*, engine_interface_v1*) /home/daver/repos/couchbase/server/ep-engine/tests/ep_testsuite.cc:11593 (ep_testsuite.so+0x0000000a3124) #11 execute_test /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1042 (engine_testapp+0x0000004ba8ff) #12 main /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1296 (engine_testapp+0x0000004b8861) Mutex M36848 acquired here while holding mutex M36787 in thread T31: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 ExecutorThread::setCurrentTask(SingleThreadedRCPtr) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:162 (ep.so+0x0000001e8326) #6 TaskQueue::_fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:141 (ep.so+0x000000259c4e) #7 TaskQueue::fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:161 (ep.so+0x00000025a54f) #8 ExecutorPool::_nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:217 (ep.so+0x0000001aebb7) #9 ExecutorPool::nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:232 (ep.so+0x0000001aed8f) #10 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:81 (ep.so+0x0000001e6b19) #11 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #12 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Mutex M36787 previously acquired by the same thread here: #0 pthread_cond_timedwait (engine_testapp+0x0000004561c0) #1 cb_cond_timedwait /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:156 (libplatform.so.0.1.0+0x000000003e5c) #2 SyncObject::wait(double) /home/daver/repos/couchbase/server/ep-engine/src/syncobject.h:44 (ep.so+0x0000001b894c) #3 TaskQueue::_doSleep(ExecutorThread&) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:92 (ep.so+0x0000002594d8) #4 TaskQueue::_fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:117 (ep.so+0x000000259727) #5 TaskQueue::fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:161 (ep.so+0x00000025a54f) #6 ExecutorPool::_nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:217 (ep.so+0x0000001aebb7) #7 ExecutorPool::nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:232 (ep.so+0x0000001aed8f) #8 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:81 (ep.so+0x0000001e6b19) #9 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #10 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Mutex M43515 acquired here while holding mutex M36848 in thread T31: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 ~ActiveStream /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.h:177 (ep.so+0x0000002a62d4) #6 ~ActiveStream /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.h:176 (ep.so+0x0000002a6417) #7 ~RCPtr /home/daver/repos/couchbase/server/ep-engine/src/atomic.h:348 (ep.so+0x000000286647) #8 ~DCPBackfill /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.cc:114 (ep.so+0x0000002a69dd) #9 ~DCPBackfill /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.cc:114 (ep.so+0x0000002a6a47) #10 SingleThreadedRCPtr::swap(GlobalTask*) /home/daver/repos/couchbase/server/ep-engine/src/atomic.h:483 (ep.so+0x000000111db5) #11 SingleThreadedRCPtr::reset(GlobalTask*) /home/daver/repos/couchbase/server/ep-engine/src/atomic.h:438 (ep.so+0x0000001e8930) #12 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:75 (ep.so+0x0000001e6a58) #13 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #14 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Mutex M36848 acquired here while holding mutex M36787 in thread T31: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 ExecutorThread::setCurrentTask(SingleThreadedRCPtr) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:162 (ep.so+0x0000001e8326) #6 TaskQueue::_fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:141 (ep.so+0x000000259c4e) #7 TaskQueue::fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:161 (ep.so+0x00000025a54f) #8 ExecutorPool::_nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:217 (ep.so+0x0000001aebb7) #9 ExecutorPool::nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:232 (ep.so+0x0000001aed8f) #10 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:81 (ep.so+0x0000001e6b19) #11 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #12 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Mutex M36787 previously acquired by the same thread here: #0 pthread_cond_timedwait (engine_testapp+0x0000004561c0) #1 cb_cond_timedwait /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:156 (libplatform.so.0.1.0+0x000000003e5c) #2 SyncObject::wait(double) /home/daver/repos/couchbase/server/ep-engine/src/syncobject.h:44 (ep.so+0x0000001b894c) #3 TaskQueue::_doSleep(ExecutorThread&) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:92 (ep.so+0x0000002594d8) #4 TaskQueue::_fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:117 (ep.so+0x000000259727) #5 TaskQueue::fetchNextTask(ExecutorThread&, bool) /home/daver/repos/couchbase/server/ep-engine/src/taskqueue.cc:161 (ep.so+0x00000025a54f) #6 ExecutorPool::_nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:217 (ep.so+0x0000001aebb7) #7 ExecutorPool::nextTask(ExecutorThread&, unsigned char) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:232 (ep.so+0x0000001aed8f) #8 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:81 (ep.so+0x0000001e6b19) #9 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #10 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Mutex M43515 acquired here while holding mutex M36848 in thread T31: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 ~ActiveStream /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.h:177 (ep.so+0x0000002a62d4) #6 ~ActiveStream /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.h:176 (ep.so+0x0000002a6417) #7 ~RCPtr /home/daver/repos/couchbase/server/ep-engine/src/atomic.h:348 (ep.so+0x000000286647) #8 ~DCPBackfill /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.cc:114 (ep.so+0x0000002a69dd) #9 ~DCPBackfill /home/daver/repos/couchbase/server/ep-engine/src/dcp-stream.cc:114 (ep.so+0x0000002a6a47) #10 SingleThreadedRCPtr::swap(GlobalTask*) /home/daver/repos/couchbase/server/ep-engine/src/atomic.h:483 (ep.so+0x000000111db5) #11 SingleThreadedRCPtr::reset(GlobalTask*) /home/daver/repos/couchbase/server/ep-engine/src/atomic.h:438 (ep.so+0x0000001e8930) #12 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:75 (ep.so+0x0000001e6a58) #13 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #14 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Mutex M36848 previously acquired by the same thread here: #0 pthread_mutex_lock (engine_testapp+0x000000474420) #1 cb_mutex_enter /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:85 (libplatform.so.0.1.0+0x000000003820) #2 Mutex::acquire() /home/daver/repos/couchbase/server/ep-engine/src/mutex.cc:31 (ep.so+0x0000001e5ace) #3 LockHolder::lock() /home/daver/repos/couchbase/server/ep-engine/src/locks.h:71 (ep.so+0x000000084053) #4 LockHolder /home/daver/repos/couchbase/server/ep-engine/src/locks.h:48 (ep.so+0x000000083cc2) #5 ExecutorThread::run() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:74 (ep.so+0x0000001e6a3d) #6 launch_executor_thread(void*) /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:34 (ep.so+0x0000001e65ca) #7 platform_thread_wrap /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:19 (libplatform.so.0.1.0+0x0000000035dc) Thread T29 (tid=47526, running) created by main thread at: #0 pthread_create (engine_testapp+0x0000004555d1) #1 cb_create_thread /home/daver/repos/couchbase/server/platform/src/cb_pthreads.c:48 (libplatform.so.0.1.0+0x00000000344e) #2 ExecutorThread::start() /home/daver/repos/couchbase/server/ep-engine/src/executorthread.cc:47 (ep.so+0x0000001e6218) #3 ExecutorPool::_startWorkers() /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:481 (ep.so+0x0000001b2d06) #4 ExecutorPool::_registerBucket(EventuallyPersistentEngine*) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:438 (ep.so+0x0000001b2036) #5 ExecutorPool::registerBucket(EventuallyPersistentEngine*) /home/daver/repos/couchbase/server/ep-engine/src/executorpool.cc:443 (ep.so+0x0000001b3543) #6 EventuallyPersistentStore /home/daver/repos/couchbase/server/ep-engine/src/ep.cc:220 (ep.so+0x0000000d28bd) #7 EventuallyPersistentEngine::initialize(char const*) /home/daver/repos/couchbase/server/ep-engine/src/ep_engine.cc:2018 (ep.so+0x00000013f685) #8 EvpInitialize(engine_interface*, char const*) /home/daver/repos/couchbase/server/ep-engine/src/ep_engine.cc:135 (ep.so+0x0000001395f5) #9 init_engine /home/daver/repos/couchbase/server/memcached/utilities/engine_loader.c:108 (libmcd_util.so.1.0.0+0x000000003dbb) #10 start_your_engines /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:918 (engine_testapp+0x0000004b9abf) #11 reload_engine /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1007 (engine_testapp+0x0000004b980d) #12 test_failover_log_dcp(engine_interface*, engine_interface_v1*) /home/daver/repos/couchbase/server/ep-engine/tests/ep_testsuite.cc:11580 (ep_testsuite.so+0x0000000a2eb9) #13 execute_test /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1042 (engine_testapp+0x0000004ba8ff) #14 main /home/daver/repos/couchbase/server/memcached/programs/engine_testapp/engine_testapp.c:1296 (engine_testapp+0x0000004b8861) SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) ??:0 pthread_mutex_lock