(gdb) thread apply all bt Thread 27 (LWP 20008): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f146dff8dd0, __mutex=, __cond=0x7f14a72973a8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297380, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297380) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297380, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=83 'S') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=83 'S') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221960) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac810) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac810) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f146dffb700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 26 (LWP 20007): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f146e7f9dd0, __mutex=, __cond=0x7f14a72973a8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297380, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297380) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297380, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=26 '\032') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=26 '\032') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221880) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac7e0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac7e0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f146e7fc700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 25 (LWP 20006): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f146effadd0, __mutex=, __cond=0x7f14a7297228) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297200, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297200) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297200, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=50 '2') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=50 '2') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a72217a0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac7b0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac7b0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f146effd700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 24 (LWP 20004): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f146fffcdd0, __mutex=, __cond=0x7f14a7296f28) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7296f00, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7296f00) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7296f00, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=27 '\033') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=27 '\033') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a72215e0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac750) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac750) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f146ffff700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 23 (LWP 20003): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f1478ff6dd0, __mutex=, __cond=0x7f14a7296f28) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7296f00, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7296f00) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7296f00, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=78 'N') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=78 'N') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221500) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac720) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac720) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f1478ff9700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 22 (LWP 20002): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f14797f7dd0, __mutex=, __cond=0x7f14a7296f28) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7296f00, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 --Type for more, q to quit, c to continue without paging-- #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7296f00) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7296f00, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=28 '\034') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=28 '\034') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221420) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac6c0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac6c0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f14797fa700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 21 (LWP 20001): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f1479ff8dd0, __mutex=, __cond=0x7f14a72970a8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297080, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297080) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297080, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=22 '\026') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=22 '\026') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221340) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac690) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac690) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f1479ffb700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 20 (LWP 20000): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f147a7f9dd0, __mutex=, __cond=0x7f14a72970a8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297080, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297080) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297080, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=9 '\t') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=9 '\t') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221260) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac660) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac660) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f147a7fc700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 19 (LWP 19999): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f147affadd0, __mutex=, __cond=0x7f14a72970a8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297080, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297080) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297080, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=9 '\t') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=9 '\t') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a7221180) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac630) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac630) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f147affd700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 18 (LWP 19998): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeefbc2b in __gthread_cond_timedwait (__abs_timeout=0x7f147b7fbdd0, __mutex=, __cond=0x7f14a72970a8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:143 #5 SyncObject::wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f14a7297080, t=..., lock=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f14aeefd09f in TaskQueue::_sleepThenFetchNextTask (t=..., this=0x7f14a7297080) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f14a7297080, thread=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f14aee8ceb7 in ExecutorPool::_nextTask (this=0x7f14a71cc800, t=..., tick=tick@entry=9 '\t') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f14aee8cf38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=9 '\t') at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f14aee968d6 in ExecutorThread::run (this=0x7f14a72210a0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac600) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f14aa3ac600) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #14 0x00007f14ac13e6ba in start_thread (arg=0x7f147b7fe700) at pthread_create.c:333 #15 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 17 (LWP 19997): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aaa7ddd1 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f147bffd7c8) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f14ac13e6ba in start_thread (arg=0x7f147bfff700) at pthread_create.c:333 #6 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 16 (LWP 19947): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f14ac93e1ec in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x00000000004a8dc1 in Executor::run (this=0x7f14a7145540) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/executor.cc:44 #3 0x00007f14ae2765bf in Couchbase::Thread::thread_entry (this=0x7f14a7145540) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/thread.cc:45 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70acf70) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70acf70) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a3072700) at pthread_create.c:333 --Type for more, q to quit, c to continue without paging-- #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 15 (LWP 19946): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f14ac93e1ec in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x00000000004a8dc1 in Executor::run (this=0x7f14a7145380) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/executor.cc:44 #3 0x00007f14ae2765bf in Couchbase::Thread::thread_entry (this=0x7f14a7145380) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/thread.cc:45 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70acf40) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70acf40) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a3873700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 14 (LWP 19945): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f14ac93e1ec in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x00000000004a8dc1 in Executor::run (this=0x7f14a71451c0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/executor.cc:44 #3 0x00007f14ae2765bf in Couchbase::Thread::thread_entry (this=0x7f14a71451c0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/thread.cc:45 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70acf10) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70acf10) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a4074700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 13 (LWP 19944): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f14ac93e1ec in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x00000000004a8dc1 in Executor::run (this=0x7f14a7145000) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/executor.cc:44 #3 0x00007f14ae2765bf in Couchbase::Thread::thread_entry (this=0x7f14a7145000) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/thread.cc:45 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70acfa0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70acfa0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a4875700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 12 (LWP 19943): #0 0x00007f14abe74a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x00007f14ada21365 in epoll_dispatch (base=0x7f14aa3e4c00, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f14ada18306 in event_base_loop (base=0x7f14aa3e4c00, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004720d9 in worker_libevent (arg=0x7f14a7086ff8) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/thread.cc:187 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70c91b0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70c91b0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a5076700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 11 (LWP 19942): #0 0x00007f14abe74a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x00007f14ada21365 in epoll_dispatch (base=0x7f14aa3e4900, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f14ada18306 in event_base_loop (base=0x7f14aa3e4900, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004720d9 in worker_libevent (arg=0x7f14a7086550) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/thread.cc:187 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70c9000) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70c9000) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a5877700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 10 (LWP 19941): #0 0x00007f14abe74a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x00007f14ada21365 in epoll_dispatch (base=0x7f14aa3e4600, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f14ada18306 in event_base_loop (base=0x7f14aa3e4600, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004720d9 in worker_libevent (arg=0x7f14a7085aa8) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/thread.cc:187 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70acd90) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70acd90) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a6078700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 9 (LWP 19940): #0 0x00007f14abe74a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x00007f14ada21365 in epoll_dispatch (base=0x7f14aa3e4300, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f14ada18306 in event_base_loop (base=0x7f14aa3e4300, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004720d9 in worker_libevent (arg=0x7f14a7085000) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/thread.cc:187 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14a70c9060) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14a70c9060) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a6879700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 8 (LWP 19939): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00000000004f2926 in __gthread_cond_timedwait (__abs_timeout=0x7f14a7078020, __mutex=, __cond=0x7f14aa3c03f8) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0x7f14aa3c03f8) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0x7f14aa3c03f8) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=0x7f14aa3c03f8) at /usr/include/c++/7/condition_variable:143 #5 AuditImpl::consume_events (this=0x7f14aa3c0000) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/auditd/src/audit.cc:412 #6 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac180) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #7 platform_thread_wrap (arg=0x7f14aa3ac180) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #8 0x00007f14ac13e6ba in start_thread (arg=0x7f14a707a700) at pthread_create.c:333 #9 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 7 (LWP 19938): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00000000004981b0 in __gthread_cond_timedwait (__abs_timeout=0x7f14a7b79000, __mutex=, __cond=0x7f14aa249378) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0x7f14aa249378) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0x7f14aa249378) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=0x7f14aa249378) at /usr/include/c++/7/condition_variable:143 #5 ExternalAuthManagerThread::run (this=0x7f14aa249280) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/external_auth_manager_thread.cc:229 #6 0x00007f14ae2765bf in Couchbase::Thread::thread_entry (this=0x7f14aa249280) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/thread.cc:45 #7 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac6f0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #8 platform_thread_wrap (arg=0x7f14aa3ac6f0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #9 0x00007f14ac13e6ba in start_thread (arg=0x7f14a7b7b700) at pthread_create.c:333 #10 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 6 (LWP 19937): #0 0x00007f14abe6874d in poll () at ../sysdeps/unix/syscall-template.S:84 --Type for more, q to quit, c to continue without paging-- #1 0x000000000047147e in poll (__timeout=60000, __nfds=1, __fds=0x7f14a8379ff8) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46 #2 get_command (buffsize=80, buffer=0x7f14a837a000 "(U\356\347\376\177") at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/stdin_check.cc:49 #3 check_stdin_thread (arg=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/stdin_check.cc:90 #4 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac090) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f14aa3ac090) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a837c700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 5 (LWP 19936): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeb270ea in __gthread_cond_timedwait (__abs_timeout=0x7f14a8b7b020, __mutex=, __cond=) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_until >, spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}::operator()() const::{lambda()#1}>(std::unique_lock&, std::chrono::time_point > > const&, spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}::operator()() const::{lambda()#1}) (__p=..., __atime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:129 #5 std::condition_variable::wait_for, spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}::operator()() const::{lambda()#1}>(std::unique_lock&, std::chrono::duration > const&, spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}::operator()() const::{lambda()#1}) (__p=..., __rtime=..., __lock=..., this=) at /usr/include/c++/7/condition_variable:156 #6 spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}::operator()() const (__closure=0x7f14aa283d08) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/third_party/spdlog/include/spdlog/details/periodic_worker.h:38 #7 std::__invoke_impl const&, std::chrono::duration >)::{lambda()#1}>(std::__invoke_other, spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}&&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60 #8 std::__invoke const&, std::chrono::duration >)::{lambda()#1}>(std::__invoke_result&&, (spdlog::details::periodic_worker::periodic_worker(std::function const&, std::chrono::duration >)::{lambda()#1}&&)...) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95 #9 std::thread::_Invoker const&, std::chrono::duration >)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x7f14aa283d08) at /usr/include/c++/7/thread:234 #10 std::thread::_Invoker const&, std::chrono::duration >)::{lambda()#1}> >::operator()() (this=0x7f14aa283d08) at /usr/include/c++/7/thread:243 #11 std::thread::_State_impl const&, std::chrono::duration >)::{lambda()#1}> > >::_M_run() (this=0x7f14aa283d00) at /usr/include/c++/7/thread:186 #12 0x00007f14ac943b10 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #13 0x00007f14ac13e6ba in start_thread (arg=0x7f14a8b7d700) at pthread_create.c:333 #14 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 4 (LWP 19935): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aeb27aba in __gthread_cond_timedwait (__abs_timeout=0x7f14a937bca0, __mutex=, __cond=0x7f14aa25c438) at /usr/include/x86_64-linux-gnu/c++/7/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0x7f14aa25c438) at /usr/include/c++/7/condition_variable:178 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0x7f14aa25c438) at /usr/include/c++/7/condition_variable:106 #4 std::condition_variable::wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for(spdlog::details::async_msg&, std::chrono::duration >)::{lambda()#1}>(std::unique_lock&, std::chrono::time_point > > const&, spdlog::details::mpmc_blocking_queue::dequeue_for(spdlog::details::async_msg&, std::chrono::duration >)::{lambda()#1}) (__p=..., __atime=..., __lock=..., this=0x7f14aa25c438) at /usr/include/c++/7/condition_variable:129 #5 std::condition_variable::wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for(spdlog::details::async_msg&, std::chrono::duration >)::{lambda()#1}>(std::unique_lock&, std::chrono::duration > const&, spdlog::details::mpmc_blocking_queue::dequeue_for(spdlog::details::async_msg&, std::chrono::duration >)::{lambda()#1}) (__p=..., __rtime=..., __lock=..., this=0x7f14aa25c438) at /usr/include/c++/7/condition_variable:156 #6 spdlog::details::mpmc_blocking_queue::dequeue_for (wait_duration=..., popped_item=..., this=0x7f14aa25c410) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/third_party/spdlog/include/spdlog/details/mpmc_blocking_q.h:61 #7 spdlog::details::thread_pool::process_next_msg_ (this=this@entry=0x7f14aa25c410) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/third_party/spdlog/include/spdlog/details/thread_pool.h:193 #8 0x00007f14aeb28040 in spdlog::details::thread_pool::worker_loop_ (this=0x7f14aa25c410) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/third_party/spdlog/include/spdlog/details/thread_pool.h:184 #9 0x00007f14ac943b10 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x00007f14ac13e6ba in start_thread (arg=0x7f14a937e700) at pthread_create.c:333 #11 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 3 (LWP 19934): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x00007f14aaa7e527 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7f14a9dfd7c8) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7f14a9dfd7c8) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007f14ac13e6ba in start_thread (arg=0x7f14a9dff700) at pthread_create.c:333 #7 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 2 (LWP 19933): #0 0x00007f14abe74a13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x00007f14ada21365 in epoll_dispatch (base=0x7f14aa3e4000, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f14ada18306 in event_base_loop (base=0x7f14aa3e4000, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x000000000042c74d in memcached_main (argc=, argv=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/daemon/memcached.cc:1945 #4 0x00007f14abd8d830 in __libc_start_main (main=0x41fd90 , argc=3, argv=0x7ffee7ee57f8, init=, fini=, rtld_fini=, stack_end=0x7ffee7ee57e8) at ../csu/libc-start.c:291 #5 0x00000000004230e9 in _start () Thread 1 (LWP 20005): #0 0x00007f14abda2428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54 #1 0x00007f14abda402a in __GI_abort () at abort.c:89 #2 0x00007f14ac90cdde in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #3 0x000000000055b002 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/utilities/terminate_handler.cc:86 #4 0x00007f14ac918896 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007f14ac918901 in std::terminate() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #6 0x00007f14ac918b55 in __cxa_throw () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #7 0x00007f14aeda4567 in cb::ThrowExceptionUnderflowPolicy::underflow (this=, desired=, current=, arg=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/include/platform/non_negative_counter.h:48 #8 0x00007f14aef88108 in cb::NonNegativeCounter::fetch_add (arg=-1, this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/include/platform/non_negative_counter.h:125 #9 cb::NonNegativeCounter::operator+= (rhs=18446744073709551615, this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/include/platform/non_negative_counter.h:182 #10 Collections::VB::ManifestEntry::updateDiskSize (delta=-1, this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/collections/vbucket_manifest_entry.h:107 #11 Collections::VB::Manifest::updateDiskSize (delta=-1, entry=..., this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/collections/vbucket_manifest.h:1055 #12 Collections::VB::Manifest::CachingReadHandle::updateDiskSize (delta=-1, this=0x7f146f7fafc0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/collections/vbucket_manifest.h:393 #13 Collections::VB::Flush::updateDiskSize (this=, key=..., delta=-1) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/collections/flush.cc:58 #14 0x00007f14aef5617a in saveDocsCallback (oldInfo=0x7f14a01a9940, newInfo=0x7f146cc71680, context=0x7f146f7fb7a0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2111 #15 0x00007f14abb1a6ab in do_save_callback (valueNew=, valueOld=0x7f146f7fb180, key=0x7f14a02305a0, rq=0x7f146f7fb360) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/couchstore/src/btree_modify.cc:431 #16 modify_node (rq=rq@entry=0x7f146f7fb360, nptr=nptr@entry=0x7f146ca68f00, start=start@entry=0, end=1, dst=dst@entry=0x7f14a0335010) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/couchstore/src/btree_modify.cc:537 #17 0x00007f14abb1b019 in modify_btree (rq=rq@entry=0x7f146f7fb360, root=0x7f146ca68f00, errcode=errcode@entry=0x7f146f7fb294) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/couchstore/src/btree_modify.cc:809 #18 0x00007f14abb301cc in update_indexes (save_callback_ctx=, save_callback=, numdocs=, idvals=, ids=, seqvals=, seqs=0x7f14a0230590, db=, this=, this=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/couchstore/src/couch_save.cc:238 #19 couchstore_save_documents_and_callback (db=, docs=, infos=, numdocs=, options=options@entry=3, save_cb=save_cb@entry=0x7f14aef56040 , save_cb_ctx=) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/couchstore/src/couch_save.cc:424 #20 0x00007f14aef624d0 in CouchKVStore::saveDocs (this=this@entry=0x7f14a04b4000, vbid=..., docs=std::vector of length 1, capacity 1 = {...}, docinfos=std::vector of length 1, capacity 1 = {...}, kvctx=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2161 #21 0x00007f14aef63423 in CouchKVStore::commit2couchstore (this=0x7f14a04b4000, commitData=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2016 #22 0x00007f14aef63ac5 in CouchKVStore::commit (this=0x7f14a04b4000, commitData=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1295 #23 0x00007f14aee3f9c2 in EPBucket::commit (this=this@entry=0x7f14a04a0000, vbid=..., kvstore=..., commitData=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/ep_bucket.cc:875 #24 0x00007f14aee4372a in EPBucket::flushVBucket (this=0x7f14a04a0000, vbid=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/ep_bucket.cc:744 #25 0x00007f14aee9c4bc in Flusher::flushVB (this=this@entry=0x7f14a04ed700) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/flusher.cc:306 #26 0x00007f14aee9d7c1 in Flusher::step (this=0x7f14a04ed700, task=0x7f14a0457110) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/flusher.cc:207 #27 0x00007f14aeea0a41 in GlobalTask::execute (this=0x7f14a0457110) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/globaltask.cc:73 #28 0x00007f14aee96a4f in ExecutorThread::run (this=0x7f14a72216c0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/executorthread.cc:188 #29 0x00007f14ae26b3e7 in CouchbaseThread::run (this=0x7f14aa3ac780) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:58 #30 platform_thread_wrap (arg=0x7f14aa3ac780) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/platform/src/cb_pthreads.cc:71 --Type for more, q to quit, c to continue without paging-- #31 0x00007f14ac13e6ba in start_thread (arg=0x7f146f7fe700) at pthread_create.c:333 #32 0x00007f14abe7441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109