(gdb) thread apply all bt Thread 73 (LWP 9421): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f882849ae11 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f87413ee220) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f8828f86ea5 in start_thread (arg=0x7f87413ff700) at pthread_create.c:307 #6 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 72 (LWP 9303): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f882849ae11 in background_thread_sleep (tsdn=, interval=3175000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f8745fee220) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f8828f86ea5 in start_thread (arg=0x7f8745fff700) at pthread_create.c:307 #6 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 71 (LWP 9267): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f88251929d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f87477fe700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 70 (LWP 9266): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825192850) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f8747fff700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 69 (LWP 9265): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f88251927c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f8764ff9700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 68 (LWP 9264): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f88251926d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f87657fa700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 67 (LWP 9263): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 ---Type to continue, or q to quit--- #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825192580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f8765ffb700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 66 (LWP 9262): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825192550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f87667fc700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 65 (LWP 9261): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825192460) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f8766ffd700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 64 (LWP 9260): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f88251923d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f87677fe700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 63 (LWP 9259): #0 0x00007f882849564d in extent_list_remove (extent=0x7f853fae53c0, list=0x7f88200097d0) at include/jemalloc/internal/extent_inlines.h:440 #1 arena_bin_slabs_full_remove (slab=0x7f853fae53c0, bin=0x7f8820009750, arena=0x7f88200008c0) at src/arena.c:1061 #2 arena_dalloc_bin_locked_impl (junked=true, ptr=, slab=0x7f853fae53c0, binind=, bin=0x7f8820009750, arena=0x7f88200008c0, tsdn=) at src/arena.c:1701 #3 je_arena_dalloc_bin_junked_locked (tsdn=tsdn@entry=0x7f8767fee220, arena=arena@entry=0x7f88200008c0, bin=bin@entry=0x7f8820009750, binind=binind@entry=25, extent=0x7f853fae53c0, ptr=) at src/arena.c:1714 #4 0x00007f88285046ad in je_tcache_bin_flush_small (tsd=tsd@entry=0x7f8767fee220, tcache=tcache@entry=0x7f87344fc000, tbin=tbin@entry=0x7f87344fc268, binind=binind@entry=25, rem=) at src/tcache.c:189 #5 0x00007f88285058f0 in je_tcache_event_hard (tsd=tsd@entry=0x7f8767fee220, tcache=tcache@entry=0x7f87344fc000) at src/tcache.c:55 #6 0x00007f8828488504 in tcache_event (tcache=0x7f87344fc000, tsd=) at include/jemalloc/internal/tcache_inlines.h:37 #7 tcache_alloc_small (slow_path=false, zero=, binind=, size=32, tcache=0x7f87344fc000, arena=, tsd=) at include/jemalloc/internal/tcache_inlines.h:99 #8 arena_malloc (slow_path=false, tcache=0x7f87344fc000, zero=, ind=, size=32, arena=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:165 #9 iallocztm (slow_path=false, arena=, is_internal=false, tcache=0x7f87344fc000, zero=, ind=, size=32, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:53 #10 imalloc_no_sample (ind=, usize=32, size=32, tsd=0x7f8767fee220, dopts=, sopts=) at src/jemalloc.c:1949 #11 imalloc_body (tsd=0x7f8767fee220, dopts=, sopts=) at src/jemalloc.c:2123 #12 imalloc (dopts=, sopts=) at src/jemalloc.c:2258 #13 je_mallocx (size=size@entry=32, flags=) at src/jemalloc.c:3137 #14 0x0000000000bcb7fd in cb::_JEArenaMalloc::malloc (size=size@entry=32) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:266 #15 0x000000000056bb7f in malloc (size=32) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39 #16 cb_malloc (size=32) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39 #17 0x0000000000bbe1e9 in operator new(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:63 #18 0x00000000009fbd1d in _M_clone (__source=..., __dest=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:88 #19 _M_manager (__op=, __source=..., __dest=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:196 #20 std::_Function_handler::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation) (__dest=..., __source=..., __op=) ---Type to continue, or q to quit--- at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:283 #21 0x00000000009f85d0 in function (__x=..., this=0x7f8767fecd88) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #22 greaterThan (this=0x7f8767fecd80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.h:65 #23 indirect_cmp (this=0x7f8767fecd80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/detail/mutable_heap.hpp:115 #24 heap_base (rhs=..., this=0x7f8767fecd80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/detail/stable_heap.hpp:188 #25 top_child_index (index=0, this=0x7f8767fed0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:339 #26 siftdown (index=0, this=0x7f8767fed0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:289 #27 decrease (index=0, this=0x7f8767fed0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:414 #28 decrease (handle=..., this=0x7f8767fed0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/detail/mutable_heap.hpp:454 #29 decrease (handle=..., this=0x7f8767fed0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:729 #30 Next (this=0x7f8767fed070) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:88 #31 magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:83 #32 0x0000000000a0bc5e in magma::KVReader::ReadKVs (this=this@entry=0x7f8767fed130) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:70 #33 0x0000000000a35adc in magma::LSMTree::mergeSSTables(std::vector, std::allocator > >&, magma::ItemGCFilter&, unsigned long, double, std::vector, std::allocator > >*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:858 #34 0x0000000000a034f5 in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock&, magma::LSMTree::CompactionTrigger, std::vector, std::allocator > >&, std::vector, std::allocator > >&, std::function > (bool)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/compaction.cc:235 #35 0x0000000000a05bbf in magma::LSMTree::compactLevel(int, std::unique_lock&, magma::LSMTree::CompactionTrigger, std::function > (bool)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #36 0x0000000000a05e91 in magma::LSMTree::compact(std::unique_lock&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/compaction.cc:411 #37 0x0000000000a27b08 in doCompactionWork (this=0x7f87c45d7410) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:417 #38 operator() (__closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:428 #39 __invoke_impl&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #40 __invoke_r&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #41 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #42 0x00000000009ec0b8 in operator() (this=0x7f87a44f96c0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #43 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158 #44 0x00000000009eda31 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180 #45 0x0000000000baf899 in run (this=0x7f88251923a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #46 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #47 0x00007f8828f86ea5 in start_thread (arg=0x7f8767fff700) at pthread_create.c:307 #48 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 62 (LWP 9258): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f88251921c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f876cff9700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 61 (LWP 9257): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825192100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f876d7fa700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 60 (LWP 9256): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f88251920a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f876dffb700) at pthread_create.c:307 ---Type to continue, or q to quit--- #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 59 (LWP 9255): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825192070) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f876e7fc700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 58 (LWP 9254): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825191ec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f876effd700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 57 (LWP 9253): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009ed638 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:105 #4 0x00000000009eda21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:179 #5 0x0000000000baf899 in run (this=0x7f8825191dd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007f8828f86ea5 in start_thread (arg=0x7f876f7fe700) at pthread_create.c:307 #8 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 56 (LWP 9252): #0 0x00007f88268c12e7 in unlink () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f882683bdf9 in __GI_remove (file=0x7f871ba05680
) at ../sysdeps/posix/remove.c:33 #2 0x0000000000bb3f45 in cb::io::rmrf(std::__cxx11::basic_string, std::allocator > const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/dirutils.cc:224 #3 0x00000000009d9bf8 in magma::RemoveAll(std::__cxx11::basic_string, std::allocator > const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:98 #4 0x00000000009d9c4d in magma::DefaultFileRemover(std::__cxx11::basic_string, std::allocator > const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:78 #5 0x00000000009dda71 in __invoke_impl const&, bool), std::__cxx11::basic_string, std::allocator > const&, bool> (__f=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:88 #6 __invoke_r const&, bool), std::__cxx11::basic_string, std::allocator > const&, bool> (__fn=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #7 std::_Function_handler, std::allocator > const&, bool), magma::Status (*)(std::__cxx11::basic_string, std::allocator > const&, bool)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string, std::allocator > const&, bool&&) (__functor=..., __args#0=..., __args#1=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #8 0x00000000009d84e7 in operator() (__args#1=, __args#0=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #9 operator() (__closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:53 #10 __invoke_impl&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #11 __invoke_r&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #12 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #13 0x00000000009ec0b8 in operator() (this=0x7f8690f00be0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #14 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158 #15 0x00000000009eda31 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180 #16 0x0000000000baf899 in run (this=0x7f8825191da0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #17 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #18 0x00007f8828f86ea5 in start_thread (arg=0x7f876ffff700) at pthread_create.c:307 #19 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 55 (LWP 9251): #0 0x00007f8828f8defd in open64 () at ../sysdeps/unix/syscall-template.S:81 #1 0x00000000009da8f5 in magma::BasicFile::Open(int) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:94 #2 0x0000000000a4627e in magma::FileWithStats::Open (this=, openFlags=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:26 #3 0x00000000009dea15 in magma::BufferedFileWriter::Open() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:297 ---Type to continue, or q to quit--- #4 0x0000000000a51073 in magma::SSTableWriter::Init() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable_writer.cc:78 #5 0x0000000000a5416a in magma::SSTableWriter::AddKV(magma::Slice const&, magma::Slice const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable_writer.cc:125 #6 0x0000000000a2aa06 in AddKVs (pairs=..., w=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:54 #7 magma::LSMTree::writeSSTable(magma::SSTableWriter&, magma::Iterator*, long, std::function) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:753 #8 0x0000000000a2b11c in magma::LSMTree::writeSSTable(bool, magma::Iterator*, unsigned long, long, bool, std::function) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:711 #9 0x0000000000a2b5da in magma::LSMTree::writeMemtable(magma::Memtable*, bool)::{lambda(magma::Iterator*)#3}::operator()(magma::Iterator*) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/memtable/memtable.h:69 #10 0x0000000000a2becb in magma::LSMTree::writeMemtable(magma::Memtable*, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:487 #11 0x0000000000a30f03 in magma::LSMTree::doMemtableFlushWork() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:545 #12 0x0000000000a318ed in operator() (__closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:929 #13 __invoke_impl, magma::LSMTree::newFlush()::&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #14 __invoke_r, magma::LSMTree::newFlush()::&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #15 std::_Function_handler(), magma::LSMTree::newFlush():: >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #16 0x0000000000a153df in operator() (this=0x7f8739429090) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #17 magma::FlushWork::Execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/flush_work.cc:61 #18 0x00000000009bda78 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:522 #19 0x00000000009bf421 in __invoke_impl&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:289 #20 __invoke_r&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #21 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #22 0x00000000009ec0b8 in operator() (this=0x7f87393c2b90) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #23 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158 #24 0x00000000009eda31 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180 #25 0x0000000000baf899 in run (this=0x7f8825191b30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #26 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #27 0x00007f8828f86ea5 in start_thread (arg=0x7f8784ff9700) at pthread_create.c:307 #28 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 54 (LWP 9250): #0 0x00007f88268c12e7 in unlink () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f882683bdf9 in __GI_remove (file=0x7f873f27c9a0
) at ../sysdeps/posix/remove.c:33 #2 0x0000000000bb3f45 in cb::io::rmrf(std::__cxx11::basic_string, std::allocator > const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/dirutils.cc:224 #3 0x00000000009d9bf8 in magma::RemoveAll(std::__cxx11::basic_string, std::allocator > const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:98 #4 0x00000000009d9c4d in magma::DefaultFileRemover(std::__cxx11::basic_string, std::allocator > const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:78 #5 0x00000000009dda71 in __invoke_impl const&, bool), std::__cxx11::basic_string, std::allocator > const&, bool> (__f=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:88 #6 __invoke_r const&, bool), std::__cxx11::basic_string, std::allocator > const&, bool> (__fn=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #7 std::_Function_handler, std::allocator > const&, bool), magma::Status (*)(std::__cxx11::basic_string, std::allocator > const&, bool)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string, std::allocator > const&, bool&&) (__functor=..., __args#0=..., __args#1=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #8 0x00000000009d8bf7 in operator() (__args#1=, __args#0=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #9 magma::ASyncFileRemover::operator()(std::__cxx11::basic_string, std::allocator > const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:60 #10 0x00000000009b3617 in operator() (blocking=, path=..., __closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:94 #11 __invoke_impl&, const std::__cxx11::basic_string, std::allocator >&, bool> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #12 __invoke_r&, const std::__cxx11::basic_string, std::allocator >&, bool> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #13 std::_Function_handler, std::allocator >&, bool), magma::KVStore::KVStore(magma::KVStoreConfig&, magma::Magma::KVStoreID, magma::Magma::KVStoreRevision):: >::_M_invoke(const std::_Any_data &, const std::__cxx11::basic_string, std::allocator > &, ) (__functor=..., __args#0=..., __args#1=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #14 0x0000000000a0f698 in operator() (__args#1=true, __args#0=..., this=0x7f873ff4f788) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #15 magma::TreeSnapshot::RemoveFile(magma::Directory&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/tree_snapshot.cc:661 #16 0x0000000000a3efaa in deleteFile (dirToSync=..., this=0x7f873fd091c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:515 #17 magma::CheckpointManager::processPendingDeletes() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:244 #18 0x0000000000a3f646 in magma::CheckpointManager::cleanupQueue (this=this@entry=0x7f873ff9efc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:399 #19 0x0000000000a3f77b in magma::CheckpointManager::Resize(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:565 #20 0x0000000000a3f7fc in magma::LSMTree::SetMaxCheckpoints (this=this@entry=0x7f873ff4f010, newSize=newSize@entry=4) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:71 #21 0x00000000009bd854 in operator() (tree=0x7f873ff4f010, __closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:311 #22 magma::KVStore::cleanupCheckpoints(magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:317 #23 0x00000000009be432 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:459 #24 0x00000000009bee28 in magma::KVStore::FlushMemTables (this=, wal=, flushMode=flushMode@entry=magma::Persistent, blockMode=blockMode@entry=magma::NonWaiting) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:394 #25 0x000000000095ed7f in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1434 #26 0x000000000095f101 in magma::Magma::Impl::heartbeatCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1626 #27 0x00000000009ec0b8 in operator() (this=0x7f86becc6340) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #28 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158 ---Type to continue, or q to quit--- #29 0x00000000009eda31 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180 #30 0x0000000000baf899 in run (this=0x7f8825191bc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #31 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #32 0x00007f8828f86ea5 in start_thread (arg=0x7f87857fa700) at pthread_create.c:307 #33 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 53 (LWP 9249): #0 0x00007f88268c77f7 in ftruncate64 () at ../sysdeps/unix/syscall-template.S:81 #1 0x00000000009dc192 in magma::BasicFile::Truncate(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:364 #2 0x0000000000a4639f in magma::FileWithStats::Truncate (this=, size=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:112 #3 0x00000000009df09b in magma::BufferedFileWriter::Close() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:322 #4 0x00000000009df136 in magma::BufferedFileWriter::~BufferedFileWriter (this=0x7f873728b800, __in_chrg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:328 #5 0x0000000000a512e8 in operator() (__ptr=0x7f873728b800, this=0x7f8785fe92b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable_writer.cc:95 #6 reset (__p=0x0, this=0x7f8785fe92b0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:182 #7 reset (__p=0x0, this=0x7f8785fe92b0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:456 #8 magma::SSTableWriter::~SSTableWriter (this=0x7f8785fe92b0, __in_chrg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable_writer.cc:95 #9 0x0000000000a2b1f7 in magma::LSMTree::writeSSTable(bool, magma::Iterator*, unsigned long, long, bool, std::function) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:720 #10 0x0000000000a2b79e in magma::LSMTree::writeMemtable(magma::Memtable*, bool)::{lambda(magma::Iterator*)#4}::operator()(magma::Iterator*) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/memtable/memtable.h:69 #11 0x0000000000a2bfe3 in magma::LSMTree::writeMemtable(magma::Memtable*, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:504 #12 0x0000000000a30f03 in magma::LSMTree::doMemtableFlushWork() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:545 #13 0x0000000000a318ed in operator() (__closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:929 #14 __invoke_impl, magma::LSMTree::newFlush()::&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #15 __invoke_r, magma::LSMTree::newFlush()::&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #16 std::_Function_handler(), magma::LSMTree::newFlush():: >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #17 0x0000000000a153df in operator() (this=0x7f8737052370) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #18 magma::FlushWork::Execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/flush_work.cc:61 #19 0x00000000009bd9cf in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:515 #20 0x00000000009bf421 in __invoke_impl&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:289 #21 __invoke_r&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #22 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #23 0x00000000009ec0b8 in operator() (this=0x7f8736fe3f90) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #24 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158 #25 0x00000000009eda31 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180 #26 0x0000000000baf899 in run (this=0x7f8825191b00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #27 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #28 0x00007f8828f86ea5 in start_thread (arg=0x7f8785ffb700) at pthread_create.c:307 #29 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 52 (LWP 9248): #0 0x00007f88268c12e7 in unlink () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f882683bdf9 in __GI_remove (file=0x7f8723963790
) at ../sysdeps/posix/remove.c:33 #2 0x0000000000bb3f45 in cb::io::rmrf(std::__cxx11::basic_string, std::allocator > const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/dirutils.cc:224 #3 0x00000000009d9bf8 in magma::RemoveAll(std::__cxx11::basic_string, std::allocator > const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:98 #4 0x00000000009d9c4d in magma::DefaultFileRemover(std::__cxx11::basic_string, std::allocator > const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:78 #5 0x00000000009dda71 in __invoke_impl const&, bool), std::__cxx11::basic_string, std::allocator > const&, bool> (__f=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:88 #6 __invoke_r const&, bool), std::__cxx11::basic_string, std::allocator > const&, bool> (__fn=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #7 std::_Function_handler, std::allocator > const&, bool), magma::Status (*)(std::__cxx11::basic_string, std::allocator > const&, bool)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string, std::allocator > const&, bool&&) (__functor=..., __args#0=..., __args#1=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #8 0x00000000009d8bf7 in operator() (__args#1=, __args#0=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #9 magma::ASyncFileRemover::operator()(std::__cxx11::basic_string, std::allocator > const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_default.cc:60 #10 0x00000000009b3617 in operator() (blocking=, path=..., __closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:94 #11 __invoke_impl&, const std::__cxx11::basic_string, std::allocator >&, bool> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #12 __invoke_r&, const std::__cxx11::basic_string, std::allocator >&, bool> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115 #13 std::_Function_handler, std::allocator >&, bool), magma::KVStore::KVStore(magma::KVStoreConfig&, magma::Magma::KVStoreID, magma::Magma::KVStoreRevision):: >::_M_invoke(const std::_Any_data &, const std::__cxx11::basic_string, std::allocator > &, ) (__functor=..., __args#0=..., __args#1=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292 #14 0x0000000000a0f698 in operator() (__args#1=true, __args#0=..., this=0x7f87306b4788) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #15 magma::TreeSnapshot::RemoveFile(magma::Directory&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/tree_snapshot.cc:661 #16 0x0000000000a3efaa in deleteFile (dirToSync=..., this=0x7f8730f50280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:515 #17 magma::CheckpointManager::processPendingDeletes() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:244 ---Type to continue, or q to quit--- #18 0x0000000000a3f646 in magma::CheckpointManager::cleanupQueue (this=this@entry=0x7f873061b600) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:399 #19 0x0000000000a3f77b in magma::CheckpointManager::Resize(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:565 #20 0x0000000000a3f7fc in magma::LSMTree::SetMaxCheckpoints (this=this@entry=0x7f87306b4010, newSize=newSize@entry=4) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:71 #21 0x00000000009bd854 in operator() (tree=0x7f87306b4010, __closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:311 #22 magma::KVStore::cleanupCheckpoints(magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:317 #23 0x00000000009be432 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:459 #24 0x00000000009bee28 in magma::KVStore::FlushMemTables (this=, wal=, flushMode=flushMode@entry=magma::Persistent, blockMode=blockMode@entry=magma::NonWaiting) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:394 #25 0x000000000095ed7f in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1434 #26 0x000000000095f101 in magma::Magma::Impl::heartbeatCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1626 #27 0x00000000009ec0b8 in operator() (this=0x7f86083e6aa0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #28 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158 #29 0x00000000009eda31 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180 #30 0x0000000000baf899 in run (this=0x7f8825191860) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #31 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #32 0x00007f8828f86ea5 in start_thread (arg=0x7f87867fc700) at pthread_create.c:307 #33 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 51 (LWP 9247): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f882849ae11 in background_thread_sleep (tsdn=, interval=3175000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f8786fec220) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f8828f86ea5 in start_thread (arg=0x7f8786ffd700) at pthread_create.c:307 #6 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 50 (LWP 7604): #0 lock (this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/MicroSpinLock.h:89 #1 lock (this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/SpinLock.h:48 #2 folly::NotificationQueue >::Consumer::consumeMessages(bool, unsigned long*) (this=0x7f88255e1100, isDrain=, numConsumed=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/NotificationQueue.h:695 #3 0x00007f8828258f9a in event_persist_closure (ev=, base=0x7f882508de00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1629 #4 event_process_active_single_queue (base=base@entry=0x7f882508de00, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1688 #5 0x00007f882825953f in event_process_active (base=0x7f882508de00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #6 event_base_loop (base=0x7f882508de00, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #7 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f882509ff00, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #8 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f882509ff00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #9 0x0000000000c2cc46 in folly::EventBase::loopForever (this=0x7f882509ff00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #10 0x0000000000c18ef9 in folly::IOThreadPoolExecutor::threadRun (this=0x7f882509f780, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/IOThreadPoolExecutor.cpp:195 #11 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #12 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #13 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #14 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #15 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #16 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #17 0x00007f8828f86ea5 in start_thread (arg=0x7f87877fe700) at pthread_create.c:307 #18 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 49 (LWP 7603): #0 0x00007f8828f8d6fd in write () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000c29b76 in folly::NotificationQueue >::ensureSignalLocked() const (this=this@entry=0x7f88255e6280) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/NotificationQueue.h:517 #2 0x0000000000c307b8 in ensureSignalLocked (this=0x7f88255e6280) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/NotificationQueue.h:505 #3 folly::NotificationQueue >::putMessageImpl >(folly::Function&&, unsigned long, bool) (this=0x7f88255e6280, message=, maxSize=maxSize@entry=0, throws=throws@entry=true) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/NotificationQueue.h:595 #4 0x0000000000c29c9b in putMessage > (message=, this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/NotificationQueue.h:409 #5 folly::EventBase::runInEventBaseThread(folly::Function) (this=0x7f882509ff00, fn=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:640 #6 0x0000000000abb8e0 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f8787fed650) at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000ac356e in operator() (this=0x7f8787fed650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 ---Type to continue, or q to quit--- #8 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #9 0x0000000000c216d0 in operator() (this=0x7f8787fed840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #10 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f8825151d00, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #11 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151d00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255e6240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255e6240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f8787fff700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 48 (LWP 7602): #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135 #1 0x00007f8828f88e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f8828f88d68 in __GI___pthread_mutex_lock (mutex=0x7f8739098f30) at ../nptl/pthread_mutex_lock.c:78 #3 0x00000000007e02ec in __gthread_mutex_lock (__mutex=0x7f8739098f30) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #4 lock (this=0x7f8739098f30) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 #5 lock_guard (__m=..., this=0x7f87a4fe71d8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:159 #6 CheckpointManager::removeClosedUnrefCheckpoints() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:608 #7 0x00000000007e90ac in CheckpointMemRecoveryTask::attemptCheckpointRemoval() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:100 #8 0x00000000007ea307 in CheckpointMemRecoveryTask::runInner() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:254 #9 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #10 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87a4fe7650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #11 0x0000000000ac356e in operator() (this=0x7f87a4fe7650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #12 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #13 0x0000000000c216d0 in operator() (this=0x7f87a4fe7840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #14 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f8825151d00, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #15 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151d00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #16 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #17 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #18 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #19 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #20 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #21 0x0000000000abb594 in operator() (this=0x7f88255e61c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #22 operator() (__closure=0x7f88255e61c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #23 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #24 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #25 0x00007f8828f86ea5 in start_thread (arg=0x7f87a4ff9700) at pthread_create.c:307 #26 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 47 (LWP 7601): #0 __atomic_add (__val=1, __mem=0x7f8721229f88) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:96 #1 __atomic_add_dispatch (__val=1, __mem=0x7f8721229f88) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:96 #2 _M_add_ref_copy (this=0x7f8721229f80) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:142 #3 __shared_count (__r=..., this=0x7f87a57e82b8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:740 #4 __shared_ptr (this=0x7f87a57e82b0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:1181 #5 shared_ptr (this=0x7f87a57e82b0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr.h:149 #6 DcpProducer::findStreams (this=0x7f87387a9f80, vbid=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:2051 ---Type to continue, or q to quit--- #7 0x00000000008ee2e9 in ActiveStreamCheckpointProcessorTask::queuePop() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream_checkpoint_processor_task.cc:109 #8 0x00000000008ee39d in ActiveStreamCheckpointProcessorTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream_checkpoint_processor_task.cc:47 #9 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #10 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87a57e8650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #11 0x0000000000ac356e in operator() (this=0x7f87a57e8650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #12 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #13 0x0000000000c216d0 in operator() (this=0x7f87a57e8840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #14 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f8825151d00, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #15 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151d00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #16 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #17 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #18 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #19 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #20 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #21 0x0000000000abb594 in operator() (this=0x7f88255e6180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #22 operator() (__closure=0x7f88255e6180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #23 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #24 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #25 0x00007f8828f86ea5 in start_thread (arg=0x7f87a57fa700) at pthread_create.c:307 #26 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 46 (LWP 7599): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=2793317912, absSteadyTime=absSteadyTime@entry=0x7f87a67ea5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027de228) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027de228, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87a67ea618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027de228) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027de228, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027de228, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255e6000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255e6000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87a67fc700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ---Type to continue, or q to quit--- Thread 45 (LWP 7598): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=2801710616, absSteadyTime=absSteadyTime@entry=0x7f87a6feb5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027df0f8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027df0f8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87a6feb618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027df0f8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027df0f8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027df0f8, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255dcfc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255dcfc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87a6ffd700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 44 (LWP 7597): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=2810103320, absSteadyTime=absSteadyTime@entry=0x7f87a77ec5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027db750) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027db750, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87a77ec618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027db750) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027db750, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027db750, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 ---Type to continue, or q to quit--- #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255dcf80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255dcf80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87a77fe700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 43 (LWP 7596): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=2818496024, absSteadyTime=absSteadyTime@entry=0x7f87a7fed5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dfbf0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dfbf0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87a7fed618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dfbf0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dfbf0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dfbf0, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255dcf40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255dcf40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87a7fff700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 42 (LWP 7595): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3305010712, absSteadyTime=absSteadyTime@entry=0x7f87c4fe75f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027e04c0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027e04c0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c4fe7618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027e04c0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027e04c0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027e04c0, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 ---Type to continue, or q to quit--- #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255dcf00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255dcf00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c4ff9700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 41 (LWP 7594): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3313403416, absSteadyTime=absSteadyTime@entry=0x7f87c57e85f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dd118) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dd118, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c57e8618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dd118) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dd118, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dd118, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255dcec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255dcec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c57fa700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 40 (LWP 7593): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3321796120, absSteadyTime=absSteadyTime@entry=0x7f87c5fe95f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dc8a8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dc8a8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c5fe9618, waitMask=waitMask@entry=4294967295) ---Type to continue, or q to quit--- at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dc8a8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dc8a8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dc8a8, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255dce80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255dce80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c5ffb700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 39 (LWP 7592): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3330188824, absSteadyTime=absSteadyTime@entry=0x7f87c67ea5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dd088) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dd088, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c67ea618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dd088) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dd088, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dd088, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6e40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6e40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 ---Type to continue, or q to quit--- #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c67fc700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 38 (LWP 7591): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3338581528, absSteadyTime=absSteadyTime@entry=0x7f87c6feb5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027e1240) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027e1240, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c6feb618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027e1240) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027e1240, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027e1240, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c6ffd700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 37 (LWP 7590): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3346974232, absSteadyTime=absSteadyTime@entry=0x7f87c77ec5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027de720) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027de720, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c77ec618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027de720) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027de720, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027de720, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 ---Type to continue, or q to quit--- #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6dc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6dc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c77fe700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 36 (LWP 7589): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3355366936, absSteadyTime=absSteadyTime@entry=0x7f87c7fed5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027e0dd8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027e0dd8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87c7fed618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027e0dd8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027e0dd8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027e0dd8, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87c7fff700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 35 (LWP 7588): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3908990488, absSteadyTime=absSteadyTime@entry=0x7f87e8fe75f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dcc20) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dcc20, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87e8fe7618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dcc20) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dcc20, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 ---Type to continue, or q to quit--- #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dcc20, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6d80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6d80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87e8ff9700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 34 (LWP 7587): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3917383192, absSteadyTime=absSteadyTime@entry=0x7f87e97e85f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dc560) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dc560, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87e97e8618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dc560) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dc560, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dc560, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87e97fa700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ---Type to continue, or q to quit--- Thread 33 (LWP 7586): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=3925775896, absSteadyTime=absSteadyTime@entry=0x7f87e9fe95f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027e06b8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027e06b8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f87e9fe9618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027e06b8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027e06b8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027e06b8, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a6a340, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a6a340) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a6a300, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88255d6cc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88255d6cc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f87e9ffb700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 32 (LWP 7585): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f873e2f5340) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f873e2f5340, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009b875f in magma::KVStore::acquireFlushBarrier() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1759 #4 0x00000000009be186 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:410 #5 0x00000000009bee28 in magma::KVStore::FlushMemTables (this=, wal=, flushMode=, blockMode=blockMode@entry=magma::Blocking) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:394 #6 0x000000000095c985 in magma::Magma::Impl::syncKVStore(unsigned short, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1464 #7 0x000000000095cb4a in SyncKVStore (kvID=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:347 #8 magma::Magma::SyncKVStore(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:351 #9 0x00000000008b91cd in MagmaMemoryTrackingProxy::SyncKVStore(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:489 #10 0x000000000089aef8 in MagmaKVStore::makeFileHandle(Vbid) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #11 0x00000000008a29f6 in MagmaKVStore::initByIdScanContext(std::unique_ptr, std::default_delete > >, std::unique_ptr, std::default_delete > >, Vbid, std::vector > const&, DocumentFilter, ValueFilter) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1812 #12 0x00000000008efe9d in DCPBackfillByIdDisk::create() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:70 #13 0x00000000008f61a2 in virtual thunk to DCPBackfillDisk::run() () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:104 #14 0x00000000008f8063 in BackfillManager::backfill() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:345 #15 0x00000000008f82f4 in BackfillManagerTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:78 #16 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #17 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87ea7ea650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #18 0x0000000000ac356e in operator() (this=0x7f87ea7ea650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #19 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #20 0x0000000000c216d0 in operator() (this=0x7f87ea7ea840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #21 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f8825151100, thread=..., task=task@entry=) ---Type to continue, or q to quit--- at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #22 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #23 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #24 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #25 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #26 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #27 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #28 0x0000000000abb594 in operator() (this=0x7f88255d6c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #29 operator() (__closure=0x7f88255d6c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #30 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #31 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #32 0x00007f8828f86ea5 in start_thread (arg=0x7f87ea7fc700) at pthread_create.c:307 #33 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 31 (LWP 7584): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f8737100340) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f8737100340, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009b875f in magma::KVStore::acquireFlushBarrier() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1759 #4 0x00000000009be186 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:410 #5 0x00000000009bee28 in magma::KVStore::FlushMemTables (this=, wal=, flushMode=, blockMode=blockMode@entry=magma::Blocking) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:394 #6 0x000000000095c985 in magma::Magma::Impl::syncKVStore(unsigned short, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1464 #7 0x000000000095cb4a in SyncKVStore (kvID=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:347 #8 magma::Magma::SyncKVStore(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:351 #9 0x00000000008b91cd in MagmaMemoryTrackingProxy::SyncKVStore(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:489 #10 0x000000000089aef8 in MagmaKVStore::makeFileHandle(Vbid) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #11 0x00000000008a29f6 in MagmaKVStore::initByIdScanContext(std::unique_ptr, std::default_delete > >, std::unique_ptr, std::default_delete > >, Vbid, std::vector > const&, DocumentFilter, ValueFilter) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1812 #12 0x00000000008efe9d in DCPBackfillByIdDisk::create() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:70 #13 0x00000000008f61a2 in virtual thunk to DCPBackfillDisk::run() () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:104 #14 0x00000000008f8063 in BackfillManager::backfill() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:345 #15 0x00000000008f82f4 in BackfillManagerTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:78 #16 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #17 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87eafeb650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #18 0x0000000000ac356e in operator() (this=0x7f87eafeb650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #19 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #20 0x0000000000c216d0 in operator() (this=0x7f87eafeb840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #21 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f8825151100, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #22 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151100, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #23 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #24 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #25 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #26 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #27 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #28 0x0000000000abb594 in operator() (this=0x7f88255d6c40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #29 operator() (__closure=0x7f88255d6c40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #30 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #31 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #32 0x00007f8828f86ea5 in start_thread (arg=0x7f87eaffd700) at pthread_create.c:307 #33 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 30 (LWP 7583): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f87407d38e0) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 ---Type to continue, or q to quit--- #2 std::condition_variable::wait (this=this@entry=0x7f87407d38e0, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009d2291 in magma::WaitGroup::WaitOnSignal (this=this@entry=0x7f87407d38a8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/common.cc:85 #4 0x000000000096001d in magma::Magma::Impl::tryThrottleForWriteCache() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1392 #5 0x000000000098689f in magma::Magma::Impl::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:52 #6 0x0000000000986a4f in magma::Magma::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #7 0x00000000008b9363 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector > const&, unsigned int, magma::Magma::HistoryMode, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #8 0x00000000008ac110 in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #9 0x0000000000894965 in MagmaKVStore::commit(std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.h:808 #10 0x000000000083ad05 in EPBucket::commit(KVStoreIface&, std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:964 #11 0x0000000000842999 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:836 #12 0x000000000084348f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:381 #13 0x00000000006ee2e6 in Flusher::flushVB (this=0x7f87409f1100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:293 #14 0x00000000006eeca0 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:203 #15 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #16 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87eb7ec650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #17 0x0000000000ac356e in operator() (this=0x7f87eb7ec650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #18 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #19 0x0000000000c216d0 in operator() (this=0x7f87eb7ec840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #20 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f882509b500, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #21 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509b500, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #22 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #23 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #24 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #25 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #26 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #27 0x0000000000abb594 in operator() (this=0x7f88255d6b80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #28 operator() (__closure=0x7f88255d6b80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #29 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #30 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #31 0x00007f8828f86ea5 in start_thread (arg=0x7f87eb7fe700) at pthread_create.c:307 #32 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 29 (LWP 7582): #0 0x00007fffc71e16c2 in ?? () #1 0x0000000000000022 in ?? () #2 0x00007f87ebfebfc0 in ?? () #3 0x00007f859d6b82b0 in ?? () #4 0x00007f859d6b82b0 in ?? () #5 0x0000000000000022 in ?? () #6 0x00007f88268e4a2d in __GI___clock_gettime (clock_id=, tp=) at ../sysdeps/unix/clock_gettime.c:115 #7 0x00007f882717dca1 in std::chrono::_V2::steady_clock::now () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/chrono.cc:88 #8 0x00000000009ce6a9 in resize (scaleFactor=0, preserve=0, size=, this=0x7f87ebfebfc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/buffer.cc:108 #9 magma::Buffer::Buffer(unsigned long, unsigned long, std::function) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/buffer.cc:20 #10 0x0000000000a565fb in magma::MemtableIterator::Seek(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/include/libmagma/slice.h:46 #11 0x0000000000a2911f in magma::LSMTree::Get(magma::Slice const&, std::shared_ptr&, magma::LookupBuffer&, magma::Slice&, bool&, bool, magma::ReaderStats*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1120 #12 0x00000000009b8e14 in magma::KVStore::KeyExists(magma::Slice const&, magma::LookupBuffer&, magma::ReaderStats*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:650 #13 0x00000000009c8898 in operator() (buffer=..., index=1593, __closure=0x7f87ebfec410) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/write.cc:87 #14 magma::KVStore::WriteDocs(magma::WAL*, std::vector > const&, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/write.cc:130 #15 0x00000000009867fc in magma::Magma::Impl::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #16 0x0000000000986a4f in magma::Magma::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 ---Type to continue, or q to quit--- #17 0x00000000008b9363 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector > const&, unsigned int, magma::Magma::HistoryMode, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #18 0x00000000008ac110 in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #19 0x0000000000894965 in MagmaKVStore::commit(std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.h:808 #20 0x000000000083ad05 in EPBucket::commit(KVStoreIface&, std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:964 #21 0x0000000000842999 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:836 #22 0x000000000084348f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:381 #23 0x00000000006ee2e6 in Flusher::flushVB (this=0x7f87e84fac00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:293 #24 0x00000000006eeca0 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:203 #25 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #26 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87ebfed650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #27 0x0000000000ac356e in operator() (this=0x7f87ebfed650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #28 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #29 0x0000000000c216d0 in operator() (this=0x7f87ebfed840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #30 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f882509b500, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #31 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509b500, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #32 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #33 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #34 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #35 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #36 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #37 0x0000000000abb594 in operator() (this=0x7f88250f8f40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #38 operator() (__closure=0x7f88250f8f40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #39 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #40 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #41 0x00007f8828f86ea5 in start_thread (arg=0x7f87ebfff700) at pthread_create.c:307 #42 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 28 (LWP 7581): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f87c41c68e0) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f87c41c68e0, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009d2291 in magma::WaitGroup::WaitOnSignal (this=this@entry=0x7f87c41c68a8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/common.cc:85 #4 0x000000000096001d in magma::Magma::Impl::tryThrottleForWriteCache() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1392 #5 0x000000000098689f in magma::Magma::Impl::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:52 #6 0x0000000000986a4f in magma::Magma::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #7 0x00000000008b9363 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector > const&, unsigned int, magma::Magma::HistoryMode, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #8 0x00000000008ac110 in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #9 0x0000000000894965 in MagmaKVStore::commit(std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.h:808 #10 0x000000000083ad05 in EPBucket::commit(KVStoreIface&, std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:964 #11 0x0000000000842999 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:836 #12 0x000000000084348f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:381 #13 0x00000000006ee2e6 in Flusher::flushVB (this=0x7f87c43e9300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:293 #14 0x00000000006eeca0 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:203 #15 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #16 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f8800fc5650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #17 0x0000000000ac356e in operator() (this=0x7f8800fc5650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #18 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #19 0x0000000000c216d0 in operator() (this=0x7f8800fc5840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #20 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f882509b500, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #21 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509b500, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 ---Type to continue, or q to quit--- #22 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #23 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #24 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #25 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #26 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #27 0x0000000000abb594 in operator() (this=0x7f88250f8f00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #28 operator() (__closure=0x7f88250f8f00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #29 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #30 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #31 0x00007f8828f86ea5 in start_thread (arg=0x7f8800fd7700) at pthread_create.c:307 #32 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 27 (LWP 7580): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f88271808bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f87a41af0e0) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f87a41af0e0, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009d2291 in magma::WaitGroup::WaitOnSignal (this=this@entry=0x7f87a41af0a8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/common.cc:85 #4 0x000000000096001d in magma::Magma::Impl::tryThrottleForWriteCache() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1392 #5 0x000000000098689f in magma::Magma::Impl::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:52 #6 0x0000000000986a4f in magma::Magma::WriteDocs(unsigned short, std::vector > const&, unsigned int, std::function, std::function > const*> ()>, magma::Magma::HistoryMode) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #7 0x00000000008b9363 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector > const&, unsigned int, magma::Magma::HistoryMode, std::function, std::function > const*> ()>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #8 0x00000000008ac110 in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #9 0x0000000000894965 in MagmaKVStore::commit(std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.h:808 #10 0x000000000083ad05 in EPBucket::commit(KVStoreIface&, std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:964 #11 0x0000000000842999 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:836 #12 0x000000000084348f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:381 #13 0x00000000006ee2e6 in Flusher::flushVB (this=0x7f8743bee000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:293 #14 0x00000000006eeca0 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:203 #15 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #16 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f88017c6650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #17 0x0000000000ac356e in operator() (this=0x7f88017c6650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #18 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #19 0x0000000000c216d0 in operator() (this=0x7f88017c6840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #20 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f882509b500, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #21 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509b500, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #22 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #23 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #24 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #25 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #26 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #27 0x0000000000abb594 in operator() (this=0x7f88250f8ec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #28 operator() (__closure=0x7f88250f8ec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #29 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #30 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #31 0x00007f8828f86ea5 in start_thread (arg=0x7f88017d8700) at pthread_create.c:307 #32 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 26 (LWP 7579): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=33322520, absSteadyTime=absSteadyTime@entry=0x7f8801fc75f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027db960) ---Type to continue, or q to quit--- at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027db960, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f8801fc7618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027db960) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027db960, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11eb7 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > ( timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027db960, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000abb594 in operator() (this=0x7f88250f8dc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 operator() (__closure=0x7f88250f8dc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007f8828f86ea5 in start_thread (arg=0x7f8801fd9700) at pthread_create.c:307 #23 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 25 (LWP 7578): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=41715224, absSteadyTime=absSteadyTime@entry=0x7f88027c85f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dd8c8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dd8c8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f88027c8618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dd8c8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dd8c8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dd8c8, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 ---Type to continue, or q to quit--- #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88250f8d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88250f8d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f88027da700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 24 (LWP 7577): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=280925720, absSteadyTime=absSteadyTime@entry=0x7f8810be95f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dcfb0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dcfb0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f8810be9618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dcfb0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dcfb0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11d7d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dcfb0, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000abb594 in operator() (this=0x7f88250f8c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #18 operator() (__closure=0x7f88250f8c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007f8828f86ea5 in start_thread (arg=0x7f8810bfb700) at pthread_create.c:307 #22 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 23 (LWP 7576): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=289318424, absSteadyTime=absSteadyTime@entry=0x7f88113ea5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027db7c8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027db7c8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f88113ea618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027db7c8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027db7c8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11eb7 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > ( timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027db7c8, deadline=..., opt=...) ---Type to continue, or q to quit--- at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000abb594 in operator() (this=0x7f88250f8d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 operator() (__closure=0x7f88250f8d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007f8828f86ea5 in start_thread (arg=0x7f88113fc700) at pthread_create.c:307 #23 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 22 (LWP 7575): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=297711128, absSteadyTime=absSteadyTime@entry=0x7f8811beb5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dd280) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dd280, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f8811beb618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dd280) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dd280, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11eb7 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > ( timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dd280, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000abb594 in operator() (this=0x7f88250f8cc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 operator() (__closure=0x7f88250f8cc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007f8828f86ea5 in start_thread (arg=0x7f8811bfd700) at pthread_create.c:307 #23 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ---Type to continue, or q to quit--- Thread 21 (LWP 7574): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=306103832, absSteadyTime=absSteadyTime@entry=0x7f88123ec5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027deff0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027deff0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f88123ec618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027deff0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027deff0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11eb7 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > ( timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027deff0, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000abb594 in operator() (this=0x7f88250f8c40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 operator() (__closure=0x7f88250f8c40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007f8828f86ea5 in start_thread (arg=0x7f88123fe700) at pthread_create.c:307 #23 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 20 (LWP 7573): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=314496536, absSteadyTime=absSteadyTime@entry=0x7f8812bed5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dc0b0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dc0b0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f8812bed618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dc0b0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dc0b0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11eb7 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > ( timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dc0b0, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) ---Type to continue, or q to quit--- at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000abb594 in operator() (this=0x7f88250f8c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 operator() (__closure=0x7f88250f8c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007f8828f86ea5 in start_thread (arg=0x7f8812bff700) at pthread_create.c:307 #23 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 19 (LWP 7572): #0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38 #1 0x0000000000c03e8d in nativeFutexWaitImpl (waitMask=waitMask@entry=416171544, absSteadyTime=absSteadyTime@entry=0x7f8818ce45f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f88027dcbd8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f88027dcbd8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f8818ce4618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000c0d74a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88027dcbd8) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88027dcbd8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000c11eb7 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > ( timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > (this=this@entry=0x7f88027dcbd8, deadline=..., opt=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000c123e6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x1a67f40, deadline=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000c14ea0 in try_wait_for > (timeout=, this=0x1a67f40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x1a67f00, time=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000c0c0f7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f882509a900, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000abb594 in operator() (this=0x7f88250f8b00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #19 operator() (__closure=0x7f88250f8b00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007f8828f86ea5 in start_thread (arg=0x7f8818cf6700) at pthread_create.c:307 #23 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 18 (LWP 7571): #0 0x000000000081137e in enqueue (updateSpinCutoff=, spinCutoff=..., turn=, this=) at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #1 enqueueWithTicketBase (stride=, cap=, slots=, ticket=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/MPMCQueue.h:1308 #2 write (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/MPMCQueue.h:859 #3 VBReadyQueue::pushUnique(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_ready_queue.cc:82 ---Type to continue, or q to quit--- #4 0x00000000007feff0 in DcpProducer::notifyStreamReady (this=0x7f8744ced080, vbucket=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:1895 #5 0x00000000008dcb6a in ActiveStream::notifyStreamReady(bool, DcpProducer*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:2343 #6 0x00000000008dccd0 in ActiveStream::notifySeqnoAvailable (this=, producer=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:1648 #7 0x0000000000803212 in DcpProducer::notifySeqnoAvailable(Vbid, unsigned long, SyncWriteOperation) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:1649 #8 0x00000000007f722c in DcpConnMap::notifyVBConnections(Vbid, unsigned long, SyncWriteOperation) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/dcpconnmap.cc:407 #9 0x000000000073cda8 in notifyNewSeqno (notifyCtx=..., this=0x7f8745656200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:448 #10 VBucket::set(Item&, CookieIface const*, EventuallyPersistentEngine&, std::function const&, cb::vbucket_info)>, Collections::VB::CachingReadHandle const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1708 #11 0x0000000000704ef4 in KVBucket::set(Item&, CookieIface const*, std::function const&, cb::vbucket_info)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #12 0x00000000006c0bef in EventuallyPersistentEngine::storeIfInner(CookieIface const*, Item&, unsigned long, StoreSemantics, std::function const&, cb::vbucket_info)> const&, bool) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #13 0x00000000006d1cad in EventuallyPersistentEngine::store_if(CookieIface const&, ItemIface&, unsigned long, StoreSemantics, std::function const&, cb::vbucket_info)> const&, std::optional const&, DocumentState, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:122 #14 0x0000000000595bf9 in bucket_store_if(Cookie&, gsl::not_null, unsigned long, StoreSemantics, std::function const&, cb::vbucket_info)>, std::optional, DocumentState, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:161 #15 0x000000000064a5cb in MutationCommandContext::storeItem() () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #16 0x000000000064ae70 in MutationCommandContext::step() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/mutation_context.cc:53 #17 0x000000000065e8a1 in SteppableCommandContext::drive() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:36 #18 0x000000000060f67b in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #19 0x000000000060f710 in Cookie::execute (this=this@entry=0x7f8712227380, useStartTime=useStartTime@entry=false) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:168 #20 0x00000000005f0282 in Connection::processNotifiedCookie(Cookie&, cb::engine_errc) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:674 #21 0x00000000005be604 in operator() (__closure=0x7f88005dced0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:167 #22 void folly::detail::function::FunctionTraits::callSmall(folly::detail::function::Data&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:387 #23 0x0000000000c3017a in operator() (this=0x7f88005dced0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #24 messageAvailable (msg=, this=0x7f88255e5d00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:137 #25 folly::NotificationQueue >::Consumer::consumeMessages(bool, unsigned long*) (this=0x7f88255e5d00, isDrain=, numConsumed=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/NotificationQueue.h:729 #26 0x00007f8828258f9a in event_persist_closure (ev=, base=0x7f882508db00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1629 #27 event_process_active_single_queue (base=base@entry=0x7f882508db00, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1688 #28 0x00007f882825953f in event_process_active (base=0x7f882508db00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #29 event_base_loop (base=0x7f882508db00, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #30 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f8825288930, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #31 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f8825288930) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #32 0x0000000000c2cc46 in folly::EventBase::loopForever (this=this@entry=0x7f8825288930) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #33 0x00000000005bd759 in worker_libevent (arg=0x7f8825288920) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 #34 0x0000000000baf899 in run (this=0x7f88250f0840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #35 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #36 0x00007f8828f86ea5 in start_thread (arg=0x7f88194f7700) at pthread_create.c:307 #37 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 17 (LWP 7570): #0 0x00007f88268cf0e3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f88282623b5 in epoll_dispatch (base=0x7f882508d800, tv=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f8828259366 in event_base_loop (base=0x7f882508d800, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f8825287d00, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f8825287d00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000c2cc46 in folly::EventBase::loopForever (this=this@entry=0x7f8825287d00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005bd759 in worker_libevent (arg=0x7f8825287cf0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 #7 0x0000000000baf899 in run (this=0x7f88255b7550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f8828f86ea5 in start_thread (arg=0x7f8819cf8700) at pthread_create.c:307 #10 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 16 (LWP 7569): #0 CheckpointManager::getQueuedItemsMemUsage (this=0x7f873424d8c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:1574 #1 0x00000000007362bc in VBucket::getCMQueuedItemsMemUsage (this=this@entry=0x7f873290b600) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:300 #2 0x0000000000733a0f in VBucketCountVisitor::visitBucket (this=0x7f86f798b240, vb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_count_visitor.cc:37 #3 0x0000000000733cc0 in VBucketStatAggregator::visitBucket (this=, vb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_count_visitor.cc:112 #4 0x00000000007034fa in KVBucket::visit(VBucketVisitor&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2383 #5 0x00000000007142ac in KVBucket::getAggregatedVBucketStats(BucketStatCollector const&, cb::prometheus::Cardinality) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1333 ---Type to continue, or q to quit--- #6 0x00000000006d2710 in EventuallyPersistentEngine::doEngineStatsLowCardinality(BucketStatCollector const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:3190 #7 0x00000000006d493b in doEngineStats (collector=..., this=0x7f8735000000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:5104 #8 EventuallyPersistentEngine::getStats(void const*, std::basic_string_view >, std::basic_string_view >, std::function >, std::basic_string_view >, void const*)> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:5104 #9 0x00000000006d5719 in EventuallyPersistentEngine::get_stats(CookieIface const&, std::basic_string_view >, std::basic_string_view >, std::function >, std::basic_string_view >, void const*)> const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #10 0x0000000000597778 in bucket_get_stats(Cookie&, std::basic_string_view >, cb::sized_buffer, std::function >, std::basic_string_view >, void const*)> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:372 #11 0x00000000006581fa in stat_all_stats(std::__cxx11::basic_string, std::allocator > const&, Cookie&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:472 #12 0x000000000065913f in StatsCommandContext::doStats() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:758 #13 0x000000000065a770 in StatsCommandContext::step() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:668 #14 0x000000000065e8a1 in SteppableCommandContext::drive() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:36 #15 0x000000000060f67b in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #16 0x000000000060f710 in Cookie::execute (this=this@entry=0x7f8709e2cf00, useStartTime=useStartTime@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:168 #17 0x00000000005f0c33 in Connection::executeCommandPipeline() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:472 #18 0x00000000005f0ee2 in Connection::executeCommandsCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:801 #19 0x00000000005f156b in Connection::rw_callback(bufferevent*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:950 #20 0x00007f882824fd7e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f87431edd80) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #21 0x00007f8828258cf1 in event_process_active_single_queue (base=base@entry=0x7f882508d500, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #22 0x00007f882825953f in event_process_active (base=0x7f882508d500) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #23 event_base_loop (base=0x7f882508d500, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #24 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f88252870d0, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #25 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f88252870d0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #26 0x0000000000c2cc46 in folly::EventBase::loopForever (this=this@entry=0x7f88252870d0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #27 0x00000000005bd759 in worker_libevent (arg=0x7f88252870c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 #28 0x0000000000baf899 in run (this=0x7f88250f0870) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #29 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #30 0x00007f8828f86ea5 in start_thread (arg=0x7f881affd700) at pthread_create.c:307 #31 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 15 (LWP 7568): #0 0x00007f88268cf0e3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f88282623b5 in epoll_dispatch (base=0x7f882508d200, tv=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f8828259366 in event_base_loop (base=0x7f882508d200, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f88252864a0, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f88252864a0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000c2cc46 in folly::EventBase::loopForever (this=this@entry=0x7f88252864a0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005bd759 in worker_libevent (arg=0x7f8825286490) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 #7 0x0000000000baf899 in run (this=0x7f88250f08a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f8828f86ea5 in start_thread (arg=0x7f881b7fe700) at pthread_create.c:307 #10 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 14 (LWP 7567): #0 0x00007fffc71e16c2 in ?? () #1 0x00007f87a418fbf8 in ?? () #2 0x0000000000000001 in ?? () #3 0x00007f8739800000 in ?? () #4 0x00007f87346a6600 in ?? () #5 0x00007f881bfeca90 in ?? () #6 0x00007f88268e4a2d in __GI___clock_gettime (clock_id=, tp=) at ../sysdeps/unix/clock_gettime.c:115 #7 0x00007f882717dca1 in std::chrono::_V2::steady_clock::now () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/chrono.cc:88 #8 0x00000000006c083f in ScopeTimer2, std::tuple > (args2=, args1=, this=0x7f881bfec920) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/trace_helpers.h:18 #9 EventuallyPersistentEngine::storeIfInner(CookieIface const*, Item&, unsigned long, StoreSemantics, std::function const&, cb::vbucket_info)> const&, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2603 Backtrace stopped: previous frame identical to this frame (corrupt stack?) Thread 13 (LWP 7566): #0 0x00007f882824f183 in bufferevent_write (bufev=, data=, size=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:454 #1 0x00000000005e3dd3 in Connection::copyToOutputStream(gsl::span > >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:1278 #2 0x00000000006592ba in copyToOutputStream (data3=..., data2=..., data1=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.h:396 ---Type to continue, or q to quit--- #3 append_stats(std::basic_string_view >, std::basic_string_view >, void const*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:100 #4 0x00000000006d712f in operator() (__args#2=, __args#1=..., __args#0=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #5 __invoke_impl, std::basic_string_view, void const*)>&, std::basic_string_view >, std::basic_string_view >, void const*> ( __f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #6 __invoke, std::basic_string_view, void const*)>&, std::basic_string_view >, std::basic_string_view >, void const*> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95 #7 operator() >, std::basic_string_view >, void const*> (this=0x7f8820ce3868) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/refwrap.h:349 #8 operator() >, std::basic_string_view >, void const*> (this=0x7f8820ce3868) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:346 #9 __invoke_impl [with auto:53 = std::reference_wrapper, std::basic_string_view, void const*)> >]::&, std::basic_string_view >, std::basic_string_view >, void const*> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #10 __invoke_r [with auto:53 = std::reference_wrapper, std::basic_string_view, void const*)> >]::&, std::basic_string_view >, std::basic_string_view >, void const*> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #11 _ZNSt17_Function_handlerIFvSt17basic_string_viewIcSt11char_traitsIcEES3_PKvEZNK19makeExitBorderGuardMUlOT_E_clISt17reference_wrapperIKSt8functionIS6_EEEEDaS8_EUlDpOT_E_E9_M_invokeERKSt9_Any_dataOS3_SO_OS5_ (__functor=..., __args#0=, __args#1=, __args#2=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #12 0x0000000000ad7353 in operator() (__args#2=, __args#1=..., __args#0=..., this=0x7f8820ce3868) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #13 CBStatCollector::addStat(cb::stats::StatDef const&, std::basic_string_view >, std::unordered_map >, std::hash > >, std::equal_to > >, std::allocator > > > > const&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/statistics/cbstat_collector.cc:43 #14 0x0000000000ad634c in CBStatCollector::addStat(cb::stats::StatDef const&, unsigned long, std::unordered_map >, std::hash > >, std::equal_to > >, std::allocator > > > > const&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/statistics/cbstat_collector.cc:66 #15 0x00000000006d9789 in addStat&, unsigned long> (k=, v=, k=, v=, this=0x7f8820ce3860) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322 #16 add_casted_stat(std::basic_string_view >, unsigned long&&, std::function >, std::basic_string_view >, void const*)> const&, void const*) (k=..., v=v@entry=, add_stat=..., cookie=0x7f8820ce38c0, cookie@entry=0x7f873cd08c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/statistics/cbstat_collector.h:128 #17 0x00000000006c2019 in EventuallyPersistentEngine::addSeqnoVbStats(CookieIface const*, std::function >, std::basic_string_view >, void const*)> const&, std::shared_ptr const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:357 #18 0x00000000006c6745 in EventuallyPersistentEngine::doSeqnoStats(CookieIface const*, std::function >, std::basic_string_view >, void const*)> const&, char const*, int) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4782 #19 0x00000000006d4ac3 in EventuallyPersistentEngine::getStats(void const*, std::basic_string_view >, std::basic_string_view >, std::function >, std::basic_string_view >, void const*)> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:5147 #20 0x00000000006d5719 in EventuallyPersistentEngine::get_stats(CookieIface const&, std::basic_string_view >, std::basic_string_view >, std::function >, std::basic_string_view >, void const*)> const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #21 0x0000000000597778 in bucket_get_stats(Cookie&, std::basic_string_view >, cb::sized_buffer, std::function >, std::basic_string_view >, void const*)> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:372 #22 0x00000000006590df in StatsCommandContext::doStats() () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322 #23 0x000000000065a770 in StatsCommandContext::step() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:668 #24 0x000000000065e8a1 in SteppableCommandContext::drive() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:36 #25 0x000000000060f67b in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #26 0x000000000060f710 in Cookie::execute (this=this@entry=0x7f873cd08c00, useStartTime=useStartTime@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:168 #27 0x00000000005f0c33 in Connection::executeCommandPipeline() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:472 #28 0x00000000005f0ee2 in Connection::executeCommandsCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:801 #29 0x00000000005f156b in Connection::rw_callback(bufferevent*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:950 #30 0x00007f882824fd7e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f873cd09380) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #31 0x00007f8828258cf1 in event_process_active_single_queue (base=base@entry=0x7f882508cc00, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #32 0x00007f882825953f in event_process_active (base=0x7f882508cc00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #33 event_base_loop (base=0x7f882508cc00, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #34 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f8825284c40, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #35 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f8825284c40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #36 0x0000000000c2cc46 in folly::EventBase::loopForever (this=this@entry=0x7f8825284c40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #37 0x00000000005bd759 in worker_libevent (arg=0x7f8825284c30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 #38 0x0000000000baf899 in run (this=0x7f88255b73d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #39 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #40 0x00007f8828f86ea5 in start_thread (arg=0x7f8820cf6700) at pthread_create.c:307 #41 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 12 (LWP 7565): #0 0x00000000006fb016 in store (__m=std::memory_order_relaxed, __i=0, this=0x7f86b4979c08) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/atomic.h:115 #1 RefCount (value=0, this=0x7f86b4979c08) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/atomic.h:115 #2 RCValue (this=0x7f86b4979c08) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/atomic.h:139 ---Type to continue, or q to quit--- #3 Item::Item(DocKey const&, unsigned int, long, SingleThreadedRCPtr, Blob::Deleter> const&, unsigned char, unsigned long, long, Vbid, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/item.cc:48 #4 0x000000000072d94e in make_unique, Blob::Deleter> const, unsigned char, unsigned long, cb::RelaxedAtomic const&, Vbid&, unsigned long> () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/dockey.h:341 #5 StoredValue::toItemBase(Vbid, StoredValue::HideLockedCas, StoredValue::IncludeValue) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/stored-value.cc:318 #6 0x000000000072da48 in StoredValue::toItem(Vbid, StoredValue::HideLockedCas, StoredValue::IncludeValue, std::optional) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/stored-value.cc:190 #7 0x000000000073ff88 in VBucket::queueDirty(HashTable::HashBucketLock const&, StoredValue&, VBQueueItemCtx const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:448 #8 0x0000000000848066 in EPVBucket::addNewStoredValue (this=0x7f8742faa200, hbl=..., itm=..., queueItmCtx=..., genRevSeqno=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:716 #9 0x000000000073a5ca in VBucket::processSetInner(HashTable::FindUpdateResult&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.h:741 #10 0x000000000073abb4 in VBucket::processSet(HashTable::FindUpdateResult&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:3414 #11 0x000000000073d89f in VBucket::setWithMeta(Item&, unsigned long, unsigned long*, CookieIface const*, EventuallyPersistentEngine&, CheckConflicts, bool, GenerateBySeqno, GenerateCas, Collections::VB::CachingReadHandle const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:2118 #12 0x0000000000706f38 in KVBucket::setWithMeta(Item&, unsigned long, unsigned long*, CookieIface const*, cb::bitset<4ul, vbucket_state_t, PermittedVBStatesMap>, CheckConflicts, bool, GenerateBySeqno, GenerateCas, ExtendedMetaData*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1786 #13 0x0000000000907ec8 in PassiveStream::processMessage(MutationConsumerMessage*, PassiveStream::MessageType) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/response.h:478 #14 0x0000000000908f74 in processMutation (mutation=, this=0x7f874374df10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:752 #15 PassiveStream::messageReceived(std::unique_ptr >, UpdateFlowControl&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:366 #16 0x00000000008fc3bc in DcpConsumer::lookupStreamAndDispatchMessage(UpdateFlowControl&, Vbid, unsigned int, std::unique_ptr >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1911 #17 0x00000000008fc5cc in DcpConsumer::processMutationOrPrepare(Vbid, unsigned int, DocKey const&, SingleThreadedRCPtr >, cb::sized_buffer, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:487 #18 0x00000000008fc757 in DcpConsumer::mutation(unsigned int, DocKey const&, cb::sized_buffer, unsigned long, unsigned char, unsigned long, Vbid, unsigned int, unsigned long, unsigned long, unsigned int, unsigned int, cb::sized_buffer, unsigned char) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/dockey.h:382 #19 0x00000000006b656f in EventuallyPersistentEngine::mutation(CookieIface const&, unsigned int, DocKey const&, cb::sized_buffer, unsigned long, unsigned char, unsigned long, Vbid, unsigned int, unsigned long, unsigned long, unsigned int, unsigned int, cb::sized_buffer, unsigned char) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6724 #20 0x000000000059940d in dcpMutation(Cookie&, unsigned int, DocKey const&, cb::sized_buffer, unsigned long, unsigned char, unsigned long, Vbid, unsigned int, unsigned long, unsigned long, unsigned int, unsigned int, cb::sized_buffer, unsigned char) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:591 #21 0x000000000063c2e9 in do_dcp_mutation (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/dockey.h:341 #22 dcp_mutation_executor(Cookie&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/dcp_mutation.cc:65 #23 0x000000000060f67b in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #24 0x000000000060f710 in Cookie::execute (this=this@entry=0x7f8818262200, useStartTime=useStartTime@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:168 #25 0x00000000005f0c33 in Connection::executeCommandPipeline() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:472 #26 0x00000000005f0ee2 in Connection::executeCommandsCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:801 #27 0x00000000005f156b in Connection::rw_callback(bufferevent*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:950 #28 0x00007f882824fd7e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f8818278080) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #29 0x00007f8828258cf1 in event_process_active_single_queue (base=base@entry=0x7f882508c900, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #30 0x00007f882825953f in event_process_active (base=0x7f882508c900) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #31 event_base_loop (base=0x7f882508c900, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #32 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f8825284010, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #33 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f8825284010) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #34 0x0000000000c2cc46 in folly::EventBase::loopForever (this=this@entry=0x7f8825284010) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #35 0x00000000005bd759 in worker_libevent (arg=0x7f8825284000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 #36 0x0000000000baf899 in run (this=0x7f88255b72b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #37 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #38 0x00007f8828f86ea5 in start_thread (arg=0x7f88214f7700) at pthread_create.c:307 #39 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 11 (LWP 7564): #0 0x00007f88268c3ddd in poll () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000aecc58 in master_thread_run (ctx=0x1a62a00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19291 #2 master_thread (thread_func_param=0x1a62a00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19386 #3 0x00007f8828f86ea5 in start_thread (arg=0x7f88234fb700) at pthread_create.c:307 #4 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 10 (LWP 7563): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000000000af5ceb in consume_socket (thread_index=, sp=0x1a61b98, ctx=0x1a62a00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 worker_thread_run (conn=0x1a612a8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18987 #3 worker_thread (thread_func_param=0x1a612a8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 ---Type to continue, or q to quit--- #4 0x00007f8828f86ea5 in start_thread (arg=0x7f8822cfa700) at pthread_create.c:307 #5 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 9 (LWP 7562): #0 0x00007f8828f88d5d in __GI___pthread_mutex_lock (mutex=0x7f871a3c60a8) at ../nptl/pthread_mutex_lock.c:78 #1 0x00000000009f7aac in __gthread_mutex_lock (__mutex=0x7f871a3c60a8) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #2 lock (this=0x7f871a3c60a8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 #3 lock_guard (__m=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:159 #4 magma::LSMTree::GetStats() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/stats.cc:109 #5 0x000000000099f2ea in magma::KVStore::GetKVStoreStats() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/stats.cc:42 #6 0x000000000097007a in magma::Magma::GetKVStoreStats(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_stats.cc:462 #7 0x00000000008b85b6 in MagmaMemoryTrackingProxy::GetStatsForDbInfo(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:296 #8 0x0000000000897803 in MagmaKVStore::getDbFileInfo(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #9 0x00000000008484d6 in EPVBucket::getHistoryDiskSize() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:448 #10 0x0000000000733bfe in VBucketCountVisitor::visitBucket (this=0x7f870a09ec40, vb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_count_visitor.cc:92 #11 0x0000000000733cc0 in VBucketStatAggregator::visitBucket (this=, vb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_count_visitor.cc:112 #12 0x00000000007034fa in KVBucket::visit(VBucketVisitor&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2383 #13 0x00000000007142ac in KVBucket::getAggregatedVBucketStats(BucketStatCollector const&, cb::prometheus::Cardinality) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1333 #14 0x00000000006d2710 in EventuallyPersistentEngine::doEngineStatsLowCardinality(BucketStatCollector const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:3190 #15 0x00000000006d61e1 in EventuallyPersistentEngine::get_prometheus_stats(BucketStatCollector const&, cb::prometheus::Cardinality) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:399 #16 0x00000000005b7e2d in operator() (bucket=..., __closure=0x7f881a4e3390) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stats.cc:249 #17 __invoke_impl&, Bucket&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #18 __invoke_r&, Bucket&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113 #19 std::_Function_handler::_M_invoke(std::_Any_data const&, Bucket&) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #20 0x00000000005d02a4 in operator() (__args#0=..., this=0x7f881a4e3390) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #21 BucketManager::forEach(std::function) (this=, fn=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:365 #22 0x00000000005b6fbc in server_prometheus_stats(StatCollector const&, cb::prometheus::Cardinality) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #23 0x0000000000ae344a in operator() (__args#1=, __args#0=..., this=0x7f88250f82a0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #24 cb::prometheus::MetricServer::KVCollectable::Collect (this=0x7f88250f8290) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/statistics/prometheus.cc:122 #25 0x0000000000afb74b in prometheus::detail::CollectMetrics (collectables=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/metrics_collector.cc:18 #26 0x0000000000afa788 in prometheus::detail::MetricsHandler::handleGet (this=0x7f88255e4800, conn=0x1a608c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/handler.cc:129 #27 0x0000000000ae8927 in CivetServer::requestHandler (conn=0x1a608c0, cbdata=0x7f88255e4800) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/CivetServer.cpp:152 #28 0x0000000000af56cd in handle_request (conn=conn@entry=0x1a608c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:14657 #29 0x0000000000af6729 in process_new_connection (conn=0x1a608c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18673 #30 worker_thread_run (conn=0x1a608c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19090 #31 worker_thread (thread_func_param=0x1a608c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 #32 0x00007f8828f86ea5 in start_thread (arg=0x7f881a4f9700) at pthread_create.c:307 #33 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 8 (LWP 7551): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x000000000068c217 in __gthread_cond_timedwait (__abs_timeout=0x7f8821ce69b0, __mutex=, __cond=0x7f882511d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f882511d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f882511d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f882511d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 AuditImpl::consume_events() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/auditd/src/audit.cc:414 #6 0x0000000000baf899 in run (this=0x7f88255b6fb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #7 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #8 0x00007f8828f86ea5 in start_thread (arg=0x7f8821cf8700) at pthread_create.c:307 #9 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 7 (LWP 7545): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00000000005ffe06 in __gthread_cond_timedwait (__abs_timeout=0x7f88224e7980, __mutex=0x7f882509f350, __cond=0x7f882509f378) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=, __lock=, __atime=, this=0x7f882509f378) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=, this=0x7f882509f378) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=, __lock=, this=0x7f882509f378) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 ExternalAuthManagerThread::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/external_auth_manager_thread.cc:91 #6 0x0000000000bc7593 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:38 #7 0x0000000000baf899 in run (this=0x7f88255b7010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f8828f86ea5 in start_thread (arg=0x7f88224f9700) at pthread_create.c:307 #10 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 ---Type to continue, or q to quit--- Thread 6 (LWP 7541): #0 0x00007f88268c3ddd in poll () at ../sysdeps/unix/syscall-template.S:81 #1 0x00000000005b8461 in get_command (buffsize=80, buffer=0x7f8823cea990 "") at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:57 #2 check_stdin_thread (arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:98 #3 0x0000000000baf899 in run (this=0x7f88255b6c50) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #4 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #5 0x00007f8828f86ea5 in start_thread (arg=0x7f8823cfc700) at pthread_create.c:307 #6 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 5 (LWP 7540): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000b7678a in __gthread_cond_timedwait (__abs_timeout=0x7f88244eb9b0, __mutex=, __cond=0x7f88255b82d8) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7f88255b82d8) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7f88255b82d8) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __atime=..., __lock=..., this=0x7f88255b82d8) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __rtime=..., __lock=..., this=0x7f88255b82d8) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 operator() (__closure=0x7f88255c2ac8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/periodic_worker-inl.h:25 #7 __invoke_impl&, std::chrono::seconds):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #8 __invoke&, std::chrono::seconds):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 _M_invoke<0> (this=0x7f88255c2ac8) at /usr/local/include/c++/7.3.0/thread:234 #10 operator() (this=0x7f88255c2ac8) at /usr/local/include/c++/7.3.0/thread:243 #11 std::thread::_State_impl&, std::chrono::seconds):: > > >::_M_run(void) (this=0x7f88255c2ac0) at /usr/local/include/c++/7.3.0/thread:186 #12 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #13 0x00007f8828f86ea5 in start_thread (arg=0x7f88244fd700) at pthread_create.c:307 #14 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 4 (LWP 7539): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000b769ca in __gthread_cond_timedwait (__abs_timeout=0x7f8824cec800, __mutex=, __cond=0x7f88255e5038) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7f88255e5038) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7f88255e5038) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > ( __p=..., __atime=..., __lock=..., this=0x7f88255e5038) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __rtime=, __lock=..., this=0x7f88255e5038) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 dequeue_for (wait_duration=..., popped_item=..., this=0x7f88255e5010) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/mpmc_blocking_q.h:58 #7 spdlog::details::thread_pool::process_next_msg_ (this=this@entry=0x7f88255e5010) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:99 #8 0x0000000000b76d18 in worker_loop_ (this=0x7f88255e5010) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:90 #9 operator() (__closure=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:28 #10 __invoke_impl):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #11 __invoke):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #12 _M_invoke<0> (this=) at /usr/local/include/c++/7.3.0/thread:234 #13 operator() (this=) at /usr/local/include/c++/7.3.0/thread:243 #14 std::thread::_State_impl):: > > >::_M_run(void) (this=) at /usr/local/include/c++/7.3.0/thread:186 #15 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #16 0x00007f8828f86ea5 in start_thread (arg=0x7f8824cfe700) at pthread_create.c:307 #17 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 3 (LWP 7534): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f882849b567 in background_thread_sleep (tsdn=, interval=3800000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7f8825dee220) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7f8825dee220) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007f8828f86ea5 in start_thread (arg=0x7f8825dff700) at pthread_create.c:307 #7 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 2 (LWP 7533): #0 0x00007f88268cf0e3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f88282623b5 in epoll_dispatch (base=0x7f882508c300, tv=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 ---Type to continue, or q to quit--- #2 0x00007f8828259366 in event_base_loop (base=0x7f882508c300, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000c2adb7 in folly::EventBase::loopBody (this=this@entry=0x7f882509f500, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000c2b286 in folly::EventBase::loop (this=this@entry=0x7f882509f500) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000c2cc46 in folly::EventBase::loopForever (this=0x7f882509f500) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x0000000000573a8c in memcached_main(int, char**) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1063 #7 0x00007f88267f2555 in __libc_start_main (main=0x546e30 , argc=3, argv=0x7fffc7087488, init=, fini=, rtld_fini=, stack_end=0x7fffc7087478) at ../csu/libc-start.c:266 #8 0x000000000056bab7 in _start () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322 Thread 1 (LWP 7600): #0 0x00007f8826806387 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:55 #1 0x00007f8826807a78 in __GI_abort () at abort.c:90 #2 0x00007f882715163c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95 #3 0x0000000000b53bab in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88 #4 0x00007f882715c8f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48 #5 0x00007f882715c961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58 #6 0x00007f882715cbf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f8788000980, tinfo=0x10a05b0 , dest=0x445400 <_ZNSt11logic_errorD1Ev@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95 #7 0x00000000004c1f61 in CheckpointManager::queueDirty(SingleThreadedRCPtr >&, GenerateBySeqno, GenerateCas, PreLinkDocumentContext*, std::function) (this=0x7f8739098e80, qi=..., generateBySeqno=, generateCas=, preLinkDocumentContext=, assignedSeqnoCallback=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:915 #8 0x000000000073fa09 in VBucket::queueItem(SingleThreadedRCPtr >&, VBQueueItemCtx const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1304 #9 0x000000000074003c in VBucket::queueDirty(HashTable::HashBucketLock const&, StoredValue&, VBQueueItemCtx const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1394 #10 0x0000000000848066 in EPVBucket::addNewStoredValue (this=0x7f87390e1600, hbl=..., itm=..., queueItmCtx=..., genRevSeqno=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:716 #11 0x000000000073a5ca in VBucket::processSetInner(HashTable::FindUpdateResult&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.h:741 #12 0x000000000073abb4 in VBucket::processSet(HashTable::FindUpdateResult&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:3414 #13 0x000000000073e084 in VBucket::prepare(Item&, unsigned long, unsigned long*, CookieIface const*, EventuallyPersistentEngine&, CheckConflicts, bool, GenerateBySeqno, GenerateCas, Collections::VB::CachingReadHandle const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1969 #14 0x000000000070721a in KVBucket::prepare(Item&, CookieIface const*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1832 #15 0x0000000000907cd6 in PassiveStream::processMessage(MutationConsumerMessage*, PassiveStream::MessageType) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/response.h:478 #16 0x00000000009082f8 in PassiveStream::processPrepare(MutationConsumerMessage*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:767 #17 0x000000000090872b in PassiveStream::processBufferedMessages(unsigned int&, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:499 #18 0x00000000008f9372 in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f8738eb1080, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1276 #19 0x00000000008ff937 in DcpConsumer::processBufferedItems() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100 #20 0x0000000000902ee5 in DcpConsumerTask::run (this=0x7f8739428e10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:84 #21 0x0000000000ac21b9 in GlobalTask::execute(std::basic_string_view >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98 #22 0x0000000000abb89a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87a5fe9650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309 #23 0x0000000000ac356e in operator() (this=0x7f87a5fe9650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #24 CancellableCPUExecutor::add(GlobalTask*, folly::Function)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42 #25 0x0000000000c216d0 in operator() (this=0x7f87a5fe9840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #26 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f8825151d00, thread=..., task=task@entry=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #27 0x0000000000c0c16a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8825151d00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #28 0x0000000000c24689 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #29 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #30 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #31 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #32 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #33 0x0000000000abb594 in operator() (this=0x7f88255e6140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #34 operator() (__closure=0x7f88255e6140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49 #35 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #36 0x00007f8827185d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #37 0x00007f8828f86ea5 in start_thread (arg=0x7f87a5ffb700) at pthread_create.c:307 #38 0x00007f88268ceb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb)