GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /opt/couchbase/bin/memcached...Reading symbols from /usr/lib/debug/opt/couchbase/bin/memcached.debug...done. done. [New LWP 124414] [New LWP 124372] [New LWP 124374] [New LWP 124379] [New LWP 124380] [New LWP 124381] [New LWP 124385] [New LWP 124386] [New LWP 124394] [New LWP 124396] [New LWP 124397] [New LWP 124398] [New LWP 124399] [New LWP 124400] [New LWP 124401] [New LWP 124402] [New LWP 124404] [New LWP 124405] [New LWP 124406] [New LWP 124407] [New LWP 124408] [New LWP 124409] [New LWP 124410] [New LWP 124411] [New LWP 124412] [New LWP 124413] [New LWP 124415] [New LWP 124416] [New LWP 124417] [New LWP 124418] [New LWP 124419] [New LWP 124420] [New LWP 124421] [New LWP 124890] [New LWP 124891] [New LWP 124892] [New LWP 124893] [New LWP 124894] [New LWP 124895] [New LWP 124896] [New LWP 124897] [New LWP 124898] [New LWP 124899] [New LWP 124900] [New LWP 124901] [New LWP 124902] [New LWP 124903] [New LWP 124904] [New LWP 124905] [New LWP 124906] [New LWP 124907] [New LWP 124908] [New LWP 124909] [New LWP 124910] [New LWP 124913] To enable execution of this file add add-auto-load-safe-path /usr/lib64/libthread_db-1.0.so line to your configuration file "/root/.gdbinit". To completely disable this security protection add set auto-load safe-path / line to your configuration file "/root/.gdbinit". For more information about this security protection see the "Auto-loading safe path" section in the GDB manual. E.g., run from the shell: info "(gdb)Auto-loading safe path" Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'. #0 __atomic_add (__val=1, __mem=0x20) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:96 Thread 55 (LWP 124913): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007fb047bede51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7fafaa3ee220) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007fb0486e8ea5 in start_thread (arg=0x7fafaa3ff700) at pthread_create.c:307 #6 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #7 0x0000000000000000 in ?? () Thread 54 (LWP 124910): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafb87f8700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 53 (LWP 124909): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c4e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafb8ff9700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 52 (LWP 124908): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c4b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafb97fa700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 51 (LWP 124907): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c480) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafb9ffb700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 50 (LWP 124906): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c450) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafba7fc700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 49 (LWP 124905): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c420) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafbaffd700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 48 (LWP 124904): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c3f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafbb7fe700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 47 (LWP 124903): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c3c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafbbfff700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 46 (LWP 124902): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c390) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc0ff9700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 45 (LWP 124901): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c360) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc17fa700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 44 (LWP 124900): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c330) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc1ffb700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 43 (LWP 124899): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c2d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc27fc700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 42 (LWP 124898): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc2ffd700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 41 (LWP 124897): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb044958f80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc37fe700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 40 (LWP 124896): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c2a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafc3fff700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 39 (LWP 124895): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fb0468bf8bc in __gthread_cond_wait (__mutex=, __cond=) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000009598a8 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #4 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #5 0x0000000000b01e39 in run (this=0x7fb04495c270) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #7 0x00007fb0486e8ea5 in start_thread (arg=0x7fafe8ff9700) at pthread_create.c:307 #8 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #9 0x0000000000000000 in ?? () Thread 38 (LWP 124894): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000959b3f in __gthread_cond_timedwait (__abs_timeout=0x7fafe97e8960, __mutex=, __cond=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b01e39 in run (this=0x7fb04495c210) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007fb0486e8ea5 in start_thread (arg=0x7fafe97fa700) at pthread_create.c:307 #10 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #11 0x0000000000000000 in ?? () Thread 37 (LWP 124893): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000959b3f in __gthread_cond_timedwait (__abs_timeout=0x7fafe9fe9960, __mutex=, __cond=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b01e39 in run (this=0x7fb04495c1e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007fb0486e8ea5 in start_thread (arg=0x7fafe9ffb700) at pthread_create.c:307 #10 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #11 0x0000000000000000 in ?? () Thread 36 (LWP 124892): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000959b3f in __gthread_cond_timedwait (__abs_timeout=0x7fafea7ea960, __mutex=, __cond=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b01e39 in run (this=0x7fb044a36e20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007fb0486e8ea5 in start_thread (arg=0x7fafea7fc700) at pthread_create.c:307 #10 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #11 0x0000000000000000 in ?? () Thread 35 (LWP 124891): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000959b3f in __gthread_cond_timedwait (__abs_timeout=0x7fafeafeb960, __mutex=, __cond=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7fb044978a58) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x0000000000959c91 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b01e39 in run (this=0x7fb0449444a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007fb0486e8ea5 in start_thread (arg=0x7fafeaffd700) at pthread_create.c:307 #10 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #11 0x0000000000000000 in ?? () Thread 34 (LWP 124890): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007fb047bede51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7fafeb7ed220) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007fb0486e8ea5 in start_thread (arg=0x7fafeb7fe700) at pthread_create.c:307 #6 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #7 0x0000000000000000 in ?? () Thread 33 (LWP 124421): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 32 (LWP 124420): #0 0x0000000000873e93 in ActiveStream::getOutstandingItems(VBucket&) () at /opt/gcc-10.2.0/include/c++/10.2.0/atomic:104 #1 0x0000000000879086 in ActiveStream::nextCheckpointItemTask(std::lock_guard const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:962 #2 0x0000000000879124 in ActiveStream::nextCheckpointItemTask() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:947 #3 0x000000000087f939 in ActiveStreamCheckpointProcessorTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream_checkpoint_processor_task.cc:52 #4 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #5 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb008fe7840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #6 0x0000000000b649a0 in operator() (this=0x7fb008fe7840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #7 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb04494dc00, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #8 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb04494dc00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #9 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x0000000000a19ce4 in operator() (this=0x7fb044de1540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #15 operator() (__closure=0x7fb044de1540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007fb0486e8ea5 in start_thread (arg=0x7fb008ff9700) at pthread_create.c:307 #19 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #20 0x0000000000000000 in ?? () Thread 31 (LWP 124419): #0 0x00000000007cef3e in incrementAge (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/blob.h:88 #1 DefragmentVisitor::visit(HashTable::HashBucketLock const&, StoredValue&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/defragmenter_visitor.cc:59 #2 0x00000000006cd74d in HashTable::pauseResumeVisit(HashTableVisitor&, HashTable::Position&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.cc:1242 #3 0x000000000070b02b in PauseResumeVBAdapter::visit (this=0x7faf8f6148f0, vb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_visitors.cc:51 #4 0x00000000006dc6ea in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2297 #5 0x00000000007cdbb5 in DefragmenterTask::defrag() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.h:636 #6 0x00000000007cece8 in DefragmenterTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/defragmenter.cc:56 #7 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #8 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb0097e8840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #9 0x0000000000b649a0 in operator() (this=0x7fb0097e8840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #10 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb04494dc00, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #11 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb04494dc00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #12 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000a19ce4 in operator() (this=0x7fb044de1500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #18 operator() (__closure=0x7fb044de1500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #21 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0097fa700) at pthread_create.c:307 #22 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #23 0x0000000000000000 in ?? () Thread 30 (LWP 124418): #0 0x00007fb046007bf9 in closelog_internal () at ../misc/syslog.c:433 #1 closelog () at ../misc/syslog.c:445 #2 0x00000000ffffffff in ?? () #3 0x0000000000b4448d in nativeFutexWaitImpl (waitMask=waitMask@entry=167679512, absSteadyTime=absSteadyTime@entry=0x7fb009fe95f8, absSystemTime=absSystemTime@entry=0x0, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:135 #4 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7fb009fe9618, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #5 0x0000000000b4e54a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #6 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #7 0x0000000000b543ad in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fb02a7de4f8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #8 0x0000000000b54a16 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > ( this=0x1a7de40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 0x0000000000b58170 in folly::UnboundedBlockingQueue::try_take_for (this=0x1a7de00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 0x0000000000b4c6e7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb04494d000, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #11 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #12 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #13 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #14 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #15 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #16 0x0000000000a19ce4 in operator() (this=0x7fb044de1400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #17 operator() (__closure=0x7fb044de1400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #18 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #19 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #20 0x00007fb0486e8ea5 in start_thread (arg=0x7fb009ffb700) at pthread_create.c:307 #21 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #22 0x0000000000000000 in ?? () Thread 29 (LWP 124417): #0 0x00007fb046004bdd in fsync () at ../sysdeps/unix/syscall-template.S:81 #1 0x000000000094a50f in magma::BasicFile::Sync() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:191 #2 0x00000000009a7ba0 in magma::FileWithStats::Sync() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:60 #3 0x000000000094c466 in magma::BufferedFileWriter::flush(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:385 #4 0x000000000094c5fd in magma::BufferedFileWriter::Write(char const*, unsigned long, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:423 #5 0x00000000009b2a38 in magma::SSTableWriter::writeFooter(magma::BlockPointer const&, magma::BlockPointer const&, magma::BlockPointer const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable_writer.cc:245 #6 0x00000000009b548f in magma::SSTableWriter::GenerateTable() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable_writer.cc:367 #7 0x00000000009976d9 in magma::LSMTree::mergeSSTables(std::vector, std::allocator > >&, magma::ItemGCFilter&, unsigned long, double, std::vector, std::allocator > >*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:709 #8 0x000000000096c417 in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock&, std::vector, std::allocator > >&, std::vector, std::allocator > >&, bool, std::function > (bool)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/compaction.cc:172 #9 0x000000000097015e in magma::LSMTree::CompactRange(magma::Slice const&, magma::Slice const&, std::function > (bool)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #10 0x000000000092ce20 in magma::KVStore::CompactRange(magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #11 0x00000000008ddfc6 in magma::Magma::Impl::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:797 #12 0x00000000008de0dd in magma::Magma::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #13 0x000000000085629e in MagmaMemoryTrackingProxy::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #14 0x000000000084c9d2 in MagmaKVStore::compactDBInternal(std::unique_lock&, std::shared_ptr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #15 0x000000000084da06 in MagmaKVStore::compactDB(std::unique_lock&, std::shared_ptr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:2032 #16 0x00000000007e9ada in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2599 #17 0x00000000007eb0f1 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1359 #18 0x00000000007089c6 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:73 #19 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #20 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb00a7ea840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #21 0x0000000000b649a0 in operator() (this=0x7fb00a7ea840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #22 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044899400, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #23 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044899400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #24 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #25 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #26 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #27 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #28 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #29 0x0000000000a19ce4 in operator() (this=0x7fb044de1340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #30 operator() (__closure=0x7fb044de1340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #31 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #32 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #33 0x00007fb0486e8ea5 in start_thread (arg=0x7fb00a7fc700) at pthread_create.c:307 #34 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #35 0x0000000000000000 in ?? () Thread 28 (LWP 124416): #0 fetch_and (__m=std::memory_order_seq_cst, __i=254 '\376', this=0x7fafa990ebb3) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/stored-value.h:206 #1 reset (order=std::memory_order_seq_cst, idx=0, this=0x7fafa990ebb3) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/3rd_party/folly/AtomicBitSet.h:146 #2 set (value=false, order=std::memory_order_seq_cst, idx=0, this=0x7fafa990ebb3) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/3rd_party/folly/AtomicBitSet.h:153 #3 set (order=std::memory_order_seq_cst, value=false, idx=0, this=0x7fafa990ebb3) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/3rd_party/folly/AtomicBitSet.h:150 #4 markClean (this=0x7fafa990eb80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/stored-value.h:206 #5 EPPersistenceCallback::operator()(Item const&, FlushStateMutation) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/persistence_callback.cc:55 #6 0x000000000083a1b9 in MagmaKVStore::commitCallback(MagmaKVStoreTransactionContext&, int, kvstats_ctx&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/kvstore_transaction_context.h:39 #7 0x000000000083a622 in MagmaKVStore::commit(std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:661 #8 0x00000000007e8045 in EPBucket::commit(KVStoreIface&, std::unique_ptr >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:929 #9 0x00000000007eefd5 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:801 #10 0x00000000007ef59f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:378 #11 0x00000000006c80b0 in Flusher::flushVB (this=0x7fb0382c0d00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:285 #12 0x00000000006c8a00 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:200 #13 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #14 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb00afeb840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #15 0x0000000000b649a0 in operator() (this=0x7fb00afeb840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #16 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044899400, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #17 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044899400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #18 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #19 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #20 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #21 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #22 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #23 0x0000000000a19ce4 in operator() (this=0x7fb044de1300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #24 operator() (__closure=0x7fb044de1300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #25 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #26 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #27 0x00007fb0486e8ea5 in start_thread (arg=0x7fb00affd700) at pthread_create.c:307 #28 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #29 0x0000000000000000 in ?? () Thread 27 (LWP 124415): #0 __normal_iterator (__i=@0x7faf9d5d7350: , this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:822 #1 end (this=0x7faf9d5d7348) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:830 #2 back (this=0x7faf9d5d7348) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:1146 #3 magma::SSTableIterator::GetKey (this=0x7faf9d5d7280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:822 #4 0x0000000000973bf6 in magma::KVReader::ReadKVs (this=0x7fb00b7eb480) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:53 #5 0x0000000000997583 in magma::LSMTree::mergeSSTables(std::vector, std::allocator > >&, magma::ItemGCFilter&, unsigned long, double, std::vector, std::allocator > >*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:741 #6 0x000000000096c417 in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock&, std::vector, std::allocator > >&, std::vector, std::allocator > >&, bool, std::function > (bool)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/compaction.cc:172 #7 0x000000000097015e in magma::LSMTree::CompactRange(magma::Slice const&, magma::Slice const&, std::function > (bool)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #8 0x000000000092ce20 in magma::KVStore::CompactRange(magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #9 0x00000000008ddfc6 in magma::Magma::Impl::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:797 #10 0x00000000008de0dd in magma::Magma::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #11 0x000000000085629e in MagmaMemoryTrackingProxy::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #12 0x000000000084c9d2 in MagmaKVStore::compactDBInternal(std::unique_lock&, std::shared_ptr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #13 0x000000000084da06 in MagmaKVStore::compactDB(std::unique_lock&, std::shared_ptr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:2032 #14 0x00000000007e9ada in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2599 #15 0x00000000007eb0f1 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1359 #16 0x00000000007089c6 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:73 #17 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #18 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb00b7ec840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #19 0x0000000000b649a0 in operator() (this=0x7fb00b7ec840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #20 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044899400, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #21 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044899400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #22 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #23 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #24 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #25 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #26 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #27 0x0000000000a19ce4 in operator() (this=0x7fb044de12c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #28 operator() (__closure=0x7fb044de12c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #29 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #30 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #31 0x00007fb0486e8ea5 in start_thread (arg=0x7fb00b7fe700) at pthread_create.c:307 #32 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #33 0x0000000000000000 in ?? () Thread 26 (LWP 124413): #0 0x00007fb047c1f399 in je_extent_heap_remove_first (ph=ph@entry=0x7fafb0008530) at src/extent.c:283 #1 0x00007fb047be49e4 in arena_bin_slabs_nonfull_tryget (bin=0x7fafb00084b8) at src/arena.c:1031 #2 arena_bin_nonfull_slab_get (binshard=0, binind=8, bin=0x7fafb00084b8, arena=, tsdn=0x7fb028fc6220) at src/arena.c:1272 #3 arena_bin_malloc_hard (tsdn=tsdn@entry=0x7fb028fc6220, arena=arena@entry=0x7fafb00008c0, bin=bin@entry=0x7fafb00084b8, binind=binind@entry=8, binshard=binshard@entry=0) at src/arena.c:1319 #4 0x00007fb047be79e2 in je_arena_tcache_fill_small (tsdn=, arena=, tcache=, tbin=, binind=, prof_accumbytes=) at src/arena.c:1407 #5 0x00007fb047c56f45 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7fb028fc6220, arena=, tcache=tcache@entry=0x7faf8c178000, tbin=tbin@entry=0x7faf8c1780d0, binind=, tcache_success=tcache_success@entry=0x7fb028fc4c10) at src/tcache.c:94 #6 0x00007fb047bdba0b in tcache_alloc_small (slow_path=false, zero=, binind=, size=128, tcache=0x7faf8c178000, arena=, tsd=) at include/jemalloc/internal/tcache_inlines.h:60 #7 arena_malloc (slow_path=false, tcache=0x7faf8c178000, zero=, ind=, size=128, arena=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:165 #8 iallocztm (slow_path=false, arena=, is_internal=false, tcache=0x7faf8c178000, zero=, ind=, size=128, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:53 #9 imalloc_no_sample (ind=, usize=128, size=128, tsd=0x7fb028fc6220, dopts=, sopts=) at src/jemalloc.c:1949 #10 imalloc_body (tsd=0x7fb028fc6220, dopts=, sopts=) at src/jemalloc.c:2123 #11 imalloc (dopts=, sopts=) at src/jemalloc.c:2258 #12 je_mallocx (size=size@entry=120, flags=) at src/jemalloc.c:3137 #13 0x0000000000b1216d in cb::_JEArenaMalloc::malloc (size=size@entry=120) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:266 #14 0x00000000005af02f in malloc (size=120) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39 #15 cb_malloc (size=120) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39 #16 0x0000000000b07209 in operator new(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:63 #17 0x000000000083a89a in MagmaKVStore::makeItem(Vbid, magma::Slice const&, magma::Slice const&, magma::Slice const&, ValueFilter) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/kv_magma_common/magma-kvstore_metadata.h:102 #18 0x000000000083b2d0 in MagmaKVStore::makeGetValue(Vbid, magma::Slice const&, magma::Slice const&, magma::Slice const&, ValueFilter) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1101 #19 0x000000000083f6c1 in MagmaKVStore::getMulti(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) const::{lambda(bool, magma::Status, magma::Magma::GetOperation const&, magma::Slice const&, magma::Slice const&)#1}::operator()(bool, magma::Status, magma::Magma::GetOperation const&, magma::Slice const&, magma::Slice const&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:887 #20 0x000000000083fc25 in std::_Function_handler, std::equal_to, std::allocator > >&) const::{lambda(bool, magma::Status, magma::Magma::GetOperation const&, magma::Slice const&, magma::Slice const&)#1}>::_M_invoke(std::_Any_data const&, bool&&, magma::Status&&, magma::Magma::GetOperation const&, magma::Slice const&, magma::Slice const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:88 #21 0x00000000008dce6c in magma::Magma::Impl::GetDocs(magma::Magma::Snapshot&, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #22 0x00000000008dd1d0 in magma::Magma::GetDocs(magma::Magma::Snapshot&, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #23 0x00000000008dd345 in magma::Magma::GetDocs(unsigned short, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #24 0x00000000008564c3 in MagmaMemoryTrackingProxy::GetDocs(unsigned short, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #25 0x00000000008445d1 in MagmaKVStore::getMulti(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #26 0x0000000000795615 in BgFetcher::doFetch(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:78 #27 0x0000000000795bc0 in BgFetcher::run(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/bgfetcher.cc:151 #28 0x0000000000708add in MultiBGFetcherTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:145 #29 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #30 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb028fc5840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #31 0x0000000000b649a0 in operator() (this=0x7fb028fc5840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #32 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044898800, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #33 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044898800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #34 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #35 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #36 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #37 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #38 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #39 0x0000000000a19ce4 in operator() (this=0x7fb044de1140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #40 operator() (__closure=0x7fb044de1140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #41 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #42 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #43 0x00007fb0486e8ea5 in start_thread (arg=0x7fb028fd7700) at pthread_create.c:307 #44 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #45 0x0000000000000000 in ?? () Thread 25 (LWP 124412): #0 0x00007fb046007bf9 in closelog_internal () at ../misc/syslog.c:433 #1 closelog () at ../misc/syslog.c:445 #2 0x00000000ffffffff in ?? () #3 0x9ddfea08eb382d69 in ?? () #4 0x0000000000135ad1 in ?? () #5 0x000000000e9a82a4 in ?? () #6 0x0000000000000000 in ?? () Thread 24 (LWP 124411): #0 0x00007fb046007bf9 in closelog_internal () at ../misc/syslog.c:433 #1 closelog () at ../misc/syslog.c:445 #2 0x00000000ffffffff in ?? () #3 0x9ddfea08eb382d69 in ?? () #4 0x0000000000135ad1 in ?? () #5 0x000000001fee73e0 in ?? () #6 0x00048199e0864d5b in ?? () #7 0x0000000000b4e54a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #8 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #9 0x0000000000b544e7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fb02a7de6c0, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #10 0x0000000000b54a16 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > ( this=0x1a7bac0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #11 0x0000000000b58170 in folly::UnboundedBlockingQueue::try_take_for (this=0x1a7ba80, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #12 0x0000000000b4c6e7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044898800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a19ce4 in operator() (this=0x7fb044de1040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7fb044de1040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007fb0486e8ea5 in start_thread (arg=0x7fb029fd9700) at pthread_create.c:307 #23 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #24 0x0000000000000000 in ?? () Thread 23 (LWP 124410): #0 magma::LSMTree::Get(magma::Slice const&, std::shared_ptr&, magma::LookupBuffer&, magma::Slice&, bool&, bool, magma::ReaderStats*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:975 #1 0x0000000000932450 in magma::KVStore::Read(magma::Slice const&, std::shared_ptr&, magma::KVStore::ReadBuffer&, magma::Slice&, magma::Slice&, bool&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1131 #2 0x00000000008dce17 in magma::Magma::Impl::GetDocs(magma::Magma::Snapshot&, magma::Operations&, std::function) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:443 #3 0x00000000008dd1d0 in magma::Magma::GetDocs(magma::Magma::Snapshot&, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #4 0x00000000008dd345 in magma::Magma::GetDocs(unsigned short, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #5 0x00000000008564c3 in MagmaMemoryTrackingProxy::GetDocs(unsigned short, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000008445d1 in MagmaKVStore::getMulti(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #7 0x0000000000795615 in BgFetcher::doFetch(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:78 #8 0x0000000000795bc0 in BgFetcher::run(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/bgfetcher.cc:151 #9 0x0000000000708add in MultiBGFetcherTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:145 #10 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #11 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb02a7c8840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #12 0x0000000000b649a0 in operator() (this=0x7fb02a7c8840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #13 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044898800, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #14 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044898800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #15 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #16 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #17 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #18 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #19 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #20 0x0000000000a19ce4 in operator() (this=0x7fb044de10c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #21 operator() (__closure=0x7fb044de10c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #22 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #23 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #24 0x00007fb0486e8ea5 in start_thread (arg=0x7fb02a7da700) at pthread_create.c:307 #25 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #26 0x0000000000000000 in ?? () Thread 22 (LWP 124409): #0 0x00007fb0468abe37 in std::__detail::_List_node_base::_M_transfer (this=0x7fb0382cb068, __first=0x7fb03852bfe0, __last=0x7fb028531a60) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++98/list.cc:102 #1 0x000000000095503a in magma::Cache::AcquireObject (this=0x7fb0382cb040, id=0x7faf90e3d000, doCaching=true) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100 #2 0x000000000094f5d5 in magma::ManagedFile::AcquireHandle() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file_manager.cc:276 #3 0x00000000009ab4ee in magma::SSTableIterator::initFileHandle() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:698 #4 0x00000000009af48f in magma::SSTableIterator::initStacks(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:704 #5 0x00000000009aff11 in magma::SSTableIterator::Seek (this=0x7faf9e950e00, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:786 #6 0x000000000098b9cc in magma::LSMTree::Get(magma::Slice const&, std::shared_ptr&, magma::LookupBuffer&, magma::Slice&, bool&, bool, magma::ReaderStats*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1040 #7 0x0000000000932450 in magma::KVStore::Read(magma::Slice const&, std::shared_ptr&, magma::KVStore::ReadBuffer&, magma::Slice&, magma::Slice&, bool&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1131 #8 0x00000000008dce17 in magma::Magma::Impl::GetDocs(magma::Magma::Snapshot&, magma::Operations&, std::function) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:443 #9 0x00000000008dd1d0 in magma::Magma::GetDocs(magma::Magma::Snapshot&, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #10 0x00000000008dd345 in magma::Magma::GetDocs(unsigned short, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #11 0x00000000008564c3 in MagmaMemoryTrackingProxy::GetDocs(unsigned short, magma::Operations&, std::function) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #12 0x00000000008445d1 in MagmaKVStore::getMulti(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #13 0x0000000000795615 in BgFetcher::doFetch(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:78 #14 0x0000000000795bc0 in BgFetcher::run(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/bgfetcher.cc:151 #15 0x0000000000708add in MultiBGFetcherTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:145 #16 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #17 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb030dea840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #18 0x0000000000b649a0 in operator() (this=0x7fb030dea840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #19 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044898800, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #20 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044898800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #21 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #22 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #23 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #24 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #25 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #26 0x0000000000a19ce4 in operator() (this=0x7fb044de1080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #27 operator() (__closure=0x7fb044de1080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #28 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #29 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #30 0x00007fb0486e8ea5 in start_thread (arg=0x7fb030dfc700) at pthread_create.c:307 #31 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #32 0x0000000000000000 in ?? () Thread 21 (LWP 124408): #0 0x00007fb046007bf9 in closelog_internal () at ../misc/syslog.c:433 #1 closelog () at ../misc/syslog.c:445 #2 0x00000000ffffffff in ?? () #3 0x9ddfea08eb382d69 in ?? () #4 0x0000000000135ad1 in ?? () #5 0x000000001efaa657 in ?? () #6 0x000481997de89710 in ?? () #7 0x0000000000b4e54a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #8 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #9 0x0000000000b544e7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fb02a7df020, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #10 0x0000000000b54a16 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > ( this=0x1a7bac0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #11 0x0000000000b58170 in folly::UnboundedBlockingQueue::try_take_for (this=0x1a7ba80, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #12 0x0000000000b4c6e7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044898800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a19ce4 in operator() (this=0x7fb044de1000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7fb044de1000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #22 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0315fd700) at pthread_create.c:307 #23 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #24 0x0000000000000000 in ?? () Thread 20 (LWP 124407): #0 0x00007ffd41be36c2 in ?? () #1 0x0000000000000001 in ?? () #2 0x00007fb031dec840 in ?? () #3 0x00007fb031dec5e0 in ?? () #4 0x00007faf9ea00848 in ?? () #5 0x00007fb031dec730 in ?? () #6 0x00007fb0460237fd in __get_clockfreq () at ../sysdeps/unix/sysv/linux/i386/get_clockfreq.c:82 #7 0x00007fb044b02800 in ?? () #8 0x0000000000000000 in ?? () Thread 19 (LWP 124406): #0 0x00007fb047be85f2 in bitmap_unset (binfo=, bit=, bitmap=0x7fb045c8c3c0) at include/jemalloc/internal/bitmap.h:344 #1 arena_slab_reg_dalloc (ptr=, slab_data=0x7fb045c8c3c0, slab=0x7fb045c8c380) at src/arena.c:368 #2 arena_dalloc_bin_locked_impl (junked=true, ptr=, slab=0x7fb045c8c380, binind=4, bin=0x7fb045c08118, arena=0x7fb045c00980, tsdn=0x7fb0325ee220, tsdn@entry=) at src/arena.c:1695 #3 je_arena_dalloc_bin_junked_locked (tsdn=tsdn@entry=0x7fb0325ee220, arena=arena@entry=0x7fb045c00980, bin=bin@entry=0x7fb045c08118, binind=binind@entry=4, extent=0x7fb045c8c380, ptr=) at src/arena.c:1714 #4 0x00007fb047c575fd in je_tcache_bin_flush_small (tsd=tsd@entry=0x7fb0325ee220, tcache=tcache@entry=0x7fb0325ee410, tbin=0x7fb0325ee480, binind=, rem=) at src/tcache.c:189 #5 0x00007fb047be1f56 in tcache_dalloc_small (slow_path=false, binind=, ptr=0x7fafa5986200, tcache=0x7fb0325ee410, tsd=) at include/jemalloc/internal/tcache_inlines.h:179 #6 arena_sdalloc (slow_path=false, alloc_ctx=, tcache=0x7fb0325ee410, size=, ptr=0x7fafa5986200, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:420 #7 isdalloct (slow_path=false, alloc_ctx=, tcache=0x7fb0325ee410, size=, ptr=0x7fafa5986200, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:131 #8 isfree (slow_path=false, tcache=0x7fb0325ee410, usize=, ptr=0x7fafa5986200, tsd=0x7fb0325ee220) at src/jemalloc.c:2644 #9 je_sdallocx_default (ptr=0x7fafa5986200, size=, flags=) at src/jemalloc.c:3583 #10 0x0000000000b123c6 in cb::_JEArenaMalloc::sized_free (ptr=ptr@entry=0x7fafa5986200, size=size@entry=64) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:324 #11 0x00000000005af196 in sized_free (size=64, ptr=0x7fafa5986200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:75 #12 cb_sized_free (ptr=ptr@entry=0x7fafa5986200, size=64) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:75 #13 0x0000000000b07295 in operator delete (ptr=ptr@entry=0x7fafa5986200, size=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:103 #14 0x0000000000a66b5b in deallocate (__t=, __p=0x7fafa5986200, this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_uninitialized.h:1013 #15 deallocate (__n=, __p=0x7fafa5986200, __a=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:492 #16 _M_deallocate (__n=, __p=0x7fafa5986200, this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:354 #17 _M_realloc_insert > > const&> (__position=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/vector.tcc:500 #18 emplace_back > > const&> (this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/vector.tcc:121 #19 operator() > (spans=..., __closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/tracing/tracer.cc:23 #20 withLock > (function=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:443 #21 cb::tracing::Tracer::begin(cb::tracing::Code, std::chrono::time_point > >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/tracing/tracer.cc:22 #22 0x00000000007f4b65 in EPVBucket::completeBGFetchForSingleItem(DiskDocKey const&, FrontEndBGFetchItem const&, std::chrono::time_point > >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:215 #23 0x000000000080cace in FrontEndBGFetchItem::complete (this=0x7faf9cca7bc0, engine=..., vb=..., startTime=..., key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket_bgfetch_item.cc:38 #24 0x00000000006e4747 in KVBucket::completeBGFetchMulti(Vbid, std::vector, std::allocator > >&, std::chrono::time_point > >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1446 #25 0x00000000007957ec in BgFetcher::doFetch(Vbid, std::unordered_map, std::equal_to, std::allocator > >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/bgfetcher.cc:104 #26 0x0000000000795bc0 in BgFetcher::run(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/bgfetcher.cc:151 #27 0x0000000000708add in MultiBGFetcherTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:145 #28 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #29 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb0325ed840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #30 0x0000000000b649a0 in operator() (this=0x7fb0325ed840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #31 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044898800, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #32 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044898800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #33 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #34 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #35 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #36 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #37 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #38 0x0000000000a19ce4 in operator() (this=0x7fb0448d5ec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #39 operator() (__closure=0x7fb0448d5ec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #40 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #41 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #42 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0325ff700) at pthread_create.c:307 #43 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #44 0x0000000000000000 in ?? () Thread 18 (LWP 124405): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 17 (LWP 124404): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 16 (LWP 124402): #0 0x00007fb047c1f399 in je_extent_heap_remove_first (ph=ph@entry=0x7fafb0208530) at src/extent.c:283 #1 0x00007fb047be49e4 in arena_bin_slabs_nonfull_tryget (bin=0x7fafb02084b8) at src/arena.c:1031 #2 arena_bin_nonfull_slab_get (binshard=0, binind=8, bin=0x7fafb02084b8, arena=, tsdn=0x7fb039fea220) at src/arena.c:1272 #3 arena_bin_malloc_hard (tsdn=tsdn@entry=0x7fb039fea220, arena=arena@entry=0x7fafb02008c0, bin=bin@entry=0x7fafb02084b8, binind=binind@entry=8, binshard=binshard@entry=0) at src/arena.c:1319 #4 0x00007fb047be79e2 in je_arena_tcache_fill_small (tsdn=, arena=, tcache=, tbin=, binind=, prof_accumbytes=) at src/arena.c:1407 #5 0x00007fb047c56f45 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7fb039fea220, arena=, tcache=tcache@entry=0x7fafa59c1000, tbin=tbin@entry=0x7fafa59c10d0, binind=, tcache_success=tcache_success@entry=0x7fb039fe8230) at src/tcache.c:94 #6 0x00007fb047bdba0b in tcache_alloc_small (slow_path=false, zero=, binind=, size=128, tcache=0x7fafa59c1000, arena=, tsd=) at include/jemalloc/internal/tcache_inlines.h:60 #7 arena_malloc (slow_path=false, tcache=0x7fafa59c1000, zero=, ind=, size=128, arena=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:165 #8 iallocztm (slow_path=false, arena=, is_internal=false, tcache=0x7fafa59c1000, zero=, ind=, size=128, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:53 #9 imalloc_no_sample (ind=, usize=128, size=128, tsd=0x7fb039fea220, dopts=, sopts=) at src/jemalloc.c:1949 #10 imalloc_body (tsd=0x7fb039fea220, dopts=, sopts=) at src/jemalloc.c:2123 #11 imalloc (dopts=, sopts=) at src/jemalloc.c:2258 #12 je_mallocx (size=size@entry=120, flags=) at src/jemalloc.c:3137 #13 0x0000000000b1216d in cb::_JEArenaMalloc::malloc (size=size@entry=120) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:266 #14 0x00000000005af02f in malloc (size=120) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39 #15 cb_malloc (size=120) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39 #16 0x0000000000b07209 in operator new(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:63 #17 0x0000000000703975 in StoredValue::toItemBase(Vbid, StoredValue::HideLockedCas, StoredValue::IncludeValue) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/dockey.h:341 #18 0x0000000000703ab8 in StoredValue::toItem(Vbid, StoredValue::HideLockedCas, StoredValue::IncludeValue, std::optional) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/stored-value.cc:190 #19 0x0000000000713d68 in VBucket::queueDirty(HashTable::HashBucketLock const&, StoredValue&, VBQueueItemCtx const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:406 #20 0x00000000007f2c16 in EPVBucket::addNewStoredValue (this=0x7fafa4b4f700, hbl=..., itm=..., queueItmCtx=..., genRevSeqno=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:656 #21 0x0000000000711c9a in VBucket::processSetInner(HashTable::FindUpdateResult&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.h:772 #22 0x0000000000712221 in VBucket::processSet(HashTable::FindUpdateResult&, StoredValue*&, Item&, unsigned long, bool, bool, VBQueueItemCtx const&, cb::StoreIfStatus, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:3268 #23 0x000000000071c648 in VBucket::prepare(Item&, unsigned long, unsigned long*, CookieIface const*, EventuallyPersistentEngine&, CheckConflicts, bool, GenerateBySeqno, GenerateCas, Collections::VB::CachingReadHandle const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1895 #24 0x00000000006dfd51 in KVBucket::prepare(Item&, CookieIface const*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1713 #25 0x000000000089713d in PassiveStream::processMessage(MutationConsumerMessage*, PassiveStream::MessageType) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/response.h:443 #26 0x00000000008976f8 in PassiveStream::processPrepare(MutationConsumerMessage*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:733 #27 0x00000000008982bc in PassiveStream::messageReceived(std::unique_ptr >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:343 #28 0x000000000088c335 in DcpConsumer::lookupStreamAndDispatchMessage(DcpConsumer::UpdateFlowControl&, Vbid, unsigned int, std::unique_ptr >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1839 #29 0x000000000088c57c in DcpConsumer::processMutationOrPrepare(Vbid, unsigned int, DocKey const&, SingleThreadedRCPtr >, cb::sized_buffer, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:462 #30 0x000000000088f403 in DcpConsumer::prepare(unsigned int, DocKey const&, cb::sized_buffer, unsigned long, unsigned char, unsigned long, Vbid, unsigned int, unsigned long, unsigned long, unsigned int, unsigned int, unsigned char, DocumentState, cb::durability::Level) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1816 #31 0x0000000000692834 in EventuallyPersistentEngine::prepare(CookieIface const&, unsigned int, DocKey const&, cb::sized_buffer, unsigned long, unsigned char, unsigned long, Vbid, unsigned int, unsigned long, unsigned long, unsigned int, unsigned int, unsigned char, DocumentState, cb::durability::Level) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6406 #32 0x000000000057f767 in dcpPrepare(Cookie&, unsigned int, DocKey const&, cb::sized_buffer, unsigned long, unsigned char, unsigned long, Vbid, unsigned int, unsigned long, unsigned long, unsigned int, unsigned int, unsigned char, DocumentState, cb::durability::Level) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:786 #33 0x000000000061dc42 in dcp_prepare_executor(Cookie&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:86 #34 0x00000000005efd9b in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #35 0x00000000005f10e0 in Cookie::execute (this=0x7fb02877e600, useStartTime=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:169 #36 0x00000000005cf483 in Connection::executeCommandPipeline() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:469 #37 0x00000000005cf723 in Connection::executeCommandsCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:806 #38 0x00000000005cfdab in Connection::rw_callback(bufferevent*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:955 #39 0x00007fb0479a2d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7fb02877e880) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #40 0x00007fb0479abd01 in event_process_active_single_queue (base=base@entry=0x7fb044890500, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #41 0x00007fb0479ac54f in event_process_active (base=0x7fb044890500) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #42 event_base_loop (base=0x7fb044890500, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #43 0x0000000000b6e087 in folly::EventBase::loopBody (this=this@entry=0x7fb044a80f50, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #44 0x0000000000b6e556 in folly::EventBase::loop (this=this@entry=0x7fb044a80f50) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #45 0x0000000000b6ff16 in folly::EventBase::loopForever (this=this@entry=0x7fb044a80f50) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #46 0x000000000059fa59 in worker_libevent (arg=0x7fb044a80f40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #47 0x0000000000b01e39 in run (this=0x7fb044dbb650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #48 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #49 0x00007fb0486e8ea5 in start_thread (arg=0x7fb039ffb700) at pthread_create.c:307 #50 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #51 0x0000000000000000 in ?? () Thread 15 (LWP 124401): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 14 (LWP 124400): #0 HashTable::Statistics::epilogue(HashTable::Statistics::StoredValueProperties, StoredValue const*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.cc:652 #1 0x00000000006ce57a in HashTable::unlocked_addNewStoredValue(HashTable::HashBucketLock const&, Item const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/hash_table.cc:453 #2 0x000000000071ed55 in VBucket::addTempStoredValue(HashTable::HashBucketLock const&, DocKey const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:3780 #3 0x00000000007f3785 in EPVBucket::addTempItemAndBGFetch (this=0x7faf9bbe9e00, hbl=..., key=..., cookie=0x7faf8bca5e00, engine=..., metadataOnly=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:761 #4 0x000000000071ad7b in VBucket::getInternal(CookieIface const*, EventuallyPersistentEngine&, get_options_t, VBucket::GetKeyOnly, Collections::VB::CachingReadHandle const&, ForGetReplicaOp) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest_handles.h:275 #5 0x00000000006e4ca0 in KVBucket::getInternal(DocKey const&, Vbid, CookieIface const*, ForGetReplicaOp, get_options_t) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1528 #6 0x00000000006d7e87 in KVBucket::get (this=, key=..., vbucket=..., cookie=, options=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:856 #7 0x0000000000698f9c in EventuallyPersistentEngine::getInner(CookieIface const*, DocKey const&, Vbid, get_options_t) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2255 #8 0x00000000006990fa in EventuallyPersistentEngine::get(CookieIface const&, DocKey const&, Vbid, DocStateFilter) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:120 #9 0x000000000057ac50 in bucket_get(Cookie&, DocKey const&, Vbid, DocStateFilter) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:191 #10 0x00000000006215c8 in GetCommandContext::getItem() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:26 #11 0x0000000000621af8 in GetCommandContext::step (this=0x7fb044883fc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:154 #12 0x000000000063e0c1 in SteppableCommandContext::drive() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:36 #13 0x00000000005efd9b in Cookie::doExecute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 #14 0x00000000005f10e0 in Cookie::execute (this=0x7faf8bca5e00, useStartTime=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:169 #15 0x00000000005cf483 in Connection::executeCommandPipeline() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:469 #16 0x00000000005cf723 in Connection::executeCommandsCallback() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:806 #17 0x00000000005cfdab in Connection::rw_callback(bufferevent*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:955 #18 0x00007fb0479a2d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7faf8bca6080) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #19 0x00007fb0479abd01 in event_process_active_single_queue (base=base@entry=0x7fb04488ff00, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #20 0x00007fb0479ac54f in event_process_active (base=0x7fb04488ff00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #21 event_base_loop (base=0x7fb04488ff00, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #22 0x0000000000b6e087 in folly::EventBase::loopBody (this=this@entry=0x7fb044a7f7b0, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #23 0x0000000000b6e556 in folly::EventBase::loop (this=this@entry=0x7fb044a7f7b0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #24 0x0000000000b6ff16 in folly::EventBase::loopForever (this=this@entry=0x7fb044a7f7b0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #25 0x000000000059fa59 in worker_libevent (arg=0x7fb044a7f7a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #26 0x0000000000b01e39 in run (this=0x7fb044dbbe60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #27 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #28 0x00007fb0486e8ea5 in start_thread (arg=0x7fb03affd700) at pthread_create.c:307 #29 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #30 0x0000000000000000 in ?? () Thread 13 (LWP 124399): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 12 (LWP 124398): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 11 (LWP 124397): #0 0x00007fb046002c3d in fts_children (sp=0x1a76608, instr=27739168) at fts.c:546 #1 0x0000000001a74420 in ?? () #2 0x00007fb042ce9880 in ?? () #3 0x00007fb042ce9884 in ?? () #4 0x0000000000000000 in ?? () Thread 10 (LWP 124396): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000000000a4a8fb in worker_thread (thread_func_param=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0424fa700) at pthread_create.c:307 #3 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #4 0x0000000000000000 in ?? () Thread 9 (LWP 124394): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000000000a4a8fb in worker_thread (thread_func_param=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 0x00007fb0486e8ea5 in start_thread (arg=0x7fb041cf9700) at pthread_create.c:307 #3 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #4 0x0000000000000000 in ?? () Thread 8 (LWP 124386): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000668877 in AuditImpl::consume_events() () at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 0x0000000000b01e39 in run (this=0x7fb044dbb920) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #3 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #4 0x00007fb0486e8ea5 in start_thread (arg=0x7fb040cf7700) at pthread_create.c:307 #5 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #6 0x0000000000000000 in ?? () Thread 7 (LWP 124385): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00000000005e1596 in ExternalAuthManagerThread::run() () at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 0x0000000000b0f343 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:38 #3 0x0000000000b01e39 in run (this=0x7fb044dbb980) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #4 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #5 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0414f8700) at pthread_create.c:307 #6 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #7 0x0000000000000000 in ?? () Thread 6 (LWP 124381): #0 0x00007fb046002c3d in fts_children (sp=0x1, instr=1953851507) at fts.c:546 #1 0x0000000000000001 in ?? () #2 0x00007fb048d236c0 in phosphor::TraceLog::getInstance()::log_instance () from /opt/couchbase/bin/../lib/libphosphor.so #3 0x0000000000000050 in ?? () #4 0x000000000001e5dd in ?? () #5 0x00007fb0486f17bf in pthread_setname_np (th=4294967296, name=0x7fb0434fc700
) at ../nptl/sysdeps/unix/sysv/linux/pthread_setname.c:43 #6 0x0000000000b01e39 in run (this=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #7 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #8 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0434fc700) at pthread_create.c:307 #9 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #10 0x0000000000000000 in ?? () Thread 5 (LWP 124380): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000acfcca in __gthread_cond_timedwait (__abs_timeout=0x7fb043ceb9b0, __mutex=, __cond=0x7fb044dbd1f8) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7fb044dbd1f8) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7fb044dbd1f8) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > ( __p=..., __atime=..., __lock=..., this=0x7fb044dbd1f8) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __rtime=..., __lock=..., this=0x7fb044dbd1f8) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 operator() (__closure=0x7fb044dc8d48) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/periodic_worker-inl.h:25 #7 __invoke_impl&, std::chrono::seconds):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #8 __invoke&, std::chrono::seconds):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 _M_invoke<0> (this=0x7fb044dc8d48) at /usr/local/include/c++/7.3.0/thread:234 #10 operator() (this=0x7fb044dc8d48) at /usr/local/include/c++/7.3.0/thread:243 #11 std::thread::_State_impl&, std::chrono::seconds):: > > >::_M_run(void) (this=0x7fb044dc8d40) at /usr/local/include/c++/7.3.0/thread:186 #12 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #13 0x00007fb0486e8ea5 in start_thread (arg=0x7fb043cfd700) at pthread_create.c:307 #14 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #15 0x0000000000000000 in ?? () Thread 4 (LWP 124379): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x0000000000acff0a in __gthread_cond_timedwait (__abs_timeout=0x7fb0444ec800, __mutex=, __cond=0x7fb044dc7138) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7fb044dc7138) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7fb044dc7138) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __atime=..., __lock=..., this=0x7fb044dc7138) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __rtime=, __lock=..., this=0x7fb044dc7138) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 dequeue_for (wait_duration=..., popped_item=..., this=0x7fb044dc7110) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/mpmc_blocking_q.h:58 #7 spdlog::details::thread_pool::process_next_msg_ (this=this@entry=0x7fb044dc7110) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:99 #8 0x0000000000ad0258 in worker_loop_ (this=0x7fb044dc7110) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:90 #9 operator() (__closure=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:28 #10 __invoke_impl):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #11 __invoke):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #12 _M_invoke<0> (this=) at /usr/local/include/c++/7.3.0/thread:234 #13 operator() (this=) at /usr/local/include/c++/7.3.0/thread:243 #14 std::thread::_State_impl):: > > >::_M_run(void) (this=) at /usr/local/include/c++/7.3.0/thread:186 #15 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #16 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0444fe700) at pthread_create.c:307 #17 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #18 0x0000000000000000 in ?? () Thread 3 (LWP 124374): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007fb047bee5a7 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7fb0455ee220) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7fb0455ee220) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007fb0486e8ea5 in start_thread (arg=0x7fb0455ff700) at pthread_create.c:307 #7 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #8 0x0000000000000000 in ?? () Thread 2 (LWP 124372): #0 0x00007fb04600deb3 in create_module () at ../sysdeps/unix/syscall-template.S:81 #1 0x0000000000000000 in ?? () Thread 1 (LWP 124414): #0 __atomic_add (__val=1, __mem=0x20) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:96 #1 __atomic_add_dispatch (__val=1, __mem=0x20) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:96 #2 _M_add_ref_copy (this=0x18) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:142 #3 __shared_count (__r=..., this=0x7fb00bfec708) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:740 #4 __shared_ptr (this=0x7fb00bfec700) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:1181 #5 shared_ptr (this=0x7fb00bfec700) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr.h:149 #6 Checkpoint (this=0x7fb00bfec6f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.h:46 #7 construct (this=, __p=0x7faf9e9511c0) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/new_allocator.h:150 #8 construct (__p=0x7faf9e9511c0, __a=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:512 #9 std::vector >::_M_realloc_insert (this=this@entry=0x7fb00bfec8b0, __position=__position@entry=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/vector.tcc:449 #10 0x000000000092f6af in emplace_back (this=0x7fb00bfec8b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1424 #11 magma::KVStore::getAllCheckpoints(std::vector >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1424 #12 0x00000000009308c4 in magma::KVStore::verifyCheckpoints() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1469 #13 0x0000000000931236 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:503 #14 0x0000000000931a0d in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:514 #15 0x0000000000931f48 in magma::KVStore::FlushMemTables (this=, wal=, flushMode=, blockMode=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:336 #16 0x00000000008dd5c5 in magma::Magma::Impl::syncKVStore(unsigned short, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1133 #17 0x00000000008dd740 in std::_Function_handler > (unsigned short)>)::{lambda()#2}>::_M_invoke(std::_Any_data const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:784 #18 0x00000000008e40ae in magma::defer::~defer (this=0x7fb00bfecd50, __in_chrg=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #19 0x00000000008de00e in magma::Magma::Impl::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:798 #20 0x00000000008de0dd in magma::Magma::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #21 0x000000000085629e in MagmaMemoryTrackingProxy::CompactKVStore(unsigned short, magma::Slice const&, magma::Slice const&, std::function > (unsigned short)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #22 0x000000000084c5ea in MagmaKVStore::compactDBInternal(std::unique_lock&, std::shared_ptr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #23 0x000000000084da06 in MagmaKVStore::compactDB(std::unique_lock&, std::shared_ptr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:2032 #24 0x00000000007e9ada in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2599 #25 0x00000000007eb0f1 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector >&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1359 #26 0x00000000007089c6 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:73 #27 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #28 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb00bfed840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #29 0x0000000000b649a0 in operator() (this=0x7fb00bfed840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #30 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fb044899400, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #31 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb044899400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #32 0x0000000000b67959 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #33 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #34 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #35 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #36 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #37 0x0000000000a19ce4 in operator() (this=0x7fb044de1280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #38 operator() (__closure=0x7fb044de1280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #39 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #40 0x00007fb0468c4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #41 0x00007fb0486e8ea5 in start_thread (arg=0x7fb00bfff700) at pthread_create.c:307 #42 0x00007fb04600d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81 #43 0x0000000000000000 in ?? () No core file now.