GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-114.el7 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /opt/couchbase/bin/memcached...Reading symbols from /usr/lib/debug/opt/couchbase/bin/memcached.debug...done. done. [New LWP 85591] [New LWP 84689] [New LWP 84690] [New LWP 84691] [New LWP 84692] [New LWP 84693] [New LWP 84694] [New LWP 84695] [New LWP 84696] [New LWP 84697] [New LWP 84698] [New LWP 84699] [New LWP 84700] [New LWP 84701] [New LWP 84702] [New LWP 84703] [New LWP 84704] [New LWP 84705] [New LWP 84706] [New LWP 85580] [New LWP 85581] [New LWP 85582] [New LWP 85583] [New LWP 85584] [New LWP 85586] [New LWP 85587] [New LWP 85589] [New LWP 85590] [New LWP 85592] [New LWP 85593] [New LWP 85594] [New LWP 85607] [New LWP 85625] To enable execution of this file add add-auto-load-safe-path /usr/lib64/libthread_db-1.0.so line to your configuration file "/root/.gdbinit". To completely disable this security protection add set auto-load safe-path / line to your configuration file "/root/.gdbinit". For more information about this security protection see the "Auto-loading safe path" section in the GDB manual. E.g., run from the shell: info "(gdb)Auto-loading safe path" Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'. #0 0x00000000008efce0 in pread_bin_internal (max_header_size=0, ret_ptr=0x7f88fe7e9ba8, pos=, file=0x7f88d588ec60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_read.cc:185 Thread 33 (LWP 85625): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f8957760e51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f88dd1ee000) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 32 (LWP 85607): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f8957760e51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f88df2ed000) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 31 (LWP 85594): #0 0x00007f895596a483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f89575283c5 in epoll_dispatch (base=0x7f894193d000, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f895751f376 in event_base_loop (base=0x7f894193d000, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000adf607 in folly::EventBase::loopBody (this=this@entry=0x7f894190d480, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000adfad6 in folly::EventBase::loop (this=this@entry=0x7f894190d480) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000ae1496 in folly::EventBase::loopForever (this=0x7f894190d480) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x0000000000aceab9 in folly::IOThreadPoolExecutor::threadRun (this=0x7f8941854c80, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/IOThreadPoolExecutor.cpp:195 #7 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #8 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #10 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #11 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #12 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #13 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #14 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 30 (LWP 85593): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f88fd7e7d3c) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7f88fd7e7d3c, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000adae6f in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::Baton::tryWaitSlow > > ( this=this@entry=0x7f88fd7e7d3c, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/Baton.h:308 #8 0x0000000000ade73f in wait (opt=..., this=0x7f88fd7e7d3c) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/Baton.h:178 #9 folly::EventBase::runInEventBaseThreadAndWait(folly::Function) (this=this@entry=0x7f894190d480, fn=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:673 #10 0x0000000000691fe9 in FollyExecutorPool::schedule(std::shared_ptr) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #11 0x00000000006b2431 in KVBucket::visitAsync(std::unique_ptr >, char const*, TaskId, std::chrono::duration >) () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100 #12 0x00000000007e78de in DurabilityTimeoutTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability_timeout_task.cc:55 #13 0x000000000069bee7 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:68 #14 0x0000000000697030 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}::operator()() const (__closure=0x7f88fd7e8630) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:190 #15 0x0000000000697030 in FollyExecutorPool::TaskProxy::timeoutExpired (this=) #16 0x0000000000ad7290 in operator() (this=0x7f88fd7e8630) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #17 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=0x7f895454c400, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #18 0x0000000000abef6a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f895454c400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #19 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #20 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #21 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #22 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #23 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #24 0x00000000006955bd in operator() (this=0x7f89482eb580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #25 operator() (__closure=0x7f89482eb580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #26 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #27 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #28 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #29 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 29 (LWP 85592): #0 0x00007f8955c42c8d in pthread_mutex_lock () from /lib64/libpthread.so.0 #1 0x00000000006c5992 in KVShard::getBucket(Vbid) const () at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 #2 0x00000000006ffee5 in VBucketMap::getBucket (this=, id=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62 #3 0x00000000006a9a3f in VBCBAdaptor::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.h:227 #4 0x000000000069bee7 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:68 #5 0x0000000000697030 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}::operator()() const (__closure=0x7f88fdfe9630) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:190 #6 0x0000000000697030 in FollyExecutorPool::TaskProxy::timeoutExpired (this=) #7 0x0000000000ad7290 in operator() (this=0x7f88fdfe9630) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #8 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=0x7f895454c400, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #9 0x0000000000abef6a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f895454c400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #10 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #11 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #12 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #13 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #14 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #15 0x00000000006955bd in operator() (this=0x7f89482eb540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 operator() (__closure=0x7f89482eb540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #17 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #18 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #19 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #20 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 28 (LWP 85590): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6dd7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197db3a0, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914001b40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914001b00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f895454a000, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 27 (LWP 85589): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6dd7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197d9ea0, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914001b40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914001b00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f895454a000, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 26 (LWP 85587): #0 atomic_store_p (mo=atomic_memory_order_release, val=0x307f88d5c29580, a=0x7f88fcaa9f90) at include/jemalloc/internal/atomic.h:62 #1 rtree_leaf_elm_write (slab=, szind=3586142632, extent=, elm=0x7f88fcaa9f90, rtree=0x7f89579fe660 , tsdn=) at include/jemalloc/internal/rtree.h:301 #2 extent_rtree_write_acquired (tsdn=0x7f8957bfead0 , slab=, szind=3586142632, extent=, elm_b=0x7f88fcaaa088, elm_a=0x7f88fcaa9f90) at src/extent.c:723 #3 extent_split_impl (tsdn=tsdn@entry=0x7f88fffee000, arena=arena@entry=0x7f88d5c008c0, r_extent_hooks=r_extent_hooks@entry=0x7f88fffdbef8, extent=, size_a=size_a@entry=131072, szind_a=szind_a@entry=48, slab_a=, size_b=, szind_b=, slab_b=, growing_retained=false) at src/extent.c:2156 #4 0x00007f895779166f in extent_split_interior (tsdn=tsdn@entry=0x7f88fffee000, arena=arena@entry=0x7f88d5c008c0, r_extent_hooks=r_extent_hooks@entry=0x7f88fffdbef8, rtree_ctx=rtree_ctx@entry=0x7f88fffee030, extent=extent@entry=0x7f88fffdbc68, lead=lead@entry=0x7f88fffdbc70, trail=0x7f88fffdbc78, to_leak=0x7f88fffdbc80, to_salvage=0x7f88fffdbc88, size=131072, pad=0, alignment=64, slab=false, szind=48, growing_retained=false, new_addr=0x0) at src/extent.c:1017 #5 0x00007f8957794b21 in extent_recycle_split (growing_retained=, extent=, szind=48, slab=false, alignment=64, pad=0, size=131072, new_addr=0x0, extents=0x7f88d5c031a8, rtree_ctx=0x7f88fffee030, r_extent_hooks=0x7f88fffdbef8, arena=0x7f88d5c008c0, tsdn=0x7f88fffee000) at src/extent.c:1066 #6 extent_recycle (tsdn=tsdn@entry=0x7f88fffee000, arena=arena@entry=0x7f88d5c008c0, r_extent_hooks=r_extent_hooks@entry=0x7f88fffdbef8, extents=extents@entry=0x7f88d5c031a8, new_addr=new_addr@entry=0x0, size=size@entry=131072, pad=, alignment=, slab=, szind=, zero=, commit=, growing_retained=) at src/extent.c:1148 #7 0x00007f8957795978 in je_extents_alloc (tsdn=tsdn@entry=0x7f88fffee000, arena=arena@entry=0x7f88d5c008c0, r_extent_hooks=r_extent_hooks@entry=0x7f88fffdbef8, extents=extents@entry=0x7f88d5c031a8, new_addr=new_addr@entry=0x0, size=size@entry=131072, pad=0, alignment=64, slab=false, szind=48, zero=0x7f88fffdbf5f, commit=0x7f88fffdbef7) at src/extent.c:545 #8 0x00007f8957759113 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7f88fffee000, arena=arena@entry=0x7f88d5c008c0, usize=usize@entry=131072, alignment=alignment@entry=64, zero=zero@entry=0x7f88fffdbf5f) at src/arena.c:438 #9 0x00007f8957798d35 in je_large_palloc (zero=, alignment=64, usize=131072, arena=0x7f88d5c008c0, tsdn=0x7f88fffee000) at src/large.c:47 #10 je_large_malloc (tsdn=0x7f88fffee000, arena=, usize=131072, zero=) at src/large.c:17 #11 0x00007f895775d236 in je_arena_palloc (tsdn=tsdn@entry=0x7f88fffee000, arena=0x7f88d5c008c0, usize=usize@entry=131072, alignment=alignment@entry=8, zero=, tcache=tcache@entry=0x7f88d7ae9000) at src/arena.c:1544 #12 0x00007f895774e804 in ipallocztm (arena=, is_internal=false, tcache=0x7f88d7ae9000, zero=, alignment=8, usize=131072, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:78 #13 ipalloct (arena=, tcache=0x7f88d7ae9000, zero=, alignment=8, usize=131072, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:89 #14 imalloc_no_sample (ind=, usize=131072, size=131072, tsd=0x7f88fffee000, dopts=, sopts=) at src/jemalloc.c:1945 #15 imalloc_body (tsd=0x7f88fffee000, dopts=, sopts=) at src/jemalloc.c:2123 #16 imalloc (dopts=, sopts=) at src/jemalloc.c:2258 #17 je_mallocx (size=, flags=) at src/jemalloc.c:3137 #18 0x0000000000a4534f in aligned_alloc (size=131072, alignment=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:66 #19 cb_aligned_alloc (alignment=, size=size@entry=131072) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:66 #20 0x00000000008f6051 in FileBuffer (_mprotect_enabled=false, _write_validation_enabled=, _tracing_enabled=false, _capacity=, _owner=..., this=0x7f88d551dfc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/iobuffer.cc:65 #21 make_unique () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:962 #22 BufferedFileOps::allocate_write_buffer(couch_file_handle_opaque*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/iobuffer.cc:566 #23 0x00000000008f6c1b in pwrite (offset=41059, nbyte=8, buf=0x7f88fffdc1d8, handle=0x7f88d5517d20, errinfo=0x7f88d5524b60, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/iobuffer.cc:678 #24 BufferedFileOps::pwrite(couchstore_error_info_t*, couch_file_handle_opaque*, void const*, unsigned long, long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/iobuffer.cc:662 #25 0x00000000008efdda in write_entire_buffer (file=file@entry=0x7f88d5524b40, buf=buf@entry=0x7f88fffdc1d8, nbytes=nbytes@entry=8, offset=offset@entry=41059) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_write.cc:45 #26 0x00000000008f0127 in raw_write (pos=41059, buf=, file=0x7f88d5524b40, diskBlockType=Data) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_write.cc:81 #27 db_write_buf (file=file@entry=0x7f88d5524b40, buf=buf@entry=0x7f88fffdc220, pos=pos@entry=0x7f88fffdc2c0, disk_size=disk_size@entry=0x7f88fffdc2c8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_write.cc:155 #28 0x00000000008f0290 in db_write_buf_compressed(tree_file*, sized_buf const*, long*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_write.cc:203 #29 0x00000000008f9a70 in flush_mr_partial (res=0x7f88d562e290, res@entry=, mr_quota=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_modify.cc:253 Thread 25 (LWP 85586): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6dd7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197daef0, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914001b40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914001b00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f895454a000, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 24 (LWP 85584): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6c9d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197db940, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914000e40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914000e00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8954546400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 23 (LWP 85583): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6dd7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197dc0a8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914000e40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914000e00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8954546400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 22 (LWP 85582): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f8957760e51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f891b7ed000) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 21 (LWP 85581): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6c9d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197db2c8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914000e40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914000e00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8954546400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb0c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb0c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 20 (LWP 85580): #0 0x00007f89559641c9 in syscall () from /lib64/libc.so.6 #1 0x0000000000ab9a9d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ac0d5a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000ac6c9d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7f89197db550, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x0000000000ac7306 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7f8914000e40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x0000000000acaa60 in folly::UnboundedBlockingQueue::try_take_for (this=0x7f8914000e00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x0000000000abeef7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8954546400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00000000006955bd in operator() (this=0x7f89482eb080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #15 operator() (__closure=0x7f89482eb080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #18 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #19 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 19 (LWP 84706): #0 0x00007f8955c44965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f89564378bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f8954333698) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f8954333698, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000005c9431 in Executor::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:38 #4 0x00007f895827d0d3 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f895826cc19 in run (this=0x7f8954551f50) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 18 (LWP 84705): #0 0x00007f8955c44965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f89564378bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f89543334d8) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f89543334d8, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000005c9431 in Executor::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:38 #4 0x00007f895827d0d3 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f895826cc19 in run (this=0x7f8954551d70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 17 (LWP 84704): #0 0x00007f8955c44965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f89564378bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f8954333318) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f8954333318, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000005c9431 in Executor::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:38 #4 0x00007f895827d0d3 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f895826cc19 in run (this=0x7f8954551d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 16 (LWP 84703): #0 0x00007f8955c44965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f89564378bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f8954333158) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865 #2 std::condition_variable::wait (this=this@entry=0x7f8954333158, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000005c9431 in Executor::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:38 #4 0x00007f895827d0d3 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f895826cc19 in run (this=0x7f8954551f80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 15 (LWP 84702): #0 0x00007f895596a483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f89575283c5 in epoll_dispatch (base=0x7f89544acc00, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f895751f376 in event_base_loop (base=0x7f89544acc00, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000585e4b in worker_libevent (arg=0x7f89544f5ff8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:202 #4 0x00007f895826cc19 in run (this=0x7f8954551da0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 14 (LWP 84701): #0 0x00007f895596a483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f89575283c5 in epoll_dispatch (base=0x7f89544ac900, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f895751f376 in event_base_loop (base=0x7f89544ac900, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000585e4b in worker_libevent (arg=0x7f89544f5550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:202 #4 0x00007f895826cc19 in run (this=0x7f8954551f20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 13 (LWP 84700): #0 0x00007f895596a483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f89575283c5 in epoll_dispatch (base=0x7f89544ac600, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f895751f376 in event_base_loop (base=0x7f89544ac600, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000585e4b in worker_libevent (arg=0x7f89544f4aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:202 #4 0x00007f895826cc19 in run (this=0x7f8954551e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 12 (LWP 84699): #0 0x00007f895596a483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f89575283c5 in epoll_dispatch (base=0x7f89544ac300, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f895751f376 in event_base_loop (base=0x7f89544ac300, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000585e4b in worker_libevent (arg=0x7f89544f4000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:202 #4 0x00007f895826cc19 in run (this=0x7f8954551c20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 11 (LWP 84698): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000624c57 in AuditImpl::consume_events() () at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 0x00007f895826cc19 in run (this=0x7f8954464810) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #3 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #4 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 10 (LWP 84697): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000005b6506 in ExternalAuthManagerThread::run() () at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 0x00007f895827d0d3 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #3 0x00007f895826cc19 in run (this=0x7f8954464ab0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #4 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #5 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 9 (LWP 84696): #0 0x00007f895595f20d in poll () from /lib64/libc.so.6 #1 0x00000000008bfba8 in master_thread (thread_func_param=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19291 #2 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 8 (LWP 84695): #0 0x00007f8955c44965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000008c8c3b in worker_thread (thread_func_param=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 7 (LWP 84694): #0 0x00007f8955c44965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000008c8c3b in worker_thread (thread_func_param=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 6 (LWP 84693): #0 0x00007f895595f20d in poll () from /lib64/libc.so.6 #1 0x00000000005858c1 in get_command (buffsize=80, buffer=0x7f895306a790 "@v#X\211\177") at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:57 #2 check_stdin_thread (arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:98 #3 0x00007f895826cc19 in run (this=0x7f8954464360) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #4 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #5 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 5 (LWP 84692): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a82cab in __gthread_cond_timedwait (__abs_timeout=0x7f895386b7b0, __mutex=, __cond=) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_until >, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __atime=..., __lock=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:158 #5 wait_for, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__rtime=..., __rtime=..., __p=..., __lock=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:185 #6 operator() (__closure=0x7f8954468748) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/periodic_worker.h:38 #7 __invoke_impl&, std::chrono::seconds):: > ( __f=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #8 __invoke&, std::chrono::seconds):: > ( __fn=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95 #9 _M_invoke<0> (this=0x7f8954468748) at /opt/gcc-10.2.0/include/c++/10.2.0/thread:264 #10 operator() (this=0x7f8954468748) at /opt/gcc-10.2.0/include/c++/10.2.0/thread:271 #11 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN6spdlog7details15periodic_workerC4ERKSt8functionIFvvEENSt6chrono8durationIlSt5ratioILl1ELl1EEEEEUlvE_EEEEE6_M_runEv (this=0x7f8954468740) at /opt/gcc-10.2.0/include/c++/10.2.0/thread:215 #12 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #13 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #14 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 4 (LWP 84691): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a834f5 in __gthread_cond_timedwait (__abs_timeout=0x7f895406c560, __mutex=, __cond=0x7f8955212638) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f8955212638) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f8955212638) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for:: > (__p=..., __atime=..., __lock=..., this=0x7f8955212638) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:158 #5 wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for:: > ( __rtime=, __rtime=, __p=..., __lock=..., this=0x7f8955212638) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:185 #6 dequeue_for (wait_duration=..., popped_item=..., this=0x7f8955212610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/mpmc_blocking_q.h:61 #7 spdlog::details::thread_pool::process_next_msg_ (this=this@entry=0x7f8955212610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/thread_pool.h:193 #8 0x0000000000a83b10 in spdlog::details::thread_pool::worker_loop_ (this=0x7f8955212610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/thread_pool.h:184 #9 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #10 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 3 (LWP 84690): #0 0x00007f8955c44d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f89577615a7 in background_thread_sleep (tsdn=, interval=1925000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7f8954dee000) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7f8954dee000) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f8955969ead in clone () from /lib64/libc.so.6 Thread 2 (LWP 84689): #0 0x00007f895596a483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f89575283c5 in epoll_dispatch (base=0x7f89544ac000, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f895751f376 in event_base_loop (base=0x7f89544ac000, flags=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000005363a6 in memcached_main(int, char**) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1525 #4 0x00007f895588e3d5 in __libc_start_main () from /lib64/libc.so.6 #5 0x000000000052c67b in _start () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/compression/allocator.h:75 Thread 1 (LWP 85591): #0 0x00000000008efce0 in pread_bin_internal (max_header_size=0, ret_ptr=0x7f88fe7e9ba8, pos=, file=0x7f88d588ec60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_read.cc:185 #1 pread_bin (file=file@entry=0x7f88d588ec60, pos=, ret_ptr=ret_ptr@entry=0x7f88fe7e9ba8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_read.cc:239 #2 0x00000000008e8669 in bp_to_doc (options=, bp=, db=0x7f88d588ec60, pDoc=0x7f88fe7e9c60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_db.cc:850 #3 couchstore_open_doc_with_docinfo () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_db.cc:996 #4 0x000000000082ef9f in bySeqnoScanCallback(Db*, DocInfo*, void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2740 #5 0x00000000008e7519 in lookup_callback (rq=, k=0x7f88fe7e9df0, v=0x7f88fe7e9e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_db.cc:1095 #6 0x00000000008fc349 in btree_lookup_inner(couchfile_lookup_request*, unsigned long, int, int) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:122 #7 0x00000000008fc04e in btree_lookup_inner(couchfile_lookup_request*, unsigned long, int, int) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:75 #8 0x00000000008fc4b1 in btree_lookup (rq=rq@entry=0x7f88fe7e9f90, root_pointer=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:162 #9 0x00000000008e9726 in couchstore_changes_since () at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_db.cc:1148 #10 0x000000000082d8c1 in CouchKVStore::scan(BySeqnoScanContext&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore-db-holder.h:50 #11 0x000000000088a933 in DCPBackfillBySeqnoDisk::scan() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_by_seqno_disk.cc:180 #12 0x000000000088d54a in DCPBackfillDisk::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_disk.cc:134 #13 0x000000000088f891 in BackfillManager::backfill() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:332 #14 0x000000000088fb04 in BackfillManagerTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:70 #15 0x000000000069bee7 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:68 #16 0x0000000000697030 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}::operator()() const (__closure=0x7f88fe7ea630) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:190 #17 0x0000000000697030 in FollyExecutorPool::TaskProxy::timeoutExpired (this=) #18 0x0000000000ad7290 in operator() (this=0x7f88fe7ea630) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #19 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=0x7f895454b800, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #20 0x0000000000abef6a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f895454b800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #21 0x0000000000ada249 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #22 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #23 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #24 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #25 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #26 0x00000000006955bd in operator() (this=0x7f89482eb440) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #27 operator() (__closure=0x7f89482eb440) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:48 #28 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #29 0x00007f895643cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 #30 0x00007f8955c40dd5 in start_thread () from /lib64/libpthread.so.0 #31 0x00007f8955969ead in clone () from /lib64/libc.so.6 No core file now.