(gdb) thread apply all bt Thread 31 (Thread 0x7fefd5ef0440 (LWP 67462)): #0 0x00007fefd11364ed in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fefd1131dcb in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007fefd1131c98 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000000000041d417 in __gthread_mutex_lock (__mutex=0x8784c0 ) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 #4 lock (this=0x8784c0 ) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 #5 lock_guard (__m=..., this=) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 #6 bucketsForEach(std::function, void*) (fn=..., arg=arg@entry=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:96 #7 0x00000000004a519e in mc_gather_timing_samples () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mc_time.cc:259 #8 mc_time_clock_tick () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mc_time.cc:214 #9 0x00000000004a5385 in mc_time_clock_event_handler (fd=, which=, arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mc_time.cc:198 #10 0x00007fefd29c1f58 in event_process_active_single_queue (base=base@entry=0x7fefcf8b2000, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1697 #11 0x00007fefd29c254f in event_process_active (base=0x7fefcf8b2000) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #12 event_base_loop (base=0x7fefcf8b2000, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #13 0x0000000000423674 in memcached_main (argc=, argv=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1547 #14 0x00007fefd0d7d3d5 in __libc_start_main () from /lib64/libc.so.6 #15 0x000000000041c6ce in _start () Thread 30 (Thread 0x7fef637fe700 (LWP 67668)): #0 0x00007fefd0e59483 in epoll_wait () from /lib64/libc.so.6 #1 0x00007fefd29cb3c5 in epoll_dispatch (base=0x7fefc469c000, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007fefd29c2376 in event_base_loop (base=0x7fefc469c000, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00007fefd52a04d7 in folly::EventBase::loopBody (this=this@entry=0x7fefc45fc780, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x00007fefd52a09a6 in folly::EventBase::loop (this=this@entry=0x7fefc45fc780) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x00007fefd52a2366 in folly::EventBase::loopForever (this=0x7fefc45fc780) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00007fefd528f989 in folly::IOThreadPoolExecutor::threadRun (this=0x7fefc4576780, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/IOThreadPoolExecutor.cpp:195 #7 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #8 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #10 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #11 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #12 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774410) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #13 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #14 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x7fef63fff700 (LWP 67667)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7fef63fe598c) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=futex@entry=0x7fef63fe598c, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd529bd3f in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::Baton::tryWaitSlow > > (this=this@entry=0x7fef63fe598c, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/Baton.h:308 #8 0x00007fefd529f60f in wait (opt=..., this=0x7fef63fe598c) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/Baton.h:178 #9 folly::EventBase::runInEventBaseThreadAndWait(folly::Function) (this=this@entry=0x7fefc45fc780, fn=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:673 #10 0x00007fefd51187ac in FollyExecutorPool::schedule (this=, task=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:937 #11 0x00007fefd514680e in KVBucket::visitAsync (this=this@entry=0x7fefc431b000, visitor=..., lbl=lbl@entry=0x7fefd52e0599 "DurabilityTimeoutVisitor", id=id@entry=DurabilityTimeoutVisitor, maxExpectedDuration=maxExpectedDuration@entry=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2276 #12 0x00007fefd50bf081 in DurabilityTimeoutTask::run (this=0x7fefc42c8870) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability_timeout_task.cc:61 #13 0x00007fefd51230c3 in GlobalTask::execute (this=0x7fefc42c8870) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:74 #14 0x00007fefd511d3bf in operator() (__closure=0x7fef63fe6000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:196 #15 folly::detail::function::FunctionTraits::callSmall(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:387 #16 0x00007fefd5298160 in operator() (this=0x7fef63fe6000) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #17 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fefc4601c00, thread=..., task=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #18 0x00007fefd528063a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc4601c00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #19 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #20 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #21 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #22 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #23 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #24 0x00007fefd511b928 in operator() (this=0x7fefc44c7e80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #25 operator() (__closure=0x7fefc44c7e80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #26 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #27 0x00007fefd511b803 in operator() (this=0x7fefcf9d1110) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #28 operator() (__closure=0x7fefcf9d1100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #29 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #30 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf7743a0) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #31 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #32 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x7fef80ff9700 (LWP 67666)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287ca7 in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d7300, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef90002d40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90002d00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc4601c00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7e40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7e40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d10c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d10b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774330) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x7fef817fa700 (LWP 67665)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d70d8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef900027c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90002780, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc4601000, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0fd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0fc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf7742c0) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7fef81ffb700 (LWP 67664)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d72e8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef900018c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90001880, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453d400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0f30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0f20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774250) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 25 (Thread 0x7fef827fc700 (LWP 67662)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d73a8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef900018c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90001880, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453d400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0ee0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0ed0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf7741e0) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 24 (Thread 0x7fef82ffd700 (LWP 67661)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d70a8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef900018c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90001880, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453d400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7bc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7bc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0e90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0e80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774170) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7fef837fe700 (LWP 67660)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d7390, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef900018c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90001880, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453d400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7b80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7b80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0e40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0e30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774100) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7fef83fff700 (LWP 67659)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d7120, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef90000a40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90000a00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453c800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7a80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7a80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0620) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774090) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7fef967fc700 (LWP 67658)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d7048, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef90000a40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90000a00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453c800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7a00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7a00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d05d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d05c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf774020) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x7fef96ffd700 (LWP 67657)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d7138, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef90000a40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90000a00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453c800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c79c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c79c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0570) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf773fb0) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7fef977fe700 (LWP 67656)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd528225a in futexWaitImpl const, std::chrono::time_point > > > (waitMask=, deadline=..., expected=, futex=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > (futex=, expected=, deadline=..., waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x00007fefd5287b6d in folly::SaturatingSemaphore::tryWaitSlow > > (this=0x7fef947d7018, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 0x00007fefd52881d6 in folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=0x7fef90000a40, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #7 0x00007fefd528b930 in folly::UnboundedBlockingQueue::try_take_for (this=0x7fef90000a00, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #8 0x00007fefd52805c7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fefc453c800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #9 0x00007fefd529b119 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #10 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #11 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #12 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #13 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #14 0x00007fefd511b928 in operator() (this=0x7fefc44c7980) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #15 operator() (__closure=0x7fefc44c7980) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54 #16 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #17 0x00007fefd511b803 in operator() (this=0x7fefcf9d0530) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416 #18 operator() (__closure=0x7fefcf9d0520) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf773f40) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #21 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 18 (Thread 0x7fef97fff700 (LWP 67655)): #0 0x00007fefd1133965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd2c03fa2 in background_thread_sleep (tsdn=, interval=, info=) at src/background_thread.c:232 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7fef97fe6a30) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x7fefbeffd700 (LWP 67491)): #0 0x00007fefd1133965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd19111bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7fefcf720698) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7fefcf720698, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000004a2351 in Executor::run (this=0x7fefcf720540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007fefd390ecdf in Couchbase::Thread::thread_entry (this=0x7fefcf720540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007fefd38f8767 in run (this=0x7fefd061ed10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7fefd061ed10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7fefbf7fe700 (LWP 67490)): #0 0x00007fefd1133965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd19111bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7fefcf7204d8) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7fefcf7204d8, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000004a2351 in Executor::run (this=0x7fefcf720380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007fefd390ecdf in Couchbase::Thread::thread_entry (this=0x7fefcf720380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007fefd38f8767 in run (this=0x7fefd061eb30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7fefd061eb30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7fefbffff700 (LWP 67489)): #0 0x00007fefd1133965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd19111bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7fefcf720318) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7fefcf720318, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000004a2351 in Executor::run (this=0x7fefcf7201c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007fefd390ecdf in Couchbase::Thread::thread_entry (this=0x7fefcf7201c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007fefd38f8767 in run (this=0x7fefd061ed70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7fefd061ed70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7fefc4ff9700 (LWP 67488)): #0 0x00007fefd1133965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd19111bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7fefcf720158) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7fefcf720158, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x00000000004a2351 in Executor::run (this=0x7fefcf720000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007fefd390ecdf in Couchbase::Thread::thread_entry (this=0x7fefcf720000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007fefd38f8767 in run (this=0x7fefd061ed40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7fefd061ed40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7fefc57fa700 (LWP 67486)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd503f767 in folly::SharedMutexImpl::waitForZeroBits::WaitForever> (this=0x7fefc42794e8, state=@0x7fefc57e09dc: 4294965633, goal=4294965248, waitMask=16, ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/detail/Futex-inl.h:101 #4 0x00007fefd5057998 in lockExclusiveImpl::WaitForever> (ctx=..., preconditionGoalMask=, state=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1002 #5 folly::SharedMutexImpl::lockExclusiveImpl::WaitForever> (this=this@entry=0x7fefc42794e8, preconditionGoalMask=preconditionGoalMask@entry=224, ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:955 #6 0x00007fefd525de6f in lock (this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:371 #7 lock (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:124 #8 lock > (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:479 #9 LockedPtrBase (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1097 #10 LockedPtr (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1402 #11 wlock (this=0x7fefc42794e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:115 #12 HdrHistogram::reset (this=0x7fefc42794d8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.cc:118 #13 0x00007fefd5169139 in EPStats::reset (this=this@entry=0x7fefc4279018) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/stats.cc:320 #14 0x00007fefd50d5464 in resetStats (this=0x7fefc4279000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4788 #15 EventuallyPersistentEngine::reset_stats (this=0x7fefc4279000, cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:475 #16 0x00000000004d5348 in stat_reset_executor (arg=..., cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:158 #17 0x00000000004d4e9d in StatsCommandContext::doStats (this=this@entry=0x7fefcf8777d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:727 #18 0x00000000004d5fc8 in StatsCommandContext::step (this=0x7fefcf8777d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/stats_context.cc:649 #19 0x00000000004d8cc0 in SteppableCommandContext::drive (this=this@entry=0x7fefcf8777d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:42 #20 0x00000000004acb96 in stat_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:192 #21 0x00000000004afad6 in operator() (__args#0=..., this=) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #22 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:919 #23 0x000000000049d094 in Cookie::doExecute (this=this@entry=0x7fefcf98e180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:157 #24 0x000000000049d1af in Cookie::execute (this=0x7fefcf98e180, useStartTime=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:170 #25 0x00000000004810c1 in Connection::executeCommandPipeline (this=this@entry=0x7fefcf98df00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:568 #26 0x00000000004882d5 in Connection::executeCommandsCallback (this=this@entry=0x7fefcf98df00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:695 #27 0x0000000000488a9f in Connection::rw_callback (bev=, ctx=0x7fefcf98df00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:892 #28 0x00007fefd29b8d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7fefcf98e400) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #29 0x00007fefd29c1d01 in event_process_active_single_queue (base=base@entry=0x7fefcf8b2c00, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #30 0x00007fefd29c254f in event_process_active (base=0x7fefcf8b2c00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #31 event_base_loop (base=0x7fefcf8b2c00, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #32 0x0000000000468ad9 in worker_libevent (arg=0x7fefcf8e8ff8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:195 #33 0x00007fefd38f8767 in run (this=0x7fefd061eb60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #34 platform_thread_wrap (arg=0x7fefd061eb60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #35 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #36 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7fefc5ffb700 (LWP 67485)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd503f767 in folly::SharedMutexImpl::waitForZeroBits::WaitForever> (this=0x7fefc42794e8, state=@0x7fefc5fe1898: 4294965649, goal=128, waitMask=1, ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/detail/Futex-inl.h:101 #4 0x00007fefd525caa4 in lockSharedImpl::WaitForever> (ctx=..., token=0x0, state=@0x7fefc5fe1898: 4294965649, this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1669 #5 folly::SharedMutexImpl::lockSharedImpl::WaitForever> (this=this@entry=0x7fefc42794e8, ctx=..., token=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1298 #6 0x00007fefd525cc34 in lock_shared (this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:413 #7 lock_shared (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:157 #8 lock > (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:499 #9 LockedPtrBase (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1097 #10 LockedPtr (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1402 #11 rlock (this=0x7fefc42794e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:141 #12 HdrHistogram::addValueAndCount (this=0x7fefc42794d8, v=5, count=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.cc:102 #13 0x00007fefd50dafee in ~ScopeTimer2 (this=0x7fefc5fe1950, __in_chrg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.h:407 #14 EventuallyPersistentEngine::getInner (this=this@entry=0x7fefc4279000, cookie=0x7fef621a4400, key=..., vbucket=..., options=options@entry=(TRACK_STATISTICS | QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP | HIDE_LOCKED_CAS)) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2233 #15 0x00007fefd50db0f5 in EventuallyPersistentEngine::get (this=0x7fefc4279000, cookie=..., key=..., vbucket=..., documentStateFilter=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:296 #16 0x000000000044e6c9 in bucket_get (cookie=..., key=..., vbucket=..., documentStateFilter=, documentStateFilter@entry=Alive) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:197 #17 0x00000000004c98e4 in GetCommandContext::getItem (this=this@entry=0x7fefc46aa8e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:32 #18 0x00000000004c9bc8 in GetCommandContext::step (this=0x7fefc46aa8e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:161 #19 0x00000000004d8cc0 in SteppableCommandContext::drive (this=this@entry=0x7fefc46aa8e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:42 #20 0x00000000004ade6c in process_bin_get (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:86 #21 get_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:184 #22 0x00000000004afad6 in operator() (__args#0=..., this=) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #23 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:919 #24 0x000000000049d094 in Cookie::doExecute (this=this@entry=0x7fef621a4400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:157 #25 0x000000000049d1af in Cookie::execute (this=0x7fef621a4400, useStartTime=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:170 #26 0x00000000004810c1 in Connection::executeCommandPipeline (this=this@entry=0x7fef6219b580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:568 #27 0x00000000004882d5 in Connection::executeCommandsCallback (this=this@entry=0x7fef6219b580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:695 #28 0x0000000000488a9f in Connection::rw_callback (bev=, ctx=0x7fef6219b580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:892 #29 0x00007fefd29b8d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7fef621a4680) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #30 0x00007fefd29c1d01 in event_process_active_single_queue (base=base@entry=0x7fefcf8b2900, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #31 0x00007fefd29c254f in event_process_active (base=0x7fefcf8b2900) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #32 event_base_loop (base=0x7fefcf8b2900, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #33 0x0000000000468ad9 in worker_libevent (arg=0x7fefcf8e8550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:195 #34 0x00007fefd38f8767 in run (this=0x7fefd061ece0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #35 platform_thread_wrap (arg=0x7fefd061ece0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #36 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #37 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7fefc67fc700 (LWP 67484)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd503f767 in folly::SharedMutexImpl::waitForZeroBits::WaitForever> (this=0x7fefc42794e8, state=@0x7fefc67e2898: 4294965632, goal=128, waitMask=1, ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/detail/Futex-inl.h:101 #4 0x00007fefd525caa4 in lockSharedImpl::WaitForever> (ctx=..., token=0x0, state=@0x7fefc67e2898: 4294965632, this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1669 #5 folly::SharedMutexImpl::lockSharedImpl::WaitForever> (this=this@entry=0x7fefc42794e8, ctx=..., token=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1298 #6 0x00007fefd525cc34 in lock_shared (this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:413 #7 lock_shared (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:157 #8 lock > (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:499 #9 LockedPtrBase (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1097 #10 LockedPtr (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1402 #11 rlock (this=0x7fefc42794e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:141 #12 HdrHistogram::addValueAndCount (this=0x7fefc42794d8, v=5, count=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.cc:102 #13 0x00007fefd50dafee in ~ScopeTimer2 (this=0x7fefc67e2950, __in_chrg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.h:407 #14 EventuallyPersistentEngine::getInner (this=this@entry=0x7fefc4279000, cookie=0x7fef6219cc00, key=..., vbucket=..., options=options@entry=(TRACK_STATISTICS | QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP | HIDE_LOCKED_CAS)) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2233 #15 0x00007fefd50db0f5 in EventuallyPersistentEngine::get (this=0x7fefc4279000, cookie=..., key=..., vbucket=..., documentStateFilter=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:296 #16 0x000000000044e6c9 in bucket_get (cookie=..., key=..., vbucket=..., documentStateFilter=, documentStateFilter@entry=Alive) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:197 #17 0x00000000004c98e4 in GetCommandContext::getItem (this=this@entry=0x7fefcf8829e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:32 #18 0x00000000004c9bc8 in GetCommandContext::step (this=0x7fefcf8829e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:161 #19 0x00000000004d8cc0 in SteppableCommandContext::drive (this=this@entry=0x7fefcf8829e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:42 #20 0x00000000004ade6c in process_bin_get (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:86 #21 get_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:184 #22 0x00000000004afad6 in operator() (__args#0=..., this=) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #23 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:919 #24 0x000000000049d094 in Cookie::doExecute (this=this@entry=0x7fef6219cc00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:157 #25 0x000000000049d1af in Cookie::execute (this=0x7fef6219cc00, useStartTime=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:170 #26 0x00000000004810c1 in Connection::executeCommandPipeline (this=this@entry=0x7fef6219c980) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:568 #27 0x00000000004882d5 in Connection::executeCommandsCallback (this=this@entry=0x7fef6219c980) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:695 #28 0x0000000000488a9f in Connection::rw_callback (bev=, ctx=0x7fef6219c980) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:892 #29 0x00007fefd29b8d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7fef6219ce80) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #30 0x00007fefd29c1d01 in event_process_active_single_queue (base=base@entry=0x7fefcf8b2600, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #31 0x00007fefd29c254f in event_process_active (base=0x7fefcf8b2600) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #32 event_base_loop (base=0x7fefcf8b2600, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #33 0x0000000000468ad9 in worker_libevent (arg=0x7fefcf8e7aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:195 #34 0x00007fefd38f8767 in run (this=0x7fefd061ebc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #35 platform_thread_wrap (arg=0x7fefd061ebc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #36 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #37 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7fefc6ffd700 (LWP 67483)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd527b04d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd503f767 in folly::SharedMutexImpl::waitForZeroBits::WaitForever> (this=0x7fefc42794e8, state=@0x7fefc6fe3898: 4294965649, goal=128, waitMask=1, ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/detail/Futex-inl.h:101 #4 0x00007fefd525caa4 in lockSharedImpl::WaitForever> (ctx=..., token=0x0, state=@0x7fefc6fe3898: 4294965649, this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1669 #5 folly::SharedMutexImpl::lockSharedImpl::WaitForever> (this=this@entry=0x7fefc42794e8, ctx=..., token=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1298 #6 0x00007fefd525cc34 in lock_shared (this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:413 #7 lock_shared (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:157 #8 lock > (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:499 #9 LockedPtrBase (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1097 #10 LockedPtr (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1402 #11 rlock (this=0x7fefc42794e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:141 #12 HdrHistogram::addValueAndCount (this=0x7fefc42794d8, v=5, count=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.cc:102 #13 0x00007fefd50dafee in ~ScopeTimer2 (this=0x7fefc6fe3950, __in_chrg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.h:407 #14 EventuallyPersistentEngine::getInner (this=this@entry=0x7fefc4279000, cookie=0x7fef6219ba80, key=..., vbucket=..., options=options@entry=(TRACK_STATISTICS | QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP | HIDE_LOCKED_CAS)) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2233 #15 0x00007fefd50db0f5 in EventuallyPersistentEngine::get (this=0x7fefc4279000, cookie=..., key=..., vbucket=..., documentStateFilter=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:296 #16 0x000000000044e6c9 in bucket_get (cookie=..., key=..., vbucket=..., documentStateFilter=, documentStateFilter@entry=Alive) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:197 #17 0x00000000004c98e4 in GetCommandContext::getItem (this=this@entry=0x7fefcf881f60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:32 #18 0x00000000004c9bc8 in GetCommandContext::step (this=0x7fefcf881f60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/get_context.cc:161 #19 0x00000000004d8cc0 in SteppableCommandContext::drive (this=this@entry=0x7fefcf881f60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:42 #20 0x00000000004ade6c in process_bin_get (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:86 #21 get_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:184 #22 0x00000000004afad6 in operator() (__args#0=..., this=) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #23 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:919 #24 0x000000000049d094 in Cookie::doExecute (this=this@entry=0x7fef6219ba80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:157 #25 0x000000000049d1af in Cookie::execute (this=0x7fef6219ba80, useStartTime=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:170 #26 0x00000000004810c1 in Connection::executeCommandPipeline (this=this@entry=0x7fef6219b800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:568 #27 0x00000000004882d5 in Connection::executeCommandsCallback (this=this@entry=0x7fef6219b800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:695 #28 0x0000000000488a9f in Connection::rw_callback (bev=, ctx=0x7fef6219b800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:892 #29 0x00007fefd29b8d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7fef6219bd00) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 #30 0x00007fefd29c1d01 in event_process_active_single_queue (base=base@entry=0x7fefcf8b2300, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 #31 0x00007fefd29c254f in event_process_active (base=0x7fefcf8b2300) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 #32 event_base_loop (base=0x7fefcf8b2300, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 #33 0x0000000000468ad9 in worker_libevent (arg=0x7fefcf8e7000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:195 #34 0x00007fefd38f8767 in run (this=0x7fefd061e9e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #35 platform_thread_wrap (arg=0x7fefd061e9e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #36 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #37 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7fefc77fe700 (LWP 67480)): #0 0x00007fefd1133d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000004f10f6 in __gthread_cond_timedwait (__abs_timeout=0x7fefc77e51b0, __mutex=, __cond=0x7fefcf8698f8) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7fefcf8698f8) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7fefcf8698f8) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=0x7fefcf8698f8) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 AuditImpl::consume_events (this=0x7fefcf869500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/auditd/src/audit.cc:408 #6 0x00007fefd38f8767 in run (this=0x7fefd0781720) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #7 platform_thread_wrap (arg=0x7fefd0781720) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #8 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #9 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7fefc7fff700 (LWP 67471)): #0 0x00007fefd1133d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000004918e0 in __gthread_cond_timedwait (__abs_timeout=0x7fefc7fe6190, __mutex=, __cond=0x7fefd06280f8) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=, this=0x7fefd06280f8) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=, this=0x7fefd06280f8) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=, __lock=, this=0x7fefd06280f8) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 ExternalAuthManagerThread::run (this=0x7fefd0628000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/external_auth_manager_thread.cc:229 #6 0x00007fefd390ecdf in Couchbase::Thread::thread_entry (this=0x7fefd0628000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #7 0x00007fefd38f8767 in run (this=0x7fefd07819c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #8 platform_thread_wrap (arg=0x7fefd07819c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #9 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7fefccc79700 (LWP 67470)): #0 0x00007fefd1133965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd4ce469b in produce_socket (sp=0x7fefccc60100, ctx=0x1e571a0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18890 #2 accept_new_connection (ctx=0x1e571a0, listener=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19232 #3 master_thread_run (ctx=0x1e571a0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19300 #4 master_thread (thread_func_param=0x1e571a0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19386 #5 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7fefcd47a700 (LWP 67469)): #0 0x00007fefd11364ed in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fefd1131dcb in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007fefd1131c98 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000000000041d417 in __gthread_mutex_lock (__mutex=0x8784c0 ) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 #4 lock (this=0x8784c0 ) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 #5 lock_guard (__m=..., this=) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 #6 bucketsForEach(std::function, void*) (fn=..., arg=arg@entry=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:96 #7 0x00000000004649fe in server_prometheus_stats (collector=..., cardinality=cb::prometheus::Low) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stats.cc:192 #8 0x00007fefd4cd41d2 in cb::prometheus::MetricServer::KVCollectable::Collect (this=0x7fefcf863010) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #9 0x00007fefd4cf2ffb in prometheus::detail::CollectMetrics (collectables=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/metrics_collector.cc:18 #10 0x00007fefd4cf1e78 in prometheus::detail::MetricsHandler::handleGet (this=0x7fefd0627780, conn=0x1e587b8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/handler.cc:129 #11 0x00007fefd4cdfec7 in CivetServer::requestHandler (conn=0x1e587b8, cbdata=0x7fefd0627780) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/CivetServer.cpp:152 #12 0x00007fefd4cecdbd in handle_request (conn=conn@entry=0x1e587b8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:14657 #13 0x00007fefd4cede19 in process_new_connection (conn=0x1e587b8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18673 #14 worker_thread_run (conn=0x1e587b8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19090 #15 worker_thread (thread_func_param=0x1e587b8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 #16 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7fefcdc7b700 (LWP 67468)): #0 0x00007fefd0e531c9 in syscall () from /lib64/libc.so.6 #1 0x00007fefd4d17b2d in nativeFutexWaitImpl (waitMask=, absSteadyTime=, absSystemTime=, expected=, addr=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=, expected=, absSystemTime=, absSteadyTime=, waitMask=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x00007fefd4cd1cb7 in folly::SharedMutexImpl::waitForZeroBits::WaitForever> (this=0x7fefc42794e8, state=@0x7fefcdc5d658: 4294965649, goal=128, waitMask=1, ctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/detail/Futex-inl.h:101 #4 0x00007fefd4cd8db4 in lockSharedImpl::WaitForever> (ctx=..., token=0x0, state=@0x7fefcdc5d658: 4294965649, this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1669 #5 folly::SharedMutexImpl::lockSharedImpl::WaitForever> (this=this@entry=0x7fefc42794e8, ctx=..., token=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:1298 #6 0x00007fefd4cd8f9c in lock_shared (this=0x7fefc42794e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/SharedMutex.h:413 #7 lock_shared (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:157 #8 lock > (mutex=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/LockTraits.h:499 #9 LockedPtrBase (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1097 #10 LockedPtr (parent=0x7fefc42794e0, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:1402 #11 rlock (this=0x7fefc42794e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Synchronized.h:144 #12 HdrHistogram::getValueCount (this=this@entry=0x7fefc42794d8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/hdrhistogram.cc:106 #13 0x00007fefd4cd4dd5 in PrometheusStatCollector::addStat (this=0x7fefcdc5dc10, spec=..., v=..., labels=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/statistics/prometheus_collector.cc:60 #14 0x00007fefd4cc9da1 in LabelledStatCollector::addStat (this=0x7fefcdc5da70, k=..., v=..., labels=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/statistics/labelled_collector.h:134 #15 0x00007fefd50da350 in StatCollector::addStat(cb::stats::Key&&, Hdr1sfMicroSecHistogram&) const (this=this@entry=0x7fefcdc5da70, v=..., k=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/statistics/collector.h:323 #16 0x00007fefd50da440 in EventuallyPersistentEngine::doTimingStats (this=0x7fefc4279000, collector=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:4035 #17 0x00007fefd50f20b5 in EventuallyPersistentEngine::get_prometheus_stats (this=0x7fefc4279000, collector=..., cardinality=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:379 #18 0x0000000000467ae8 in operator() (bucket=..., __closure=0x7fefcdc5db30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stats.cc:201 #19 std::_Function_handler >::_M_invoke(const std::_Any_data &, Bucket &, ) (__functor=..., __args#0=..., __args#1=) at /usr/local/include/c++/7.3.0/bits/std_function.h:302 #20 0x000000000041d4d2 in operator() (__args#1=, __args#0=..., this=0x7fefcdc5db30) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #21 bucketsForEach(std::function, void*) (fn=..., arg=arg@entry=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:118 #22 0x00000000004649fe in server_prometheus_stats (collector=..., cardinality=cb::prometheus::High) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stats.cc:192 #23 0x00007fefd4cd41d2 in cb::prometheus::MetricServer::KVCollectable::Collect (this=0x7fefcf863590) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 #24 0x00007fefd4cf2ffb in prometheus::detail::CollectMetrics (collectables=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/metrics_collector.cc:18 #25 0x00007fefd4cf1e78 in prometheus::detail::MetricsHandler::handleGet (this=0x7fefd0627080, conn=0x1e57dd0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/pull/src/handler.cc:129 #26 0x00007fefd4cdfec7 in CivetServer::requestHandler (conn=0x1e57dd0, cbdata=0x7fefd0627080) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/CivetServer.cpp:152 #27 0x00007fefd4cecdbd in handle_request (conn=conn@entry=0x1e57dd0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:14657 #28 0x00007fefd4cede19 in process_new_connection (conn=0x1e57dd0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18673 #29 worker_thread_run (conn=0x1e57dd0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19090 #30 worker_thread (thread_func_param=0x1e57dd0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 #31 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #32 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7fefce47c700 (LWP 67467)): #0 0x00007fefd0e4e20d in poll () from /lib64/libc.so.6 #1 0x0000000000467d4e in get_command (buffsize=80, buffer=0x7fefce463190 "") at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:49 #2 check_stdin_thread (arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:90 #3 0x00007fefd38f8767 in run (this=0x7fefd0781270) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #4 platform_thread_wrap (arg=0x7fefd0781270) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #5 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7fefcec7d700 (LWP 67466)): #0 0x00007fefd1133d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd442df7a in __gthread_cond_timedwait (__abs_timeout=0x7fefcec641b0, __mutex=, __cond=) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 operator() (__closure=0x7fefcf855a88) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/periodic_worker.h:38 #7 __invoke_impl&, std::chrono::seconds):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #8 __invoke&, std::chrono::seconds):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 _M_invoke<0> (this=0x7fefcf855a88) at /usr/local/include/c++/7.3.0/thread:234 #10 operator() (this=0x7fefcf855a88) at /usr/local/include/c++/7.3.0/thread:243 #11 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN6spdlog7details15periodic_workerC4ERKSt8functionIFvvEENSt6chrono8durationIlSt5ratioILl1ELl1EEEEEUlvE_EEEEE6_M_runEv (this=0x7fefcf855a80) at /usr/local/include/c++/7.3.0/thread:186 #12 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefcf855a80) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #13 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #14 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7fefcf47e700 (LWP 67465)): #0 0x00007fefd1133d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd442e8bd in __gthread_cond_timedwait (__abs_timeout=0x7fefcf464f60, __mutex=, __cond=0x7fefd0612538) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7fefd0612538) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7fefd0612538) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __atime=..., __lock=..., this=0x7fefd0612538) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __rtime=, __lock=..., this=0x7fefd0612538) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 dequeue_for (wait_duration=..., popped_item=..., this=0x7fefd0612510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/mpmc_blocking_q.h:61 #7 spdlog::details::thread_pool::process_next_msg_ (this=this@entry=0x7fefd0612510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/thread_pool.h:193 #8 0x00007fefd442ee10 in spdlog::details::thread_pool::worker_loop_ (this=0x7fefd0612510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/thread_pool.h:184 #9 0x00007fefd1916dcf in std::execute_native_thread_routine (__p=0x7fefd0645140) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #10 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007fefd0e58ead in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7fefd01ff700 (LWP 67463)): #0 0x00007fefd1133d12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fefd2c045a7 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7fefd01e6a30) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7fefd01e6a30) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007fefd112fdd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fefd0e58ead in clone () from /lib64/libc.so.6