############################## 172.23.121.126: 1 core dump seen 2021-11-24 08:48:27,519 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: rm -rf /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core running: //opt/couchbase/bin/minidump-2-core /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.dmp > /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core 2021-11-24 08:48:27,538 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: //opt/couchbase/bin/minidump-2-core /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.dmp > /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core running: gdb --batch /opt/couchbase/bin/memcached -c /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core -ex "bt full" -ex quit 2021-11-24 08:48:27,572 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: gdb --batch /opt/couchbase/bin/memcached -c /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core -ex "bt full" -ex quit 172.23.121.126: Stack Trace of first crash - e299ceb3-8985-4ce0-0c4601b6-3fda1379.dmp Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'. #0 0x00007f56e2643387 in raise () from /lib64/libc.so.6 #0 0x00007f56e2643387 in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00007f56e2644a78 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x00007f56e2f8e63c in __gnu_cxx::__verbose_terminate_handler() [clone .cold] () from /opt/couchbase/bin/../lib/libstdc++.so.6 No symbol table info available. #3 0x0000000000afd3db in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88 No locals. #4 0x00007f56e2f998f6 in __cxxabiv1::__terminate(void (*)()) () from /opt/couchbase/bin/../lib/libstdc++.so.6 No symbol table info available. #5 0x00007f56e2f99961 in std::terminate() () from /opt/couchbase/bin/../lib/libstdc++.so.6 No symbol table info available. #6 0x00007f56e2f99bf4 in __cxa_throw () from /opt/couchbase/bin/../lib/libstdc++.so.6 No symbol table info available. #7 0x00000000004c272e in ActiveDurabilityMonitor::State::updateHighCompletedSeqno (this=0x7f55ef0d6e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:1620 e = @0x7f5678000940: #8 0x00000000007f784c in ActiveDurabilityMonitor::commit(DurabilityMonitor::ActiveSyncWrite const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:919 s = {holder_ = { >, folly::SharedMutexImpl >, folly::SharedMutexImpl, folly::LockPolicyExclusive>> = {parent_ = 0x7f54ec53f1e8}, static AllowsConcurrentAccess = false}} key = cHandle = { = {readLock = {lock_ = 0x7f56d4395158, token_ = {type_ = folly::SharedMutexToken::DEFERRED_SHARED, slot_ = 8}}, manifest = 0x7f56d43950c0}, itr = { const*, std::pair*>> = {}, underlying_ = {itemPtr_ = 0x7f5650d1ad50, index_ = 8}}, key = {> = {}, buffer = {static npos = , buf = 0x7f55f3c60d20
, len = 19}, encoding = Yes}} prepareEnd = {__d = {__r = 8069434606129862}} cookie = result = __func__ = "commit" index = commitDuration = #9 0x00000000007fe273 in ActiveDurabilityMonitor::processCompletedSyncWriteQueue() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:488 sw = {storage_ = {{emptyState = 0 '\000', value = { = {item = {value = 0x7f55e5e75f00}, startTime = {__d = {__r = 8069346703426060}}, status = ToCommit}, cookie = 0x0, firstChain = {ackCount = {> = {label = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x7f5683fed2e0 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000&V\177\000", _M_allocated_capacity = 7812730926796664437}}}, val = 0 '\000'}, chainPtr = 0x0}, secondChain = {ackCount = {> = {label = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x7f5683fed310 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000&V\177\000", _M_allocated_capacity = 7812730926796664437}}}, val = 0 '\000'}, chainPtr = 0x0}, expiryTime = { > >, true, true>> = { > >, std::_Optional_base > >, true, true> >> = {}, _M_payload = { > > >> = {_M_payload = {_M_empty = {}, _M_value = {__d = {__r = 0}}}, _M_engaged = false}, }}, > > > >> = {}, }}}, hasValue = true}} lock = #10 0x000000000071f484 in VBucket::processResolvedSyncWrites() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:710 rlh = {lock_ = 0x7f55e6baf7f8, token_ = {type_ = folly::SharedMutexToken::INLINE_SHARED, slot_ = 115}} #11 0x0000000000840158 in VBNotifiableTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_notifiable_task.cc:53 vb = {> = {> = {}, _M_ptr = 0x7f55e6baf600, _M_refcount = {_M_pi = 0x7f56439d2f60}}, } runtime = startTime = {__d = {__r = 8069434606121978}} pendingVb = {vbid = 494} #12 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 guard = {previous = 0x0} #13 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f5683fed840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 scheduleOverhead = start = {__d = {__r = 8069434606120059}} runAgain = false proxy = @0x7f56e15c3c10: #14 0x0000000000bbf750 in operator() (this=0x7f5683fed840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 fn = @0x7f5683fed840: {> = {}, data_ = {big = 0x7f56e15c3c10, tiny = {__data = "\020<\\\341V\177\000\000\000\000\000\000\000\000\000\000\260\"}}}, call_ = 0xa6b2a0 ::callSmall(folly::detail::function::Data&)>, exec_ = 0xa699f0 (folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>} #15 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f56e1157c00, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 rctx = {prev_ = {> = {> = {}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, }} startTime = {__d = {__r = 8069434606115758}} stats = {expired = false, waitTime = {__r = 14806}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 8069434606100952}}, requestId = 0} #16 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f56e1157c00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 task = {storage_ = {{emptyState = 16 '\020', value = { = {func_ = {> = {}, data_ = {big = 0x7f56e15c3c10, tiny = {__data = "\020<\\\341V\177\000\000\000\000\000\000\000\000\000\000\260\"}}}, call_ = 0xa6b2a0 ::callSmall(folly::detail::function::Data&)>, exec_ = 0xa699f0 (folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 8069434606100952}}, expiration_ = {__r = 0}, expireCallback_ = {> = {}, data_ = {big = 0x1cb33, tiny = {__data = "3\313\001\000\000\000\000\000\333)\242", '\000' , "_\276\037\345V\177\000\000p\352>\316V\177\000\000@V@\345V\177\000", __align = {}}}, call_ = 0x463cfd ::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0}, context_ = {> = {> = {}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, }}, poison = false, priority_ = 0 '\000', queueObserverPayload_ = 140011140538112}}, hasValue = true}} guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = , b_ = 0xc7b62b "CPUThreadPoolExecutor", e_ = 0xc7b640 ""}}}} #17 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 No locals. #18 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 No locals. #19 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 No locals. #20 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 No locals. #21 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 fn = #22 0x0000000000a6a874 in operator() (this=0x7f56e15e4080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 fn = @0x7f56e15e4080: #23 operator() (__closure=0x7f56e15e4080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x7f5683fed9a0 "NonIoPool0"}, _M_string_length = 10, {_M_local_buf = "NonIoPool0\000\000\000\000\000", _M_allocated_capacity = 8029725099528449870}}}, hasValue = true}} func = #24 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 fn = @0x7f56e15e4080: #25 0x00007f56e2fc2d40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 No symbol table info available. #26 0x00007f56e4dcaea5 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #27 0x00007f56e270b8dd in clone () from /lib64/libc.so.6 No symbol table info available. ############################## running: gdb -p `(pidof memcached)` -ex "thread apply all bt" -ex detach -ex quit 2021-11-24 08:48:36,246 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: gdb -p `(pidof memcached)` -ex "thread apply all bt" -ex detach -ex quit 5 queries submitted, 0 failed, 5080 passed, 0 rejected, 0 cancelled, 0 timeout [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Loaded symbols for /lib64/libpthread.so.0 Reading symbols from /lib64/librt.so.1...(no debugging symbols found)...done. Loaded symbols for /lib64/librt.so.1 Reading symbols from /opt/couchbase/bin/../lib/liblz4.so.1...Reading symbols from /usr/lib/debug/opt/couchbase/lib/liblz4.so.1.9.2-7.1.0-1745.x86_64.debug...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /opt/couchbase/bin/../lib/liblz4.so.1 Reading symbols from /opt/couchbase/bin/../lib/libsnappy.so.1...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libsnappy.so.1.1.8-7.1.0-1745.x86_64.debug...done. done. Loaded symbols for /opt/couchbase/bin/../lib/libsnappy.so.1 Reading symbols from /opt/couchbase/bin/../lib/libjemalloc.so.2...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libjemalloc.so.2-7.1.0-1745.x86_64.debug...done. done. Loaded symbols for /opt/couchbase/bin/../lib/libjemalloc.so.2 Reading symbols from /opt/couchbase/bin/../lib/libevent_core-2.1.so.7...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libevent_core-2.1.so.7.0.0-7.1.0-1745.x86_64.debug...done. done. Loaded symbols for /opt/couchbase/bin/../lib/libevent_core-2.1.so.7 Reading symbols from /opt/couchbase/bin/../lib/libevent_extra-2.1.so.7...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libevent_extra-2.1.so.7.0.0-7.1.0-1745.x86_64.debug...done. done. Loaded symbols for /opt/couchbase/bin/../lib/libevent_extra-2.1.so.7 Reading symbols from /opt/couchbase/bin/../lib/libevent_pthreads-2.1.so.7...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libevent_pthreads-2.1.so.7.0.0-7.1.0-1745.x86_64.debug...done. done. Loaded symbols for /opt/couchbase/bin/../lib/libevent_pthreads-2.1.so.7 Reading symbols from /opt/couchbase/bin/../lib/libevent_openssl-2.1.so.7...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libevent_openssl-2.1.so.7.0.0-7.1.0-1745.x86_64.debug...done. done. Loaded symbols for /opt/couchbase/bin/../lib/libevent_openssl-2.1.so.7 Reading symbols from /opt/couchbase/bin/../lib/libssl.so.1.1...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libssl.so.1.1-7.1.0-1745.x86_64.debug...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /opt/couchbase/bin/../lib/libssl.so.1.1 Reading symbols from /opt/couchbase/bin/../lib/libcrypto.so.1.1...Reading symbols from /usr/lib/debug/opt/couchbase/lib/libcrypto.so.1.1-7.1.0-1745.x86_64.debug...(no debugging symbols found)...done. (no debugging symbols found)...done. Loaded symbols for /opt/couchbase/bin/../lib/libcrypto.so.1.1 Reading symbols from /opt/couchbase/bin/../lib/libstdc++.so.6...(no debugging symbols found)...done. Loaded symbols for /opt/couchbase/bin/../lib/libstdc++.so.6 Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libm.so.6 Reading symbols from /opt/couchbase/bin/../lib/libgcc_s.so.1...(no debugging symbols found)...done. Loaded symbols for /opt/couchbase/bin/../lib/libgcc_s.so.1 Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done. Loaded symbols for /lib64/libc.so.6 Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done. Loaded symbols for /lib64/ld-linux-x86-64.so.2 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 To enable execution of this file add add-auto-load-safe-path /opt/couchbase/lib/libstdc++.so.6.0.28-gdb.py line to your configuration file "/root/.gdbinit". To completely disable this security protection add set auto-load safe-path / line to your configuration file "/root/.gdbinit". For more information about this security protection see the "Auto-loading safe path" section in the GDB manual. E.g., run from the shell: info "(gdb)Auto-loading safe path" Thread 78 (Thread 0x7f51f1dff700 (LWP 128432)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f44715a7 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7f51f1dee220) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7f51f1dee220) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 77 (Thread 0x7f51f0cfe700 (LWP 128433)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000b229ca in __gthread_cond_timedwait ( __abs_timeout=0x7f51f0cec800, __mutex=, __cond=0x7f51f15c7138) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7f51f15c7138) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > ( __atime=..., __lock=..., this=0x7f51f15c7138) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __atime=..., __lock=..., this=0x7f51f15c7138) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __rtime=, __lock=..., this=0x7f51f15c7138) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 dequeue_for (wait_duration=..., popped_item=..., this=0x7f51f15c7110) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/mpmc_blocking_q.h:58 #7 spdlog::details::thread_pool::process_next_msg_ ( this=this@entry=0x7f51f15c7110) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:99 #8 0x0000000000b22d18 in worker_loop_ (this=0x7f51f15c7110) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:90 #9 operator() (__closure=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/thread_pool-inl.h:28 #10 __invoke_impl):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #11 __invoke):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #12 _M_invoke<0> (this=) at /usr/local/include/c++/7.3.0/thread:234 #13 operator() (this=) at /usr/local/include/c++/7.3.0/thread:243 #14 std::thread::_State_impl):: > > >::_M_run(void) (this=) at /usr/local/include/c++/7.3.0/thread:186 #15 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #16 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 76 (Thread 0x7f51f04fd700 (LWP 128434)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000b2278a in __gthread_cond_timedwait ( __abs_timeout=0x7f51f04eb9b0, __mutex=, __cond=0x7f51f15bd268) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7f51f15bd268) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > ( __atime=..., __lock=..., this=0x7f51f15bd268) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __atime=..., __lock=..., this=0x7f51f15bd268) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __rtime=..., __lock=..., this=0x7f51f15bd268) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 operator() (__closure=0x7f51f15c8dc8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/spdlog/spdlog-prefix/src/spdlog/include/spdlog/details/periodic_worker-inl.h:25 #7 __invoke_impl&, std::chrono::seconds):: > ( __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #8 __invoke&, std::chrono::seconds):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 _M_invoke<0> (this=0x7f51f15c8dc8) at /usr/local/include/c++/7.3.0/thread:234 #10 operator() (this=0x7f51f15c8dc8) at /usr/local/include/c++/7.3.0/thread:243 #11 std::thread::_State_impl&, std::chrono::seconds):: > > >::_M_run(void) (this=0x7f51f15c8dc0) at /usr/local/include/c++/7.3.0/thread:186 #12 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #13 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #14 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 75 (Thread 0x7f51efcfc700 (LWP 128435)): #0 0x00007f51f2899c3d in poll () from /lib64/libc.so.6 #1 0x00000000005a4181 in get_command (buffsize=80, buffer=0x7f51efcea990 "@\346Y\365Q\177") at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:57 #2 check_stdin_thread (arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:98 #3 0x0000000000b54b79 in run (this=0x7f51f15bb740) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #4 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #5 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 74 (Thread 0x7f51edcf8700 (LWP 128439)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000005e9dc6 in __gthread_cond_timedwait ( __abs_timeout=0x7f51edce6980, __mutex=0x7f51f15eb350, __cond=0x7f51f15eb378) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=, __lock=, __atime=, this=0x7f51f15eb378) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=, this=0x7f51f15eb378) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=, __lock=, this=0x7f51f15eb378) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 ExternalAuthManagerThread::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/external_auth_manager_thread.cc:91 #6 0x0000000000b6a233 in Couchbase::Thread::thread_entry() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:38 #7 0x0000000000b54b79 in run (this=0x7f51f15bbb00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 73 (Thread 0x7f51ed4f7700 (LWP 128441)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000673ce7 in __gthread_cond_timedwait ( __abs_timeout=0x7f51ed4e59b0, __mutex=, __cond=0x7f51f111d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f111d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f111d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f111d3f8) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 AuditImpl::consume_events() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/auditd/src/audit.cc:401 #6 0x0000000000b54b79 in run (this=0x7f51f15bbaa0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #7 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #8 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #9 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 72 (Thread 0x7f51ee4f9700 (LWP 128442)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a9c78b in consume_socket (thread_index=, sp=0x2f745a0, ctx=0x2f75df0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 worker_thread_run (conn=0x2f73cb0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18987 #3 worker_thread (thread_func_param=0x2f73cb0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 #4 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 71 (Thread 0x7f51eecfa700 (LWP 128443)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x0000000000a9c78b in consume_socket (thread_index=, sp=0x2f74f88, ctx=0x2f75df0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18846 #2 worker_thread_run (conn=0x2f74698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:18987 #3 worker_thread (thread_func_param=0x2f74698) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19145 #4 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 70 (Thread 0x7f51ef4fb700 (LWP 128444)): #0 0x00007f51f2899c3d in poll () from /lib64/libc.so.6 #1 0x0000000000a936f8 in master_thread_run (ctx=0x2f75df0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19291 #2 master_thread (thread_func_param=0x2f75df0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/prometheus-cpp/prometheus-cpp-prefix/src/prometheus-cpp/3rdparty/civetweb/src/civetweb.c:19386 #3 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 69 (Thread 0x7f51eccf6700 (LWP 128445)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f108f900, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f108f900, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f112c010, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f112c010) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f112c010) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f112c000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bbda0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 68 (Thread 0x7f51e7fff700 (LWP 128446)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f108fc00, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f108fc00, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f112cbe0, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f112cbe0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f112cbe0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f112cbd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bbec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 67 (Thread 0x7f51e77fe700 (LWP 128447)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f108ff00, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f108ff00, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f112d7b0, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f112d7b0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f112d7b0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f112d7a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bbfe0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 66 (Thread 0x7f51e6ffd700 (LWP 128448)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f1090200, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f1090200, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f112e380, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f112e380) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f112e380) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f112e370) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bc0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 65 (Thread 0x7f51e67fc700 (LWP 128449)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f1090500, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f1090500, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f112ef50, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f112ef50) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f112ef50) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f112ef40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bc0a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 64 (Thread 0x7f51e5ffb700 (LWP 128450)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f1090800, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f1090800, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f112fb20, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f112fb20) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f112fb20) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f112fb10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bc040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 63 (Thread 0x7f51e57fa700 (LWP 128451)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f1090b00, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f1090b00, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f11306f0, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f11306f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever ( this=this@entry=0x7f51f11306f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005a93a9 in worker_libevent (arg=0x7f51f11306e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:115 #7 0x0000000000b54b79 in run (this=0x7f51f15bc070) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 62 (Thread 0x7f51e4ff9700 (LWP 128452)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x00000000009a80db in magma::Cache::makeObject(magma::BlockCache::BlockID) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009a6f85 in magma::Cache::AcquireObject (this=this@entry=0x7f515c00b6c0, id=..., doCaching=doCaching@entry=false) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #7 0x000000000099ef29 in AcquireObject (doCaching=, id=..., this=0x7f515c0e7c60) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #8 Read (buffer=, transform=..., doCaching=, readBytes=, outSize=, outData=..., size=, offset=, fileID=, file=, this=0x7f515c0e7c60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.h:60 #9 magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file_manager.cc:114 #10 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #11 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #12 0x0000000000a0072c in magma::SSTableIterator::Seek (this=0x7f515489af00, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:810 #13 0x00000000009b5b45 in Next (this=0x7f515c039e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:84 #14 magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:81 #15 0x00000000009c5358 in magma::KVReader::ReadKVs (this=0x7f515c039d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:69 #16 0x00000000009b57d7 in magma::SnapshotIterator::readKV() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:309 #17 0x00000000009b5ae5 in magma::SnapshotIterator::Next (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:252 #18 0x00000000009d3842 in Next (this=0x7f51561aad80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:171 #19 magma::BySeqIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:169 #20 0x00000000008707fa in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1651 #21 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #22 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #23 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f515bf0cc10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #24 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #25 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51e4fe7840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #26 0x0000000000bbf750 in operator() (this=0x7f51e4fe7840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #27 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #28 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #29 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #30 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #31 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #32 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #33 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #34 0x0000000000a6a874 in operator() (this=0x7f51f10d5fc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #35 operator() (__closure=0x7f51f10d5fc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #36 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #37 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #38 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #39 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 61 (Thread 0x7f51debff700 (LWP 128453)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x00000000009b3fde in magma::JoinIterator::Next (this=0x7f515c03a340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:168 #9 0x00000000009b5b45 in Next (this=0x7f515c03ae80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:84 #10 magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:81 #11 0x00000000009c5358 in magma::KVReader::ReadKVs (this=0x7f515c03a580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:69 #12 0x00000000009b57d7 in magma::SnapshotIterator::readKV() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:309 #13 0x00000000009b5ae5 in magma::SnapshotIterator::Next (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:252 #14 0x00000000009d3842 in Next (this=0x7f5151107200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:171 #15 magma::BySeqIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:169 #16 0x00000000008707fa in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1651 #17 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #18 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #19 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f515c1b5210) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #20 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #21 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51debed840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #22 0x0000000000bbf750 in operator() (this=0x7f51debed840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #23 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #24 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #25 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #26 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #27 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #28 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #29 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #30 0x0000000000a6a874 in operator() (this=0x7f51f15e00c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #31 operator() (__closure=0x7f51f15e00c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #32 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #33 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #34 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #35 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 60 (Thread 0x7f51de3fe700 (LWP 128454)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x0000000000a0072c in magma::SSTableIterator::Seek (this=0x7f515489a000, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:810 #9 0x00000000009b5f9b in magma::MergeIterator::Seek (this=0x7f5150cf85c0, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:69 #10 0x00000000009b5aca in magma::SnapshotIterator::Seek ( this=this@entry=0x7f5148e48100, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:243 #11 0x00000000009d39f6 in magma::BySeqIterator::Seek(unsigned long, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:62 #12 0x000000000087069d in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1650 #13 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #14 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #15 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f51564cec10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #16 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #17 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51de3ec840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #18 0x0000000000bbf750 in operator() (this=0x7f51de3ec840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #19 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #20 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #21 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #22 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #23 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #24 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #25 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #26 0x0000000000a6a874 in operator() (this=0x7f51f15e0100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #27 operator() (__closure=0x7f51f15e0100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #28 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #29 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #30 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #31 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 59 (Thread 0x7f51ddbfd700 (LWP 128455)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x0000000000a0072c in magma::SSTableIterator::Seek (this=0x7f51546e8e80, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:810 #9 0x00000000009b5f9b in magma::MergeIterator::Seek (this=0x7f515c039140, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:69 #10 0x00000000009b5aca in magma::SnapshotIterator::Seek ( this=this@entry=0x7f5148e03c80, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:243 #11 0x00000000009d39f6 in magma::BySeqIterator::Seek(unsigned long, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:62 #12 0x000000000087069d in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1650 #13 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #14 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #15 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f515c1b5310) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #16 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #17 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51ddbeb840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #18 0x0000000000bbf750 in operator() (this=0x7f51ddbeb840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #19 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #20 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #21 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #22 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #23 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #24 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #25 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #26 0x0000000000a6a874 in operator() (this=0x7f51f15e0180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #27 operator() (__closure=0x7f51f15e0180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #28 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #29 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #30 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #31 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 58 (Thread 0x7f51dd3fc700 (LWP 128456)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x00000000009b3fde in magma::JoinIterator::Next (this=0x7f515be2d340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:168 #9 0x00000000009b5b45 in Next (this=0x7f5150cf8140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:84 #10 magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:81 #11 0x00000000009c5358 in magma::KVReader::ReadKVs (this=0x7f515be2df40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:69 #12 0x00000000009b57d7 in magma::SnapshotIterator::readKV() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:309 #13 0x00000000009b5ae5 in magma::SnapshotIterator::Next (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:252 #14 0x00000000009d3842 in Next (this=0x7f5150ba3500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:171 #15 magma::BySeqIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:169 #16 0x00000000008707fa in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1651 #17 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #18 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #19 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f515bf0ce10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #20 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #21 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51dd3ea840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #22 0x0000000000bbf750 in operator() (this=0x7f51dd3ea840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #23 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #24 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #25 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #26 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #27 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #28 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #29 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #30 0x0000000000a6a874 in operator() (this=0x7f51f15e01c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #31 operator() (__closure=0x7f51f15e01c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #32 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #33 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #34 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #35 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 57 (Thread 0x7f51dcbfb700 (LWP 128457)): #0 0x00007fff6295c6c2 in clock_gettime () #1 0x00007f51f28ba7fd in clock_gettime () from /lib64/libc.so.6 #2 0x00007f51f3153ca1 in std::chrono::_V2::steady_clock::now() () from /opt/couchbase/bin/../lib/libstdc++.so.6 #3 0x000000000073a752 in LoadStorageKVPairCallback::callback(GetValue&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:950 #4 0x0000000000870763 in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/kvstore.h:389 #5 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #6 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #7 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f515bf0cd10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #8 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #9 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51dcbe9840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #10 0x0000000000bbf750 in operator() (this=0x7f51dcbe9840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #11 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #12 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e0140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e0140) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 56 (Thread 0x7f51d27da700 (LWP 128458)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x0000000000a0072c in magma::SSTableIterator::Seek (this=0x7f515538ea00, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:810 #9 0x00000000009b5f9b in magma::MergeIterator::Seek (this=0x7f5150cbbf40, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:69 #10 0x00000000009b5aca in magma::SnapshotIterator::Seek ( this=this@entry=0x7f5153011380, key=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:243 #11 0x00000000009d39f6 in magma::BySeqIterator::Seek(unsigned long, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:62 #12 0x000000000087069d in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1650 #13 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #14 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #15 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f5156199110) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #16 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #17 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51d27c8840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #18 0x0000000000bbf750 in operator() (this=0x7f51d27c8840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #19 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #20 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #21 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #22 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #23 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #24 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #25 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #26 0x0000000000a6a874 in operator() (this=0x7f51f15e0200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #27 operator() (__closure=0x7f51f15e0200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #28 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #29 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #30 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #31 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 55 (Thread 0x7f51d1fd9700 (LWP 128459)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x00000000009b3fde in magma::JoinIterator::Next (this=0x7f51564c7640) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:168 #9 0x00000000009b5b45 in Next (this=0x7f5149af6880) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:84 #10 magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:81 #11 0x00000000009c5358 in magma::KVReader::ReadKVs (this=0x7f51564c77c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:69 #12 0x00000000009b57d7 in magma::SnapshotIterator::readKV() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:309 #13 0x00000000009b5ae5 in magma::SnapshotIterator::Next (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:252 #14 0x00000000009d3842 in Next (this=0x7f5154ebcc00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:171 #15 magma::BySeqIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/iterator.cc:169 #16 0x00000000008707fa in MagmaKVStore::scan(BySeqnoScanContext&) const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1651 #17 0x000000000073893f in WarmupVbucketVisitor::visit(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:541 #18 0x00000000006e9c92 in KVBucket::pauseResumeVisit(PauseResumeVBVisitor&, KVBucketIface::Position&, VBucketFilter*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:2329 #19 0x000000000074c120 in WarmupBackfillTask::run (this=0x7f51564ce610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:445 #20 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68 #21 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f51d1fc7840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189 #22 0x0000000000bbf750 in operator() (this=0x7f51d1fc7840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 #23 folly::ThreadPoolExecutor::runTask(std::shared_ptr const&, folly::ThreadPoolExecutor::Task&&) ( this=this@entry=0x7f51f112a800, thread=..., task=task@entry=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 #24 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 #25 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #26 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #27 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #28 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #29 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #30 0x0000000000a6a874 in operator() (this=0x7f51f15e0240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #31 operator() (__closure=0x7f51f15e0240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #32 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #33 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #34 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #35 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 54 (Thread 0x7f51d17d8700 (LWP 128460)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3514590744, absSteadyTime=absSteadyTime@entry=0x7f51d17c65f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db0d8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db0d8, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51d17c6618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db0d8) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db0d8, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf15d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db0d8, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000a6a874 in operator() (this=0x7f51f15e0340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #18 operator() (__closure=0x7f51f15e0340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #21 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 53 (Thread 0x7f51d0fd7700 (LWP 128461)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3506198040, absSteadyTime=absSteadyTime@entry=0x7f51d0fc55f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db0f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db0f0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51d0fc5618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db0f0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db0f0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf15d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db0f0, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000a6a874 in operator() (this=0x7f51f15e0380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #18 operator() (__closure=0x7f51f15e0380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #21 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 52 (Thread 0x7f51b7fff700 (LWP 128462)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3086931480, absSteadyTime=absSteadyTime@entry=0x7f51b7fed5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db108) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db108, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b7fed618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db108) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db108, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf15d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db108, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000a6a874 in operator() (this=0x7f51f15e03c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #18 operator() (__closure=0x7f51f15e03c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #21 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 51 (Thread 0x7f51b77fe700 (LWP 128463)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3078538776, absSteadyTime=absSteadyTime@entry=0x7f51b77ec5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db120) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db120, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b77ec618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db120) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db120, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf15d in futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:164 #6 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db120, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #7 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #8 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #9 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #10 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #11 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #12 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #13 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #14 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #15 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #16 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #17 0x0000000000a6a874 in operator() (this=0x7f51f10d5900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #18 operator() (__closure=0x7f51f10d5900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #20 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #21 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 50 (Thread 0x7f51b6ffd700 (LWP 128464)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3070146072, absSteadyTime=absSteadyTime@entry=0x7f51b6feb5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db138) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db138, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b6feb618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db138) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db138, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db138, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e0440) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e0440) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 49 (Thread 0x7f51b67fc700 (LWP 128465)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3061753368, absSteadyTime=absSteadyTime@entry=0x7f51b67ea5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db150) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db150, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b67ea618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db150) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db150, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db150, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e0400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e0400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 48 (Thread 0x7f51b5ffb700 (LWP 128466)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3053360664, absSteadyTime=absSteadyTime@entry=0x7f51b5fe95f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db168) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db168, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b5fe9618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db168) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db168, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db168, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e0480) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e0480) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 47 (Thread 0x7f51b57fa700 (LWP 128467)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3044967960, absSteadyTime=absSteadyTime@entry=0x7f51b57e85f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db180) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db180, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b57e8618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db180) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db180, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db180, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7c9c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7c9c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7c980, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f112b400, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e04c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e04c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 46 (Thread 0x7f51b4ff9700 (LWP 128468)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=3036575256, absSteadyTime=absSteadyTime@entry=0x7f51b4fe75f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db198) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db198, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51b4fe7618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db198) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db198, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db198, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7e0c0, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7e0c0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7e080, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f1157000, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e05c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e05c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 45 (Thread 0x7f5193fff700 (LWP 128469)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=2482951704, absSteadyTime=absSteadyTime@entry=0x7f5193fed5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db1b0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db1b0, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f5193fed618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db1b0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db1b0, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db1b0, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7e640, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7e640) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7e600, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f1157c00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e3080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e3080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 44 (Thread 0x7f51937fe700 (LWP 128470)): #0 0x00007f51f289ebf9 in syscall () from /lib64/libc.so.6 #1 0x0000000000b9f22d in nativeFutexWaitImpl ( waitMask=waitMask@entry=2474559000, absSteadyTime=absSteadyTime@entry=0x7f51937ec5f8, absSystemTime=absSystemTime@entry=0x0, expected=expected@entry=2, addr=addr@entry=0x7f51d27db090) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:123 #2 folly::detail::futexWaitImpl (futex=futex@entry=0x7f51d27db090, expected=expected@entry=2, absSystemTime=absSystemTime@entry=0x0, absSteadyTime=absSteadyTime@entry=0x7f51937ec618, waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex.cpp:254 #3 0x0000000000ba92fa in futexWaitImpl const, std::chrono::time_point > > > (waitMask=4294967295, deadline=..., expected=2, futex=0x7f51d27db090) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:85 #4 folly::detail::futexWaitUntil, std::chrono::_V2::steady_clock, std::chrono::duration > > ( futex=futex@entry=0x7f51d27db090, expected=expected@entry=2, deadline=..., waitMask=waitMask@entry=4294967295) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/Futex-inl.h:123 #5 0x0000000000baf297 in futexWaitPreIdle, std::chrono::time_point > >, std::chrono::duration > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=..., _ret=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:194 #6 futexWaitUntil, std::chrono::time_point > > > (timeoutVariationFrac=0.5, stackToRetain=1024, idleTimeout=..., waitMask=4294967295, deadline=..., expected=2, fut=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/detail/MemoryIdler.h:151 #7 folly::SaturatingSemaphore::tryWaitSlow > > ( this=this@entry=0x7f51d27db090, deadline=..., opt=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:318 #8 0x0000000000baf7c6 in try_wait_until > > (opt=..., deadline=..., this=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/SaturatingSemaphore.h:179 #9 folly::detail::LifoSemBase, std::atomic>::try_wait_until > > (this=this@entry=0x2f7e640, deadline=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:521 #10 0x0000000000bb2f20 in try_wait_for > ( timeout=, this=0x2f7e640) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/synchronization/LifoSem.h:498 #11 folly::UnboundedBlockingQueue::try_take_for (this=0x2f7e600, time=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/task_queue/UnboundedBlockingQueue.h:41 #12 0x0000000000ba7497 in folly::CPUThreadPoolExecutor::threadRun ( this=0x7f51f1157c00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:243 #13 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #14 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #15 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #16 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #17 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #18 0x0000000000a6a874 in operator() (this=0x7f51f15e30c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #19 operator() (__closure=0x7f51f15e30c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47 #20 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401 #21 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #22 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 43 (Thread 0x7f5192ffd700 (LWP 128471)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f1090e00, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f1090e00, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f15ec180, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f15ec180) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever (this=0x7f51f15ec180) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x0000000000bb6f79 in folly::IOThreadPoolExecutor::threadRun ( this=0x7f51f15eb780, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/IOThreadPoolExecutor.cpp:195 #7 0x0000000000bc2709 in __invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> ( __t=, __f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:73 #8 __invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 __call (__args=, this=) at /usr/local/include/c++/7.3.0/functional:467 #10 operator()<> (this=) at /usr/local/include/c++/7.3.0/functional:551 #11 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 #12 0x00007f51f315bd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6 #13 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #14 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 42 (Thread 0x7f51927fc700 (LWP 128472)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f4470e51 in background_thread_sleep (tsdn=, interval=1600000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f51927eb220) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 41 (Thread 0x7f5191ffb700 (LWP 128473)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f5191fe9960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f51f117d6c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 40 (Thread 0x7f51917fa700 (LWP 128474)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f51917e8960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f51f117d6f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 39 (Thread 0x7f5190ff9700 (LWP 128475)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f5190fe7960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f51f117d720) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 38 (Thread 0x7f515ffff700 (LWP 128476)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f515ffed960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f51f117d750) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 37 (Thread 0x7f515f7fe700 (LWP 128477)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d780) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 36 (Thread 0x7f515effd700 (LWP 128478)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d7b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 35 (Thread 0x7f515e7fc700 (LWP 128479)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d7e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 34 (Thread 0x7f515dffb700 (LWP 128480)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d810) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 33 (Thread 0x7f515cfff700 (LWP 128492)): #0 0x00007f51f4f6afc3 in pread64 () from /lib64/libpthread.so.0 #1 0x0000000000996858 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145 #2 0x00000000009f7f8a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39 #3 0x000000000099aae6 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273 #4 0x00000000009a7ec4 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function >&, unsigned long&)>, std::unique_ptr >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39 #5 0x000000000099f0d2 in magma::ManagedFile::Read(std::shared_ptr::ObjectHandleImpl> const&, long, unsigned long, std::shared_ptr&, unsigned long&, unsigned long&, magma::CacheReadMode, std::function >&, unsigned long&)>, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #6 0x00000000009fd0ed in magma::SSTable::readBlock(std::shared_ptr::ObjectHandleImpl> const&, magma::BlockPointer const&, magma::Block*, magma::ReaderStats&, bool, magma::BufferedFileReader*) () at /opt/gcc-10.2.0/include/c++/10.2.0/new:175 #7 0x0000000000a0004f in magma::SSTableIterator::initIterators(magma::Slice const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/sstable/sstable.cc:764 #8 0x00000000009b5b45 in Next (this=0x7f515cfecfe0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:84 #9 magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:81 #10 0x00000000009c5358 in magma::KVReader::ReadKVs ( this=this@entry=0x7f515cfecf30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:69 #11 0x00000000009e7863 in magma::LSMTree::mergeSSTables(std::vector, std::allocator > >&, magma::ItemGCFilter&, unsigned long, double, std::vector, std::allocator > >*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:756 #12 0x00000000009bdd1d in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock&, magma::LSMTree::CompactionTrigger, std::vector, std::allocator > >&, std::vector, std::allocator > >&, std::function > (bool)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/compaction.cc:202 #13 0x00000000009c055f in magma::LSMTree::compactLevel(int, std::unique_lock&, magma::LSMTree::CompactionTrigger, std::function > (bool)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #14 0x00000000009c0811 in magma::LSMTree::compact(std::unique_lock&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/compaction.cc:384 #15 0x00000000009d8e18 in doCompactionWork (this=0x7f5151fa0010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:385 #16 operator() (__closure=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:395 #17 __invoke_impl&> ( __f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60 #18 __invoke_r&> ( __fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110 #19 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291 #20 0x00000000009a8734 in operator() (this=0x7f5148da22d0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248 #21 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:144 #22 0x00000000009a9f21 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:161 #23 0x0000000000b54b79 in run (this=0x7f51f117d840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #24 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #25 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 32 (Thread 0x7f515bdd3700 (LWP 128493)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d8a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 31 (Thread 0x7f515b5d2700 (LWP 128494)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d8d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 30 (Thread 0x7f515add1700 (LWP 128495)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x7f515a5d0700 (LWP 128496)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d930) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x7f5159dcf700 (LWP 128497)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51f117d960) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x7f51595ce700 (LWP 128498)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51b43b5b50) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7f5158dcd700 (LWP 128499)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51b43b5b80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 25 (Thread 0x7f51585cc700 (LWP 128500)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51b43b5bb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 24 (Thread 0x7f5157dcb700 (LWP 128501)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51b43b5be0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7f51575ca700 (LWP 128502)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51b43b5c10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7f5156dc9700 (LWP 128503)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51b43b5c40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7f514d760700 (LWP 128605)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f514d74e960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f515444ea30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x7f514cf5f700 (LWP 128606)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f514cf4d960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f515444ea90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7f514c75e700 (LWP 128607)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f514c74c960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f515384f8c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 18 (Thread 0x7f514bf5d700 (LWP 128608)): #0 0x00007f51f4f67de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00000000009a9dbf in __gthread_cond_timedwait ( __abs_timeout=0x7f514bf4b960, __mutex=, __cond=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:872 #2 __wait_until_impl > > (__lock=..., __lock=..., __atime=, this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:232 #3 wait_until > > (__atime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:141 #4 wait_for > (__rtime=..., __rtime=..., __lock=..., this=0x7f51f11a1498) at /opt/gcc-10.2.0/include/c++/10.2.0/condition_variable:172 #5 magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:99 #6 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #7 0x0000000000b54b79 in run (this=0x7f5153870130) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #8 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #9 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x7f514b75c700 (LWP 128609)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f5153870160) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7f514af5b700 (LWP 128610)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f5153870190) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7f51470d9700 (LWP 128611)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f5153881a00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7f51468d8700 (LWP 128612)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f5153881a30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7f51460d7700 (LWP 128613)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f5153881a60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7f51458d6700 (LWP 128614)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538aa2d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7f51450d5700 (LWP 128615)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538aa300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7f51448d4700 (LWP 128616)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538aa330) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7f51440d3700 (LWP 128617)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538c4ba0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7f51438d2700 (LWP 128618)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538c4bd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7f51430d1700 (LWP 128619)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538c4c00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7f51428d0700 (LWP 128620)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538d7470) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7f51420cf700 (LWP 128621)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538d74a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7f51418ce700 (LWP 128622)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538d74d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7f51410cd700 (LWP 128623)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538f0d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 2 (Thread 0x7f51408cc700 (LWP 128624)): #0 0x00007f51f4f67a35 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f51f31568bc in std::condition_variable::wait(std::unique_lock&) () from /opt/couchbase/bin/../lib/libstdc++.so.6 #2 0x00000000009a9b28 in magma::TaskQueue::dequeue(std::atomic&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:101 #3 0x00000000009a9f11 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:160 #4 0x0000000000b54b79 in run (this=0x7f51538f0d70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 #5 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 #6 0x00007f51f4f63ea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f51f28a48dd in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7f51f5ddac40 (LWP 128431)): #0 0x00007f51f28a4eb3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007f51f42383c5 in epoll_dispatch (base=0x7f51f108f300, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f51f422f376 in event_base_loop (base=0x7f51f108f300, flags=1) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x0000000000bc8e37 in folly::EventBase::loopBody ( this=this@entry=0x7f51f15eb500, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 #4 0x0000000000bc9306 in folly::EventBase::loop ( this=this@entry=0x7f51f15eb500) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 #5 0x0000000000bcacc6 in folly::EventBase::loopForever (this=0x7f51f15eb500) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 #6 0x00000000005630eb in memcached_main(int, char**) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1010 #7 0x00007f51f27c8555 in __libc_start_main () from /lib64/libc.so.6 #8 0x000000000055b587 in _start () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322 Detaching from program: /opt/couchbase/bin/memcached, process 128431 [Inferior 1 (process 128431) detached] 2021-11-24 08:48:42,134 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: ls /opt/couchbase/var/lib/couchbase/logs/memcached.log.* 2021-11-24 08:48:42,154 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: grep -r ' CRITICAL ' /opt/couchbase/var/lib/couchbase/logs/memcached.log.000454.txt | grep -v 'Rollback point not found' | grep -v 'No space left on device' | grep -v 'Permission denied' | grep -v 'write traffic will be disabled for this node' 2021-11-24 08:48:42,267 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: grep -r ' ERROR ' /opt/couchbase/var/lib/couchbase/logs/memcached.log.000454.txt | grep -v 'XERROR' | grep -v 'compaction failed for vb' | grep -v 'failed - Already exists' 2021-11-24 08:48:42,296 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: grep -r 'exception occurred in runloop' /opt/couchbase/var/lib/couchbase/logs/memcached.log.000454.txt 2021-11-24 08:48:42,316 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: grep -r 'Stream request failed because the snap start seqno' /opt/couchbase/var/lib/couchbase/logs/memcached.log.000454.txt 2021-11-24 08:48:42,342 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: ls /opt/couchbase/var/lib/couchbase/logs/sanitizers.log.* 2021-11-24 08:48:42,361 | test | DEBUG | MainThread | [basetestcase:check_coredump_exist:892] 172.23.121.126: No 'sanitizers.log.*' files found 2021-11-24 08:48:42,362 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: ls /opt/couchbase/var/lib/couchbase/logs/babysitter.log 2021-11-24 08:48:42,380 | infra | DEBUG | MainThread | [remote_util:execute_command_raw_jsch:3307] Running command on 172.23.121.126: grep -r ' CRITICAL ' /opt/couchbase/var/lib/couchbase/logs/babysitter.log | grep -v 'Rollback point not found' | grep -v 'No space left on device' | grep -v 'Permission denied' | grep -v 'write traffic will be disabled for this node' 2021-11-24 08:48:42,401 | test | INFO | MainThread | [basetestcase:check_logs:832] Comparing timestamps: Log's latest timestamp: 2021-11-24 08:47:54, Test's start timestamp is 2021-11-23 20:57:23.174000 2021-11-24 08:48:42,404 | test | INFO | MainThread | [basetestcase:check_logs:832] Comparing timestamps: Log's latest timestamp: 2021-11-24 08:47:54, Test's start timestamp is 2021-11-23 20:57:23.174000 2021-11-24 08:48:42,404 | test | INFO | MainThread | [basetestcase:check_coredump_exist:941] unwanted messages in /opt/couchbase/var/lib/couchbase/logs/babysitter.log 2021-11-24 08:48:42,405 | test | CRITICAL | MainThread | [basetestcase:check_coredump_exist:943] 172.23.121.126: Found ' CRITICAL ' logs - ['[ns_server:info,2021-11-23T10:19:08.264-08:00,babysitter_of_ns_1@cb.local:<0.133.0>:ns_port_server:log:226]memcached<0.133.0>: 2021-11-23T10:19:08.060307-08:00 CRITICAL *** Fatal error encountered during exception handling ***\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061478-08:00 CRITICAL Caught unhandled std::exception-derived exception. what(): ThrowExceptionUnderflowPolicy current:5009856 arg:5010384\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061498-08:00 CRITICAL Exception thrown from:\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061548-08:00 CRITICAL #0 /opt/couchbase/bin/memcached() [0x400000+0x1341b6]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061569-08:00 CRITICAL #1 /opt/couchbase/bin/memcached() [0x400000+0x4aa388]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061588-08:00 CRITICAL #2 /opt/couchbase/bin/memcached() [0x400000+0x4aac78]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061607-08:00 CRITICAL #3 /opt/couchbase/bin/memcached() [0x400000+0x4ad110]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061625-08:00 CRITICAL #4 /opt/couchbase/bin/memcached() [0x400000+0x3c4da8]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061639-08:00 CRITICAL #5 /opt/couchbase/bin/memcached() [0x400000+0x3c52cb]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061653-08:00 CRITICAL #6 /opt/couchbase/bin/memcached() [0x400000+0x3c822c]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061665-08:00 CRITICAL #7 /opt/couchbase/bin/memcached() [0x400000+0x3c9248]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061689-08:00 CRITICAL #8 /opt/couchbase/bin/memcached() [0x400000+0x66d9d2]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061714-08:00 CRITICAL #9 /opt/couchbase/bin/memcached() [0x400000+0x66abe5]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061741-08:00 CRITICAL #10 /opt/couchbase/bin/memcached() [0x400000+0x7bf750]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061763-08:00 CRITICAL #11 /opt/couchbase/bin/memcached() [0x400000+0x7a750a]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061782-08:00 CRITICAL #12 /opt/couchbase/bin/memcached() [0x400000+0x7c2709]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061801-08:00 CRITICAL #13 /opt/couchbase/bin/memcached() [0x400000+0x66a874]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061880-08:00 CRITICAL #14 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc5970ef000+0xcdd40]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061887-08:00 CRITICAL #15 /lib64/libpthread.so.0() [0x7fc598fbd000+0x7ea5]\n', 'memcached<0.133.0>: 2021-11-23T10:19:08.061921-08:00 CRITICAL #16 /lib64/libc.so.6(clone+0x6d) [0x7fc596807000+0xfe8dd]\n', '[ns_server:info,2021-11-23T10:19:09.056-08:00,babysitter_of_ns_1@cb.local:<0.133.0>:ns_port_server:log:226]memcached<0.133.0>: CRITICAL Breakpad caught a crash (Couchbase version 7.1.0-1745). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/e70e0955-3bc7-4477-3ffebfaf-dd52a39b.dmp before terminating.\n', '[ns_server:info,2021-11-23T10:19:10.202-08:00,babysitter_of_ns_1@cb.local:<0.21966.2>:ns_port_server:log:226]memcached<0.21966.2>: 2021-11-23T10:19:10.001179-08:00 CRITICAL Detected previous crash\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001266-08:00 CRITICAL Breakpad caught a crash (Couchbase version 7.1.0-1745). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/e70e0955-3bc7-4477-3ffebfaf-dd52a39b.dmp before terminating.\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001279-08:00 CRITICAL Stack backtrace of crashed thread:\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001281-08:00 CRITICAL #0 /opt/couchbase/bin/memcached() [0x400000+0x6f0268]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001284-08:00 CRITICAL #1 /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x74045a]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001286-08:00 CRITICAL #2 /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x740798]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001288-08:00 CRITICAL #3 /lib64/libpthread.so.0() [0x7fc598fbd000+0xf630]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001289-08:00 CRITICAL #4 /lib64/libc.so.6(gsignal+0x37) [0x7fc596807000+0x36387]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001291-08:00 CRITICAL #5 /lib64/libc.so.6(abort+0x148) [0x7fc596807000+0x37a78]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001293-08:00 CRITICAL #6 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc5970ef000+0x9963c]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001294-08:00 CRITICAL #7 /opt/couchbase/bin/memcached() [0x400000+0x6fd3db]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001308-08:00 CRITICAL #8 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc5970ef000+0xa48f6]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001317-08:00 CRITICAL #9 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc5970ef000+0xa4961]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001321-08:00 CRITICAL #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc5970ef000+0xa4bf4]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001324-08:00 CRITICAL #11 /opt/couchbase/bin/memcached() [0x400000+0x1342ca]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001326-08:00 CRITICAL #12 /opt/couchbase/bin/memcached() [0x400000+0x4aa388]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001328-08:00 CRITICAL #13 /opt/couchbase/bin/memcached() [0x400000+0x4aac78]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001331-08:00 CRITICAL #14 /opt/couchbase/bin/memcached() [0x400000+0x4ad110]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001335-08:00 CRITICAL #15 /opt/couchbase/bin/memcached() [0x400000+0x3c4da8]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001337-08:00 CRITICAL #16 /opt/couchbase/bin/memcached() [0x400000+0x3c52cb]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001342-08:00 CRITICAL #17 /opt/couchbase/bin/memcached() [0x400000+0x3c822c]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001343-08:00 CRITICAL #18 /opt/couchbase/bin/memcached() [0x400000+0x3c9248]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001381-08:00 CRITICAL #19 /opt/couchbase/bin/memcached() [0x400000+0x66d9d2]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001385-08:00 CRITICAL #20 /opt/couchbase/bin/memcached() [0x400000+0x66abe5]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001389-08:00 CRITICAL #21 /opt/couchbase/bin/memcached() [0x400000+0x7bf750]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001392-08:00 CRITICAL #22 /opt/couchbase/bin/memcached() [0x400000+0x7a750a]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001394-08:00 CRITICAL #23 /opt/couchbase/bin/memcached() [0x400000+0x7c2709]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001395-08:00 CRITICAL #24 /opt/couchbase/bin/memcached() [0x400000+0x66a874]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001399-08:00 CRITICAL #25 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc5970ef000+0xcdd40]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001401-08:00 CRITICAL #26 /lib64/libpthread.so.0() [0x7fc598fbd000+0x7ea5]\n', 'memcached<0.21966.2>: 2021-11-23T10:19:10.001404-08:00 CRITICAL #27 /lib64/libc.so.6(clone+0x6d) [0x7fc596807000+0xfe8dd]\n', 'memcached<0.13782.2>: 2021-11-24T08:47:52.533964-08:00 CRITICAL *** Fatal error encountered during exception handling ***\n', 'memcached<0.13782.2>: 2021-11-24T08:47:52.534064-08:00 CRITICAL Caught unhandled std::exception-derived exception. what(): std::exception\n', '[ns_server:info,2021-11-24T08:47:54.054-08:00,babysitter_of_ns_1@cb.local:<0.13782.2>:ns_port_server:log:226]memcached<0.13782.2>: CRITICAL Breakpad caught a crash (Couchbase version 7.1.0-1745). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.dmp before terminating.\n', '[ns_server:info,2021-11-24T08:47:55.133-08:00,babysitter_of_ns_1@cb.local:<0.29312.3>:ns_port_server:log:226]memcached<0.29312.3>: 2021-11-24T08:47:54.930536-08:00 CRITICAL Detected previous crash\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930637-08:00 CRITICAL Breakpad caught a crash (Couchbase version 7.1.0-1745). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.dmp before terminating.\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930641-08:00 CRITICAL Stack backtrace of crashed thread:\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930646-08:00 CRITICAL #0 /opt/couchbase/bin/memcached() [0x400000+0x6f0268]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930648-08:00 CRITICAL #1 /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x74045a]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930650-08:00 CRITICAL #2 /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x740798]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930652-08:00 CRITICAL #3 /lib64/libpthread.so.0() [0x7f56e4dc3000+0xf630]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930697-08:00 CRITICAL #4 /lib64/libc.so.6(gsignal+0x37) [0x7f56e260d000+0x36387]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930701-08:00 CRITICAL #5 /lib64/libc.so.6(abort+0x148) [0x7f56e260d000+0x37a78]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930703-08:00 CRITICAL #6 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56e2ef5000+0x9963c]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930889-08:00 CRITICAL #7 /opt/couchbase/bin/memcached() [0x400000+0x6fd3db]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930891-08:00 CRITICAL #8 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56e2ef5000+0xa48f6]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930893-08:00 CRITICAL #9 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56e2ef5000+0xa4961]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930895-08:00 CRITICAL #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56e2ef5000+0xa4bf4]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930897-08:00 CRITICAL #11 /opt/couchbase/bin/memcached() [0x400000+0xc272e]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930899-08:00 CRITICAL #12 /opt/couchbase/bin/memcached() [0x400000+0x3f784c]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930900-08:00 CRITICAL #13 /opt/couchbase/bin/memcached() [0x400000+0x3fe273]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930903-08:00 CRITICAL #14 /opt/couchbase/bin/memcached() [0x400000+0x31f484]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930905-08:00 CRITICAL #15 /opt/couchbase/bin/memcached() [0x400000+0x440158]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930907-08:00 CRITICAL #16 /opt/couchbase/bin/memcached() [0x400000+0x66d9d2]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930908-08:00 CRITICAL #17 /opt/couchbase/bin/memcached() [0x400000+0x66abe5]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930910-08:00 CRITICAL #18 /opt/couchbase/bin/memcached() [0x400000+0x7bf750]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930911-08:00 CRITICAL #19 /opt/couchbase/bin/memcached() [0x400000+0x7a750a]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930913-08:00 CRITICAL #20 /opt/couchbase/bin/memcached() [0x400000+0x7c2709]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930914-08:00 CRITICAL #21 /opt/couchbase/bin/memcached() [0x400000+0x66a874]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930927-08:00 CRITICAL #22 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56e2ef5000+0xcdd40]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930929-08:00 CRITICAL #23 /lib64/libpthread.so.0() [0x7f56e4dc3000+0x7ea5]\n', 'memcached<0.29312.3>: 2021-11-24T08:47:54.930930-08:00 CRITICAL #24 /lib64/libc.so.6(clone+0x6d) [0x7f56e260d000+0xfe8dd]\n']