Thread 28 (LWP 47981): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59c9d22e51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f59bdff5a90) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f59c804dea5 in start_thread (arg=0x7f59bdfff700) at pthread_create.c:307 #6 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 27 (LWP 47957): #0 __atomic_add (__val=1, __mem=0x7f59799f7a88) at /usr/local/include/c++/7.3.0/ext/atomicity.h:53 #1 __atomic_add_dispatch (__val=1, __mem=0x7f59799f7a88) at /usr/local/include/c++/7.3.0/ext/atomicity.h:96 #2 _M_add_ref_copy (this=0x7f59799f7a80) at /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:138 #3 __shared_count (__r=..., this=) at /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:691 #4 __shared_ptr (this=) at /usr/local/include/c++/7.3.0/bits/shared_ptr_base.h:1121 #5 shared_ptr (this=) at /usr/local/include/c++/7.3.0/bits/shared_ptr.h:119 #6 getBytesAllocated (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/memory_tracking_allocator.h:120 #7 MemoryTrackingAllocator > > > (other=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/memory_tracking_allocator.h:74 #8 get_allocator (this=) at /usr/local/include/c++/7.3.0/bits/stl_list.h:843 #9 Checkpoint::getMemoryOverhead (this=0x7f595efa8f00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.h:613 #10 0x00007f59cbf1d230 in getMemConsumption (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.h:592 #11 getMemoryUsage_UNLOCKED (this=0x7f5979dfa400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:1479 #12 CheckpointManager::getMemoryUsage (this=0x7f5979dfa400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:1494 #13 0x00007f59cc0627cc in VBucket::getChkMgrMemUsage (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:297 #14 0x00007f59cc07eb32 in VBucketMap::getVBucketsTotalCheckpointMemoryUsage (this=0x7f5988189018) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucketmap.cc:137 #15 0x00007f59cbf2a46f in ClosedUnrefCheckpointRemoverTask::isReductionInCheckpointMemoryNeeded (this=0x7f598816b490) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:92 #16 0x00007f59cbf2aee8 in ClosedUnrefCheckpointRemoverTask::run (this=0x7f598816b490) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:216 #17 0x00007f59cc006be3 in GlobalTask::execute (this=0x7f598816b490) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73 #18 0x00007f59cbfff48f in ExecutorThread::run (this=0x7f59c69bb880) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188 #19 0x00007f59caa10777 in run (this=0x7f59c764c0a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #20 platform_thread_wrap (arg=0x7f59c764c0a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #21 0x00007f59c804dea5 in start_thread (arg=0x7f598b7f6700) at pthread_create.c:307 #22 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 26 (LWP 47956): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cc05d69b in __gthread_cond_timedwait (__abs_timeout=0x7f598bfed000, __mutex=, __cond=0x7f59c675a4a8) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f59c675a480, t=..., lock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f59cc05e9fb in _sleepThenFetchNextTask (t=..., this=0x7f59c675a480) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f59c675a480, thread=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f59cbff5eb7 in ExecutorPool::_nextTask (this=0x7f59c6868e00, t=..., tick=tick@entry=97 'a') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f59cbff5f38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=97 'a') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f59cbfff316 in ExecutorThread::run (this=0x7f59c69bb7a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f59caa10777 in run (this=0x7f59c764c070) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f59c764c070) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #14 0x00007f59c804dea5 in start_thread (arg=0x7f598bff7700) at pthread_create.c:307 #15 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 25 (LWP 47955): #0 0x00007ffdb28df6c2 in ?? () #1 0x00007f598c7ec390 in ?? () #2 0x00007f598c7ec380 in ?? () #3 0x00007f598815fb40 in ?? () #4 0x00007f59cbeb0d48 in __bss_start () from /opt/couchbase/bin/../lib/libcouchstore.so #5 0x00007f598815fb40 in ?? () #6 0x00007f59c7d8c7fd in __GI___clock_gettime (clock_id=clock_id@entry=1, tp=tp@entry=0x7f598c7ec2e0) at ../sysdeps/unix/clock_gettime.c:115 #7 0x00007f59c882d111 in std::chrono::_V2::steady_clock::now () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/chrono.cc:90 #8 0x00007f59cbfb05e6 in GenericBlockTimer::~GenericBlockTimer (this=0x7f598c7ec380, __in_chrg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/histogram.h:449 #9 0x00007f59cc0ca1fd in StatsOps::pread (this=0x7f5988161860, errinfo=0x7f5979d89600, h=0x7f598815fb40, buf=0x7f5979c54000, sz=4096, off=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-fs-stats.cc:116 #10 0x00007f59cbc8e48b in load_buffer_from (nbyte=, offset=, buf=0x7f5970686940, errinfo=0x7f5979d89600) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/iobuffer.cc:379 #11 BufferedFileOps::pread (errinfo=0x7f5979d89600, handle=0x7f5979b9d000, buf=0x7f5979bf7bbd, nbyte=88, offset=45057, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/iobuffer.cc:554 #12 0x00007f59cbc820b3 in read_skipping_prefixes (dst=0x7f5979bf7bbd, len=88, pos=, file=0x7f5979d895e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_read.cc:115 #13 pread_bin_internal (file=0x7f5979d895e0, pos=45057, pos@entry=44731, ret_ptr=ret_ptr@entry=0x7f598c7ec578, max_header_size=max_header_size@entry=0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_read.cc:169 #14 0x00007f59cbc825ea in pread_compressed (file=, pos=pos@entry=44731, ret_ptr=ret_ptr@entry=0x7f598c7ec618) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/couch_file_read.cc:204 #15 0x00007f59cbc7071a in btree_lookup_inner (rq=rq@entry=0x7f598c7ec8f0, diskpos=44731, current=current@entry=0, end=end@entry=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:43 #16 0x00007f59cbc708c3 in btree_lookup_inner (rq=rq@entry=0x7f598c7ec8f0, diskpos=, current=current@entry=0, end=end@entry=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:76 #17 0x00007f59cbc708c3 in btree_lookup_inner (rq=rq@entry=0x7f598c7ec8f0, diskpos=, current=current@entry=0, end=1) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:76 #18 0x00007f59cbc70ce1 in btree_lookup (rq=rq@entry=0x7f598c7ec8f0, root_pointer=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/btree_read.cc:162 #19 0x00007f59cbc87112 in compact_seq_tree (ctx=0x7f598c7ec970, target=0x7f5979c2c440, source=0x7f5979d895e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/db_compact.cc:407 #20 cb::couchstore::compact(_db&, char const*, unsigned long, std::function, std::function, FileOpsInterface*) (source=..., target_filename=, flags=flags@entry=251658242, filterCallback=..., rewriteDocInfoCallback=..., ops=ops@entry=0x7f5988161860) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/db_compact.cc:180 #21 0x00007f59cc0bef52 in CouchKVStore::compactDBInternal(compaction_ctx*, std::function) (this=this@entry=0x7f5988186300, hook_ctx=, docinfo_hook=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1063 #22 0x00007f59cc0bffb9 in CouchKVStore::compactDB (this=0x7f5988186300, hook_ctx=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:927 #23 0x00007f59cbfab8af in EPBucket::compactInternal (this=0x7f5988189000, config=..., purgeSeqno=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1140 #24 0x00007f59cbfabcad in EPBucket::doCompact (this=0x7f5988189000, config=..., purgeSeqno=0, cookie=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1205 #25 0x00007f59cc05c4a8 in CompactTask::run (this=0x7f5978ca7450) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:67 #26 0x00007f59cc006be3 in GlobalTask::execute (this=0x7f5978ca7450) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73 #27 0x00007f59cbfff48f in ExecutorThread::run (this=0x7f59c69bb6c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188 #28 0x00007f59caa10777 in run (this=0x7f59c764c040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #29 platform_thread_wrap (arg=0x7f59c764c040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #30 0x00007f59c804dea5 in start_thread (arg=0x7f598c7f8700) at pthread_create.c:307 #31 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 24 (LWP 47954): #0 0x00007f59c7d69147 in unlink () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f59c7ce3db9 in __GI_remove (file=0x7f597201b010
) at ../sysdeps/posix/remove.c:33 #2 0x00007f59cbc92e7f in TreeWriterFree (writer=0x7f597201b000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/tree_writer.cc:79 #3 0x00007f59cbc86a8c in cb::couchstore::compact(_db&, char const*, unsigned long, std::function, std::function, FileOpsInterface*) (source=..., target_filename=, flags=flags@entry=251658242, filterCallback=..., rewriteDocInfoCallback=..., ops=ops@entry=0x7f59bc86d240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/couchstore/src/db_compact.cc:202 #4 0x00007f59cc0bef52 in CouchKVStore::compactDBInternal(compaction_ctx*, std::function) (this=this@entry=0x7f59bc893500, hook_ctx=, docinfo_hook=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1063 #5 0x00007f59cc0bffb9 in CouchKVStore::compactDB (this=0x7f59bc893500, hook_ctx=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:927 #6 0x00007f59cbfab8af in EPBucket::compactInternal (this=0x7f59bc89f000, config=..., purgeSeqno=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1140 #7 0x00007f59cbfabcad in EPBucket::doCompact (this=0x7f59bc89f000, config=..., purgeSeqno=0, cookie=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1205 #8 0x00007f59cc05c4a8 in CompactTask::run (this=0x7f5989e47290) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:67 ---Type to continue, or q to quit--- #9 0x00007f59cc006be3 in GlobalTask::execute (this=0x7f5989e47290) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73 #10 0x00007f59cbfff48f in ExecutorThread::run (this=0x7f59c69bb5e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188 #11 0x00007f59caa10777 in run (this=0x7f59c764c010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #12 platform_thread_wrap (arg=0x7f59c764c010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #13 0x00007f59c804dea5 in start_thread (arg=0x7f598cff9700) at pthread_create.c:307 #14 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 23 (LWP 47953): #0 0x00007f59cc07e405 in _M_local_data (this=) at /usr/local/include/c++/7.3.0/bits/basic_string.h:182 #1 basic_string (__str=, this=) at /usr/local/include/c++/7.3.0/bits/basic_string.h:532 #2 construct, std::__cxx11::basic_string, std::allocator > > (this=, __p=) at /usr/local/include/c++/7.3.0/ext/new_allocator.h:136 #3 construct, std::__cxx11::basic_string, std::allocator > > (__a=..., __p=) at /usr/local/include/c++/7.3.0/bits/alloc_traits.h:475 #4 nlohmann::basic_json, std::allocator >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer>::create, std::allocator >, std::__cxx11::basic_string, std::allocator > >(std::__cxx11::basic_string, std::allocator >&&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:834 #5 0x00007f59cc07be18 in json_value (value=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:968 #6 construct > (s=, j=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/detail/conversions/to_json.hpp:53 #7 to_json > (s=, j=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/detail/conversions/to_json.hpp:213 #8 operator(), std::__cxx11::basic_string > (this=, val=, j=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/detail/conversions/to_json.hpp:332 #9 to_json, std::__cxx11::basic_string > (val=, j=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/adl_serializer.hpp:45 #10 basic_json > (val=, this=0x7f598d7eee90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:1253 #11 json_ref, std::allocator > > (this=0x7f598d7eee90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/detail/json_ref.hpp:35 #12 to_json (json=..., vbs=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket_state.cc:139 #13 0x00007f59cc0b4221 in operator(), const vbucket_state&> (this=, val=..., j=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/detail/conversions/to_json.hpp:332 #14 to_json, const vbucket_state&> (val=..., j=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/adl_serializer.hpp:45 #15 basic_json (val=..., this=0x7f598d7ef4c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:1253 #16 CouchKVStore::saveVBState (this=this@entry=0x7f5988185900, db=0x7f5978b17ba0, vbState=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2456 #17 0x00007f59cc0c0bc1 in CouchKVStore::saveDocs (this=this@entry=0x7f5988185900, vbid=..., docs=std::vector of length 11, capacity 11 = {...}, docinfos=std::vector of length 11, capacity 11 = {...}, kvctx=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2181 #18 0x00007f59cc0c1582 in CouchKVStore::commit2couchstore (this=0x7f5988185900, commitData=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:2014 #19 0x00007f59cc0c1c15 in CouchKVStore::commit (this=0x7f5988185900, commitData=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1293 #20 0x00007f59cbfaaba2 in EPBucket::commit (this=this@entry=0x7f5988189000, vbid=..., kvstore=..., commitData=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:875 #21 0x00007f59cbfaee51 in EPBucket::flushVBucket (this=0x7f5988189000, vbid=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:744 #22 0x00007f59cc004afc in Flusher::flushVB (this=this@entry=0x7f5979bf7380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:306 #23 0x00007f59cc005cc9 in Flusher::step (this=0x7f5979bf7380, task=0x7f59881af510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:207 #24 0x00007f59cc006be3 in GlobalTask::execute (this=0x7f59881af510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73 #25 0x00007f59cbfff48f in ExecutorThread::run (this=0x7f59c69bb500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188 #26 0x00007f59caa10777 in run (this=0x7f59c764bfe0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #27 platform_thread_wrap (arg=0x7f59c764bfe0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #28 0x00007f59c804dea5 in start_thread (arg=0x7f598d7fa700) at pthread_create.c:307 #29 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 22 (LWP 47952): #0 phosphor::TraceChunk::addEvent (this=0x7f59bf2fc048) at /home/couchbase/jenkins/workspace/couchbase-server-unix/phosphor/src/trace_buffer.cc:54 #1 0x00007f59ca7f4027 in phosphor::TraceLog::logEvent (this=, tpi=0x7f59cc4133a0 )::phosphor_internal_tpi_966>, start=..., duration=..., argA=..., argB=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/phosphor/src/trace_log.cc:153 #2 0x00007f59cc0beb79 in CouchKVStore::compactDBInternal(compaction_ctx*, std::function) (this=this@entry=0x7f59bc104900, hook_ctx=, docinfo_hook=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:966 #3 0x00007f59cc0bffb9 in CouchKVStore::compactDB (this=0x7f59bc104900, hook_ctx=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:927 #4 0x00007f59cbfab8af in EPBucket::compactInternal (this=0x7f59bc108000, config=..., purgeSeqno=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1140 #5 0x00007f59cbfabcad in EPBucket::doCompact (this=0x7f59bc108000, config=..., purgeSeqno=0, cookie=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1205 #6 0x00007f59cc05c4a8 in CompactTask::run (this=0x7f597b781850) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:67 #7 0x00007f59cc006be3 in GlobalTask::execute (this=0x7f597b781850) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73 #8 0x00007f59cbfff48f in ExecutorThread::run (this=0x7f59c69bb420) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188 #9 0x00007f59caa10777 in run (this=0x7f59c764bfb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #10 platform_thread_wrap (arg=0x7f59c764bfb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #11 0x00007f59c804dea5 in start_thread (arg=0x7f598dffb700) at pthread_create.c:307 #12 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 21 (LWP 47951): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cc05d69b in __gthread_cond_timedwait (__abs_timeout=0x7f598e7f2000, __mutex=, __cond=0x7f59c675a328) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f59c675a300, t=..., lock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f59cc05e9fb in _sleepThenFetchNextTask (t=..., this=0x7f59c675a300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f59c675a300, thread=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f59cbff5eb7 in ExecutorPool::_nextTask (this=0x7f59c6868e00, t=..., tick=tick@entry=135 '\207') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f59cbff5f38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=135 '\207') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f59cbfff316 in ExecutorThread::run (this=0x7f59c69bb340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f59caa10777 in run (this=0x7f59c764bf80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f59c764bf80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #14 0x00007f59c804dea5 in start_thread (arg=0x7f598e7fc700) at pthread_create.c:307 #15 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 20 (LWP 47950): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cc05d69b in __gthread_cond_timedwait (__abs_timeout=0x7f598eff3000, __mutex=, __cond=0x7f59c675a328) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f59c675a300, t=..., lock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f59cc05e9fb in _sleepThenFetchNextTask (t=..., this=0x7f59c675a300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f59c675a300, thread=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f59cbff5eb7 in ExecutorPool::_nextTask (this=0x7f59c6868e00, t=..., tick=tick@entry=143 '\217') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f59cbff5f38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=143 '\217') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f59cbfff316 in ExecutorThread::run (this=0x7f59c69bb260) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f59caa10777 in run (this=0x7f59c764bf50) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f59c764bf50) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #14 0x00007f59c804dea5 in start_thread (arg=0x7f598effd700) at pthread_create.c:307 #15 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 19 (LWP 47946): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cc05d69b in __gthread_cond_timedwait (__abs_timeout=0x7f598f7f4000, __mutex=, __cond=0x7f59c675a328) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f59c675a300, t=..., lock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:98 ---Type to continue, or q to quit--- #7 0x00007f59cc05e9fb in _sleepThenFetchNextTask (t=..., this=0x7f59c675a300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f59c675a300, thread=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f59cbff5eb7 in ExecutorPool::_nextTask (this=0x7f59c6868e00, t=..., tick=tick@entry=110 'n') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f59cbff5f38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=110 'n') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f59cbfff316 in ExecutorThread::run (this=0x7f59c69bb180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f59caa10777 in run (this=0x7f59c764bf20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f59c764bf20) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #14 0x00007f59c804dea5 in start_thread (arg=0x7f598f7fe700) at pthread_create.c:307 #15 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 18 (LWP 47945): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cc05d69b in __gthread_cond_timedwait (__abs_timeout=0x7f598fff5000, __mutex=, __cond=0x7f59c675a328) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 wait_for (nanoSecs=..., lock=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/syncobject.h:53 #6 TaskQueue::_doSleep (this=this@entry=0x7f59c675a300, t=..., lock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:98 #7 0x00007f59cc05e9fb in _sleepThenFetchNextTask (t=..., this=0x7f59c675a300) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:119 #8 TaskQueue::sleepThenFetchNextTask (this=this@entry=0x7f59c675a300, thread=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/taskqueue.cc:158 #9 0x00007f59cbff5eb7 in ExecutorPool::_nextTask (this=0x7f59c6868e00, t=..., tick=tick@entry=107 'k') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:190 #10 0x00007f59cbff5f38 in ExecutorPool::nextTask (this=, t=..., tick=tick@entry=107 'k') at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorpool.cc:204 #11 0x00007f59cbfff316 in ExecutorThread::run (this=0x7f59c69bb0a0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:129 #12 0x00007f59caa10777 in run (this=0x7f59c764bef0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f59c764bef0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #14 0x00007f59c804dea5 in start_thread (arg=0x7f598ffff700) at pthread_create.c:307 #15 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 17 (LWP 47943): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59c9d22e51 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=, info=, tsdn=) at src/background_thread.c:307 #3 background_work (ind=, tsd=0x7f59bd1f5a90) at src/background_thread.c:497 #4 background_thread_entry (ind_arg=) at src/background_thread.c:522 #5 0x00007f59c804dea5 in start_thread (arg=0x7f59bd1ff700) at pthread_create.c:307 #6 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 16 (LWP 47613): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f59c882f1bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f59c690b698) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7f59c690b698, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x000000000049d381 in Executor::run (this=0x7f59c690b540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007f59caa1b40f in Couchbase::Thread::thread_entry (this=0x7f59c690b540) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f59caa10777 in run (this=0x7f59c68836d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7f59c68836d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f59c804dea5 in start_thread (arg=0x7f59c0472700) at pthread_create.c:307 #8 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 15 (LWP 47612): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f59c882f1bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f59c690b4d8) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7f59c690b4d8, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x000000000049d381 in Executor::run (this=0x7f59c690b380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007f59caa1b40f in Couchbase::Thread::thread_entry (this=0x7f59c690b380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f59caa10777 in run (this=0x7f59c6883760) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7f59c6883760) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f59c804dea5 in start_thread (arg=0x7f59c0c73700) at pthread_create.c:307 #8 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 14 (LWP 47611): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f59c882f1bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f59c690b318) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7f59c690b318, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x000000000049d381 in Executor::run (this=0x7f59c690b1c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007f59caa1b40f in Couchbase::Thread::thread_entry (this=0x7f59c690b1c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f59caa10777 in run (this=0x7f59c6883730) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7f59c6883730) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f59c804dea5 in start_thread (arg=0x7f59c1474700) at pthread_create.c:307 #8 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 13 (LWP 47610): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f59c882f1bc in __gthread_cond_wait (__mutex=, __cond=__cond@entry=0x7f59c690b158) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:864 #2 std::condition_variable::wait (this=this@entry=0x7f59c690b158, __lock=...) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/condition_variable.cc:53 #3 0x000000000049d381 in Executor::run (this=0x7f59c690b000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/executor.cc:44 #4 0x00007f59caa1b40f in Couchbase::Thread::thread_entry (this=0x7f59c690b000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #5 0x00007f59caa10777 in run (this=0x7f59c6883700) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #6 platform_thread_wrap (arg=0x7f59c6883700) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #7 0x00007f59c804dea5 in start_thread (arg=0x7f59c1c75700) at pthread_create.c:307 #8 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 12 (LWP 47609): #0 0x00007f59c7d76eb3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f59c9aea3c5 in epoll_dispatch (base=0xd68670, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f59c9ae1376 in event_base_loop (base=0xd68670, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004692a9 in worker_libevent (arg=0x7f59c6844ff8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210 #4 0x00007f59caa10777 in run (this=0x7f59c6883610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f59c6883610) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f59c804dea5 in start_thread (arg=0x7f59c2476700) at pthread_create.c:307 #7 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 11 (LWP 47608): #0 0x00007f59c7d76eb3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f59c9aea3c5 in epoll_dispatch (base=0xd67f40, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f59c9ae1376 in event_base_loop (base=0xd67f40, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004692a9 in worker_libevent (arg=0x7f59c6844550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210 #4 0x00007f59caa10777 in run (this=0x7f59c6883850) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f59c6883850) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f59c804dea5 in start_thread (arg=0x7f59c2c77700) at pthread_create.c:307 ---Type to continue, or q to quit--- #7 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 10 (LWP 47607): #0 0x00007f59c7d76eb3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f59c9aea3c5 in epoll_dispatch (base=0xd67910, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f59c9ae1376 in event_base_loop (base=0xd67910, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004692a9 in worker_libevent (arg=0x7f59c6843aa8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210 #4 0x00007f59caa10777 in run (this=0x7f59c764bd70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f59c764bd70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f59c804dea5 in start_thread (arg=0x7f59c3478700) at pthread_create.c:307 #7 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 9 (LWP 47606): #0 0x00007f59c7d76eb3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 #1 0x00007f59c9aea3c5 in epoll_dispatch (base=0xd67320, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f59c9ae1376 in event_base_loop (base=0xd67320, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004692a9 in worker_libevent (arg=0x7f59c6843000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:210 #4 0x00007f59caa10777 in run (this=0x7f59c764bb90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #5 platform_thread_wrap (arg=0x7f59c764bb90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #6 0x00007f59c804dea5 in start_thread (arg=0x7f59c3c79700) at pthread_create.c:307 #7 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 8 (LWP 47605): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00000000004e4926 in __gthread_cond_timedwait (__abs_timeout=0x7f59c4470230, __mutex=, __cond=0x7f59c67be8f8) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7f59c67be8f8) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7f59c67be8f8) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=0x7f59c67be8f8) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 AuditImpl::consume_events (this=0x7f59c67be500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/auditd/src/audit.cc:412 #6 0x00007f59caa10777 in run (this=0x7f59c764b5f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #7 platform_thread_wrap (arg=0x7f59c764b5f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #8 0x00007f59c804dea5 in start_thread (arg=0x7f59c447a700) at pthread_create.c:307 #9 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 7 (LWP 47604): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x000000000048d8b0 in __gthread_cond_timedwait (__abs_timeout=0x7f59c4c71210, __mutex=, __cond=0x7f59c763b378) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=, this=0x7f59c763b378) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=, this=0x7f59c763b378) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_for > (__rtime=, __lock=, this=0x7f59c763b378) at /usr/local/include/c++/7.3.0/condition_variable:138 #5 ExternalAuthManagerThread::run (this=0x7f59c763b280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/external_auth_manager_thread.cc:229 #6 0x00007f59caa1b40f in Couchbase::Thread::thread_entry (this=0x7f59c763b280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45 #7 0x00007f59caa10777 in run (this=0x7f59c764bda0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #8 platform_thread_wrap (arg=0x7f59c764bda0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #9 0x00007f59c804dea5 in start_thread (arg=0x7f59c4c7b700) at pthread_create.c:307 #10 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 6 (LWP 47603): #0 0x00007f59c7d6bc3d in poll () at ../sysdeps/unix/syscall-template.S:81 #1 0x000000000046869e in get_command (buffsize=80, buffer=0x7f59c5472210 "") at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:49 #2 check_stdin_thread (arg=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/stdin_check.cc:90 #3 0x00007f59caa10777 in run (this=0x7f59c764b770) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #4 platform_thread_wrap (arg=0x7f59c764b770) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #5 0x00007f59c804dea5 in start_thread (arg=0x7f59c547c700) at pthread_create.c:307 #6 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 5 (LWP 47602): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cb6f9f3a in __gthread_cond_timedwait (__abs_timeout=0x7f59c5c73230, __mutex=, __cond=) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __atime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::periodic_worker::periodic_worker(const std::function&, std::chrono::seconds):::: > (__p=..., __rtime=..., __lock=..., this=) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 operator() (__closure=0x7f59c7623d48) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/periodic_worker.h:38 #7 __invoke_impl&, std::chrono::seconds):: > (__f=) at /usr/local/include/c++/7.3.0/bits/invoke.h:60 #8 __invoke&, std::chrono::seconds):: > (__fn=) at /usr/local/include/c++/7.3.0/bits/invoke.h:95 #9 _M_invoke<0> (this=0x7f59c7623d48) at /usr/local/include/c++/7.3.0/thread:234 #10 operator() (this=0x7f59c7623d48) at /usr/local/include/c++/7.3.0/thread:243 #11 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN6spdlog7details15periodic_workerC4ERKSt8functionIFvvEENSt6chrono8durationIlSt5ratioILl1ELl1EEEEEUlvE_EEEEE6_M_runEv (this=0x7f59c7623d40) at /usr/local/include/c++/7.3.0/thread:186 #12 0x00007f59c8834dcf in std::execute_native_thread_routine (__p=0x7f59c7623d40) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #13 0x00007f59c804dea5 in start_thread (arg=0x7f59c5c7d700) at pthread_create.c:307 #14 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 4 (LWP 47601): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59cb6fa87d in __gthread_cond_timedwait (__abs_timeout=0x7f59c6473fe0, __mutex=, __cond=0x7f59c763a138) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x7f59c763a138) at /usr/local/include/c++/7.3.0/condition_variable:166 #3 wait_until > > (__atime=..., __lock=..., this=0x7f59c763a138) at /usr/local/include/c++/7.3.0/condition_variable:106 #4 wait_until >, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __atime=..., __lock=..., this=0x7f59c763a138) at /usr/local/include/c++/7.3.0/condition_variable:129 #5 wait_for, spdlog::details::mpmc_blocking_queue::dequeue_for(T&, std::chrono::milliseconds) [with T = spdlog::details::async_msg]:: > (__p=..., __rtime=, __lock=..., this=0x7f59c763a138) at /usr/local/include/c++/7.3.0/condition_variable:145 #6 dequeue_for (wait_duration=..., popped_item=..., this=0x7f59c763a110) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/mpmc_blocking_q.h:61 #7 spdlog::details::thread_pool::process_next_msg_ (this=this@entry=0x7f59c763a110) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/thread_pool.h:193 #8 0x00007f59cb6fadd0 in spdlog::details::thread_pool::worker_loop_ (this=0x7f59c763a110) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/spdlog/include/spdlog/details/thread_pool.h:184 #9 0x00007f59c8834dcf in std::execute_native_thread_routine (__p=0x7f59c7638de0) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83 #10 0x00007f59c804dea5 in start_thread (arg=0x7f59c647e700) at pthread_create.c:307 #11 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 3 (LWP 47600): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238 #1 0x00007f59c9d235a7 in background_thread_sleep (tsdn=, interval=10000000000, info=) at src/background_thread.c:255 #2 background_work_sleep_once (ind=0, info=, tsdn=) at src/background_thread.c:307 #3 background_thread0_work (tsd=0x7f59c71f5a90) at src/background_thread.c:452 #4 background_work (ind=, tsd=0x7f59c71f5a90) at src/background_thread.c:490 #5 background_thread_entry (ind_arg=) at src/background_thread.c:522 #6 0x00007f59c804dea5 in start_thread (arg=0x7f59c71ff700) at pthread_create.c:307 #7 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111 Thread 2 (LWP 47599): #0 0x00007f59c7d76eb3 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81 ---Type to continue, or q to quit--- #1 0x00007f59c9aea3c5 in epoll_dispatch (base=0xd64520, tv=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:465 #2 0x00007f59c9ae1376 in event_base_loop (base=0xd64520, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1998 #3 0x00000000004276dd in memcached_main (argc=, argv=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1942 #4 0x00007f59c7c9a555 in __libc_start_main (main=0x41bbc0 , argc=3, argv=0x7ffdb28d8998, init=, fini=, rtld_fini=, stack_end=0x7ffdb28d8988) at ../csu/libc-start.c:266 #5 0x000000000041e555 in _start () Thread 1 (LWP 47958): #0 __GI___pthread_mutex_lock (mutex=0x2e65746972776063) at ../nptl/pthread_mutex_lock.c:65 #1 0x000000000046a1b8 in __gthread_mutex_lock (__mutex=0x2e65746972776063) at /usr/local/include/c++/7.3.0/x86_64-pc-linux-gnu/bits/gthr-default.h:748 #2 lock (this=) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:103 #3 lock_guard (__m=..., this=) at /usr/local/include/c++/7.3.0/bits/std_mutex.h:162 #4 add_conn_to_pending_io_list (c=0x7f59c695f100, cookie=cookie@entry=0x7f5978524c00, status=ENGINE_SUCCESS) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:483 #5 0x000000000046a91f in notify_io_complete (void_cookie=..., status=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:349 #6 0x00007f59cbfd05e8 in EventuallyPersistentEngine::notifyIOComplete (this=0x7f5988100000, cookie=0x7f5978524c00, status=status@entry=ENGINE_SUCCESS) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6255 #7 0x00007f59cbf2fbb0 in ConnMap::processPendingNotifications (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/connmap.cc:174 #8 0x00007f59cbf2ca77 in notifyConnections (this=0x7f59882ff290) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/conn_notifier.cc:92 #9 ConnNotifierCallback::run (this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/conn_notifier.cc:39 #10 0x00007f59cc006be3 in GlobalTask::execute (this=0x7f59881548b0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73 #11 0x00007f59cbfff48f in ExecutorThread::run (this=0x7f59c69bb960) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188 #12 0x00007f59caa10777 in run (this=0x7f59c764c0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 #13 platform_thread_wrap (arg=0x7f59c764c0d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 #14 0x00007f59c804dea5 in start_thread (arg=0x7f598aff5700) at pthread_create.c:307 #15 0x00007f59c7d768dd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111