(gdb) bt full #0 hash (__seed=3339675911, __clength=, __ptr=) at /usr/local/include/c++/7.3.0/bits/functional_hash.h:192 No locals. #1 operator() (this=0x7f64ac5f0670, __s=...) at /usr/local/include/c++/7.3.0/bits/basic_string.h:6591 No locals. #2 _M_hash_code (this=0x7f64ac5f0670, __k=...) at /usr/local/include/c++/7.3.0/bits/hashtable_policy.h:1368 No locals. #3 std::_Hashtable, std::allocator >, std::pair, std::allocator > const, DurabilityMonitor::Position > > >, std::allocator, std::allocator > const, DurabilityMonitor::Position > > > >, std::__detail::_Select1st, std::equal_to, std::allocator > >, std::hash, std::allocator > >, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >::find (this=0x7f64ac5f0670, __k=...) at /usr/local/include/c++/7.3.0/bits/hashtable.h:1436 __code = __n = #4 0x00007f650df5ff80 in find (__x=..., this=) at /usr/local/include/c++/7.3.0/bits/unordered_map.h:927 No locals. #5 hasAcked (bySeqno=13298, node=..., this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/durability_monitor_impl.cc:352 itr = #6 operator<< (os=..., sw=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/durability_monitor_impl.cc:266 __for_range = acks = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x7f64f6fe3380 ""}, _M_string_length = 0, {_M_local_buf = "\000\071\071\000\000\000\000\000\320\064\376\366d\177\000", _M_allocated_capacity = 3750144}} #7 0x00007f650df591e0 in operator<< (os=..., s=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/durability_monitor_impl.h:745 __for_range = @0x7f648f670980: #8 0x00007f650df53fb7 in ActiveDurabilityMonitor::toOStream (this=0x7f648f6e5dc0, os=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:1082 No locals. #9 0x00007f650df5daa0 in operator<< (os=..., dm=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/durability_monitor.cc:21 No locals. #10 0x00007f650df63ca9 in PassiveDurabilityMonitor::PassiveDurabilityMonitor(VBucket&, ActiveDurabilityMonitor&&) (this=0x7f64863f70e0, vb=..., adm=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/passive_durability_monitor.cc:91 ss = ---Type to continue, or q to quit--- { >> = { >> = { >> = { = { _vptr.ios_base = 0x8184b8 , std::allocator >+24>, static boolalpha = std::_S_boolalpha, static dec = std::_S_dec, static fixed = std::_S_fixed, static hex = std::_S_hex, static internal = std::_S_internal, static left = std::_S_left, static oct = std::_S_oct, static right = std::_S_right, static scientific = std::_S_scientific, static showbase = std::_S_showbase, static showpoint = std::_S_showpoint, static showpos = std::_S_showpos, static skipws = std::_S_skipws, static unitbuf = std::_S_unitbuf, static uppercase = std::_S_uppercase, static adjustfield = std::_S_adjustfield, static basefield = std::_S_basefield, static floatfield = std::_S_floatfield, static badbit = std::_S_badbit, static eofbit = std::_S_eofbit, static failbit = std::_S_failbit, static goodbit = std::_S_goodbit, static app = std::_S_app, static ate = std::_S_ate, static binary = std::_S_bin, static in = std::_S_in, static out = std::_S_out, static trunc = std::_S_trunc, static beg = std::_S_beg, static cur = std::_S_cur, static end = std::_S_end, _M_precision = 0, _M_width = 8488160, _M_flags = 8490264, _M_exception = std::_S_goodbit, _M_streambuf_state = -1879878400, _M_callbacks = 0x7f648ff35500, _M_word_zero = { _M_pword = 0x7f648ff35510, _M_iword = 140069888546048}, _M_local_word = {{_M_pword = 0x7f648ff3567f, _M_iword = 140069888546560}, { _M_pword = 0x7f650aaac720 <(anonymous namespace)::c_locale_impl>, _M_iword = 3473183930213990424}, {_M_pword = 0x7f648ff35500, _M_iword = 16}, {_M_pword = 0x200, _M_iword = 7238526415436669298}, { _M_pword = 0x818508 , std::allocator >+104>, _M_iword = 6}, {_M_pword = 0x0, _M_iword = 4098}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}}, _M_word_size = 0, _M_word = 0x0, _M_ios_locale = {static none = 0, static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x0, static _S_classic = 0x0, static _S_global = 0x0, static _S_categories = 0x372fc0, static _S_once = 0, static _S_twinned_facets = 0x7f650aaa0760 }}, _M_tie = 0x0, _M_fill = 0 '\000', _M_fill_init = false, _M_streambuf = 0x0, _M_ctype = 0x0, _M_num_put = 0x0, _M_num_get = 0x0}, _vptr.basic_istream = 0x8184b8 , std::allocator >+24>, _M_gcount = 0}, >> = { _vptr.basic_ostream = 0x8184e0 , std::allocator >+64>}, }, _M_stringbuf = { >> = { _vptr.basic_streambuf = 0x818d18 , std::allocator >+16>, _M_in_beg = 0x7f648ff35500
, _M_in_cur = 0x7f648ff35500
, _M_in_end = 0x7f648ff35510
, _M_out_beg = 0x7f648ff35500
, _M_out_cur = 0x7f648ff3567f
, _M_out_end = 0x7f648ff35700
, _M_buf_locale = {static none = 0, static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x7f650aaac720 <(anonymous namespace)::c_locale_impl>, static _S_classic = 0x0, static _S_global = 0x0, static _S_categories = 0x372fc0, static _S_once = 0, static _S_twinned_facets = 0x7f650aaa0760 }}, _M_mode = 24, _M_string = { static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x7f648ff35500
}, _M_string_length = 16, { _M_local_buf = "\000\002\000\000\000\000\000\000rabilitd", _M_allocated_capacity = 512}}}} s = highPreparedSeqno = ---Type to continue, or q to quit--- highCompletedSeqno = admState = #11 0x00007f650e0308df in make_unique () at /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825 No locals. #12 VBucket::setupSyncReplication (this=this@entry=0x7f648f5a0c00, topology=topology@entry=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:649 currentPassiveDM = trackedWrites = { >, std::allocator > > >> = { _M_impl = { > >> = {<__gnu_cxx::new_allocator > >> = {}, }, _M_start = 0x7f64500df680, _M_finish = 0x7f64500df6e0, _M_end_of_storage = 0x7f64500df700}}, } #13 0x00007f650e032fc3 in VBucket::setState_UNLOCKED (this=0x7f648f5a0c00, to=to@entry=vbucket_state_replica, meta=meta@entry=0x0, vbStateLock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:551 No locals. #14 0x00007f650e040674 in VBucketMap::setState_UNLOCKED (this=this@entry=0x7f64f171d018, vb=..., newState=newState@entry=vbucket_state_replica, meta=meta@entry=0x0, vbStateLock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucketmap.cc:195 No locals. #15 0x00007f650dff0d8e in KVBucket::setVBucketState_UNLOCKED (this=this@entry=0x7f64f171d000, vb=..., to=to@entry=vbucket_state_replica, meta=meta@entry=0x0, transfer=, notify_dcp=notify_dcp@entry=true, vbset=..., vbStateLock=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:909 oldstate = #16 0x00007f650dff1444 in KVBucket::setVBucketState (this=0x7f64f171d000, vbid=..., to=vbucket_state_replica, meta=0x0, transfer=, cookie=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:865 vbStateLock = {lock_ = 0x7f648f5a0df8} lh = {_M_device = 0x7f64f171d130, _M_owns = true} vb = {> = {> = {}, _M_ptr = 0x7f648f5a0c00, _M_refcount = {_M_pi = 0x7f648f68f2a0}}, } #17 0x00007f650df82bc4 in EventuallyPersistentEngine::setVBucket (this=0x7f64f167f000, cookie=..., vbid=..., cas=, state=vbucket_state_replica, meta=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6578 No locals. #18 0x0000000000450b61 in bucket_set_vbucket (cookie=..., state=vbucket_state_replica, meta=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:947 req = @0x7f64c8758830: ret = #19 0x00000000004d6d07 in SetVbucketCommandContext::sessionLockedStep (this=0x7f65089f7bb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/session_validated_command_context.cc:150 this = 0x7f65089f7bb0 #20 0x00000000004d6ac8 in SessionValidatedCommandContext::step (this=0x7f65089f7bb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/session_validated_command_context.cc:40 ---Type to continue, or q to quit--- ret = this = 0x7f65089f7bb0 #21 0x00000000004dc4a0 in SteppableCommandContext::drive (this=this@entry=0x7f65089f7bb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/steppable_command_context.cc:42 ret = cb::success #22 0x00000000004b0766 in set_vbucket_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:566 No locals. #23 0x00000000004b4436 in operator() (__args#0=..., this=) at /usr/local/include/c++/7.3.0/bits/std_function.h:706 No locals. #24 execute_client_request_packet (cookie=..., request=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/mcbp_executors.cc:920 privilegeChains = {commandChains = {_M_elems = {{ chain = {, std::allocator > >> = { _M_impl = { >> = {<__gnu_cxx::new_allocator >> = {}, }, _M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, }} }}} res = {status = cb::rbac::PrivilegeAccess::Ok} #25 0x00000000004a24d9 in Cookie::execute (this=this@entry=0x7f6508930b80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:158 header = @0x7f64c8758830: #26 0x00000000004838ad in Connection::executeCommandPipeline (this=this@entry=0x7f6508974900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:561 status = input = 0x7f6508860360 stop = false maxActiveCommands = 32 active = false #27 0x000000000048a8d5 in Connection::executeCommandsCallback (this=this@entry=0x7f6508974900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:687 start = stop = ns = #28 0x000000000048b78f in Connection::rw_callback (ctx=0x7f6508974900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:816 instance = @0x7f6508974900: phosphor_internal_category_enabled_776 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = const*>::is_always_lock_free".>} phosphor_internal_category_enabled_temp_776 = ---Type to continue, or q to quit--- phosphor_internal_tpi_wait_776 = {category = 0x59bb98 "mutex", name = 0x59fb00 "Connection::rw_callback::threadLock.wait", type = phosphor::Complete, argument_names = {_M_elems = {0x59bfb8 "this", 0x5b4653 ""}}, argument_types = {_M_elems = {phosphor::is_pointer, phosphor::is_none}}} phosphor_internal_tpi_held_776 = {category = 0x59bb98 "mutex", name = 0x59fad0 "Connection::rw_callback::threadLock.held", type = phosphor::Complete, argument_names = {_M_elems = {0x5b4653 "", 0x5b4653 ""}}, argument_types = {_M_elems = {phosphor::is_pointer, phosphor::is_none}}} phosphor_internal_guard_776 = {tpiWait = 0x5a0000 , tpiHeld = 0x59ffc0 , enabled = true, mutex = @0x7f65088b35e8, threshold = {__r = 10000000}, start = {__d = {__r = 64422032058001}}, lockedAt = {__d = {__r = 64422032059318}}, releasedAt = {__d = { __r = 0}}} #29 0x00007f650b886d8e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f6508930680) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 readcb = 0x48b560 cbarg = 0x7f6508974900 bufev_private = 0x7f6508930680 bufev = 0x7f6508930680 #30 0x00007f650b88fd01 in event_process_active_single_queue (base=base@entry=0x7f650962a900, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 evcb_cbfinalize = ev = 0x0 evcb = 0x7f6508930820 count = 2 #31 0x00007f650b89054f in event_process_active (base=0x7f650962a900) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 activeq = i = 0 c = 0 tv = {tv_sec = 0, tv_usec = 140071936227055} maxcb = 2147483647 endtime = 0x0 limit_after_prio = 2147483647 #32 event_base_loop (base=0x7f650962a900, flags=flags@entry=0) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 evsel = 0x7f650baa3c20 tv = {tv_sec = 0, tv_usec = 0} tv_p = res = done = 0 retval = 0 __func__ = "event_base_loop" ---Type to continue, or q to quit--- #33 0x000000000046bba9 in worker_libevent (arg=0x7f65088b3550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:195 me = @0x7f65088b3550: #34 0x00007f650c7c6727 in run (this=0x7f65088547f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58 No locals. #35 platform_thread_wrap (arg=0x7f65088547f0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71 context = {_M_t = { _M_t = { >> = { >> = {, true>> = {> = {}, }, }, > = {_M_head_impl = 0x7f65088547f0}, }, }}} #36 0x00007f6509ffee25 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #37 0x00007f6509d2c34d in clone () from /lib64/libc.so.6 No symbol table info available. (gdb) (gdb)