#0 __GI___pthread_mutex_lock (mutex=0x160) at ../nptl/pthread_mutex_lock.c:67 67 ../nptl/pthread_mutex_lock.c: No such file or directory. [Current thread is 1 (LWP 15525)] (gdb) bt full #0 __GI___pthread_mutex_lock (mutex=0x160) at ../nptl/pthread_mutex_lock.c:67 type = __PRETTY_FUNCTION__ = "__pthread_mutex_lock" id = #1 0x00000000008fe128 in __gthread_mutex_lock (__mutex=0x160) at /opt/gcc-10.2.0/include/c++/10.2.0/x86_64-pc-linux-gnu/bits/gthr-default.h:749 No locals. #2 std::mutex::lock (this=0x160) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_mutex.h:100 __e = __e = #3 std::unique_lock::lock (this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_lock.h:138 No locals. #4 std::unique_lock::unique_lock (__m=..., this=) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_lock.h:68 No locals. #5 PassiveStream::moveFlowControlBytes (this=0x0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:160 lg = #6 0x00000000008f7a54 in DcpConsumer::streamEnd (this=0x7f9978b1c880, opaque=36, vbucket=..., status=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:453 stream = {> = {> = {}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, } phosphor_internal_category_enabled_401 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = } phosphor_internal_category_enabled_temp_401 = phosphor_internal_tpi_401 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}} phosphor_internal_guard_401 = {tpi = 0x1083b20 , enabled = true, arg1 = 901, arg2 = 1, start = {__d = {__r = 3714570191775009}}} ufc = {consumer = @0x7f9978b1c880, bytes = 28} stream = {> = {> = {}, _M_ptr = 0x7f9979b20290, _M_refcount = {_M_pi = 0x7f9979b20280}}, } msg = res = cb::engine_errc::success #7 0x00000000006b543b in non-virtual thunk to EventuallyPersistentEngine::stream_end(CookieIface const&, unsigned int, Vbid, cb::mcbp::DcpStreamEndStatus) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.h:236 Collections::ManifestFile = Collections::CollectionEventDebugTag = cb::rbac::PrivilegeAccessOk = folly::sorted_equivalent = Collections::VB::LegacyXattrKey = aliveVBStates = {bits = {> = {_M_w = 0}, }} cb::xattr::vattrs::DOCUMENT = NoopSeqnoAckCb = {> = {> = {}, }, = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' }, _M_manager = 0x0}, _M_invoker = 0x0} cb::xattr::vattrs::XTOC = cb::xattr::macros::VALUE_CRC32C = --Type for more, q to quit, c to continue without paging-- magmaShardFile = DcpControlKeys::ChangeStreams = Collections::DefaultCollectionName = cb::NoExpiryLimit = Collections::DefaultScopeName = nlohmann::(anonymous namespace)::to_json = globalBucketLoggerName = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x0}, _M_string_length = 0, { _M_local_buf = '\000' , _M_allocated_capacity = 0}} cb::xattr::vattrs::VBUCKET = cb::userdataStartTag = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x0}, _M_string_length = 0, { _M_local_buf = '\000' , _M_allocated_capacity = 0}} Collections::VB::LegacyMaxSeqnoKey = Collections::DefaultCollectionIdentifier = cb::userdataEndTag = {static npos = 18446744073709551615, _M_dataplus = {> = {<__gnu_cxx::new_allocator> = {}, }, _M_p = 0x0}, _M_string_length = 0, { _M_local_buf = '\000' , _M_allocated_capacity = 0}} std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true} ImmediateCkptDisposer = { >, void>&&, Vbid const&>> = { >, void>&&, Vbid const&, void>> = {}, }, = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' }, _M_manager = 0x0}, _M_invoker = 0x0} Collections::NoDataLimit = Collections::VB::LegacyJSONFormat = cb::xattr::macros::CAS = NoopSyncWriteCompleteCb = {> = {> = {}, }, = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = { _M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' }, _M_manager = 0x0}, _M_invoker = 0x0} nlohmann::(anonymous namespace)::from_json = DcpControlKeys::FlatBuffersSystemEvents = Collections::ScopeEventDebugTag = folly::sorted_unique = cb::rbac::PrivilegeAccessFail = cb::xattr::macros::SEQNO = Collections::ManifestFileName = Collections::DefaultScopeIdentifier = cb::rbac::PrivilegeAccessFailNoPrivileges = boost::system::detail::cat_holder::generic_category_instance = { = { _vptr.error_category = 0xc5d830 , id_ = 12874403176081448717}, } boost::system::detail::cat_holder::system_category_instance = { = { _vptr.error_category = 0xc5d878 , id_ = 10353725095272636571}, } --Type for more, q to quit, c to continue without paging-- makeExitBorderGuard = {} flatbuffers::flatbuffer_version_string = 0xc7616a "FlatBuffers 1.10.0" #8 0x0000000000598f3f in dcpStreamEnd (cookie=..., opaque=36, vbucket=vbucket@entry=..., status=status@entry=cb::mcbp::DcpStreamEndStatus::Closed) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/engine_wrapper.cc:699 connection = dcp = ret = #9 0x000000000063b83b in dcp_stream_end_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:73 request = payload = ret = #10 0x000000000060e0cb in Cookie::doExecute (this=0x7f9a20641580) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/header.h:110 header = #11 0x000000000060e160 in Cookie::execute (this=this@entry=0x7f9a20641580, useStartTime=useStartTime@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:168 ts = done = te = {__d = {__r = 1}} #12 0x00000000005ef543 in Connection::executeCommandPipeline (this=0x7f9a48136c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.h:472 cookie = drainSize = 28 limit = status = input = 0x7f9a4b2904a0 stop = false maxActiveCommands = 32 active = false maxSendQueueSize = 41943040 #13 0x00000000005ef7f2 in Connection::executeCommandsCallback (this=0x7f9a48136c80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:801 start = {__d = {__r = 3714570191766861}} stop = ns = #14 0x00000000005efe7b in Connection::rw_callback (bev=, ctx=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:950 instance = thread = phosphor_internal_category_enabled_945 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = } phosphor_internal_category_enabled_temp_945 = phosphor_internal_tpi_wait_945 = {category = 0xcc802b "mutex", name = 0xc65290 "Connection::rw_callback::threadLock.wait", type = phosphor::TraceEventType::Complete, argument_names = {_M_elems = {0xcc1d99 "this", 0xcc8ac2 ""}}, argument_types = {_M_elems = { phosphor::TraceArgumentType::is_pointer, phosphor::TraceArgumentType::is_none}}} --Type for more, q to quit, c to continue without paging-- phosphor_internal_tpi_held_945 = {category = 0xcc802b "mutex", name = 0xc65260 "Connection::rw_callback::threadLock.held", type = phosphor::TraceEventType::Complete, argument_names = {_M_elems = {0xcc8ac2 "", 0xcc8ac2 ""}}, argument_types = {_M_elems = { phosphor::TraceArgumentType::is_pointer, phosphor::TraceArgumentType::is_none}}} phosphor_internal_guard_945 = {tpiWait = 0xc65aa0 , tpiHeld = 0xc65a60 , enabled = true, mutex = @0x7f9a4b487e90, threshold = { __r = 10000000}, start = {__d = {__r = 3714570191763135}}, lockedAt = {__d = {__r = 3714570191764921}}, releasedAt = {__d = {__r = 0}}} #15 0x00007f9a4e0a1d7e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f9a20641800) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 readcb = 0x5efd50 cbarg = 0x7f9a48136c80 bufev_private = 0x7f9a20641800 bufev = 0x7f9a20641800 #16 0x00007f9a4e0aacf1 in event_process_active_single_queue (base=base@entry=0x7f9a4b28cc00, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0, activeq=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 evcb_cbfinalize = ev = 0x0 evcb = 0x7f9a206419a0 count = 7 #17 0x00007f9a4e0ab53f in event_process_active (base=0x7f9a4b28cc00) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 activeq = i = 0 c = 0 tv = {tv_sec = 140300508895376, tv_usec = 140300687968806} maxcb = 2147483647 endtime = 0x0 limit_after_prio = 2147483647 activeq = i = c = endtime = tv = maxcb = limit_after_prio = #18 event_base_loop (base=0x7f9a4b28cc00, flags=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 n = evsel = 0x7f9a4e2bec20 tv = {tv_sec = 1, tv_usec = 3714570188057535} tv_p = res = done = 0 retval = 0 __func__ = "event_base_loop" #19 0x0000000000c24927 in folly::EventBase::loopBody (this=this@entry=0x7f9a4b487c40, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) --Type for more, q to quit, c to continue without paging-- at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:397 callbacks = {, (boost::intrusive::link_mode_type)2, boost::intrusive::dft_tag, 1>, unsigned long, false, void>> = { static constant_time_size = false, static stateful_value_traits = , static has_container_from_iterator = , static safemode_or_autounlink = true, data_ = {, (boost::intrusive::link_mode_type)2, boost::intrusive::dft_tag, 1>> = {*, boost::intrusive::dft_tag, 1>> = {}, static link_mode = boost::intrusive::auto_unlink}, root_plus_size_ = {> = { static constant_time_size = }, m_header = {> = {next_ = 0x7f9a41fe9900, prev_ = 0x7f9a41fe9900}, }}}}, } message = 0xce2e20 "Your code just tried to loop over an event base from inside another event base loop. Since libevent is not reentrant, this leads to undefined behavior in opt builds. Please fix immediately. For the co"... SCOPE_EXIT_STATE13 = { = {dismissed_ = false}, function_ = {__this = 0x7f9a4b487c40}} res = 0 ranLoopCallbacks = blocking = true once = false prev = {__d = {__r = 3712856760614157}} idleStart = {__d = {__r = 3714570190746209}} busy = idle = prevLoopThread = {_M_thread = 0} #20 0x0000000000c24df6 in folly::EventBase::loop (this=this@entry=0x7f9a4b487c40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:315 guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = , b_ = 0xce2acb "EventBase", e_ = 0xce2ad4 ""}}}} #21 0x0000000000c267b6 in folly::EventBase::loopForever (this=this@entry=0x7f9a4b487c40) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:538 SCOPE_EXIT_STATE14 = SCOPE_EXIT_STATE15 = ret = #22 0x00000000005bbfc9 in worker_libevent (arg=0x7f9a4b487c30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:257 me = #23 0x0000000000ba9409 in CouchbaseThread::run (this=0x7f9a4b7b7340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51 No locals. #24 platform_thread_wrap (arg=0x7f9a4b7b7340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64 context = #25 0x00007f9a4ebdafa3 in start_thread (arg=) at pthread_create.c:486 ret = pd = now = unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140300508968704, -7970701374105727940, 140721147781022, 140721147781023, 140300508968704, 140300664732720, 7950791409889823804, 7950819474975214652}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} not_first_call = #26 0x00007f9a4caa2eff in __init_misc (argc=, argv=0x7f9a41ffb700, envp=0x7f99e41900c8) at init-misc.c:33 --Type for more, q to quit, c to continue without paging-- p = #27 0x0000000000000000 in ?? ()