[New LWP 48723] [New LWP 48707] [New LWP 48711] [New LWP 48712] [New LWP 48713] [New LWP 48714] [New LWP 48715] [New LWP 48716] [New LWP 48717] [New LWP 48718] [New LWP 48719] [New LWP 48720] [New LWP 48721] [New LWP 48722] [New LWP 48724] [New LWP 48725] [New LWP 48726] [New LWP 48727] [New LWP 48728] [New LWP 48729] [New LWP 48730] [New LWP 48731] [New LWP 48732] [New LWP 48733] [New LWP 48734] [New LWP 48735] [New LWP 48736] [New LWP 48737] [New LWP 48738] [New LWP 48739] [New LWP 48740] [New LWP 48741] [New LWP 48742] [New LWP 48743] [New LWP 48744] [New LWP 48745] [New LWP 48746] [New LWP 48747] [New LWP 48748] [New LWP 48749] [New LWP 48750] [New LWP 48751] [New LWP 48752] [New LWP 48753] [New LWP 48754] [New LWP 48755] [New LWP 48756] [New LWP 48757] [New LWP 48758] [New LWP 48759] [New LWP 48760] [New LWP 48761] [New LWP 48762] [New LWP 48763] [New LWP 48764] [New LWP 48765] To enable execution of this file add add-auto-load-safe-path /usr/lib/x86_64-linux-gnu/libthread_db-1.0.so line to your configuration file "/root/.gdbinit". To completely disable this security protection add set auto-load safe-path / line to your configuration file "/root/.gdbinit". For more information about this security protection see the "Auto-loading safe path" section in the GDB manual. E.g., run from the shell: info "(gdb)Auto-loading safe path" Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'. #0 0x000000000086e524 in std::__atomic_base::load (__m=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/atomic.h:503 [Current thread is 1 (LWP 48723)] #0 0x000000000086e524 in std::__atomic_base::load (__m=, this=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/atomic.h:503 __b = __b = #1 cb::AtomicNonNegativeCounter::load (this=) at /opt/gcc-13.2.0/include/c++/13.2.0/x86_64-pc-linux-gnu/bits/bitset:235 No locals. #2 cb::AtomicNonNegativeCounter::operator unsigned long (this=) at /opt/gcc-13.2.0/include/c++/13.2.0/x86_64-pc-linux-gnu/bits/bitset:231 No locals. #3 Collections::VB::ManifestEntry::getItemCount (this=) at /opt/gcc-13.2.0/include/c++/13.2.0/pstl/gthr-default.h:135 No locals. #4 Collections::VB::StatsReadHandle::getItemCount (this=this@entry=0x7f2b2e7f1090) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest_handles.cc:43 No locals. #5 0x00000000009bda9b in ActiveStream::tryAndScheduleOSOBackfill (this=0x7f2a7ff03390, producer=..., vb=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:2073 stats = { = {readLock = {lock_ = 0x7f2a92c3bb18, token_ = {type_ = folly::SharedMutexToken::Type::INLINE_SHARED, slot_ = 0}}, manifest = 0x7f2a92c3ba80}, itr = { const*, std::pair*>> = {}, underlying_ = {itemPtr_ = 0x0, index_ = 0}}} __for_range = __for_begin = __for_end = cid = sid = colItemCount = 0 colDiskSize = 0 vbItemCount = osoBackfill = {static npos = 18446744073709551615, _M_dataplus = {> = {> = {}, }, _M_p = 0x7f2b2e7f1020 "auto"}, _M_string_length = 4, {_M_local_buf = "auto\000\000\000\000a\002M:\000\000\000", _M_allocated_capacity = 1869903201}} config = #6 0x00000000009c15f5 in ActiveStream::scheduleBackfill_UNLOCKED (this=0x7f2a7ff03390, producer=..., reschedule=) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/basic_string.h:1349 vbucket = {> = {> = {}, _M_ptr = 0x7f2a92d33800, _M_refcount = {_M_pi = 0x7f2a92d217a0}}, } backfillStart = backfillEnd = tryBackfill = #7 0x00000000009bfa62 in ActiveStream::transitionState (this=0x7f2a7ff03390, newState=ActiveStream::StreamState::Backfilling) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:2413 producer = {> = {> = {}, _M_ptr = 0x7f2a8e45f580, _M_refcount = {_M_pi = 0x7f2a8e45f500}}, } logLevel = validTransition = oldState = #8 0x00000000009cb17d in ActiveStream::setActive (this=0x7f2a7ff03390) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/item.h:138 lh = {_M_device = @0x7f2a7ff033f8} #9 0x00000000008b3d7e in DcpProducer::streamRequest(unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, std::function > const&)>, std::optional > >) (this=, flags=, opaque=, vbucket=..., start_seqno=, end_seqno=18446744073709551615, vbucket_uuid=0, snap_start_seqno=0, snap_end_seqno=0, rollback_seqno=, callback=, json=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/producer.cc:612 rlh = {lock_ = 0x7f2a92d34228, token_ = {type_ = folly::SharedMutexToken::Type::DEFERRED_SHARED, slot_ = 4}} vb = {> = {> = {}, _M_ptr = 0x7f2a92d33800, _M_refcount = {_M_pi = 0x7f2a92d217a0}}, } highSeqno = filter = {filter = {_M_h = {, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits >> = {, std::__detail::_Select1st, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {, true>> = {> = {}, }, }, , true>> = {> = {> = {}, }, }, }, , std::allocator >, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits, true>> = {}, , std::allocator >, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits, false>> = {, std::allocator >, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits >> = {}, }, , std::allocator >, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits, std::integral_constant >> = {}, , std::allocator >, std::__detail::_Select1st, std::equal_to, std::hash, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits, true>> = {}, , true> > >> = {, true> >, true>> = {, true> >> = {, true> >> = {}, }, }, }, > = {}, _M_buckets = 0x7f2b2e7f1700, _M_bucket_count = 1, _M_before_begin = {_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}, scopeID = {> = { >> = {}, _M_payload = {> = {_M_payload = {_M_empty = {}, _M_value = {value = 77}}, _M_engaged = false}, }}, >> = {}, }, lastCheckedPrivilegeRevision = {> = { >> = {}, _M_payload = {> = {_M_payload = {_M_empty = {}, _M_value = 160}, _M_engaged = true}, }}, >> = {}, }, streamId = {id = 0}, scopeIsDropped = false, defaultAllowed = false, passthrough = false, systemEventsAllowed = true} callAddVBConnByVBId = true found = purgeSeqno = needsRollback = {> = { >> = {}, _M_payload = {> = {> = {_M_payload = {_M_empty = {}, _M_value = {rollbackReason = {static npos = 18446744073709551615, _M_dataplus = {> = {> = {}, }, _M_p = 0x7f2b365d2100 }, _M_string_length = 705, {_M_local_buf = "\301\002\000\000\000\000\000\000,\233L:+\177\000", _M_allocated_capacity = 705}}, rollbackSeqno = 139820691459328}}, _M_engaged = false}, }, }}, >> = {}, } failoverEntries = { >> = {_M_impl = {> = {> = {}, }, >::_Vector_impl_data> = {_M_start = 0x7f2a97867070, _M_finish = 0x7f2a97867080, _M_end_of_storage = 0x7f2a97867080}, }}, } streamID = s = {> = {> = {}, _M_ptr = 0x7f2a7ff03390, _M_refcount = {_M_pi = 0x7f2a7ff03380}}, } epe = rv = 32555 #10 0x0000000000737387 in EventuallyPersistentEngine::stream_req(CookieIface&, unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, std::function > const&)>, std::optional > >) (this=, cookie=..., flags=16, opaque=2936247296, vbucket=..., startSeqno=0, endSeqno=18446744073709551615, vbucketUuid=0, snapStartSeqno=0, snapEndSeqno=0, rollbackSeqno=0x7f2b2e7f1ae8, callback=..., json=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/range_scans/vb_visitors.h:247 engine = conn = #11 0x00000000005d2a91 in dcpStreamReq(Cookie&, unsigned int, unsigned int, Vbid, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long*, std::function > const&)>, std::optional > >) (cookie=..., flags=flags@entry=16, opaque=2936247296, vbucket=vbucket@entry=..., startSeqno=startSeqno@entry=0, endSeqno=endSeqno@entry=18446744073709551615, vbucketUuid=0, snapStartSeqno=0, snapEndSeqno=0, rollbackSeqno=0x7f2b2e7f1ae8, callback=..., json=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/mcbp/protocol/privilege_database.h:247 connection = dcp = ret = #12 0x000000000069ed98 in dcp_stream_req_executor (cookie=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/protocol/mcbp/connection.h:73 request = flags = 16 start_seqno = 0 vbucket_uuid = 0 snap_start_seqno = 0 collections = payload = end_seqno = 18446744073709551615 snap_end_seqno = 0 rollback_seqno = 0 ret = connection = #13 0x000000000061e975 in Cookie::doExecute (this=0x7f2b366bad00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/io/async/IOBuf.h:110 header = #14 0x000000000061ea0f in Cookie::execute (this=this@entry=0x7f2b366bad00, useStartTime=useStartTime@entry=true) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:189 ts = done = te = {__d = {__r = 139823573347584}} #15 0x0000000000605420 in Connection::executeCommandPipeline (this=0x7f2aab6daa00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/chrono.h:434 cookie = status = stop = serverless = false maxActiveCommands = 32 active = false now = maxSendQueueSize = 1048576 #16 0x000000000060a344 in Connection::executeCommandsCallback (this=this@entry=0x7f2aab6daa00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:985 bucketName = {value = {_M_elems = "default", '\000' }} start = stop = ns = #17 0x0000000000614547 in LibeventConnection::read_callback (this=0x7f2aab6daa00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection_libevent.cc:135 phosphor_internal_category_enabled_130 = {_M_b = {_M_p = 0x0}} phosphor_internal_category_enabled_temp_130 = phosphor_internal_tpi_wait_130 = {category = 0xe3f8d7 "mutex", name = 0xde2cb0 "LibeventConnection::read_callback::threadLock.wait", type = phosphor::TraceEventType::Complete, argument_names = {_M_elems = {0xe3c3cc "this", 0xe402ed ""}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_pointer, phosphor::TraceArgumentType::is_none}}} phosphor_internal_tpi_held_130 = {category = 0xe3f8d7 "mutex", name = 0xde2c78 "LibeventConnection::read_callback::threadLock.held", type = phosphor::TraceEventType::Complete, argument_names = {_M_elems = {0xe402ed "", 0xe402ed ""}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_pointer, phosphor::TraceArgumentType::is_none}}} phosphor_internal_guard_130 = {tpiWait = 0xe51680 , tpiHeld = 0xe51640 , enabled = true, mutex = @0x7f2b38dc97c0, threshold = {__r = 10000000}, start = {__d = {__r = 16902672764290032}}, lockedAt = {__d = {__r = 16902672764291082}}, releasedAt = {__d = {__r = 0}}} #18 0x00007f2b3b00284e in bufferevent_run_deferred_callbacks_unlocked (cb=, arg=0x7f2aab6da780) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208 readcb = 0x614720 cbarg = 0x7f2aab6daa00 bufev_private = 0x7f2aab6da780 bufev = 0x7f2aab6da780 #19 0x00007f2b3b00bb99 in event_process_active_single_queue (base=base@entry=0x7f2b38cd6800, activeq=0x7f2b38d6a030, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726 evcb_cbfinalize = ev = evcb = 0x7f2aab6da920 count = 11 #20 0x00007f2b3b00c287 in event_process_active (base=0x7f2b38cd6800) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789 activeq = i = 0 c = 0 tv = {tv_sec = 139823440404496, tv_usec = 139823633251222} maxcb = 2147483647 endtime = 0x0 limit_after_prio = 2147483647 activeq = i = c = endtime = tv = maxcb = limit_after_prio = #21 event_base_loop (base=0x7f2b38cd6800, flags=flags@entry=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012 n = evsel = 0x7f2b3b21fc20 tv = {tv_sec = 0, tv_usec = 0} tv_p = res = done = 0 retval = 0 __func__ = "event_base_loop" #22 0x0000000000d7ddd6 in (anonymous namespace)::EventBaseBackend::eb_event_base_loop (flags=1, this=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:74 No locals. #23 folly::EventBase::loopBody (this=0x7f2b38dc9560, flags=, ignoreKeepAlive=) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:381 callbacks = {, (boost::intrusive::link_mode_type)2, boost::intrusive::dft_tag, 1>, unsigned long, false, void>> = {static constant_time_size = false, static stateful_value_traits = , static has_container_from_iterator = , static safemode_or_autounlink = true, data_ = {, (boost::intrusive::link_mode_type)2, boost::intrusive::dft_tag, 1>> = {*, boost::intrusive::dft_tag, 1>> = {}, static link_mode = boost::intrusive::auto_unlink}, root_plus_size_ = {> = {static constant_time_size = }, m_header = {> = {next_ = 0x7f2b2e7f2080, prev_ = 0x7f2b2e7f2080}, }}}}, } message = 0xe2d5b8 "Your code just tried to loop over an event base from inside another event base loop. Since libevent is not reentrant, this leads to undefined behavior in opt builds. Please fix immediately. For the co"... SCOPE_EXIT_STATE8 = { = {dismissed_ = false}, function_ = {__this = 0x7f2b38dc9560}} res = 0 ranLoopCallbacks = blocking = once = prev = {__d = {__r = 16902456063351241}} idleStart = {__d = {__r = 16902672762548297}} busy = idle = prevLoopThread = {_M_thread = 0} #24 0x0000000000d7e2de in folly::EventBase::loop (this=0x7f2b38dc9560) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:305 guard = {list_ = {prev = 0x11f58a0 , curr = {forbid = true, allowTerminationOnBlocking = false, ex = 0x7f2b38dc9570, tag = {static npos = , b_ = 0x0, e_ = 0x0}}}} #25 0x0000000000d807a8 in folly::EventBase::loopForever (this=this@entry=0x7f2b38dc9560) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:524 SCOPE_EXIT_STATE9 = SCOPE_EXIT_STATE10 = ret = #26 0x00000000005c4aa9 in worker_libevent (arg=0x7f2b38dc9550) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:272 me = #27 0x0000000000cf146c in std::function::operator()() const (this=0x7f2b38c999e8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/googletest.exploded/include/gtest/std_thread.h:247 No locals. #28 operator() (__closure=0x7f2b38c999c8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:117 main = n = main = n = #29 std::__invoke_impl, std::string):: > (__f=...) at /opt/gcc-13.2.0/include/c++/13.2.0/x86_64-pc-linux-gnu/bits/exception:61 No locals. #30 std::__invoke, std::string):: > (__fn=...) at /opt/gcc-13.2.0/include/c++/13.2.0/x86_64-pc-linux-gnu/bits/exception:96 No locals. #31 std::thread::_Invoker, std::string):: > >::_M_invoke<0> (this=0x7f2b38c999c8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/thread.cc:292 No locals. #32 std::thread::_Invoker, std::string):: > >::operator() (this=0x7f2b38c999c8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/thread.cc:299 No locals. #33 std::thread::_State_impl, std::string):: > > >::_M_run(void) (this=0x7f2b38c999c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/platform/thread.cc:244 No locals. #34 0x00007f2b3a323aa3 in std::execute_native_thread_routine (__p=0x7f2b38c999c0) at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/src/c++11/thread.cc:104 __t = #35 0x00007f2b3a9a3fa3 in start_thread (arg=) at pthread_create.c:486 ret = pd = now = unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139823440447232, -6246543336363255034, 140730728226926, 140730728226927, 139823440447232, 0, 6276202656155048710, 6276228812454632198}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} not_first_call = #36 0x00007f2b39fce06f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 No locals.