172.23.106.203: 111 core dump seen
|
running: //opt/couchbase/bin/minidump-2-core /opt/couchbase/var/lib/couchbase/crash/538a2ebf-54a4-47e0-65b76c9f-d4b770b8.dmp > /opt/couchbase/var/lib/couchbase/crash/538a2ebf-54a4-47e0-65b76c9f-d4b770b8.core
|
running: gdb --batch /opt/couchbase/bin/memcached -c /opt/couchbase/var/lib/couchbase/crash/538a2ebf-54a4-47e0-65b76c9f-d4b770b8.core -ex "bt full" -ex quit
|
172.23.106.203: Stack Trace of first crash - 538a2ebf-54a4-47e0-65b76c9f-d4b770b8.dmp
|
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
|
#0 0x00007f5d2db9d387 in raise () from /lib64/libc.so.6
|
#0 0x00007f5d2db9d387 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007f5d2db9ea78 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00007f5d2e4e863c in __gnu_cxx::__verbose_terminate_handler() [clone .cold] () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#3 0x0000000000b31f5b in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88
|
No locals.
|
#4 0x00007f5d2e4f38f6 in __cxxabiv1::__terminate(void (*)()) () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#5 0x00007f5d2e4f3961 in std::terminate() () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#6 0x00007f5d2e4f3bf4 in __cxa_throw () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#7 0x000000000045013f in gsl::detail::fail_fast_throw (message=0xc7cf68 "GSL: Precondition failure: 'highestPrepareSeqno == s->highPreparedSeqno.it->getBySeqno()' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/passive_durabil"...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769
|
No locals.
|
#8 0x00000000004c88bf in PassiveDurabilityMonitor::PassiveDurabilityMonitor(VBucket&, long, long, std::vector<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >&&) (this=0x7f5c89e1c310, vb=..., highPreparedSeqno=30898, highCompletedSeqno=30896, outstandingPrepares=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1934.x86_64.debug, CU 0x4ddba58, DIE 0x4e7ca73>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:245
|
s = {holder_ = {<folly::LockedPtrBase<folly::Synchronized<std::unique_ptr<PassiveDurabilityMonitor::State, std::default_delete<PassiveDurabilityMonitor::State> >, folly::SharedMutexImpl<false, void, std::atomic, false, false> >, folly::SharedMutexImpl<false, void, std::atomic, false, false>, folly::LockPolicyExclusive>> = {parent_ = 0x7f5c89e1c340}, static AllowsConcurrentAccess = false}}
|
highestPrepareSeqno = <optimized out>
|
#9 0x000000000082bd93 in make_unique<PassiveDurabilityMonitor, EPVBucket&, unsigned long const&, unsigned long const&, std::vector<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > > > () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:977
|
No locals.
|
#10 EPVBucket::loadOutstandingPrepares(vbucket_state const&, std::vector<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >&&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc:981
|
No locals.
|
#11 0x00000000008215f7 in EPBucket::loadPreparedSyncWrites(VBucket&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:2068
|
epVb = @0x7f5c89c65000: <error reading variable>
|
kvStore = <optimized out>
|
vbState = 0x7f5c8d9ae240
|
prepares = {<std::_Vector_base<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >> = {_M_impl = {<std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<__gnu_cxx::new_allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >::_Vector_impl_data> = {_M_start = 0x7f5cb44f7c20, _M_finish = 0x7f5cb44f7c38, _M_end_of_storage = 0x7f5cb44f7c38}, <No data fields>}}, <No data fields>}
|
startSeqno = <optimized out>
|
endSeqno = <optimized out>
|
docFilter = ALL_ITEMS
|
valFilter = <optimized out>
|
scanCtx = {_M_t = {<std::__uniq_ptr_impl<BySeqnoScanContext, std::default_delete<BySeqnoScanContext> >> = {_M_t = {<std::_Tuple_impl<0, BySeqnoScanContext*, std::default_delete<BySeqnoScanContext> >> = {<std::_Tuple_impl<1, std::default_delete<BySeqnoScanContext> >> = {<std::_Head_base<1, std::default_delete<BySeqnoScanContext>, true>> = {<std::default_delete<BySeqnoScanContext>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, BySeqnoScanContext*, false>> = {_M_head_impl = 0x7f5d1832ff60}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
scanResult = <optimized out>
|
storageCB = @0x7f5cb44e63c0: <error reading variable>
|
numPrepares = <optimized out>
|
#12 0x0000000000742ec4 in Warmup::loadPreparedSyncWrites (this=0x7f5c8cc01b00, shardId=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:1577
|
itr = {<folly::detail::IteratorFacade<folly::AtomicHashMap<unsigned short, std::shared_ptr<VBucket> >::ahm_iterator<folly::AtomicHashMap<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>, std::pair<unsigned short const, std::shared_ptr<VBucket> >, folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>::aha_iterator<folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>, std::pair<unsigned short const, std::shared_ptr<VBucket> > > >, std::pair<unsigned short const, std::shared_ptr<VBucket> >, std::forward_iterator_tag>> = {<No data fields>}, ahm_ = 0x7f5c8cc01cd8, subMap_ = 0, subIt_ = {<folly::detail::IteratorFacade<folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>::aha_iterator<folly::AtomicHashArray<unsigned short, std::shared_ptr<VBucket>, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<char>, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>, std::pair<unsigned short const, std::shared_ptr<VBucket> > >, std::pair<unsigned short const, std::shared_ptr<VBucket> >, std::forward_iterator_tag>> = {<No data fields>}, aha_ = 0x7f5c8b02b000, offset_ = 803}}
|
vb = <optimized out>
|
itemsVisited = 0
|
success = true
|
preparesLoaded = 0
|
epStats = <optimized out>
|
vbid = {vbid = 803}
|
__for_range = <optimized out>
|
#13 0x000000000074f5ee in WarmupLoadPreparedSyncWrites::run (this=0x7f5c8cc852d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:309
|
phosphor_internal_category_enabled_305 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_305 = <optimized out>
|
phosphor_internal_tpi_305 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}
|
phosphor_internal_guard_305 = {tpi = 0x105e7e0 <WarmupLoadPreparedSyncWrites::run()::phosphor_internal_tpi_305>, enabled = true, arg1 = 3, arg2 = {<No data fields>}, start = {__d = {__r = 6313846476366249}}}
|
#14 0x0000000000aa2562 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#15 0x0000000000a9f775 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f5d117c6840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 6313846476364859}}
|
runAgain = false
|
proxy = @0x7f5c8c55d3e0: <error reading variable>
|
#16 0x0000000000bf4900 in operator() (this=0x7f5d117c6840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f5d117c6840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f5c8c55d3e0, tiny = {__data = "\340\323U\214\\\177\000\000\000\000\000\000\000\000\000\000p\340W,]\177\000\000\354s\001\000\000\000\000\000(\000\000\000\000\000\000\000\020i|\021]\177\000", __align = {<No data fields>}}}, call_ = 0xa9fe30 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>, exec_ = 0xa9e580 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#17 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f5d2c50a800, thread=..., task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1934.x86_64.debug, CU 0xa1b0c7f, DIE 0xa234bb4>) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 6313846476361415}}
|
stats = {expired = false, waitTime = {__r = 68010}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 6313846476293405}}, requestId = 0}
|
#18 0x0000000000bdc6ba in folly::CPUThreadPoolExecutor::threadRun (this=0x7f5d2c50a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = -32 '\340', value = {<folly::ThreadPoolExecutor::Task> = {func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f5c8c55d3e0, tiny = {__data = "\340\323U\214\\\177\000\000\000\000\000\000\000\000\000\000p\340W,]\177\000\000\354s\001\000\000\000\000\000(\000\000\000\000\000\000\000\020i|\021]\177\000", __align = {<No data fields>}}}, call_ = 0xa9fe30 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>, exec_ = 0xa9e580 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 6313846476293405}}, expiration_ = {__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x173ec, tiny = {__data = "\354s\001\000\000\000\000\000\v\357\242", '\000' <repeats 13 times>, "_^u0]\177\000\000p\312~\"]\177\000\000@\366\225\060]\177\000", __align = {<No data fields>}}}, call_ = 0x4659bd <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0}, context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000', queueObserverPayload_ = 0}}, hasValue = true}}
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcc4a0b "CPUThreadPoolExecutor", e_ = 0xcc4a20 ""}}}}
|
#19 0x0000000000bf78b9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#20 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#21 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#22 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#23 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#24 0x0000000000a9f404 in operator() (this=0x7f5d2c9e0280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
fn = @0x7f5d2c9e0280: <error reading variable>
|
#25 operator() (__closure=0x7f5d2c9e0280) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5d117c69a0 "ReaderPool7"}, _M_string_length = 11, {_M_local_buf = "ReaderPool7\000\000\000\000", _M_allocated_capacity = 8021036716148221266}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7f5d2c9e0280)>
|
#26 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = @0x7f5d2c9e0280: <error reading variable>
|
#27 0x00007f5d2e51cd40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#28 0x00007f5d30324ea5 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#29 0x00007f5d2dc658dd in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
Bug
Critical