Details
-
Bug
-
Resolution: Not a Bug
-
Critical
-
None
-
7.2.0
-
7.2.0-5312
-
Triaged
-
-
0
-
Unknown
-
KV 2023-4
Description
Steps To Recreate:
- Create a 4 node cluster
- Create a magma bucket with (bucket_history_retention_seconds=1800,bucket_history_retention_bytes=6000000000)
- Create 1 non default collection(including default collection count is 2)
- Create 40000000 items(doc size = 512) in non default collection
- Start new doc ops(Create:Delete:expiry)
- While doc ops are going keep killing memcached (Since this test is about crash during recovery process , so at every 3 seconds memcached is killed (basically for five iteration test kills memcached at every 3rd seconds, without waiting for warm up to finish) and then after these five iterations test waits for warmup to finish before next five iteration of memcached kills and test keeps doing it in a loop)
- Observed Memcached crashed in 0x00000000004c1e49 in CheckpointManager::extractItemsToExpel (this=0x7f669d41b100, lh=...)
Below Core Dump was found on node 172.23.122.155
BackTrace:
(gdb) bt full
|
#0 0x00007f6796e6b7bb in raise () from /lib/x86_64-linux-gnu/libc.so.6
|
No symbol table info available.
|
#1 0x00007f6796e56535 in abort () from /lib/x86_64-linux-gnu/libc.so.6
|
No symbol table info available.
|
#2 0x00007f679742a63c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x0000000000b4b85b in backtrace_terminate_handler ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88
|
No locals.
|
#4 0x00007f67974358f6 in __cxxabiv1::__terminate (handler=<optimized out>)
|
at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48
|
No locals.
|
#5 0x00007f6797435961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58
|
No locals.
|
#6 0x00007f6797435bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f6708007560, tinfo=0x10965b0 <typeinfo for std::logic_error>,
|
dest=0x445460 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95
|
globals = <optimized out>
|
header = 0x7f67080074e0
|
#7 0x00000000004c1e49 in CheckpointManager::extractItemsToExpel (this=0x7f669d41b100, lh=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:2047
|
ss =
|
{<std::basic_iostream<char, std::char_traits<char> >> = {<std::basic_istream<char, std::char_traits<char> >> = {<std::basic_ios<char, std::char_traits<char> >> = {<std::ios_base> = {
|
_vptr.ios_base = 0x1096018 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+24>,
|
static boolalpha = std::_S_boolalpha, static dec = std::_S_dec, static fixed = std::_S_fixed, static hex = std::_S_hex,
|
static internal = std::_S_internal, static left = std::_S_left, static oct = std::_S_oct, static right = std::_S_right,
|
static scientific = std::_S_scientific, static showbase = std::_S_showbase, static showpoint = std::_S_showpoint,
|
static showpos = std::_S_showpos, static skipws = std::_S_skipws, static unitbuf = std::_S_unitbuf,
|
static uppercase = std::_S_uppercase, static adjustfield = std::_S_adjustfield, static basefield = std::_S_basefield,
|
static floatfield = std::_S_floatfield, static badbit = std::_S_badbit, static eofbit = std::_S_eofbit,
|
static failbit = std::_S_failbit, static goodbit = std::_S_goodbit, static app = std::_S_app, static ate = std::_S_ate,
|
static binary = std::_S_bin, static in = std::_S_in, static out = std::_S_out, static trunc = std::_S_trunc,
|
static beg = std::_S_beg, static cur = std::_S_cur, static end = std::_S_end, _M_precision = 0, _M_width = 17391680,
|
_M_flags = 17390616, _M_exception = std::_S_goodbit, _M_streambuf_state = -844368768, _M_callbacks = 0x7f66cdabf480,
|
_M_word_zero = {_M_pword = 0x7f66cdabf490, _M_iword = 140079513990272}, _M_local_word = {{_M_pword = 0x7f66cdabf57a,
|
_M_iword = 140079513990784}, {_M_pword = 0x7f679775cdc0 <(anonymous namespace)::c_locale_impl>, _M_iword = 24}, {
|
_M_pword = 0x7f66cdabf480, _M_iword = 16}, {_M_pword = 0x200, _M_iword = 29099049307960430}, {
|
_M_pword = 0x1096068 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+104>,
|
_M_iword = 6}, {_M_pword = 0x0, _M_iword = 4098}, {_M_pword = 0x7f6700000000, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}},
|
--Type <RET> for more, q to quit, c to continue without paging--
|
_M_word_size = 0, _M_word = 0x0, _M_ios_locale = {static none = 0, static ctype = 1, static numeric = 2, static collate = 4,
|
static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x0, static _S_classic = 0x0,
|
static _S_global = 0x0, static _S_categories = 0x3bd680, static _S_once = 0, static _S_twinned_facets =
|
0x7f679774ff00 <std::locale::_S_twinned_facets>}}, _M_tie = 0x0, _M_fill = 0 '\000', _M_fill_init = false, _M_streambuf = 0x0, _M_ctype = 0x0,
|
_M_num_put = 0x0, _M_num_get = 0x0},
|
_vptr.basic_istream = 0x1096018 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+24>,
|
_M_gcount = 0}, <std::basic_ostream<char, std::char_traits<char> >> = {
|
_vptr.basic_ostream = 0x1096040 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+64>}, <No data fields>}, _M_stringbuf = {<std::basic_streambuf<char, std::char_traits<char> >> = {
|
_vptr.basic_streambuf = 0x1095c18 <vtable for std::__cxx11::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >+16>,
|
_M_in_beg = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>,
|
_M_in_cur = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>,
|
_M_in_end = 0x7f66cdabf490 <error: Cannot access memory at address 0x7f66cdabf490>,
|
_M_out_beg = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>,
|
_M_out_cur = 0x7f66cdabf57a <error: Cannot access memory at address 0x7f66cdabf57a>,
|
_M_out_end = 0x7f66cdabf680 <error: Cannot access memory at address 0x7f66cdabf680>, _M_buf_locale = {static none = 0,
|
static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32,
|
static all = 63, _M_impl = 0x7f679775cdc0 <(anonymous namespace)::c_locale_impl>, static _S_classic = 0x0, static _S_global = 0x0,
|
static _S_categories = 0x3bd680, static _S_once = 0, static _S_twinned_facets = 0x7f679774ff00 <std::locale::_S_twinned_facets>}},
|
_M_mode = 24, _M_string = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>}, _M_string_length = 16, {
|
_M_local_buf = "\000\002\000\000\000\000\000\000ntManag", _M_allocated_capacity = 512}}}}
|
oldestCkptIterator = <optimized out>
|
oldestCheckpoint = 0x7f667818b200
|
lowestCursor = {<std::__shared_ptr<CheckpointCursor, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CheckpointCursor, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f66aecae070, _M_refcount = {_M_pi = 0x7f66aecae060}}, <No data fields>}
|
iterator = {
|
container = {<std::_Reference_wrapper_base_memfun<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, false>> = {<std::_Reference_wrapper_base<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > > >> = {<std::_Weak_result_type<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > > >> = {<std::_Weak_result_type_memfun<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, false>> = {<std::_Weak_result_type_impl<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > > >> = {<std::_Maybe_get_result_type<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, void>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::_Ref--Type <RET> for more, q to quit, c to continue without paging--
|
wrap_base_arg1<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, void>> = {<No data fields>}, <std::_Refwrap_base_arg2<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, void>> = {<No data fields>}, <No data fields>}, <No data fields>}, _M_data = 0x7f667bdfdaa8}, iter = {m_iit = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x7f666d9d9ce0}}}}
|
distance = <optimized out>
|
expelledItems = {<boost::container::dtl::node_alloc_holder<MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >, boost::intrusive::list_impl<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>, unsigned long, true, void> >> = {<MemoryTrackingAllocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >> = {
|
baseAllocator = {<__gnu_cxx::new_allocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*> >> = {<No data fields>}, <No data fields>},
|
bytesAllocated = {<std::__shared_ptr<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {
|
_M_pi = 0x1096040 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+64>}}, <No data fields>}}, m_icont = {static constant_time_size = true, static stateful_value_traits = <optimized out>,
|
static has_container_from_iterator = <optimized out>, static safemode_or_autounlink = false,
|
data_ = {<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::normal_link},
|
root_plus_size_ = {<boost::intrusive::detail::size_holder<true, unsigned long, void>> = {static constant_time_size = <optimized out>,
|
size_ = 17390616}, m_header = {<boost::intrusive::list_node<void*>> = {next_ = 0x7f66cdabf480,
|
prev_ = 0x7f66cdabf480}, <No data fields>}}}}}, <No data fields>}
|
numExpelledItems = <optimized out>
|
name = <optimized out>
|
cursor = <optimized out>
|
#8 0x00000000007df8f2 in CheckpointManager::expelUnreferencedCheckpointItems (this=0x7f669d41b100)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:646
|
lh = {_M_device = @0x7f669d41b190}
|
overheadCheck = <optimized out>
|
extractRes = {
|
items = {<boost::container::dtl::node_alloc_holder<MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >, boost::intrusive::list_impl<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>, unsigned long, true, void> >> = {<MemoryTrackingAllocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >> = {
|
baseAllocator = {<__gnu_cxx::new_allocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> --Type <RET> for more, q to quit, c to continue without paging--
|
>, void*> >> = {<No data fields>}, <No data fields>},
|
bytesAllocated = {<std::__shared_ptr<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f6668ea9410, _M_refcount = {_M_pi = 0x7f6668ea9400}}, <No data fields>}}, m_icont = {static constant_time_size = true,
|
static stateful_value_traits = <optimized out>, static has_container_from_iterator = <optimized out>,
|
static safemode_or_autounlink = false,
|
data_ = {<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::normal_link},
|
root_plus_size_ = {<boost::intrusive::detail::size_holder<true, unsigned long, void>> = {
|
static constant_time_size = <optimized out>, size_ = 0}, m_header = {<boost::intrusive::list_node<void*>> = {
|
next_ = 0x7f67137ec140, prev_ = 0x7f67137ec140}, <No data fields>}}}}}, <No data fields>}, manager = 0x0,
|
expelCursor = {<std::__shared_ptr<CheckpointCursor, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CheckpointCursor, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}, checkpoint = 0x0}
|
numItemsExpelled = <optimized out>
|
queuedItemsMemReleased = <optimized out>
|
estimatedMemRecovered = <optimized out>
|
#9 0x00000000007e4fbc in CheckpointMemRecoveryTask::attemptItemExpelling (this=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:123
|
vbid = {vbid = 750}
|
vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f6730400500, _M_refcount = {_M_pi = 0x7f66add8fba0}}, <No data fields>}
|
expelResult = <optimized out>
|
it = <error reading variable>
|
__for_range = @0x7f67137ec230: {<std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >> = {
|
_M_impl = {<std::allocator<std::pair<Vbid, unsigned long> >> = {<__gnu_cxx::new_allocator<std::pair<Vbid, unsigned long> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >::_Vector_impl_data> = {
|
_M_start = 0x7f66745ea000, _M_finish = 0x7f66745ec010, _M_end_of_storage = 0x7f66745ee000}, <No data fields>}}, <No data fields>}
|
__for_begin = <optimized out>
|
__for_end = <optimized out>
|
|
|
bucket = <error reading variable>
|
vbuckets = {<std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >> = {
|
_M_impl = {<std::allocator<std::pair<Vbid, unsigned long> >> = {<__gnu_cxx::new_allocator<std::pair<Vbid, unsigned long> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >::_Vector_impl_data> = {
|
_M_start = 0x7f66745ea000, _M_finish = 0x7f66745ec010, _M_end_of_storage = 0x7f66745ee000}, <No data fields>}}, <No data fields>}
|
#10 0x00000000007e5d88 in CheckpointMemRecoveryTask::runInner (this=0x7f66cd3ecd10)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:265
|
phosphor_internal_category_enabled_205 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_205 = <optimized out>
|
--Type <RET> for more, q to quit, c to continue without paging--
|
phosphor_internal_tpi_205 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0,
|
0x0}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}}
|
phosphor_internal_guard_205 = {tpi = 0x107faa0 <CheckpointMemRecoveryTask::runInner()::phosphor_internal_tpi_205>, enabled = true,
|
arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 8902877171642235}}}
|
bucket = <error reading variable>
|
wasAboveBackfillThreshold = false
|
onReturn = <optimized out>
|
bytesToFree = 60769435
|
#11 0x0000000000ab9eb9 in GlobalTask::execute (this=0x7f66cd3ecd10, threadName=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98
|
guard = {previous = 0x0}
|
executedAt = <optimized out>
|
scheduleOverhead = <optimized out>
|
start = <optimized out>
|
runAgain = <optimized out>
|
end = <optimized out>
|
runtime = <optimized out>
|
#12 0x0000000000ab357a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f67137ec630)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309
|
runAgain = <optimized out>
|
proxy = <error reading variable>
|
#13 0x0000000000abb26e in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f67137ec630)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42
|
fn = @0x7f67137ec630: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f67744622a0, tiny = {
|
__data = "\240\"Ftg\177\000\000 \307~\023g\177\000\000\000\000\000\000\000\000\000\000 d\233\225g\177\000\000\001\000\000\000\000\000\000\000\200\374Z\002\000\000\000", __align = {<No data fields>}}},
|
call_ = 0xab3ab0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xab1fa0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#14 operator() (__closure=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42
|
task = {storage_ = {{emptyState = 16 '\020', value = {task = 0x7f66cd3ecd10,
|
func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f67744622a0, tiny = {
|
__data = "\240\"Ftg\177\000\000 \307~\023g\177\000\000\000\000\000\000\000\000\000\000 d\233\225g\177\000\000\001\000\000\000\000\000\000\000\200\374Z\002\000\000\000", __align = {<No data fields>}}},
|
call_ = 0xab3ab0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xab1fa0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}}}, hasValue = true}}
|
--Type <RET> for more, q to quit, c to continue without paging--
|
this = <optimized out>
|
#15 0x0000000000c19380 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f67137ec820)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f67137ec820: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6795951c00, tiny = {
|
__data = "\000\034\225\225g\177\000\000\320\367+\231g\177\000\000\060\000\000\000\000\000\000\000\064n\000\000\000\000\000\000P\000\000\000\000\000\000\000\360\310~\023g\177\000", __align = {<No data fields>}}},
|
call_ = 0xabb5f0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,
|
exec_ = 0xababa0 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}
|
fn = <optimized out>
|
#16 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f6795951d00, thread=..., task=...)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 8902877171633117}}
|
stats = {expired = false, waitTime = {__r = 273680}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 8902877171359437}}, requestId = 0}
|
#17 0x0000000000c03e1a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f6795951d00, thread=...)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {
|
func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6795951c00, tiny = {
|
__data = "\000\034\225\225g\177\000\000\320\367+\231g\177\000\000\060\000\000\000\000\000\000\000\064n\000\000\000\000\000\000P\000\000\000\000\000\000\000\360\310~\023g\177\000", __align = {<No data fields>}}},
|
call_ = 0xabb5f0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,
|
exec_ = 0xababa0 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}, enqueueTime_ = {__d = {__r = 8902877171359437}},
|
expiration_ = {__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {
|
big = 0x6e34, tiny = {
|
__data = "4n\000\000\000\000\000\000\213\242\246", '\000' <repeats 13 times>, "_>\v\231g\177\000\000p\352\376Pg\177\000\000@\326+\231g\177\000", __align = {<No data fields>}}}, call_ = 0x466ce7
|
<folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},
|
context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,
|
priority_ = 0 '\000', queueObserverPayload_ = 140082887526784}}, hasValue = true}}
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcdf473 "CPUThreadPoolExecutor",
|
e_ = 0xcdf488 ""}}}}
|
#18 0x0000000000c1c339 in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>)
|
--Type <RET> for more, q to quit, c to continue without paging--
|
at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#19 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#20 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)
|
at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#21 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#22 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#23 0x0000000000ab3274 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f6795de20c0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49
|
fn = <error reading variable>
|
#24 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f6795de20c0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49
|
threadNameOpt = {storage_ = {{emptyState = -128 '\200', value = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f67137ec980 "NonIoPool2"}, _M_string_length = 10, {_M_local_buf = "NonIoPool2\000\000\000\000\000",
|
_M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7f6795de20c0)>
|
func = <optimized out>
|
threadNameOpt = <optimized out>
|
#25 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = <error reading variable>
|
#26 0x00007f679745ed40 in std::execute_native_thread_routine (__p=0x7f6795dbe970)
|
at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
|
__t = <optimized out>
|
#27 0x00007f6799064fa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
|
No symbol table info available.
|
#28 0x00007f6796f2ceff in clone () from /lib/x86_64-linux-gnu/libc.so.6
|
No symbol table info available.
|
QE-TEST:
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.69321.ini bucket_storage=magma,rerun=false,GROUP=P0;recovery_512MB,randomize_value=true,bucket_eviction_policy=fullEviction,replicas=1,nodes_init=4,enable_dp=false,collect_pcaps=True,key_size=12,ops_rate=10000,num_collections=2,sdk_client_pool=False,init_loading=False,get-cbcollect-info=True,autoCompactionDefined=true,bucket_history_retention_seconds=1800,bucket_history_retention_bytes=6000000000,upgrade_version=7.2.0-5312 -t storage.magma.magma_crash_recovery.MagmaCrashTests.test_crash_during_recovery_new,bucket_ram_quota=512,multiplier=1,batch_size=500,num_items=40000000,doc_size=512,doc_ops=create:delete:expiry,create_perc=50,delete_perc=25,expiry_perc=25,GROUP=P0;recovery_512MB'
|
Job: http://qe-jenkins.sc.couchbase.com/job/test_suite_executor-TAF/24052/consoleFull