Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.2.0
-
7.2.0-5126
-
Untriaged
-
Centos 64-bit
-
-
0
-
No
Description
Steps to repro:
- Create a 3 node cluster (172.23.107.126 ,172.23.107.237, 172.23.107.221)
- Create a magma bucket with historyRetentionCollectionDefault=true and historyRetentionBytes=102400
- Create four non default collections
- Start data loading(only creates)
- Observed Memcached Crashed in magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)
Core found on all three node
Below backtraces are from node 172.23.107.126, dmp_file: 335819f4-1fb0-492e-85c963be-ba8f6f80.dmp
BackTrace:
#0 0x00007fea91276387 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007fea91277a78 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00007fea91bc163c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x0000000000b2f7fb in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88
|
No locals.
|
#4 0x00007fea91bcc8f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48
|
No locals.
|
#5 0x00007fea91bcc961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58
|
No locals.
|
#6 0x00007fea91bccbf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0x1072240 <typeinfo for std::bad_function_call>,
|
tinfo@entry=0x7fea91ee8370 <typeinfo for std::bad_function_call>, dest=0x4443f0 <_ZNSt17bad_function_callD1Ev@plt>,
|
dest@entry=0x7fea91bf3a70 <std::bad_function_call::~bad_function_call()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95
|
globals = <optimized out>
|
header = 0x7fe9800008c0
|
#7 0x00007fea91bc409a in std::__throw_bad_function_call () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/functional.cc:34
|
No locals.
|
#8 0x0000000000934017 in operator() (this=0x7fe9d1f6fe30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:277
|
No locals.
|
#9 operator() (time=0, size=311555, __closure=0x7fe9d1f6fe30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:264
|
currTime = <optimized out>
|
diff = <optimized out>
|
retainHistory = <optimized out>
|
maxTime = <error reading variable maxTime (Cannot access memory at address 0x7fe9d1f6fe58)>
|
maxSize = <error reading variable maxSize (Cannot access memory at address 0x7fe9d1f6fe50)>
|
timeNowFn = <error reading variable timeNowFn (Cannot access memory at address 0x7fe9d1f6fe30)>
|
#10 __invoke_impl<bool, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(size_t, uint64_t)>&, long unsigned int, long int> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60
|
No locals.
|
#11 __invoke_r<bool, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(size_t, uint64_t)>&, long unsigned int, long int> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113
|
No locals.
|
#12 std::_Function_handler<bool(long unsigned int, long int), magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(size_t, uint64_t)> >::_M_invoke(const std::_Any_data &, <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5126.x86_64.debug, CU 0x6e823bf, DIE 0x6f66cf5>, <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5126.x86_64.debug, CU 0x6e823bf, DIE 0x6f66d06>) (__functor=...,
|
__args#0=<optimized out>, __args#1=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291
|
No locals.
|
#13 0x0000000000a01cb4 in operator() (__args#1=0, __args#0=<optimized out>, this=0x7fe9d1ea3c38) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
|
No locals.
|
#14 magma::LSMTree::recomputeHistorySSTableState(std::shared_ptr<magma::TreeSnapshot>) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:2035
|
---Type <return> to continue, or q <return> to quit---
|
isHistoryEnabled = true
|
canRetainTable = <optimized out>
|
dropHistory = <optimized out>
|
table = 0x7fea8420dc80
|
tables = {<std::_Vector_base<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >> = {
|
_M_impl = {<std::allocator<std::shared_ptr<magma::Table> >> = {<__gnu_cxx::new_allocator<std::shared_ptr<magma::Table> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >::_Vector_impl_data> = {_M_start = 0x7fea840e3af0,
|
_M_finish = 0x7fea840e3b00, _M_end_of_storage = 0x7fea840e3b00}, <No data fields>}}, <No data fields>}
|
found = false
|
lastHistoryTableLowKey = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fea1d7e8620 ""},
|
_M_string_length = 0, {_M_local_buf = "\000\300\004\000\000\000\000\000\330wR`\352\177\000", _M_allocated_capacity = 311296}}
|
itr = {<std::iterator<std::random_access_iterator_tag, std::shared_ptr<magma::Table>, long, std::shared_ptr<magma::Table>*, std::shared_ptr<magma::Table>&>> = {<No data fields>}, current = {_M_current = 0x7fea840e3b00}}
|
historyStartKey = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7fea74346860 <Address 0x7fea74346860 out of bounds>}, _M_string_length = 17, {_M_local_buf = "\021", '\000' <repeats 14 times>,
|
_M_allocated_capacity = 17}}
|
dataLevel = 4
|
historyStats = {Size = 0, WastedSize = 0, Tables = 0}
|
#15 0x0000000000a04983 in magma::LSMTree::swapTreeSnapshotAndRecomputeHistory(std::shared_ptr<magma::TreeSnapshot>) ()
|
at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100
|
historyLevel = <optimized out>
|
historyStats = <optimized out>
|
#16 0x0000000000a0dea4 in magma::LSMTree::doMemtableFlushWork() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100
|
isHistoryEnabled = <optimized out>
|
status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {
|
_M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}}
|
newSnapshot = {<std::__shared_ptr<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea60527790, _M_refcount = {_M_pi = 0x7fea60527780}}, <No data fields>}
|
lock = {_M_device = 0x7fe9d1ea3500, _M_owns = true}
|
_ = {fn = {<std::_Maybe_unary_or_binary_function<void>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8,
|
_M_functor = {_M_unused = {_M_object = 0x7fe9d1ea3410, _M_const_object = 0x7fe9d1ea3410, _M_function_pointer = 0x7fe9d1ea3410,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7fe9d1ea3410},
|
_M_pod_data = "\020\064\352\321\351\177\000\000\000\000\000\000\000\000\000"},
|
_M_manager = 0x9fd870 <std::_Function_handler<void(), magma::LSMTree::doMemtableFlushWork()::<lambda()> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},
|
_M_invoker = 0x9fd610 <std::_Function_handler<void(), magma::LSMTree::doMemtableFlushWork()::<lambda()> >::_M_invoke(const std::_Any_data &)>}}
|
flushIndex = <optimized out>
|
txn = {mgr = 0x7fea60527790,
|
snapshot = {<std::__shared_ptr<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea60527780, _M_refcount = {_M_pi = 0x27}}, <No data fields>}, active = 16, commit = 214}
|
---Type <return> to continue, or q <return> to quit---
|
#17 0x0000000000a0e2dd in operator() (__closure=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1031
|
this = <optimized out>
|
#18 __invoke_impl<std::tuple<magma::Status, magma::CheckpointTransaction>, magma::LSMTree::newFlush()::<lambda()>&> (__f=...)
|
at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60
|
No locals.
|
#19 __invoke_r<std::tuple<magma::Status, magma::CheckpointTransaction>, magma::LSMTree::newFlush()::<lambda()>&> (__fn=...)
|
at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115
|
No locals.
|
#20 std::_Function_handler<std::tuple<magma::Status, magma::CheckpointTransaction>(), magma::LSMTree::newFlush()::<lambda()> >::_M_invoke(const std::_Any_data &) (
|
__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292
|
No locals.
|
#21 0x00000000009f0a2f in operator() (this=0x7fe9e0ad41d0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
|
No locals.
|
#22 magma::FlushWork::Execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/flush_work.cc:61
|
No locals.
|
#23 0x000000000099ad28 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:525
|
status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {
|
_M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}}
|
_ = {fn = {<std::_Maybe_unary_or_binary_function<void>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8,
|
_M_functor = {_M_unused = {_M_object = 0x7fea74347240, _M_const_object = 0x7fea74347240, _M_function_pointer = 0x7fea74347240,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7fea74347240, this adjustment 140643493906880},
|
_M_pod_data = "@r4t\352\177\000\000\300\211~\035\352\177\000"},
|
_M_manager = 0x990f70 <std::_Function_handler<void(), magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()> mutable::<lambda()> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},
|
_M_invoker = 0x997ec0 <std::_Function_handler<void (), magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()()::{lambda()#2}>::_M_invoke(std::_Any_data const&)>}}
|
deltaOps = {_M_t = {<std::__uniq_ptr_impl<magma::DocSequenceBuffer, std::default_delete<magma::DocSequenceBuffer> >> = {
|
_M_t = {<std::_Tuple_impl<0, magma::DocSequenceBuffer*, std::default_delete<magma::DocSequenceBuffer> >> = {<std::_Tuple_impl<1, std::default_delete<magma::DocSequenceBuffer> >> = {<std::_Head_base<1, std::default_delete<magma::DocSequenceBuffer>, true>> = {<std::default_delete<magma::DocSequenceBuffer>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::DocSequenceBuffer*, false>> = {_M_head_impl =
|
0x7fe9d1ee2a80}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
queueTime = <error reading variable queueTime (Cannot access memory at address 0x7fe9ec57bf80)>
|
kFlush = <error reading variable kFlush (Cannot access memory at address 0x7fe9ec57bf90)>
|
sFlush = <error reading variable sFlush (Cannot access memory at address 0x7fe9ec57bfa0)>
|
lFlush = <error reading variable lFlush (Cannot access memory at address 0x7fe9ec57bfb0)>
|
this = <error reading variable this (Cannot access memory at address 0x7fe9ec57bfc0)>
|
flushMode = <error reading variable flushMode (Cannot access memory at address 0x7fe9ec57bfc8)>
|
#24 0x000000000099c6d1 in __invoke_impl<magma::Status, magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()>&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:289
|
No locals.
|
#25 __invoke_r<void, magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()>&> (__fn=...)
|
---Type <return> to continue, or q <return> to quit---
|
at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110
|
No locals.
|
#26 std::_Function_handler<void(), magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291
|
No locals.
|
#27 0x00000000009c8808 in operator() (this=0x7fea8403a710) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
|
No locals.
|
#28 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158
|
wg = 0x7fe9d1d10c00
|
envGuard = {previous = {<std::__pair_base<void*, int>> = {<No data fields>}, first = 0x0, second = 2}}
|
currTask = {<std::__shared_ptr<magma::Task, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::Task, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea8403a700, _M_refcount = {_M_pi = 0x7fea8403a6f0}}, <No data fields>}
|
#29 0x00000000009ca181 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180
|
task = {<std::__shared_ptr<magma::TimedTask, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::TimedTask, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe9d28e4a30, _M_refcount = {_M_pi = 0x7fe9d28e4a20}}, <No data fields>}
|
worker = 0x7fea7c250500
|
#30 0x0000000000b8b219 in run (this=0x7fea7c253920) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51
|
No locals.
|
#31 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64
|
No locals.
|
#32 0x00007fea939fdea5 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#33 0x00007fea9133eb0d in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
QE-TEST:
git fetch https://review.couchbase.org/TAF refs/changes/72/185372/36 && git checkout -b change-185372 FETCH_HEAD
|
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/temp_vol_2.ini -p bucket_storage=magma,init_loading=False,bucket_eviction_policy=fullEviction,rerun=False -t storage.magma.magma_crash_recovery.MagmaCrashTests.test_magma_failures_with_CDC,nodes_init=3,graceful=False,skip_cleanup=True,num_items=500000,num_buckets=3,doc_size=1024,bucket_type=membase,eviction_policy=fullEviction,iterations=1,batch_size=1000,sdk_timeout=60,log_level=info,infra_log_level=info,rerun=False,process_concurrency=4,skip_cleanup=True,key_size=22,randomize_doc_size=False,randomize_value=False,num_collections=5,ops_rate=40000,ramQuota=256,key_type=SimpleKey,vbuckets=1024,replicas=2,doc_ops=update:delete,test_itr=2,bucket_history_retention_seconds=300,bucket_history_retention_bytes=1000,standard_buckets=1,magma_buckets=1,num_scopes=1,induce_failures=False,retention_flag=True,crash_during_enable_disable_history=True -m rest'
|