[CDC-Magma] Memcached crashed in magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)

#0  0x00007fea91276387 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007fea91277a78 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007fea91bc163c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000b2f7fb in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

No locals.

#4  0x00007fea91bcc8f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007fea91bcc961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

No locals.

#6  0x00007fea91bccbf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0x1072240 <typeinfo for std::bad_function_call>,

    tinfo@entry=0x7fea91ee8370 <typeinfo for std::bad_function_call>, dest=0x4443f0 <_ZNSt17bad_function_callD1Ev@plt>,

    dest@entry=0x7fea91bf3a70 <std::bad_function_call::~bad_function_call()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

        globals = <optimized out>

        header = 0x7fe9800008c0

#7  0x00007fea91bc409a in std::__throw_bad_function_call () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/functional.cc:34

No locals.

#8  0x0000000000934017 in operator() (this=0x7fe9d1f6fe30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:277

No locals.

#9  operator() (time=0, size=311555, __closure=0x7fe9d1f6fe30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:264

        currTime = <optimized out>

        diff = <optimized out>

        retainHistory = <optimized out>

        maxTime = <error reading variable maxTime (Cannot access memory at address 0x7fe9d1f6fe58)>

        maxSize = <error reading variable maxSize (Cannot access memory at address 0x7fe9d1f6fe50)>

        timeNowFn = <error reading variable timeNowFn (Cannot access memory at address 0x7fe9d1f6fe30)>

#10 __invoke_impl<bool, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(size_t, uint64_t)>&, long unsigned int, long int> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

No locals.

#11 __invoke_r<bool, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(size_t, uint64_t)>&, long unsigned int, long int> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113

No locals.

#12 std::_Function_handler<bool(long unsigned int, long int), magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(size_t, uint64_t)> >::_M_invoke(const std::_Any_data &, <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5126.x86_64.debug, CU 0x6e823bf, DIE 0x6f66cf5>, <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5126.x86_64.debug, CU 0x6e823bf, DIE 0x6f66d06>) (__functor=...,

    __args#0=<optimized out>, __args#1=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

No locals.

#13 0x0000000000a01cb4 in operator() (__args#1=0, __args#0=<optimized out>, this=0x7fe9d1ea3c38) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

No locals.

#14 magma::LSMTree::recomputeHistorySSTableState(std::shared_ptr<magma::TreeSnapshot>) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:2035

---Type <return> to continue, or q <return> to quit---

        isHistoryEnabled = true

        canRetainTable = <optimized out>

        dropHistory = <optimized out>

        table = 0x7fea8420dc80

        tables = {<std::_Vector_base<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >> = {

            _M_impl = {<std::allocator<std::shared_ptr<magma::Table> >> = {<__gnu_cxx::new_allocator<std::shared_ptr<magma::Table> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >::_Vector_impl_data> = {_M_start = 0x7fea840e3af0,

                _M_finish = 0x7fea840e3b00, _M_end_of_storage = 0x7fea840e3b00}, <No data fields>}}, <No data fields>}

        found = false

        lastHistoryTableLowKey = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fea1d7e8620 ""},

          _M_string_length = 0, {_M_local_buf = "\000\300\004\000\000\000\000\000\330wR`\352\177\000", _M_allocated_capacity = 311296}}

        itr = {<std::iterator<std::random_access_iterator_tag, std::shared_ptr<magma::Table>, long, std::shared_ptr<magma::Table>*, std::shared_ptr<magma::Table>&>> = {<No data fields>}, current = {_M_current = 0x7fea840e3b00}}

        historyStartKey = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7fea74346860 <Address 0x7fea74346860 out of bounds>}, _M_string_length = 17, {_M_local_buf = "\021", '\000' <repeats 14 times>,

            _M_allocated_capacity = 17}}

        dataLevel = 4

        historyStats = {Size = 0, WastedSize = 0, Tables = 0}

#15 0x0000000000a04983 in magma::LSMTree::swapTreeSnapshotAndRecomputeHistory(std::shared_ptr<magma::TreeSnapshot>) ()

    at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

        historyLevel = <optimized out>

        historyStats = <optimized out>

#16 0x0000000000a0dea4 in magma::LSMTree::doMemtableFlushWork() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

        isHistoryEnabled = <optimized out>

        status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {

                _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}}

        newSnapshot = {<std::__shared_ptr<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea60527790, _M_refcount = {_M_pi = 0x7fea60527780}}, <No data fields>}

        lock = {_M_device = 0x7fe9d1ea3500, _M_owns = true}

        _ = {fn = {<std::_Maybe_unary_or_binary_function<void>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8,

              _M_functor = {_M_unused = {_M_object = 0x7fe9d1ea3410, _M_const_object = 0x7fe9d1ea3410, _M_function_pointer = 0x7fe9d1ea3410,

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7fe9d1ea3410},

                _M_pod_data = "\020\064\352\321\351\177\000\000\000\000\000\000\000\000\000"},

              _M_manager = 0x9fd870 <std::_Function_handler<void(), magma::LSMTree::doMemtableFlushWork()::<lambda()> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},

            _M_invoker = 0x9fd610 <std::_Function_handler<void(), magma::LSMTree::doMemtableFlushWork()::<lambda()> >::_M_invoke(const std::_Any_data &)>}}

        flushIndex = <optimized out>

        txn = {mgr = 0x7fea60527790,

          snapshot = {<std::__shared_ptr<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::TreeSnapshot, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea60527780, _M_refcount = {_M_pi = 0x27}}, <No data fields>}, active = 16, commit = 214}

---Type <return> to continue, or q <return> to quit---

#17 0x0000000000a0e2dd in operator() (__closure=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1031

        this = <optimized out>

#18 __invoke_impl<std::tuple<magma::Status, magma::CheckpointTransaction>, magma::LSMTree::newFlush()::<lambda()>&> (__f=...)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

No locals.

#19 __invoke_r<std::tuple<magma::Status, magma::CheckpointTransaction>, magma::LSMTree::newFlush()::<lambda()>&> (__fn=...)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115

No locals.

#20 std::_Function_handler<std::tuple<magma::Status, magma::CheckpointTransaction>(), magma::LSMTree::newFlush()::<lambda()> >::_M_invoke(const std::_Any_data &) (

    __functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292

No locals.

#21 0x00000000009f0a2f in operator() (this=0x7fe9e0ad41d0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

No locals.

#22 magma::FlushWork::Execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/flush_work.cc:61

No locals.

#23 0x000000000099ad28 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()() ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:525

        status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {

                _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}}

        _ = {fn = {<std::_Maybe_unary_or_binary_function<void>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8,

              _M_functor = {_M_unused = {_M_object = 0x7fea74347240, _M_const_object = 0x7fea74347240, _M_function_pointer = 0x7fea74347240,

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7fea74347240, this adjustment 140643493906880},

                _M_pod_data = "@r4t\352\177\000\000\300\211~\035\352\177\000"},

              _M_manager = 0x990f70 <std::_Function_handler<void(), magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()> mutable::<lambda()> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},

            _M_invoker = 0x997ec0 <std::_Function_handler<void (), magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::{lambda()#2}::operator()()::{lambda()#2}>::_M_invoke(std::_Any_data const&)>}}

        deltaOps = {_M_t = {<std::__uniq_ptr_impl<magma::DocSequenceBuffer, std::default_delete<magma::DocSequenceBuffer> >> = {

              _M_t = {<std::_Tuple_impl<0, magma::DocSequenceBuffer*, std::default_delete<magma::DocSequenceBuffer> >> = {<std::_Tuple_impl<1, std::default_delete<magma::DocSequenceBuffer> >> = {<std::_Head_base<1, std::default_delete<magma::DocSequenceBuffer>, true>> = {<std::default_delete<magma::DocSequenceBuffer>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::DocSequenceBuffer*, false>> = {_M_head_impl =

    0x7fe9d1ee2a80}, <No data fields>}, <No data fields>}}, <No data fields>}}

        queueTime = <error reading variable queueTime (Cannot access memory at address 0x7fe9ec57bf80)>

        kFlush = <error reading variable kFlush (Cannot access memory at address 0x7fe9ec57bf90)>

        sFlush = <error reading variable sFlush (Cannot access memory at address 0x7fe9ec57bfa0)>

        lFlush = <error reading variable lFlush (Cannot access memory at address 0x7fe9ec57bfb0)>

        this = <error reading variable this (Cannot access memory at address 0x7fe9ec57bfc0)>

        flushMode = <error reading variable flushMode (Cannot access memory at address 0x7fe9ec57bfc8)>

#24 0x000000000099c6d1 in __invoke_impl<magma::Status, magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()>&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:289

No locals.

#25 __invoke_r<void, magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()>&> (__fn=...)

---Type <return> to continue, or q <return> to quit---

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110

No locals.

#26 std::_Function_handler<void(), magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

No locals.

#27 0x00000000009c8808 in operator() (this=0x7fea8403a710) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

No locals.

#28 magma::TimedTask::Complete(bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:158

        wg = 0x7fe9d1d10c00

        envGuard = {previous = {<std::__pair_base<void*, int>> = {<No data fields>}, first = 0x0, second = 2}}

        currTask = {<std::__shared_ptr<magma::Task, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::Task, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea8403a700, _M_refcount = {_M_pi = 0x7fea8403a6f0}}, <No data fields>}

#29 0x00000000009ca181 in magma::TaskWorker::loop(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/worker.cc:180

        task = {<std::__shared_ptr<magma::TimedTask, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::TimedTask, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe9d28e4a30, _M_refcount = {_M_pi = 0x7fe9d28e4a20}}, <No data fields>}

        worker = 0x7fea7c250500

#30 0x0000000000b8b219 in run (this=0x7fea7c253920) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:51

No locals.

#31 platform_thread_wrap(void*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:64

No locals.

#32 0x00007fea939fdea5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#33 0x00007fea9133eb0d in clone () from /lib64/libc.so.6

No symbol table info available.

git fetch https://review.couchbase.org/TAF refs/changes/72/185372/36 && git checkout -b change-185372 FETCH_HEAD

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/temp_vol_2.ini -p bucket_storage=magma,init_loading=False,bucket_eviction_policy=fullEviction,rerun=False -t storage.magma.magma_crash_recovery.MagmaCrashTests.test_magma_failures_with_CDC,nodes_init=3,graceful=False,skip_cleanup=True,num_items=500000,num_buckets=3,doc_size=1024,bucket_type=membase,eviction_policy=fullEviction,iterations=1,batch_size=1000,sdk_timeout=60,log_level=info,infra_log_level=info,rerun=False,process_concurrency=4,skip_cleanup=True,key_size=22,randomize_doc_size=False,randomize_value=False,num_collections=5,ops_rate=40000,ramQuota=256,key_type=SimpleKey,vbuckets=1024,replicas=2,doc_ops=update:delete,test_itr=2,bucket_history_retention_seconds=300,bucket_history_retention_bytes=1000,standard_buckets=1,magma_buckets=1,num_scopes=1,induce_failures=False,retention_flag=True,crash_during_enable_disable_history=True -m rest'