[Magma] Memcached crashed in magma::LSMTree::UpdateCommitInfo (Disk FULL TEST)

(gdb) bt full

#0  0x00007f80f66d3387 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007f80f66d4a78 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007f80f701e63c in __gnu_cxx::__verbose_terminate_handler ()

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000ab289b in backtrace_terminate_handler() ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

No locals.

#4  0x00007f80f70298f6 in __cxxabiv1::__terminate(void (*)()) ()

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007f80f7029961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

No locals.

#6  0x00007f80f7029bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f80d0000940, tinfo=0xfd1460 <typeinfo for std::runtime_error>,

    dest=0x4441b0 <_ZNSt13runtime_errorD1Ev@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

        globals = <optimized out>

        header = 0x7f80d00008c0

#7  0x00000000004f6330 in magma::LSMTree::UpdateCommitInfo (this=0x7f80e23eb960, meta=..., offset=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1079

        ss =

    {<std::basic_iostream<char, std::char_traits<char> >> = {<std::basic_istream<char, std::char_traits<char> >> = {<std::basic_ios<char, std::char_traits<char> >> = {<std::ios_base> = {

                  _vptr.ios_base = 0xfd10a8 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+24>, static boolalpha = std::_S_boolalpha, static dec = std::_S_dec, static fixed = std::_S_fixed, static hex = std::_S_hex,

                  static internal = std::_S_internal, static left = std::_S_left, static oct = std::_S_oct,

                  static right = std::_S_right, static scientific = std::_S_scientific, static showbase = std::_S_showbase,

                  static showpoint = std::_S_showpoint, static showpos = std::_S_showpos, static skipws = std::_S_skipws,

                  static unitbuf = std::_S_unitbuf, static uppercase = std::_S_uppercase, static adjustfield = std::_S_adjustfield,

                  static basefield = std::_S_basefield, static floatfield = std::_S_floatfield, static badbit = std::_S_badbit,

                  static eofbit = std::_S_eofbit, static failbit = std::_S_failbit, static goodbit = std::_S_goodbit,

                  static app = std::_S_app, static ate = std::_S_ate, static binary = std::_S_bin, static in = std::_S_in,

                  static out = std::_S_out, static trunc = std::_S_trunc, static beg = std::_S_beg, static cur = std::_S_cur,

                  static end = std::_S_end, _M_precision = 0, _M_width = 16584912, _M_flags = 16583832, _M_exception = std::_S_goodbit,

---Type <return> to continue, or q <return> to quit---

                  _M_streambuf_state = 1502538112, _M_callbacks = 0x7f80598ee980, _M_word_zero = {_M_pword = 0x7f80598ee990,

                    _M_iword = 140189235079552}, _M_local_word = {{_M_pword = 0x7f80598ee9eb, _M_iword = 140189235080064}, {

                      _M_pword = 0x7f80f7350dc0 <(anonymous namespace)::c_locale_impl>, _M_iword = 24}, {_M_pword = 0x7f80598ee980,

                      _M_iword = 16}, {_M_pword = 0x200, _M_iword = 27423363524534313}, {

                      _M_pword = 0xfd10f8 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+104>, _M_iword = 6}, {_M_pword = 0x0, _M_iword = 4098}, {_M_pword = 0x0, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}},

                  _M_word_size = 0, _M_word = 0x0, _M_ios_locale = {static none = 0, static ctype = 1, static numeric = 2,

                    static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x0,

                    static _S_classic = 0x0, static _S_global = 0x0, static _S_categories = 0x3bd680, static _S_once = 0,

                    static _S_twinned_facets = 0x7f80f7343f00 <std::locale::_S_twinned_facets>}}, _M_tie = 0x0, _M_fill = 0 '\000',

                _M_fill_init = false, _M_streambuf = 0x0, _M_ctype = 0x0, _M_num_put = 0x0, _M_num_get = 0x0},

              _vptr.basic_istream = 0xfd10a8 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+24>, _M_gcount = 0}, <std::basic_ostream<char, std::char_traits<char> >> = {

              _vptr.basic_ostream = 0xfd10d0 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+64>}, <No data fields>}, _M_stringbuf = {<std::basic_streambuf<char, std::char_traits<char> >> = {

              _vptr.basic_streambuf = 0xfd0c98 <vtable for std::__cxx11::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >+16>, _M_in_beg = 0x7f80598ee980 <Address 0x7f80598ee980 out of bounds>,

              _M_in_cur = 0x7f80598ee980 <Address 0x7f80598ee980 out of bounds>,

              _M_in_end = 0x7f80598ee990 <Address 0x7f80598ee990 out of bounds>,

              _M_out_beg = 0x7f80598ee980 <Address 0x7f80598ee980 out of bounds>,

              _M_out_cur = 0x7f80598ee9eb <Address 0x7f80598ee9eb out of bounds>,

              _M_out_end = 0x7f80598eeb80 <Address 0x7f80598eeb80 out of bounds>, _M_buf_locale = {static none = 0, static ctype = 1,

                static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32, static all = 63,

                _M_impl = 0x7f80f7350dc0 <(anonymous namespace)::c_locale_impl>, static _S_classic = 0x0, static _S_global = 0x0,

                static _S_categories = 0x3bd680, static _S_once = 0,

                static _S_twinned_facets = 0x7f80f7343f00 <std::locale::_S_twinned_facets>}}, _M_mode = 24, _M_string = {

              static npos = 18446744073709551615,

              _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                _M_p = 0x7f80598ee980 <Address 0x7f80598ee980 out of bounds>}, _M_string_length = 16, {

                _M_local_buf = "\000\002\000\000\000\000\000\000) magma", _M_allocated_capacity = 512}}}}

#8  0x000000000090db4e in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:382

        std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}

#9  0x000000000090e2d8 in magma::KVStore::FlushMemTables (this=<optimized out>, wal=<optimized out>, flushMode=<optimized out>,

    blockMode=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:312

        offset = {SegID = 16302677501690281856, SegOffset = 32640}

---Type <return> to continue, or q <return> to quit---

#10 0x00000000008d5e25 in magma::Magma::Impl::syncKVStore(unsigned short, bool) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1068

        std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}

#11 0x00000000008d62aa in magma::Magma::SyncKVStore(unsigned short) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:310

        std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}

#12 0x000000000086454d in MagmaKVStore::initBySeqnoScanContext(std::unique_ptr<StatusCallback<GetValue>, std::default_delete<StatusCallback<GetValue> > >, std::unique_ptr<StatusCallback<CacheLookup>, std::default_delete<StatusCallback<CacheLookup> > >, Vbid, unsigned long, DocumentFilter, ValueFilter, SnapshotSource) const ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62

        handle = {_M_t = {<std::__uniq_ptr_impl<KVFileHandle, std::default_delete<KVFileHandle> >> = {

              _M_t = {<std::_Tuple_impl<0, KVFileHandle*, std::default_delete<KVFileHandle> >> = {<std::_Tuple_impl<1, std::default_delete<KVFileHandle> >> = {<std::_Head_base<1, std::default_delete<KVFileHandle>, true>> = {<std::default_delete<KVFileHandle>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, KVFileHandle*, false>> = {

                    _M_head_impl = 0x7f805e8c07c0}, <No data fields>}, <No data fields>}}, <No data fields>}}

        snapshot = {_M_t = {<std::__uniq_ptr_impl<magma::Magma::Snapshot, std::default_delete<magma::Magma::Snapshot> >> = {

              _M_t = {<std::_Tuple_impl<0, magma::Magma::Snapshot*, std::default_delete<magma::Magma::Snapshot> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Magma::Snapshot> >> = {<std::_Head_base<1, std::default_delete<magma::Magma::Snapshot>, true>> = {<std::default_delete<magma::Magma::Snapshot>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Magma::Snapshot*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}

        status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {

                _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {_M_head_impl = 0x746c756166656428}, <No data fields>}, <No data fields>}}, <No data fields>}}}

        itr = {_M_t = {<std::__uniq_ptr_impl<magma::Magma::SeqIterator, std::default_delete<magma::Magma::SeqIterator> >> = {

              _M_t = {<std::_Tuple_impl<0, magma::Magma::SeqIterator*, std::default_delete<magma::Magma::SeqIterator> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Magma::SeqIterator> >> = {<std::_Head_base<1, std::default_delete<magma::Magma::SeqIterator>, true>> = {<std::default_delete<magma::Magma::SeqIterator>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Magma::SeqIterator*, false>> = {_M_head_impl = 0x61646174656d2029}, <No data fields>}, <No data fields>}}, <No data fields>}}

        readState = {status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {

                  _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {_M_head_impl = 0x400000100}, <No data fields>}, <No data fields>}}, <No data fields>}}}, vbstate = {static CurrentVersion = 4,

            maxDeletedSeqno = {counter = {_M_elems = "\035\000\000\000\000"}}, highSeqno = 4605624, purgeSeqno = 0,

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.83854.ini bucket_storage=magma,rerun=false,bucket_eviction_policy=fullEviction,randomize_value=True,enable_dp=false,GROUP=P0,get-cbcollect-info=True,infra_log_level=info,log_level=info,bucket_storage=magma,upgrade_version=7.1.0-1120 -t storage.magma.magma_disk_full.MagmaDiskFull.test_crash_recovery_disk_full,doc_size=2048,upgrade_version=7.1.0-1120,rerun=false,GROUP=P0,enable_dp=false,sdk_timeout=60,get-cbcollect-info=True,replicas=1,bucket_storage=magma,log_level=info,bucket_eviction_policy=fullEviction,nodes_init=4,num_items=5000000,randomize_value=True,infra_log_level=info'