Master : [MAGMA] : Memcached crashed in magma::RemoveKVStoreConfig (path=..., fs=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:28

(gdb) bt full

#0  0x00007f691690c387 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007f691690da78 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007f691725763c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000a8b03b in backtrace_terminate_handler() ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:87

No locals.

#4  0x00007f69172628f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007f6917262961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

No locals.

#6  0x00007f6917262bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f68a0000940, tinfo=0xfa0220 <typeinfo for std::runtime_error>,

    dest=0x44acc0 <_ZNSt13runtime_errorD1Ev@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

        globals = <optimized out>

        header = 0x7f68a00008c0

#7  0x00000000004e5f49 in magma::RemoveKVStoreConfig (path=..., fs=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:28

        oldPath = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7f685b9f78e0 <Address 0x7f685b9f78e0 out of bounds>}, _M_string_length = 68, {

            _M_local_buf = "D\000\000\000\000\000\000\000\033\000\000\000\000\000\000", _M_allocated_capacity = 68}}

        newPath = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7f6888f50410 <Address 0x7f6888f50410 out of bounds>}, _M_string_length = 76, {

            _M_local_buf = "L\000\000\000\000\000\000\000\033\000\000\000\000\000\000", _M_allocated_capacity = 76}}

#8  0x00000000009434f3 in magma::KVStoreSet::KVStoreInstance::Destroy(unsigned int) ()

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

        found = <optimized out>

        lock = {_M_device = @0x7f690036f030}

        status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {

                _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {

                      _M_head_impl = 0x7f68b5fe9220}, <No data fields>}, <No data fields>}}, <No data fields>}}}

        removeAll = {<std::_Maybe_unary_or_binary_function<magma::Status, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool>> = {<std::binary_function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, magma::S---Type <return> to continue, or q <return> to quit---

tatus>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {

              _M_unused = {_M_object = 0x98d4d0

     <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>,

                _M_const_object = 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>,

                _M_function_pointer = 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>,

                _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>, this adjustment 13},

              _M_pod_data = "\320\324\230\000\000\000\000\000\r\000\000\000\000\000\000"},

            _M_manager = 0x919ce0 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},

          _M_invoker = 0x919a10 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool&&)>}

        wg = {counter = {<std::__atomic_base<long>> = {static _S_alignment = 8, _M_i = 140084936150624}, static is_always_lock_free = true},

          active = false, waitMutex = {<std::__mutex_base> = {_M_mutex = {__data = {__lock = 9545392, __count = 0, __owner = 9543440,

                  __nusers = 0, __kind = 10010688, __spins = 0, __elision = 0, __list = {__prev = 0x0,

                    __next = 0x919ca0 <std::_Function_handler<std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>}},

                __size = "\260\246\221\000\000\000\000\000\020\237\221\000\000\000\000\000@\300\230", '\000' <repeats 13 times>, "\240\234\221\000\000\000\000", __align = 9545392}}, <No data fields>}, condition = {_M_cond = {__data = {__lock = 9542128, __futex = 0, __total_seq = 10015952,

                __wakeup_seq = 0, __woken_seq = 9542880,

                __mutex = 0x919a10 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool&&)>, __nwaiters = 10014384,

                __broadcast_seq = 0},

              __size = "\360\231\221\000\000\000\000\000\320\324\230", '\000' <repeats 13 times>, "\340\234\221\000\000\000\000\000\020\232\221\000\000\000\000\000\260\316\230\000\000\000\000", __align = 9542128}}, prefix = {static npos = 18446744073709551615,

            _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},

            _M_string_length = 9542944, {_M_local_buf = "0\232\221\000\000\000\000\000\b\000\000\000i\177\000", _M_allocated_capacity = 9542192}}}

        status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {

                _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {

                      _M_head_impl = 0x7f6917582790 <construction vtable for std::istream-in-std::__cxx11::basic_stringstream<char, std::char_traits---Type <return> to continue, or q <return> to quit---

<char>, std::allocator<char> >+24>}, <No data fields>}, <No data fields>}}, <No data fields>}}}

        path = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7f67d87ca780 <Address 0x7f67d87ca780 out of bounds>}, _M_string_length = 47, {

            _M_local_buf = "/\000\000\000\000\000\000\000\066\061\065\000i\177\000", _M_allocated_capacity = 47}}

        fs = {

          MakeFile = {<std::_Maybe_unary_or_binary_function<std::unique_ptr<magma::File, std::default_delete<magma::File> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>> = {<std::unary_function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::unique_ptr<magma::File, std::default_delete<magma::File> > >> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x7f6881cfabc0,

                  _M_const_object = 0x7f6881cfabc0, _M_function_pointer = 0x7f6881cfabc0,

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f6881cfabc0, this adjustment 140087706685999}, _M_pod_data = "\300\253\317\201h\177\000\000/\222\376\265h\177\000"},

              _M_manager = 0x91a6b0 <_ZNSt17_Function_handlerIFSt10unique_ptrIN5magma4FileESt14default_deleteIS2_EERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEZNS1_23FileSystemWithHistogramC4ENS1_10FileSystemEPSt6vectorIS0_INS1_14MagmaFileStatsES3_ISI_EESaISK_EEmEUlSD_E_E10_M_managerERSt9_Any_dataRKSQ_St18_Manager_operation>},

            _M_invoker = 0x919f10 <_ZNSt17_Function_handlerIFSt10unique_ptrIN5magma4FileESt14default_deleteIS2_EERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEZNS1_23FileSystemWithHistogramC4ENS1_10FileSystemEPSt6vectorIS0_INS1_14MagmaFileStatsES3_ISI_EESaISK_EEmEUlSD_E_E9_M_invokeERKSt9_Any_dataSD_>},

          MakeDirectory = {<std::_Maybe_unary_or_binary_function<std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>> = {<std::unary_function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > >> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x98c040

     <magma::DefaultDirectoryConstructor(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>,

                  _M_const_object = 0x98c040 <magma::DefaultDirectoryConstructor(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>,

                  _M_function_pointer = 0x98c040 <magma::DefaultDirectoryConstructor(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>,

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x98c040 <magma::DefaultDirectoryConstructor(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>}, _M_pod_data = "@\300\230", '\000' <repeats 12 times>},

              _M_manager = 0x919ca0 <std::_Function_handler<std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},

            _M_invoker = 0x9199f0 <std::_Function_handler<std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), std::unique_ptr<magma::Directory, std::default_delete<magma::Directory> > (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>},

          RemoveAll = {<std::_Maybe_unary_or_binary_function<magma::Status, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<---Type <return> to continue, or q <return> to quit---

char> > const&, bool>> = {<std::binary_function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, magma::Status>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {

                _M_unused = {

                  _M_object = 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>,

                  _M_const_object = 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>,

                  _M_function_pointer = 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>,

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x98d4d0 <magma::DefaultFileRemover(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>, this adjustment 6},

                _M_pod_data = "\320\324\230\000\000\000\000\000\006\000\000\000\000\000\000"},

              _M_manager = 0x919ce0 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},

            _M_invoker = 0x919a10 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)>::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool&&)>},

          Rename = {<std::_Maybe_unary_or_binary_function<magma::Status, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>> = {<std::binary_function<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, magma::Status>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8,

              _M_functor = {_M_unused = {

                  _M_object = 0x98ceb0 <magma::DefaultFileRenamer(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>,

                  _M_const_object = 0x98ceb0 <magma::DefaultFileRenamer(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>,

                  _M_function_pointer = 0x98ceb0 <magma::DefaultFileRenamer(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>,

                  _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x98ceb0 <magma::DefaultFileRenamer(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>}, _M_pod_data = "\260\316\230", '\000' <repeats 12 times>},

              _M_manager = 0x919d20 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},

            _M_invoker = 0x919a30 <std::_Function_handler<magma::Status (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&), magma::Status (*)(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>---Type <return> to continue, or q <return> to quit---

::_M_invoke(std::_Any_data const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)>}}

        target = {<std::__shared_ptr<magma::KVStore, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<magma::KVStore, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f6882ed3210, _M_refcount = {_M_pi = 0x7f6882ed3200}}, <No data fields>}

#9  0x0000000000944573 in magma::KVStoreSet::DestroyKVStore(unsigned short, unsigned int) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore_set.cc:60

        ok = <optimized out>

        lock = <optimized out>

#10 0x000000000090dc07 in magma::Magma::Impl::DeleteKVStore(unsigned short, unsigned int) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:269

        std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}

        boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_alloc_>::e = {ptr_ = {px = 0x0, pn = {

              pi_ = 0x0}}}

        boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_exception_>::e = {ptr_ = {px = 0x0, pn = {

              pi_ = 0x0}}}

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankushj.ini bucket_storage=magma,rerun=false,bucket_eviction_policy=fullEviction,randomize_value=True,enable_dp=True,log_level=debug,infra_log_level=debug,GROUP=P0,copy_data=True -t storage.magma.magma_disk_full.MagmaDiskFull.test_disk_full_reduce_replica,nodes_init=4,num_items=5000000,doc_size=2048,sdk_timeout=60,replicas=3,GROUP=P0'