[CDC-Crash Recovery] Memcached crashed in 0x00000000004c1e49 in CheckpointManager::extractItemsToExpel (this=0x7f669d41b100, lh=...)

(gdb) bt full

#0  0x00007f6796e6b7bb in raise () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#1  0x00007f6796e56535 in abort () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#2  0x00007f679742a63c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000b4b85b in backtrace_terminate_handler ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

No locals.

#4  0x00007f67974358f6 in __cxxabiv1::__terminate (handler=<optimized out>)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007f6797435961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

No locals.

#6  0x00007f6797435bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f6708007560, tinfo=0x10965b0 <typeinfo for std::logic_error>,

    dest=0x445460 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

        globals = <optimized out>

        header = 0x7f67080074e0

#7  0x00000000004c1e49 in CheckpointManager::extractItemsToExpel (this=0x7f669d41b100, lh=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:2047

        ss =

    {<std::basic_iostream<char, std::char_traits<char> >> = {<std::basic_istream<char, std::char_traits<char> >> = {<std::basic_ios<char, std::char_traits<char> >> = {<std::ios_base> = {

                  _vptr.ios_base = 0x1096018 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+24>,

                  static boolalpha = std::_S_boolalpha, static dec = std::_S_dec, static fixed = std::_S_fixed, static hex = std::_S_hex,

                  static internal = std::_S_internal, static left = std::_S_left, static oct = std::_S_oct, static right = std::_S_right,

                  static scientific = std::_S_scientific, static showbase = std::_S_showbase, static showpoint = std::_S_showpoint,

                  static showpos = std::_S_showpos, static skipws = std::_S_skipws, static unitbuf = std::_S_unitbuf,

                  static uppercase = std::_S_uppercase, static adjustfield = std::_S_adjustfield, static basefield = std::_S_basefield,

                  static floatfield = std::_S_floatfield, static badbit = std::_S_badbit, static eofbit = std::_S_eofbit,

                  static failbit = std::_S_failbit, static goodbit = std::_S_goodbit, static app = std::_S_app, static ate = std::_S_ate,

                  static binary = std::_S_bin, static in = std::_S_in, static out = std::_S_out, static trunc = std::_S_trunc,

                  static beg = std::_S_beg, static cur = std::_S_cur, static end = std::_S_end, _M_precision = 0, _M_width = 17391680,

                  _M_flags = 17390616, _M_exception = std::_S_goodbit, _M_streambuf_state = -844368768, _M_callbacks = 0x7f66cdabf480,

                  _M_word_zero = {_M_pword = 0x7f66cdabf490, _M_iword = 140079513990272}, _M_local_word = {{_M_pword = 0x7f66cdabf57a,

                      _M_iword = 140079513990784}, {_M_pword = 0x7f679775cdc0 <(anonymous namespace)::c_locale_impl>, _M_iword = 24}, {

                      _M_pword = 0x7f66cdabf480, _M_iword = 16}, {_M_pword = 0x200, _M_iword = 29099049307960430}, {

                      _M_pword = 0x1096068 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+104>,

                      _M_iword = 6}, {_M_pword = 0x0, _M_iword = 4098}, {_M_pword = 0x7f6700000000, _M_iword = 0}, {_M_pword = 0x0, _M_iword = 0}},

--Type <RET> for more, q to quit, c to continue without paging--

                  _M_word_size = 0, _M_word = 0x0, _M_ios_locale = {static none = 0, static ctype = 1, static numeric = 2, static collate = 4,

                    static time = 8, static monetary = 16, static messages = 32, static all = 63, _M_impl = 0x0, static _S_classic = 0x0,

                    static _S_global = 0x0, static _S_categories = 0x3bd680, static _S_once = 0, static _S_twinned_facets =

    0x7f679774ff00 <std::locale::_S_twinned_facets>}}, _M_tie = 0x0, _M_fill = 0 '\000', _M_fill_init = false, _M_streambuf = 0x0, _M_ctype = 0x0,

                _M_num_put = 0x0, _M_num_get = 0x0},

              _vptr.basic_istream = 0x1096018 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+24>,

              _M_gcount = 0}, <std::basic_ostream<char, std::char_traits<char> >> = {

              _vptr.basic_ostream = 0x1096040 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+64>}, <No data fields>}, _M_stringbuf = {<std::basic_streambuf<char, std::char_traits<char> >> = {

              _vptr.basic_streambuf = 0x1095c18 <vtable for std::__cxx11::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >+16>,

              _M_in_beg = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>,

              _M_in_cur = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>,

              _M_in_end = 0x7f66cdabf490 <error: Cannot access memory at address 0x7f66cdabf490>,

              _M_out_beg = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>,

              _M_out_cur = 0x7f66cdabf57a <error: Cannot access memory at address 0x7f66cdabf57a>,

              _M_out_end = 0x7f66cdabf680 <error: Cannot access memory at address 0x7f66cdabf680>, _M_buf_locale = {static none = 0,

                static ctype = 1, static numeric = 2, static collate = 4, static time = 8, static monetary = 16, static messages = 32,

                static all = 63, _M_impl = 0x7f679775cdc0 <(anonymous namespace)::c_locale_impl>, static _S_classic = 0x0, static _S_global = 0x0,

                static _S_categories = 0x3bd680, static _S_once = 0, static _S_twinned_facets = 0x7f679774ff00 <std::locale::_S_twinned_facets>}},

            _M_mode = 24, _M_string = {static npos = 18446744073709551615,

              _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                _M_p = 0x7f66cdabf480 <error: Cannot access memory at address 0x7f66cdabf480>}, _M_string_length = 16, {

                _M_local_buf = "\000\002\000\000\000\000\000\000ntManag", _M_allocated_capacity = 512}}}}

        oldestCkptIterator = <optimized out>

        oldestCheckpoint = 0x7f667818b200

        lowestCursor = {<std::__shared_ptr<CheckpointCursor, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CheckpointCursor, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f66aecae070, _M_refcount = {_M_pi = 0x7f66aecae060}}, <No data fields>}

        iterator = {

          container = {<std::_Reference_wrapper_base_memfun<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, false>> = {<std::_Reference_wrapper_base<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > > >> = {<std::_Weak_result_type<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > > >> = {<std::_Weak_result_type_memfun<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, false>> = {<std::_Weak_result_type_impl<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > > >> = {<std::_Maybe_get_result_type<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, void>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::_Ref--Type <RET> for more, q to quit, c to continue without paging--

wrap_base_arg1<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, void>> = {<No data fields>}, <std::_Refwrap_base_arg2<boost::container::list<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> > >, void>> = {<No data fields>}, <No data fields>}, <No data fields>}, _M_data = 0x7f667bdfdaa8}, iter = {m_iit = {static stateful_value_traits = false, members_ = {nodeptr_ = 0x7f666d9d9ce0}}}}

        distance = <optimized out>

        expelledItems = {<boost::container::dtl::node_alloc_holder<MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >, boost::intrusive::list_impl<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>, unsigned long, true, void> >> = {<MemoryTrackingAllocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >> = {

              baseAllocator = {<__gnu_cxx::new_allocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*> >> = {<No data fields>}, <No data fields>},

              bytesAllocated = {<std::__shared_ptr<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {

                    _M_pi = 0x1096040 <vtable for std::__cxx11::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >+64>}}, <No data fields>}}, m_icont = {static constant_time_size = true, static stateful_value_traits = <optimized out>,

              static has_container_from_iterator = <optimized out>, static safemode_or_autounlink = false,

              data_ = {<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::normal_link},

                root_plus_size_ = {<boost::intrusive::detail::size_holder<true, unsigned long, void>> = {static constant_time_size = <optimized out>,

                    size_ = 17390616}, m_header = {<boost::intrusive::list_node<void*>> = {next_ = 0x7f66cdabf480,

                      prev_ = 0x7f66cdabf480}, <No data fields>}}}}}, <No data fields>}

        numExpelledItems = <optimized out>

        name = <optimized out>

        cursor = <optimized out>

#8  0x00000000007df8f2 in CheckpointManager::expelUnreferencedCheckpointItems (this=0x7f669d41b100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_manager.cc:646

        lh = {_M_device = @0x7f669d41b190}

        overheadCheck = <optimized out>

        extractRes = {

          items = {<boost::container::dtl::node_alloc_holder<MemoryTrackingAllocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >, boost::intrusive::list_impl<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>, unsigned long, true, void> >> = {<MemoryTrackingAllocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy> >> = {

                baseAllocator = {<__gnu_cxx::new_allocator<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> --Type <RET> for more, q to quit, c to continue without paging--

>, void*> >> = {<No data fields>}, <No data fields>},

                bytesAllocated = {<std::__shared_ptr<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<cb::NonNegativeCounter<unsigned long, cb::ClampAtZeroUnderflowPolicy>, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f6668ea9410, _M_refcount = {_M_pi = 0x7f6668ea9400}}, <No data fields>}}, m_icont = {static constant_time_size = true,

                static stateful_value_traits = <optimized out>, static has_container_from_iterator = <optimized out>,

                static safemode_or_autounlink = false,

                data_ = {<boost::intrusive::bhtraits<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::bhtraits_base<boost::container::dtl::list_node<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, void*>, boost::intrusive::list_node<void*>*, boost::intrusive::dft_tag, 1>> = {<No data fields>}, static link_mode = boost::intrusive::normal_link},

                  root_plus_size_ = {<boost::intrusive::detail::size_holder<true, unsigned long, void>> = {

                      static constant_time_size = <optimized out>, size_ = 0}, m_header = {<boost::intrusive::list_node<void*>> = {

                        next_ = 0x7f67137ec140, prev_ = 0x7f67137ec140}, <No data fields>}}}}}, <No data fields>}, manager = 0x0,

          expelCursor = {<std::__shared_ptr<CheckpointCursor, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CheckpointCursor, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}, checkpoint = 0x0}

        numItemsExpelled = <optimized out>

        queuedItemsMemReleased = <optimized out>

        estimatedMemRecovered = <optimized out>

#9  0x00000000007e4fbc in CheckpointMemRecoveryTask::attemptItemExpelling (this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:123

        vbid = {vbid = 750}

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f6730400500, _M_refcount = {_M_pi = 0x7f66add8fba0}}, <No data fields>}

        expelResult = <optimized out>

        it = <error reading variable>

        __for_range = @0x7f67137ec230: {<std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >> = {

            _M_impl = {<std::allocator<std::pair<Vbid, unsigned long> >> = {<__gnu_cxx::new_allocator<std::pair<Vbid, unsigned long> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >::_Vector_impl_data> = {

                _M_start = 0x7f66745ea000, _M_finish = 0x7f66745ec010, _M_end_of_storage = 0x7f66745ee000}, <No data fields>}}, <No data fields>}

        __for_begin = <optimized out>

        __for_end = <optimized out>

        bucket = <error reading variable>

        vbuckets = {<std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >> = {

            _M_impl = {<std::allocator<std::pair<Vbid, unsigned long> >> = {<__gnu_cxx::new_allocator<std::pair<Vbid, unsigned long> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::pair<Vbid, unsigned long>, std::allocator<std::pair<Vbid, unsigned long> > >::_Vector_impl_data> = {

                _M_start = 0x7f66745ea000, _M_finish = 0x7f66745ec010, _M_end_of_storage = 0x7f66745ee000}, <No data fields>}}, <No data fields>}

#10 0x00000000007e5d88 in CheckpointMemRecoveryTask::runInner (this=0x7f66cd3ecd10)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:265

        phosphor_internal_category_enabled_205 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_205 = <optimized out>

--Type <RET> for more, q to quit, c to continue without paging--

        phosphor_internal_tpi_205 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0,

              0x0}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}}

        phosphor_internal_guard_205 = {tpi = 0x107faa0 <CheckpointMemRecoveryTask::runInner()::phosphor_internal_tpi_205>, enabled = true,

          arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 8902877171642235}}}

        bucket = <error reading variable>

        wasAboveBackfillThreshold = false

        onReturn = <optimized out>

        bytesToFree = 60769435

#11 0x0000000000ab9eb9 in GlobalTask::execute (this=0x7f66cd3ecd10, threadName=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98

        guard = {previous = 0x0}

        executedAt = <optimized out>

        scheduleOverhead = <optimized out>

        start = <optimized out>

        runAgain = <optimized out>

        end = <optimized out>

        runtime = <optimized out>

#12 0x0000000000ab357a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f67137ec630)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

        runAgain = <optimized out>

        proxy = <error reading variable>

#13 0x0000000000abb26e in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f67137ec630)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        fn = @0x7f67137ec630: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f67744622a0, tiny = {

              __data = "\240\"Ftg\177\000\000 \307~\023g\177\000\000\000\000\000\000\000\000\000\000 d\233\225g\177\000\000\001\000\000\000\000\000\000\000\200\374Z\002\000\000\000", __align = {<No data fields>}}},

          call_ = 0xab3ab0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xab1fa0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}

#14 operator() (__closure=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        task = {storage_ = {{emptyState = 16 '\020', value = {task = 0x7f66cd3ecd10,

                func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f67744622a0, tiny = {

                      __data = "\240\"Ftg\177\000\000 \307~\023g\177\000\000\000\000\000\000\000\000\000\000 d\233\225g\177\000\000\001\000\000\000\000\000\000\000\200\374Z\002\000\000\000", __align = {<No data fields>}}},

                  call_ = 0xab3ab0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

                  exec_ = 0xab1fa0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}}}, hasValue = true}}

--Type <RET> for more, q to quit, c to continue without paging--

        this = <optimized out>

#15 0x0000000000c19380 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f67137ec820)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

        fn = @0x7f67137ec820: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6795951c00, tiny = {

              __data = "\000\034\225\225g\177\000\000\320\367+\231g\177\000\000\060\000\000\000\000\000\000\000\064n\000\000\000\000\000\000P\000\000\000\000\000\000\000\360\310~\023g\177\000", __align = {<No data fields>}}},

          call_ = 0xabb5f0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

          exec_ = 0xababa0 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}

        fn = <optimized out>

#16 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f6795951d00, thread=..., task=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 8902877171633117}}

        stats = {expired = false, waitTime = {__r = 273680}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 8902877171359437}}, requestId = 0}

#17 0x0000000000c03e1a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f6795951d00, thread=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {

                  func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6795951c00, tiny = {

                        __data = "\000\034\225\225g\177\000\000\320\367+\231g\177\000\000\060\000\000\000\000\000\000\000\064n\000\000\000\000\000\000P\000\000\000\000\000\000\000\360\310~\023g\177\000", __align = {<No data fields>}}},

                    call_ = 0xabb5f0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

                    exec_ = 0xababa0 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}, enqueueTime_ = {__d = {__r = 8902877171359437}},

                  expiration_ = {__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {

                      big = 0x6e34, tiny = {

                        __data = "4n\000\000\000\000\000\000\213\242\246", '\000' <repeats 13 times>, "_>\v\231g\177\000\000p\352\376Pg\177\000\000@\326+\231g\177\000", __align = {<No data fields>}}}, call_ = 0x466ce7

     <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,

                priority_ = 0 '\000', queueObserverPayload_ = 140082887526784}}, hasValue = true}}

        guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcdf473 "CPUThreadPoolExecutor",

                e_ = 0xcdf488 ""}}}}

#18 0x0000000000c1c339 in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>)

--Type <RET> for more, q to quit, c to continue without paging--

    at /usr/local/include/c++/7.3.0/bits/invoke.h:73

No locals.

#19 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95

No locals.

#20 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)

    at /usr/local/include/c++/7.3.0/functional:467

No locals.

#21 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

No locals.

#22 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

        fn = <optimized out>

#23 0x0000000000ab3274 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f6795de20c0)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        fn = <error reading variable>

#24 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f6795de20c0)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        threadNameOpt = {storage_ = {{emptyState = -128 '\200', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = 0x7f67137ec980 "NonIoPool2"}, _M_string_length = 10, {_M_local_buf = "NonIoPool2\000\000\000\000\000",

                  _M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}

        func = <error reading variable func (Cannot access memory at address 0x7f6795de20c0)>

        func = <optimized out>

        threadNameOpt = <optimized out>

#25 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

        fn = <error reading variable>

#26 0x00007f679745ed40 in std::execute_native_thread_routine (__p=0x7f6795dbe970)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

        __t = <optimized out>

#27 0x00007f6799064fa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0

No symbol table info available.

#28 0x00007f6796f2ceff in clone () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.69321.ini bucket_storage=magma,rerun=false,GROUP=P0;recovery_512MB,randomize_value=true,bucket_eviction_policy=fullEviction,replicas=1,nodes_init=4,enable_dp=false,collect_pcaps=True,key_size=12,ops_rate=10000,num_collections=2,sdk_client_pool=False,init_loading=False,get-cbcollect-info=True,autoCompactionDefined=true,bucket_history_retention_seconds=1800,bucket_history_retention_bytes=6000000000,upgrade_version=7.2.0-5312 -t storage.magma.magma_crash_recovery.MagmaCrashTests.test_crash_during_recovery_new,bucket_ram_quota=512,multiplier=1,batch_size=500,num_items=40000000,doc_size=512,doc_ops=create:delete:expiry,create_perc=50,delete_perc=25,expiry_perc=25,GROUP=P0;recovery_512MB'