[CDC] Memcached crashed in je_arena_bin_choose_lock (tsdn=tsdn@entry=0x7f87effee220, arena=arena@entry=0x7f887be00980, binind=binind@entry=168, binshard=binshard@entry=0x7f87effeca30)

(gdb) bt full

#0  0x00007f887e7beef3 in pthread_mutex_trylock () from /lib64/libpthread.so.0

No symbol table info available.

#1  0x00007f887dcca3dc in malloc_mutex_trylock_final (mutex=0x34) at include/jemalloc/internal/mutex.h:161

No locals.

#2  malloc_mutex_lock (mutex=0x34, tsdn=0x7f87effee220) at include/jemalloc/internal/mutex.h:220

No locals.

#3  je_arena_bin_choose_lock (tsdn=tsdn@entry=0x7f87effee220, arena=arena@entry=0x7f887be00980, binind=binind@entry=168,

    binshard=binshard@entry=0x7f87effeca30) at src/arena.c:1375

        bin = 0x34

#4  0x00007f887dd3a84e in je_tcache_bin_flush_small (tsd=tsd@entry=0x7f87effee220, tcache=tcache@entry=0x7f87effee410, tbin=0x7f87effef3e0,

    binind=<optimized out>, rem=<optimized out>) at src/tcache.c:213

        binshard = 0

        bin = <optimized out>

        merged_stats = <optimized out>

        arena = <optimized out>

        nflush = <optimized out>

        item_extent = 0x7f87effec9b0

#5  0x00007f887dcc4f16 in tcache_dalloc_small (slow_path=false, binind=<optimized out>, ptr=0x7f88756e1eee, tcache=0x7f87effee410,

    tsd=<optimized out>) at include/jemalloc/internal/tcache_inlines.h:179

        bin_info = 0x7f887be00b60

#6  arena_sdalloc (slow_path=false, alloc_ctx=<optimized out>, tcache=0x7f87effee410, size=<optimized out>, ptr=0x7f88756e1eee, tsdn=<optimized out>)

    at include/jemalloc/internal/arena_inlines_b.h:420

        szind = <optimized out>

        slab = <optimized out>

        local_ctx = {szind = 3, slab = 117}

#7  isdalloct (slow_path=false, alloc_ctx=<optimized out>, tcache=0x7f87effee410, size=<optimized out>, ptr=0x7f88756e1eee, tsdn=<optimized out>)

    at include/jemalloc/internal/jemalloc_internal_inlines_c.h:131

No locals.

#8  isfree (slow_path=false, tcache=0x7f87effee410, usize=<optimized out>, ptr=0x7f88756e1eee, tsd=0x7f87effee220) at src/jemalloc.c:2644

        alloc_ctx = {szind = 168, slab = true}

        ctx = <optimized out>

#9  je_sdallocx_default (ptr=0x7f88756e1eee, size=<optimized out>, flags=<optimized out>) at src/jemalloc.c:3583

        tcache = 0x7f87effee410

#10 0x0000000000bc08e6 in cb::_JEArenaMalloc<cb::JEArenaCoreLocalTracker>::sized_free (ptr=ptr@entry=0x7f88756e1eee, size=size@entry=140224062430958)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:324

        c = <optimized out>

#11 0x0000000000569506 in sized_free (size=140224062430958, ptr=0x7f88756e1eee)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:75

No locals.

---Type <return> to continue, or q <return> to quit---

#12 cb_sized_free (ptr=0x7f88756e1eee, size=140224062430958)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:75

No locals.

#13 0x0000000000bb3105 in operator delete (ptr=<optimized out>, size=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:103

No locals.

#14 0x0000000000577881 in deallocate (__t=<optimized out>, __p=<optimized out>, this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:1122

No locals.

#15 deallocate (__n=<optimized out>, __p=<optimized out>, __a=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:492

No locals.

#16 _M_destroy (__size=<optimized out>, this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/basic_string.h:237

No locals.

#17 _M_dispose (this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/basic_string.h:232

No locals.

#18 ~basic_string (this=<optimized out>, __in_chrg=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/basic_string.h:658

No locals.

#19 destroy<std::__cxx11::basic_string<char> > (__p=<optimized out>, this=<synthetic pointer>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/ext/new_allocator.h:156

No locals.

#20 destroy<std::__cxx11::basic_string<char> > (__p=<optimized out>, __a=<synthetic pointer>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:531

No locals.

#21 nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > >::json_value::destroy (

    this=0x7f87effecd28, t=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:1192

        stack = {<std::_Vector_base<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > >, std::allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > > >> = {

            _M_impl = {<std::allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > >> = {<__gnu_cxx::new_allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > >, std::allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > > >::_---Type <return> to continue, or q <return> to quit---

Vector_impl_data> = {_M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}, <No data fields>}}, <No data fields>}

#22 0x0000000000577974 in ~basic_json (this=0x7f87effecd20, __in_chrg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:1142

No locals.

#23 nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > >::json_value::destroy (

    this=0x7f87effecdb8, t=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/json.exploded/include/nlohmann/json.hpp:1145

        current_item = {m_type = nlohmann::detail::string, m_value = {object = 0x7f887408f780, array = 0x7f887408f780, string = 0x7f887408f780,

            binary = 0x7f887408f780, boolean = 128, number_integer = 140224039024512, number_unsigned = 140224039024512,

            number_float = 6.927988040311368e-310}}

        stack = {<std::_Vector_base<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > >, std::allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > > >> = {

            _M_impl = {<std::allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > >> = {<__gnu_cxx::new_allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > >, std::allocator<nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > > >::_Vector_impl_data> = {_M_start = 0x7f887afdcb60, _M_finish = 0x7f887afdcb60, _M_end_of_storage = 0x7f887afdcb70}, <No data fields>}}, <No data fields>}

#24 0x000000000060a40d in ~basic_json (this=0x7f87effecdb0, __in_chrg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:91

No locals.

#25 Cookie::setErrorJsonExtras(nlohmann::basic_json<std::map, std::vector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, unsigned long, double, std::allocator, nlohmann::adl_serializer, std::vector<unsigned char, std::allocator<unsigned char> > > const&)

    () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:91

No locals.

#26 0x000000000060a5ad in Cookie::setUnknownCollectionErrorContext(unsigned long) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:881

        json = {m_type = nlohmann::detail::object, m_value = {object = 0x7f87b5b7c420, array = 0x7f87b5b7c420, string = 0x7f87b5b7c420,

            binary = 0x7f87b5b7c420, boolean = 32, number_integer = 140220846031904, number_unsigned = 140220846031904,

            number_float = 6.9278302855158642e-310}}

#27 0x00000000006afee7 in EventuallyPersistentEngine::setUnknownCollectionErrorContext(CookieIface const*, unsigned long) const ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:1901

        guard = {engine = 0x7f886c1fb000}

---Type <return> to continue, or q <return> to quit---

#28 0x00000000007cb025 in Collections::VB::CachingReadHandle::handleWriteStatus (this=<optimized out>, engine=..., cookie=<optimized out>,

    state=<optimized out>, nBytes=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_manifest.h:967

No locals.

#29 0x0000000000701a9b in KVBucket::setWithMeta(Item&, unsigned long, unsigned long*, CookieIface const*, cb::bitset<4ul, vbucket_state_t, PermittedVBStatesMap>, CheckConflicts, bool, GenerateBySeqno, GenerateCas, ExtendedMetaData*) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:442

        cHandle = {<Collections::VB::ReadHandle> = {readLock = {lock_ = 0x7f87ec308038, token_ = {type_ = folly::SharedMutexToken::DEFERRED_SHARED,

                slot_ = 0}}, manifest = 0x7f87ec307fa0},

          itr = {<folly::f14::detail::BaseIter<std::pair<CollectionID const, Collections::VB::ManifestEntry> const*, std::pair<CollectionID const, Collections::VB::ManifestEntry>*>> = {<No data fields>}, underlying_ = {itemPtr_ = 0x0, index_ = 0}},

          key = {<DocKeyInterface<DocKey>> = {<No data fields>}, buffer = {static npos = <optimized out>,

              buf = 0x7f879776c148 <Address 0x7f879776c148 out of bounds>, len = 13}, encoding = Yes}}

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f87b7180700, _M_refcount = {_M_pi = 0x7f880c7be880}}, <No data fields>}

        rlh = {lock_ = 0x7f87b71808e8, token_ = {type_ = folly::SharedMutexToken::DEFERRED_SHARED, slot_ = 6}}

        rv = cb::success

#30 0x00000000008fcc88 in PassiveStream::processMessage(MutationConsumerMessage*, PassiveStream::MessageType) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/response.h:478

        delCas = 140221823766529

        meta = {cas = 140224207724288, revSeqno = {counter = {_M_elems = "\020\t\340{\210\177"}}, flags = 4026458656, exptime = 140224202821398}

        taskToString = {_M_elems = {{static npos = 18446744073709551615,

              _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                _M_p = 0x7f87effed0a0 "mutation"}, _M_string_length = 8, {_M_local_buf = "mutation\000H\314}\210\177\000",

                _M_allocated_capacity = 7957695015159100781}}, {static npos = 18446744073709551615,

              _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                _M_p = 0x7f87effed0c0 "deletion"}, _M_string_length = 8, {_M_local_buf = "deletion\000\000\000\000\001\000\000",

                _M_allocated_capacity = 7957695015225681252}}, {static npos = 18446744073709551615,

              _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                _M_p = 0x7f87effed0e0 "expiration"}, _M_string_length = 10, {_M_local_buf = "expiration\000\000\000\000\000",

                _M_allocated_capacity = 7598805615304013925}}, {static npos = 18446744073709551615,

              _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                _M_p = 0x7f87effed100 "prepare"}, _M_string_length = 7, {_M_local_buf = "prepare\000(\000\000\000\000\000\000",

                _M_allocated_capacity = 28554735470867056}}}}

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f87b7180700, _M_refcount = {_M_pi = 0x7f880c7be880}}, <No data fields>}

        consumer = {<std::__shared_ptr<DcpConsumer, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<DcpConsumer, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f880c620000}}, <No data fields>}

        ret = <optimized out>

---Type <return> to continue, or q <return> to quit---

        deleteSource = Explicit

        switchComplete = false

#31 0x00000000008fd545 in processMutation (mutation=0x7f878c873450, this=0x7f87b717cd90)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:746

No locals.

#32 PassiveStream::processBufferedMessages(unsigned int&, unsigned long) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:481

        ret = cb::success

        response = <synthetic pointer>

        message_bytes = <synthetic pointer>

        seqno = {<std::_Optional_base<long, true, true>> = {<std::_Optional_base_impl<long, std::_Optional_base<long, true, true> >> = {<No data fields>}, _M_payload = {<std::_Optional_payload_base<long>> = {_M_payload = {_M_empty = {<No data fields>}, _M_value = 9074472},

                _M_engaged = true}, <No data fields>}}, <std::_Enable_copy_move<true, true, true, true, std::optional<long> >> = {<No data fields>}, <No data fields>}

        lh = {_M_device = 0x7f87b717cef0, _M_owns = false}

        count = 7

        total_bytes_processed = 7462

        failed = false

        noMem = false

#33 0x00000000008ee1f2 in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f880c620080, stream=..., yieldThreshold=10)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1270

        rval = all_processed

        bytesProcessed = 0

        iterations = 9

#34 0x00000000008f47f7 in DcpConsumer::processBufferedItems() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

        stream = {<std::__shared_ptr<PassiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<PassiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f87b717cd90, _M_refcount = {_M_pi = 0x7f87b717cd80}}, <No data fields>}

        process_ret = all_processed

        vbucket = {vbid = 7}

#35 0x00000000008f7d25 in DcpConsumerTask::run (this=0x7f882c345530)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:84

        phosphor_internal_category_enabled_73 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_73 = <optimized out>

        phosphor_internal_tpi_73 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},

          argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}

        phosphor_internal_guard_73 = {tpi = 0x107cc60 <DcpConsumerTask::run()::phosphor_internal_tpi_73>, enabled = true, arg1 = {<No data fields>},

          arg2 = {<No data fields>}, start = {__d = {__r = 42635584951029941}}}

        consumer = {<std::__shared_ptr<DcpConsumer, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<DcpConsumer, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f880c620000}}, <No data fields>}

---Type <return> to continue, or q <return> to quit---

        sleepFor = 0

        state = <optimized out>

#36 0x0000000000ab7099 in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98

        guard = {previous = 0x0}

        start = <optimized out>

        runAgain = <optimized out>

#37 0x0000000000ab075a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f87effed650)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

        runAgain = <optimized out>

        proxy = @0x7f8874196190: <error reading variable>

#38 0x0000000000ab844e in operator() (this=0x7f87effed650)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        fn = @0x7f87effed650: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f8874196190, tiny = {

              __data = "\220a\031t\210\177\000\000@\327\376\357\207\177\000\000\000\000\000\000\000\000\000\000\300\032\271z\210\177\000\000;\000\000\000\000\000\000\000\215o\300\000\000\000\000", __align = {<No data fields>}}},

          call_ = 0xab0c90 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xaaf180 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}

#39 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        task = {storage_ = {{emptyState = 48 '0', value = {task = 0x7f882c345530,

                func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f8874196190, tiny = {

                      __data = "\220a\031t\210\177\000\000@\327\376\357\207\177\000\000\000\000\000\000\000\000\000\000\300\032\271z\210\177\000\000;\000\000\000\000\000\000\000\215o\300\000\000\000\000", __align = {<No data fields>}}},

                  call_ = 0xab0c90 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

                  exec_ = 0xaaf180 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}}}, hasValue = true}}

        this = <optimized out>

#40 0x0000000000c16560 in operator() (this=0x7f87effed840)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

        fn = @0x7f87effed840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f887ab51c00, tiny = {

              __data = "\000\034\265z\210\177\000\000\320\227\337~\210\177\000\000\060\000\000\000\000\000\000\000\206\200\001\000\000\000\000\000\320\000\000\000\000\000\000\000\020\331\376\357\207\177\000", __align = {<No data fields>}}},

          call_ = 0xab87d0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

          exec_ = 0xab7d80 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::funct---Type <return> to continue, or q <return> to quit---

ion::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}

#41 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (

    this=this@entry=0x7f887ab51d00, thread=...,

    task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5212.x86_64.debug, CU 0xa8d6c0f, DIE 0xa95ab44>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 42635584951026169}}

        stats = {expired = false, waitTime = {__r = 22864}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 42635584951003305}}, requestId = 0}

#42 0x0000000000c00ffa in folly::CPUThreadPoolExecutor::threadRun (this=0x7f887ab51d00, thread=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {

                  func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f887ab51c00, tiny = {

                        __data = "\000\034\265z\210\177\000\000\320\227\337~\210\177\000\000\060\000\000\000\000\000\000\000\206\200\001\000\000\000\000\000\320\000\000\000\000\000\000\000\020\331\376\357\207\177\000", __align = {<No data fields>}}},

                    call_ = 0xab87d0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

                    exec_ = 0xab7d80 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}, enqueueTime_ = {__d = {__r = 42635584951003305}},

                  expiration_ = {__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {

                      big = 0x18086, tiny = {

                        __data = "\206\200\001\000\000\000\000\000kt\246", '\000' <repeats 13 times>, "_\336\276~\210\177\000\000p\272\376-\210\177\000\000@v\337~\210\177\000", __align = {<No data fields>}}}, call_ = 0x46752f

     <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,

                priority_ = 0 '\000', queueObserverPayload_ = 140224202794948}}, hasValue = true}}

        guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcdb90b "CPUThreadPoolExecutor",

                e_ = 0xcdb920 ""}}}}

#43 0x0000000000c19519 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>)

    at /usr/local/include/c++/7.3.0/bits/invoke.h:73

No locals.

#44 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95

No locals.

---Type <return> to continue, or q <return> to quit---

#45 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

No locals.

#46 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

No locals.

#47 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

        fn = <optimized out>

#48 0x0000000000ab0454 in operator() (this=0x7f887afe2080)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        fn = @0x7f887afe2080: <error reading variable>

#49 operator() (__closure=0x7f887afe2080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = 0x7f87effed9a0 "NonIoPool3"}, _M_string_length = 10, {_M_local_buf = "NonIoPool3\000\000\000\000\000",

                  _M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}

        func = <error reading variable func (Cannot access memory at address 0x7f887afe2080)>

#50 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

        fn = @0x7f887afe2080: <error reading variable>

#51 0x00007f887c9bbd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

No locals.

#52 0x00007f887e7bcea5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#53 0x00007f887c104b0d in clone () from /lib64/libc.so.6

No symbol table info available.

git fetch https://review.couchbase.org/TAF refs/changes/57/187357/6 && git checkout -b change-187357 FETCH_HEAD

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/temp_vol_3.ini -p bucket_storage=magma,bucket_ram_quota=1024,init_loading=True,bucket_eviction_policy=fullEviction,rerun=False -t storage.magma.magma_crash_recovery.MagmaCrashTests.test_crash_during_dedupe,nodes_init=3,skip_cleanup=True,num_items=5000000,doc_size=1024,batch_size=100,sdk_timeout=60,log_level=info,infra_log_level=info,key_size=12,num_collections=15,ops_rate=20000,key_type=RandomKey,vbuckets=16,replicas=2,test_itr=3,bucket_history_retention_seconds=86400,bucket_history_retention_bytes=99000000000,standard_buckets=1,magma_buckets=1,num_scopes=1,autoCompactionDefined=true,meta_purge_interval=120,randomize_value=True,num_collections_to_drop=5,dedupe_iterations=5000 -m rest'