Memcached crashes & Expects(res == MutationStatus::NotFound) failing

./testrunner -i /tmp/durability_volume.ini sdk_client_pool=True,rerun=False -t bucket_collections.collections_rebalance.CollectionsRebalance.test_data_load_collections_with_rebalance_in_out,nodes_init=4,nodes_in=2,nodes_out=1,override_spec_params=durability;replicas,durability=PERSIST_TO_MAJORITY,replicas=Bucket.ReplicaNum.TWO,bucket_spec=multi_bucket.buckets_all_membase_for_rebalance_tests_more_collections,data_load_spec=volume_test_load_with_CRUD_on_collections,data_load_stage=during,quota_percent=80,GROUP=rebalance_with_collection_crud_durability

(gdb) bt full

#0  0x00007f0cd8c1d207 in ?? () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007f0cd8c1e8f8 in _quicksort (pbase=0x0, total_elems=<optimized out>, size=0, cmp=0x7fa760 <stderr>, arg=0x7f0c70021130) at qsort.c:123

        __tmp = 87 'W'

        __size = 18446744073709551604

        __a = 0x12 <Address 0x12 out of bounds>

        __b = 0xb <Address 0xb out of bounds>

        left_ptr = <optimized out>

        right_ptr = <optimized out>

        mid = 0x7f0c8b611bf8 <Address 0x7f0c8b611bf8 out of bounds>

        lo = 0x0

        hi = 0x0

        stack = {{lo = 0x0, hi = 0x0}, {lo = 0x0, hi = 0x0}, {lo = 0x7f0cd8fae1c0 <__GI__IO_list_all> "", hi = 0x7f0cd8cd5ff9 <lockf+169> "f\211\064$\276\006"}, {lo = 0x0, 

            hi = 0x7f0cd8c610b3 <_IO_new_file_init+67> "\b[]\303f\017\037\204"}, {lo = 0x7f0c8b611bf8 <Address 0x7f0c8b611bf8 out of bounds>, hi = 0x7f0cd8fae1c0 <__GI__IO_list_all> ""}, {lo = 0x1 <Address 0x1 out of bounds>, 

            hi = 0x7f0cd8fae243 <_IO_2_1_stderr_+99> ""}, {lo = 0x7f0cd8faa3a0 <__GI__IO_file_jumps> "", hi = 0x7f0cd8c628ce <__GI__IO_switch_to_get_mode+126> "\037"}, {lo = 0x7f0cd8fae1c0 <__GI__IO_list_all> "", 

            hi = 0xa <Address 0xa out of bounds>}, {lo = 0x7f0c70021130 <Address 0x7f0c70021130 out of bounds>, hi = 0x7f0c8a745f18 <Address 0x7f0c8a745f18 out of bounds>}, {lo = 0x7f0c9f7f3e20 "`\253\206\213\f\177", 

            hi = 0x7f0cd8c62dc3 <__GI__IO_doallocbuf+35> ""}, {lo = 0x7f0cd8fae1c0 <__GI__IO_list_all> "", hi = 0x7f0cd9777180 <__cxa_get_globals_fast+16> "H\203\304\bH\005"}, {lo = 0x7f0c8b611bf8 <Address 0x7f0c8b611bf8 out of bounds>, 

            hi = 0x7f0cdde67500 <Address 0x7f0cdde67500 out of bounds>}, {lo = 0x7f0c9f7f4a70 "\240\020\002p\f\177", 

            hi = 0x7f0cd977a195 <__gnu_cxx::__verbose_terminate_handler()+293> "H\203\352\001H\211\307uL\350-r\377\377H\213\020H\211\307\377R\020H\213M"}, {lo = 0x0, hi = 0xd9777180 <Address 0xd9777180 out of bounds>}, {

            lo = 0x7f0cd85901d0 <Address 0x7f0cd85901d0 out of bounds>, hi = 0x7f0c70021110 <Address 0x7f0c70021110 out of bounds>}, {lo = 0x7f0cdce884c0 <gsl::fail_fast::~fail_fast()> "H\215\005\311\263O", hi = 0x549d52

     <backtrace_terminate_handler()+66> "\350\371\340\354\377\350\004\377\377\377\353\346H\215\025\253'\005"}, {lo = 0x0, hi = 0x7f0c8b611bf8 <Address 0x7f0c8b611bf8 out of bounds>}, {

            lo = 0x7f0cdce884c0 <gsl::fail_fast::~fail_fast()> "H\215\005\311\263O", hi = 0x7f0c700210f0 <Address 0x7f0c700210f0 out of bounds>}, {lo = 0x7f0cdd383850 <typeinfo for gsl::fail_fast> "", 

            hi = 0x7f0cd9777f86 <__cxxabiv1::__terminate(void (*)())+6> "\350\065\226\377\377H\211\307\350=\224\377\377\353\361f.\017\037\204"}, {lo = 0x7f0c700210f0 <Address 0x7f0c700210f0 out of bounds>, 

            hi = 0x7f0cd9777fd1 "f.\017\037\204"}, {lo = 0x333 <Address 0x333 out of bounds>, hi = 0x7f0cd9778213 "f.\017\037\204"}, {lo = 0x7f0c70021110 <Address 0x7f0c70021110 out of bounds>, 

            hi = 0x7f0c9f7f3db0 "\260>\177\237\f\177"}, {lo = 0x7f0c8b86ab68 <Address 0x7f0c8b86ab68 out of bounds>, hi = 0x7f0cdcf299f0

     <EPVBucket::loadOutstandingPrepares(folly::SharedMutexImpl<false, void, std::atomic, false, false>::WriteHolder const&, vbucket_state const&, std::vector<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >&&)+560> "H\213}\260I\203\304\020H\211\303L9\347t\005\350\313y\363\377L\211\357\350C\317\r"}, {lo = 0x2 <Address 0x2 out of bounds>, 

            hi = 0x7f0c8a745f00 <Address 0x7f0c8a745f00 out of bounds>}, {lo = 0x0, hi = 0x8 <Address 0x8 out of bounds>}, {lo = 0x502700 <mcbp::datatype::to_string[abi:cxx11](unsigned char)+784> "@H\211|$PH\307D$H", hi = 0x0}, {

            lo = 0x8 <Address 0x8 out of bounds>, hi = 0x8 <Address 0x8 out of bounds>}, {lo = 0x0, hi = 0x7f0c8b86ab68 <Address 0x7f0c8b86ab68 out of bounds>}, {lo = 0x7f0c9f7f3eb0 "P?\177\237\f\177", 

            hi = 0x7f0c8b86ab68 <Address 0x7f0c8b86ab68 out of bounds>}, {lo = 0x1 <Address 0x1 out of bounds>, hi = 0x7f0c8b86ab60 <Address 0x7f0c8b86ab60 out of bounds>}, {lo = 0x7f0c6d80f320 <Address 0x7f0c6d80f320 out of bounds>, 

            hi = 0x7f0c8a745f00 <Address 0x7f0c8a745f00 out of bounds>}, {lo = 0x7f0c9f7f3eb0 "P?\177\237\f\177", 

            hi = 0x7f0cdcf1cdf7 <EPBucket::loadPreparedSyncWrites(folly::SharedMutexImpl<false, void, std::atomic, false, false>::WriteHolder&, VBucket&)+1159> "M\213n H\213\265X\377\377\377H\205\366t\tH\213\006H\211\367\377P\bH\213\205x\377\377\377H\213\235p\377\377\377H9\330tYA\277\001"}, {lo = 0x300000100 <Address 0x300000100 out of bounds>, hi = 0x7f0c9f7f3e60 ""}, {lo = 0x131eba2ade1a3 <Address 0x131eba2ade1a3 out of bounds>, 

            hi = 0x30000000005c6 <Address 0x30000000005c6 out of bounds>}, {lo = 0x8 <Address 0x8 out of bounds>, hi = 0x7f0c9f7f3f10 "\330`t\212\f\177"}, {lo = 0x7f0c8b3033c0 <Address 0x7f0c8b3033c0 out of bounds>, 

            hi = 0x7f0c9f7f3e20 "`\253\206\213\f\177"}, {lo = 0x1f <Address 0x1f out of bounds>, hi = 0x7f0c8b862000 <Address 0x7f0c8b862000 out of bounds>}, {lo = 0x28 <Address 0x28 out of bounds>, hi = 0x0}, {

            lo = 0x7f0c8b86ab60 <Address 0x7f0c8b86ab60 out of bounds>, hi = 0x7f0c8b86ab68 <Address 0x7f0c8b86ab68 out of bounds>}, {lo = 0x7f0c8b86ab68 <Address 0x7f0c8b86ab68 out of bounds>, 

            hi = 0x7f0c9f7f3f70 "\001@\177\237\f\177"}, {lo = 0x0, hi = 0x5ce <Address 0x5ce out of bounds>}, {lo = 0x7f0c8b800000 <Address 0x7f0c8b800000 out of bounds>, hi = 0x7f0cdd029e00

     <CouchKVStore::rollback(Vbid, unsigned long, std::unique_ptr<RollbackCB, std::default_delete<RollbackCB> >)> "UH\211\345AWAVAUATI\211\375SH\211\363\211\326H\211\337M\211\307H\201\354h\002"}, {lo = 0x0, 

            hi = 0x7f0cdcf174ac <EPBucket::doRollback(Vbid, unsigned long)+124> "H\213|$\bH\205\377t\006H\213\a\377P\bH\203\304\030H\211\330[]A\\A]A^A_\303H\213|$\bH\211\303H\205\377t\006H\213\a\377P\bH\211\337\350'\261\364\377\220f\017\037D"}, {lo = 0x7f0c8b896800 <Address 0x7f0c8b896800 out of bounds>, hi = 0x0}, {lo = 0x47 <Address 0x47 out of bounds>, hi = 0x7f0c8b889000 <Address 0x7f0c8b889000 out of bounds>}, {lo = 0x5ce <Address 0x5ce out of bounds>, 

            hi = 0x7f0c8b889018 <Address 0x7f0c8b889018 out of bounds>}, {lo = 0x5fb <Address 0x5fb out of bounds>, hi = 0x7f0c9f7f3f70 "\001@\177\237\f\177"}, {lo = 0x7f0c9f7f3f50 "", hi = 0x7f0cdcf9c54d

     <KVBucket::rollback(Vbid, unsigned long)+1069> "H\213\274$\220"}, {lo = 0x0, hi = 0x7f0c9f7f3f0c "\200\001"}, {lo = 0x0, hi = 0x0}, {lo = 0x7f0cd76d1000 <Address 0x7f0cd76d1000 out of bounds>, 

            hi = 0x7f0c764f55a0 <Address 0x7f0c764f55a0 out of bounds>}, {lo = 0x7f0cd7671600 <Address 0x7f0cd7671600 out of bounds>, hi = 0x37f0cd7671600 <Address 0x37f0cd7671600 out of bounds>}, {

            lo = 0x7f0cd7671800 <Address 0x7f0cd7671800 out of bounds>, hi = 0x180764f5590 <Address 0x180764f5590 out of bounds>}, {lo = 0x7f0c8a7460d8 <Address 0x7f0c8a7460d8 out of bounds>, 

            hi = 0x7f0cce8f2f00 <Address 0x7f0cce8f2f00 out of bounds>}, {lo = 0x7f0c8b889130 <Address 0x7f0c8b889130 out of bounds>, hi = 0x7f0c9f7f4001 "@\177\237\f\177"}, {lo = 0x7f0c8b580078 <Address 0x7f0c8b580078 out of bounds>, 

            hi = 0x7f0cd76ad601 <Address 0x7f0cd76ad601 out of bounds>}, {lo = 0x7f0c8a745f00 <Address 0x7f0c8a745f00 out of bounds>, hi = 0x7f0c8aa884a0 <Address 0x7f0c8aa884a0 out of bounds>}, {

            lo = 0x7f0c8a745f00 <Address 0x7f0c8a745f00 out of bounds>, hi = 0x7f0c8aa884a0 <Address 0x7f0c8aa884a0 out of bounds>}, {lo = 0x7f0c8b580078 <Address 0x7f0c8b580078 out of bounds>, 

            hi = 0x7f0cd7671601 <Address 0x7f0cd7671601 out of bounds>}, {lo = 0x7f0c9f7f4001 "@\177\237\f\177", hi = 0x5c6 <Address 0x5c6 out of bounds>}, {lo = 0x5c6 <Address 0x5c6 out of bounds>, 

            hi = 0x5c6 <Address 0x5c6 out of bounds>}, {lo = 0x0, hi = 0x0}, {lo = 0x7f0cd7671600 <Address 0x7f0cd7671600 out of bounds>, hi = 0x7f0cd7671600 <Address 0x7f0cd7671600 out of bounds>}, {

            lo = 0x7f0cd7671800 <Address 0x7f0cd7671800 out of bounds>, hi = 0x7f0c764f5590 <Address 0x7f0c764f5590 out of bounds>}}

        top = 0x0

        base_ptr = <optimized out>

        max_thresh = 0

#2  0x00007f0cdb98e97d in setup (index=<optimized out>, mallocFlags=<optimized out>, this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:48

No locals.

#3  cb::_JEArenaMalloc<cb::JEArenaCoreLocalTracker>::switchToClient (client=..., tcache=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:214

        tcacheFlags = 4296464

#4  0x00007f0c9f7f40d8 in ?? ()

No symbol table info available.

#5  0x000000000003008b in ?? ()

No symbol table info available.

---Type <return> to continue, or q <return> to quit---

#6  0x00007f0cdcec579a in RollbackTask::run (this=0x7f0c9f7f40d8) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:908

        phosphor_internal_category_enabled_904 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <error reading variable: No global symbol "std::atomic<std::atomic<phosphor::CategoryStatus> const*>::is_always_lock_free".>}

        phosphor_internal_category_enabled_temp_904 = <optimized out>

        phosphor_internal_tpi_904 = {category = 0x292d33 <Address 0x292d33 out of bounds>, name = 0x299550 <Address 0x299550 out of bounds>, type = phosphor::Complete, argument_names = {_M_elems = {

              0x2bfb9b <Address 0x2bfb9b out of bounds>, 0x2bfb9b <Address 0x2bfb9b out of bounds>}}, argument_types = {_M_elems = {phosphor::is_none, phosphor::is_none}}}

        phosphor_internal_guard_904 = {tpi = 0x7f0cdd3803c0 <RollbackTask::run()::phosphor_internal_tpi_904>, enabled = true, arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 336362699821085}}}

#7  0x00007f0cdcf77723 in GlobalTask::execute (this=0x7f0c8b305c50) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73

        guard = {previous = 0x0}

#8  0x00007f0cdcf6e08f in ExecutorThread::run (this=0x7f0cd77af420) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188

        curTaskDescr = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

            _M_p = 0x7f0cd8426760 <Address 0x7f0cd8426760 out of bounds>}, _M_string_length = 30, {_M_local_buf = "\036\000\000\000\000\000\000\000W[\000\000\000\000\000", _M_allocated_capacity = 30}}

        woketime = <optimized out>

        scheduleOverhead = <optimized out>

        again = <optimized out>

        runtime = <optimized out>

        q = <optimized out>

        tick = 216 '\330'

        guard = {engine = 0x0}

#9  0x00007f0cdb97e777 in run (this=0x7f0cd844c220) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#10 platform_thread_wrap (arg=0x7f0cd844c220) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7f0cd844c220}, <No data fields>}, <No data fields>}}}

#11 0x00007f0cd8fbbdd5 in __nptl_deallocate_tsd () at pthread_create.c:171

        cnt = 0

#12 0x00007f0c9f7fe700 in ?? ()

No symbol table info available.

#13 0xf707fbd312a21b6e in ?? ()

No symbol table info available.

#14 0xf70774daecd81b6e in ?? ()

No symbol table info available.

#15 0x0000000000000000 in ?? ()

No symbol table info available.

grep CRITICAL memcached.log

2020-06-05T06:06:46.268488-07:00 CRITICAL *** Fatal error encountered during exception handling ***

2020-06-05T06:06:46.272433-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): GSL: Precondition failure at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_vb.cc: 879

2020-06-05T06:06:46.581539-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.0-2261). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/1d4f7c47-2301-45f3-3d476a88-558b4d7e.dmp before terminating.

2020-06-05T06:06:46.581567-07:00 CRITICAL Stack backtrace of crashed thread:

2020-06-05T06:06:46.581819-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x13952d]

2020-06-05T06:06:46.581835-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x14e85a]

2020-06-05T06:06:46.581846-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x14eb98]

2020-06-05T06:06:46.581856-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7f0cd8fb4000+0xf5d0]

2020-06-05T06:06:46.581889-07:00 CRITICAL     /lib64/libc.so.6(gsignal+0x37) [0x7f0cd8be7000+0x36207]

2020-06-05T06:06:46.581915-07:00 CRITICAL     /lib64/libc.so.6(abort+0x148) [0x7f0cd8be7000+0x378f8]

2020-06-05T06:06:46.582191-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6(_ZN9__gnu_cxx27__verbose_terminate_handlerEv+0x125) [0x7f0cd96e9000+0x91195]

2020-06-05T06:06:46.582211-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x149d52]

2020-06-05T06:06:46.582236-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f0cd96e9000+0x8ef86]

2020-06-05T06:06:46.582261-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f0cd96e9000+0x8efd1]

2020-06-05T06:06:46.582276-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f0cd96e9000+0x8f213]

2020-06-05T06:06:46.582287-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0xf89f0]

2020-06-05T06:06:46.582294-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0xebdf7]

2020-06-05T06:06:46.582545-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0x16b54d]

2020-06-05T06:06:46.582559-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0x9460b]

2020-06-05T06:06:46.582566-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0x9479a]

2020-06-05T06:06:46.582580-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0x146723]

2020-06-05T06:06:46.582586-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7f0cdce31000+0x13d08f]

2020-06-05T06:06:46.582592-07:00 CRITICAL     /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7f0cdb96e000+0x10777]

2020-06-05T06:06:46.582605-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7f0cd8fb4000+0x7dd5]

2020-06-05T06:06:46.582647-07:00 CRITICAL     /lib64/libc.so.6(clone+0x6d) [0x7f0cd8be7000+0xfdead]