Memcached crashed with segmentation fault at ActiveStream::markDiskSnapshot (this=0x7fe37822ba10, startSeqno=<optimized out>, endSeqno=103927, highCompletedSeqno=..., maxVisibleSeqno=103927, timestamp=..., snapshotType=HistoryFollowingNoHistory)

(gdb) bt full

#0  0x00007fe477ef67bb in raise () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#1  0x00007fe477ee1535 in abort () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

#2  0x00007fe4782c07fc in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000c85f16 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

No locals.

#4  0x00007fe4782cbab6 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007fe4782cbb21 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

No locals.

#6  0x00007fe4782cbdb4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0x10b4ef8 <typeinfo for std::logic_error>,

    dest=0x450840 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

        globals = <optimized out>

        header = 0x7fe400001590

#7  0x0000000000d6e6f5 in __cxxabiv1::__cxa_throw (thrownException=0x7fe400001610, type=0x10b4ef8 <typeinfo for std::logic_error>, destructor=<optimized out>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp:106

        orig_cxa_throw = 0x0

#8  0x00000000005048d5 in ActiveStream::markDiskSnapshot (this=0x7fe37822ba10, startSeqno=<optimized out>, endSeqno=103927, highCompletedSeqno=...,

    maxVisibleSeqno=103927, timestamp=..., snapshotType=HistoryFollowingNoHistory)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:482

        msg = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7fe3a79c2200 <error: Cannot access memory at address 0x7fe3a79c2200>}, _M_string_length = 355, {

            _M_local_buf = "c\001", '\000' <repeats 13 times>, _M_allocated_capacity = 355}}

        lh = {_M_device = 0x7fe37822ba78, _M_owns = true}

        chkCursorSeqno = 103927

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe3a5750a00, _M_refcount = {_M_pi = 0x7fe3a7712560}}, <No data fields>}

        hcsToSend = <optimized out>

        mvsToSend = {<std::_Optional_base<unsigned long, true, true>> = {<std::_Optional_base_impl<unsigned long, std::_Optional_base<unsigned long, true, true> >> = {<No data fields>}, _M_payload = {<std::_Optional_payload_base<unsigned long>> = {_M_payload = {_M_empty = {<No data fields>}, _M_value = 103927},

                _M_engaged = true}, <No data fields>}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}

        flags = <optimized out>

        wasFirst = 254

        sendHCS = <optimized out>

#9  0x0000000000a1009a in DCPBackfillDiskToStream::scanHistoryCreate (this=0x7fe3a0a42720, bucket=..., scanCtx=..., streamPtr=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_types.h:205

        historyScanCtx = <error reading variable>

        ctx = <optimized out>

#10 0x0000000000a10752 in DCPBackfillDiskToStream::doHistoryScan (this=0x7fe3a0a42720, bucket=..., scanCtx=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_disk_to_stream.cc:253

        stream = {<std::__shared_ptr<ActiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<ActiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe37822ba10, _M_refcount = {_M_pi = 0x7fe37822ba00}}, <No data fields>}

--Type <RET> for more, q to quit, c to continue without paging--

        historyScanCtx = <error reading variable>

        bySeqnoCtx = <optimized out>

        kvstore = <optimized out>

#11 0x00000000008db29e in DCPBackfill::run (this=0x7fe3a0a42720)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill.cc:58

        lockedState = {static AllowsConcurrentAccess = false, lock_ = {_M_device = 0x7fe3a0a42744, _M_owns = true}}

        runtimeGuard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__this = 0x7fe3a0a42720}}

        currentState = <optimized out>

        phosphor_internal_category_enabled_30 = {_M_b = {

_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_30 = <optimized out>

        phosphor_internal_tpi_30 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},

          argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}}

        phosphor_internal_guard_30 = {tpi = 0x10abd20 <DCPBackfill::run()::phosphor_internal_tpi_30>, enabled = false, arg1 = 335, arg2 = 2 '\002', start = {__d = {

              __r = 0}}}

        status = backfill_finished

        __PRETTY_FUNCTION__ = "virtual backfill_status_t DCPBackfill::run()"

#12 0x0000000000a1249b in BackfillManager::backfill (this=0x7fe3a3b62010)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:324

        lh = {_M_device = 0x7fe3a3b62080, _M_owns = false}

        backfill = {_M_t = {<std::__uniq_ptr_impl<DCPBackfillIface, std::default_delete<DCPBackfillIface> >> = {

              _M_t = {<std::_Tuple_impl<0, DCPBackfillIface*, std::default_delete<DCPBackfillIface> >> = {<std::_Tuple_impl<1, std::default_delete<DCPBackfillIface> >> = {<std::_Head_base<1, std::default_delete<DCPBackfillIface>, true>> = {<std::default_delete<DCPBackfillIface>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, DCPBackfillIface*, false>> = {_M_head_impl = 0x7fe3a0a42720}, <No data fields>}, <No data fields>}}, <No data fields>}}

        source = BackfillManager::Source::Active

        status = <optimized out>

#13 0x0000000000a12734 in BackfillManagerTask::run (this=0x7fe3a28339b0)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:78

        phosphor_internal_category_enabled_67 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_67 = <optimized out>

        phosphor_internal_tpi_67 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},

          argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}}

        phosphor_internal_guard_67 = {tpi = 0x10adec0 <BackfillManagerTask::run()::phosphor_internal_tpi_67>, enabled = false, arg1 = {<No data fields>},

          arg2 = {<No data fields>}, start = {__d = {__r = 0}}}

        manager = {<std::__shared_ptr<BackfillManager, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<BackfillManager, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7fe3a3b62000}}, <No data fields>}

        status = <optimized out>

#14 0x0000000000bd308d in GlobalTask::execute (this=0x7fe3a28339b0, threadName=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:104

        guard = {previous = 0x0}

        taskableName = {value = {_M_elems = "default", '\000' <repeats 24 times>}}

        executedAt = <optimized out>

        scheduleOverhead = <optimized out>

        start = <optimized out>

        runAgain = <optimized out>

        end = <optimized out>

        runtime = <optimized out>

--Type <RET> for more, q to quit, c to continue without paging--

#15 0x0000000000bd12ba in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fe40fff4e20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:314

        runAgain = <optimized out>

        proxy = <error reading variable>

        proxy = <optimized out>

        runAgain = <optimized out>

        _logger_ = <optimized out>

#16 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (

    p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:363

        fn = @0x7fe40fff4e20: {__proxy = @0x7fe4769edbc0}

#17 0x0000000000bd4216 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7fe40fff4e20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        fn = @0x7fe40fff4e20: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fe4769edbc0, tiny = {

              __data = "\300\333\236v\344\177\000\000\000\000\000\000\343\177\000\001", '\000' <repeats 16 times>, " O\377\017\344\177\000\000f\235\325\000\000\000\000", __align = {<No data fields>}}},

          call_ = 0xbd1230 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>

, exec_ = 0x60d0a0

     <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}

#18 operator() (__closure=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        task = {storage_ = {{emptyState = -80 '\260', value = {task = 0x7fe3a28339b0, func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>},

                  data_ = {big = 0x7fe4769edbc0, tiny = {

                      __data = "\300\333\236v\344\177\000\000\000\000\000\000\343\177\000\001", '\000' <repeats 16 times>, " O\377\017\344\177\000\000f\235\325\000\000\000\000", __align = {<No data fields>}}},

                  call_ = 0xbd1230 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

                  exec_ = 0x60d0a0 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}}}, hasValue = true}}

        this = <optimized out>

#19 0x0000000000d6a6ef in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7fe40fff50d0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

        fn = @0x7fe40fff50d0: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fe4769b7000, tiny = {

              __data = "\000p\233v\344\177\000\000\020\205\210v\344\177\000\000\000\000\000\000\000\000\000\000\020\203\031\001\000\000\000\000\060\000\000\000\000\000\000\000?\031\000\000\000\000\000", __align = {<No data fields>}}},

          call_ = 0xbd45b0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

          exec_ = 0x60d0a0 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}

#20 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7fe4769b7100, thread=..., task=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 5395213594973129}}

        stats = {expired = false, waitTime = {__r = 18411}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 5395213594954718}}, requestId = 0}

#21 0x0000000000d533e7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7fe4769b7100, thread=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

--Type <RET> for more, q to quit, c to continue without paging--

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {

                  func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fe4769b7000, tiny = {

                        __data = "\000p\233v\344\177\000\000\020\205\210v\344\177\000\000\000\000\000\000\000\000\000\000\020\203\031\001\000\000\000\000\060\000\000\000\000\000\000\000?\031\000\000\000\000\000", __align = {<No data fields>}}},

                    call_ = 0xbd45b0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

                    exec_ = 0x60d0a0 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}, enqueueTime_ = {__d = {__r = 5395213594954718}}, expiration_ = {__r = 0},

                  expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x0, tiny = {

                        __data = "\000\000\000\000\000\000\000\000?\031\000\000\000\000\000\000?\031\000\000\000\000\000\000I\256Z", '\000' <repeats 13 times>, "\322\227\065x\344\177\000", __align = {<No data fields>}}}, call_ = 0x470f55

     <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000',

                queueObserverPayload_ = -1}}, hasValue = true}}

        guard = {storage_ = {{emptyState = 32 ' ', value = {list_ = {prev = 0x1220e20 <folly::emptyList>, curr = {forbid = true, allowTerminationOnBlocking = false,

                    ex = 0x7fe4769b7100, tag = {static npos = <optimized out>, b_ = 0x7fe477a08720 <error: Cannot access memory at address 0x7fe477a08720>,

                      e_ = 0x7fe477a08739 <error: Cannot access memory at address 0x7fe477a08739>}}}}}, hasValue = true}}

        threadIDsGuard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__this = 0x7fe4769b7100}}

#22 0x0000000000d6d28a in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, __t=<optimized out>, __f=<optimized out>, __t=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

No locals.

#23 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

No locals.

#24 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

No locals.

#25 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

No locals.

#26 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

        fn = <optimized out>

#27 0x0000000000bcdf44 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7fe476dee300)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        fn = <error reading variable>

#28 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7fe476dee300)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        threadNameOpt = {storage_ = {{emptyState = 64 '@', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = 0x7fe40fff5240 "AuxIoPool1"}, _M_string_length = 10, {_M_local_buf = "AuxIoPool1\000\000\000\000\000",

                  _M_allocated_capacity = 8029725099529106753}}}, hasValue = true}}

--Type <RET> for more, q to quit, c to continue without paging--

        func = <error reading variable func (Cannot access memory at address 0x7fe476dee300)>

        func = <optimized out>

        threadNameOpt = <optimized out>

#29 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:377

        fn = <error reading variable>

#30 0x00007fe4782f4f00 in std::execute_native_thread_routine (__p=0x7fe476ab8f30) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

        __t = <optimized out>

#31 0x00007fe4788f7fa3 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0

No symbol table info available.

#32 0x00007fe477fb7eff in clone () from /lib/x86_64-linux-gnu/libc.so.6

No symbol table info available.

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.73051.ini GROUP=stop_start_rebalance,rerun=False,disk_optimized_thread_settings=True,get-cbcollect-info=True,autoCompactionDefined=true,upgrade_version=7.6.0-1452,sirius_url=http://172.23.120.103:4000 -t rebalance_new.rebalance_start_stop.RebalanceStartStopTests.test_start_stop_rebalance_with_mutations,nodes_init=4,nodes_in=1,nodes_out=0,extra_nodes_in=1,extra_nodes_out=0,bucket_spec=magma_dgm.10_percent_dgm.4_node_1_replica_magma_512,doc_size=512,randomize_value=True,default_history_retention_for_collections=false,bucket_history_retention_seconds=60,bucket_history_retention_bytes=100000000000,dedupe_update_itrs=10000,GROUP=P0_set0;stop_start_rebalance'