Memcached crashed in ItemAccessVisitor::~ItemAccessVisitor (this=0x7f6c900b2a00, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:69

(gdb) bt full

#0  0x0000000000cda263 in cb::SemaphoreGuard<cb::Semaphore*>::reset (this=0x7f6c900b2b20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/semaphore_guard.cc:95

No locals.

#1  0x0000000000cda2e9 in cb::SemaphoreGuard<cb::Semaphore*>::~SemaphoreGuard (this=<optimized out>, __in_chrg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/semaphore_guard.cc:79

No locals.

#2  0x00000000008af115 in ItemAccessVisitor::~ItemAccessVisitor (this=0x7f6c900b2a00, __in_chrg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:69

No locals.

#3  ItemAccessVisitor::~ItemAccessVisitor (this=0x7f6c900b2a00, __in_chrg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:71

No locals.

#4  0x0000000000802765 in std::default_delete<InterruptableVBucketVisitor>::operator() (__ptr=<optimized out>, this=0x7f6c60c28090)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:85

No locals.

#5  std::unique_ptr<InterruptableVBucketVisitor, std::default_delete<InterruptableVBucketVisitor> >::~unique_ptr (this=0x7f6c60c28090,

    __in_chrg=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:361

        __ptr = <error reading variable>

#6  VBCBAdaptor::~VBCBAdaptor (this=0x7f6c60c28030, __in_chrg=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_adapters.h:35

No locals.

#7  0x0000000000bc5ea9 in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x7f6c60c28020)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:158

No locals.

#8  std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x7f6c60c28020)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:151

No locals.

#9  std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count (this=<optimized out>, __in_chrg=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:733

No locals.

#10 std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=<optimized out>, __in_chrg=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:1183

No locals.

#11 std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>::reset (this=0x7f6c7dff0e20)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/shared_ptr_base.h:1301

No locals.

#12 FollyExecutorPool::TaskProxy::resetTaskPtr(std::atomic<int>&, bool)::{lambda()#2}::operator()() (__closure=0x7f6c7dff0e20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:215

        guard = {previous = 0x0}

--Type <RET> for more, q to quit, c to continue without paging--

        resetOnScheduler = false

        pendingResets = <error reading variable>

        ptrToReset = {<std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<GlobalTask, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}

        resetOnScheduler = <optimized out>

        pendingResets = <optimized out>

        ptrToReset = {<std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<GlobalTask, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = <optimized out>}}, <No data fields>}

        _logger_ = <optimized out>

        guard = {previous = <optimized out>}

#13 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::resetTaskPtr(std::atomic<int>&, bool)::{lambda()#2}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:363

        fn = @0x7f6c7dff0e20: {

          __ptrToReset = {<std::__shared_ptr<GlobalTask, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<GlobalTask, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}, __pendingResets = @0x7f6cb041ab50,

          __resetOnScheduler = false}

#14 0x0000000000bc8496 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f6c7dff0e20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        fn = @0x7f6c7dff0e20: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x0, tiny = {

              __data = '\000' <repeats 16 times>, "P\253A\260l\177\000\000\000\000\000\000\000\000\000\000 \017\377}l\177\000\000\246\312\321\000\000\000\000", __align = {<No data fields>}}},

          call_ = 0xbc5e10 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::resetTaskPtr(std::atomic<int>&, bool)::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xbc2150 <folly::detail::function::DispatchSmall::exec<FollyExecutorPool::TaskProxy::resetTaskPtr(std::atomic<int>&, bool)::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}

#15 operator() (__closure=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        task = {storage_ = {{emptyState = 0 '\000', value = {task = 0x0,

                func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x0, tiny = {

                      __data = '\000' <repeats 16 times>, "P\253A\260l\177\000\000\000\000\000\000\000\000\000\000 \017\377}l\177\000\000\246\312\321\000\000\000\000", __align = {<No data fields>}}},

                  call_ = 0xbc5e10 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::resetTaskPtr(std::atomic<int>&, bool)::{lambda()#2}>(folly::detail::function::Data&)>,

                  exec_ = 0xbc2150 <folly::detail::function::DispatchSmall::exec<FollyExecutorPool::TaskProxy::resetTaskPtr(std::atomic<int>&, bool)::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}}}, hasValue = true}}

        this = <optimized out>

#16 0x0000000000d2d42f in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f6c7dff10d0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

--Type <RET> for more, q to quit, c to continue without paging--

        fn = @0x7f6c7dff10d0: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6cb0413c00, tiny = {

              __data = "\000<A\260l\177\000\000\030\253A\260l\177\000\000\b\000\000\000\000\000\000\000\020\362\064\001\000\000\000\000\060\000\000\000\000\000\000\000\327\220\017\000\000\000\000", __align = {<No data fields>}}},

          call_ = 0xbc8830 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>, exec_ = 0x606c00

     <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}

#17 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f6cb0413d00, thread=..., task=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 370336425685684}}

        stats = {expired = false, waitTime = {__r = 28177}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 370336425657507}}, requestId = 0}

#18 0x0000000000d16127 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f6cb0413d00, thread=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {

                  func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6cb0413c00, tiny = {

                        __data = "\000<A\260l\177\000\000\030\253A\260l\177\000\000\b\000\000\000\000\000\000\000\020\362\064\001\000\000\000\000\060\000\000\000\000\000\000\000\327\220\017\000\000\000\000", __align = {<No data fields>}}},

                    call_ = 0xbc8830 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

                    exec_ = 0x606c00 <folly::detail::function::DispatchSmallTrivial::exec_<16ul>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}, enqueueTime_ = {__d = {__r = 370336425657507}}, expiration_ = {__r = 0},

                  expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f6c7dff1270, tiny = {

                        __data = "p\022\377}l\177\000\000\327\220\017\000\000\000\000\000\327\220\017\000\000\000\000\000\tIZ", '\000' <repeats 13 times>, "\022\006\315\261l\177\000", __align = {<No data fields>}}}, call_ = 0x46bc43

     <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,

                priority_ = 0 '\000', queueObserverPayload_ = 0}}, hasValue = true}}

        guard = {storage_ = {{emptyState = 32 ' ', value = {list_ = {prev = 0x13d7d20 <folly::emptyList>, curr = {forbid = true,

                    allowTerminationOnBlocking = false, ex = 0x7f6cb0413d00, tag = {static npos = <optimized out>,

                      b_ = 0x7f6cb04f65a0 <error: Cannot access memory at address 0x7f6cb04f65a0>,

                      e_ = 0x7f6cb04f65b9 <error: Cannot access memory at address 0x7f6cb04f65b9>}}}}}, hasValue = true}}

        threadIDsGuard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__this = 0x7f6cb0413d00}}

#19 0x0000000000d2ffca in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, __t=<optimized out>, __f=<optimized out>,

    __t=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:73

--Type <RET> for more, q to quit, c to continue without paging--

No locals.

#20 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

No locals.

#21 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

No locals.

#22 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

No locals.

#23 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

        fn = <optimized out>

#24 0x0000000000bc22d4 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f6cb0511080)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        fn = <error reading variable>

#25 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f6cb0511080)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        threadNameOpt = {storage_ = {{emptyState = 64 '@', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = 0x7f6c7dff1240 "AuxIoPool0"}, _M_string_length = 10, {_M_local_buf = "AuxIoPool0\000\034\207\206C\316",

                  _M_allocated_capacity = 8029725099529106753}}}, hasValue = true}}

        func = <error reading variable func (Cannot access memory at address 0x7f6cb0511080)>

        func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = <optimized out>, tiny = {__data = {

                <optimized out> <repeats 48 times>}, __align = {<No data fields>}}}, call_ = <optimized out>, exec_ = <optimized out>}

        threadNameOpt = {storage_ = {{emptyState = <optimized out>, value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = <optimized out>}, _M_string_length = <optimized out>, {_M_local_buf = {<optimized out> <repeats 16 times>},

                  _M_allocated_capacity = <optimized out>}}}, hasValue = <optimized out>}}

#26 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:377

        fn = <error reading variable>

#27 0x00007f6cb1c6bd40 in std::execute_native_thread_routine (__p=0x7f6cb05f8e90)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

        __t = {_M_t = {<std::__uniq_ptr_impl<std::thread::_State, std::default_delete<std::thread::_State> >> = {

              _M_t = {<std::_Tuple_impl<0, std::thread::_State*, std::default_delete<std::thread::_State> >> = {<std::_Tuple_impl<1, std::default_delet--Type <RET> for more, q to quit, c to continue without paging--

e<std::thread::_State> >> = {<std::_Head_base<1, std::default_delete<std::thread::_State>, true>> = {<std::default_delete<std::thread::_State>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, std::thread::_State*, false>> = {

                    _M_head_impl = <optimized out>}, <No data fields>}, <No data fields>}}, <No data fields>}}

#28 0x00007f6cb246fea7 in start_thread (arg=<optimized out>) at pthread_create.c:477

        ret = <optimized out>

        pd = <optimized out>

        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140103947106048, 3887749089748198352, 140721154368478, 140721154368479, 140103947064192, 8396800,

                -3806398424605383728, -3806267641659644976}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0,

              cleanup = 0x0, canceltype = 0}}}

        not_first_call = 0

#29 0x00007f6cb1767a2f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

No locals.

./testrunner -i /data/workspace/debian-p0-tunable-vset00-00-warmup/testexec.27999.ini -p max-dupe-result-count=10,num-tries=60,attempt-num=60,dgm_run=true,eviction_policy=fullEviction,GROUP=P0,skip_log_scan=False,bucket_storage=couchstore,sirius_url=http://172.23.120.103:4000 -t warmup.warmuptest.WarmUpTests.warmup_test,stats_monitor=vb_replica_perc_mem_resident;vb_active_perc_mem_resident,items=20000,default_bucket=False,standard_buckets=1,active_resident_threshold=60,is_partial=False,dgm_run=True,doc_ops=update;delete,GROUP=P0