Memcached crashed with segmentation fault at EventuallyPersistentEngine::getVBucketHlcNow((this=<optimized out>, vbucket=...)

(gdb) bt

#0  0x000000000078223c in EventuallyPersistentEngine::getVBucketHlcNow (this=<optimized out>, vbucket=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.h:694

#1  0x00000000009a2108 in magmakv::getHistoryTimeNow (kvid=<optimized out>, engine=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:194

#2  operator() (kvid=<optimized out>, __closure=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:639

#3  std::__invoke_impl<std::optional<std::chrono::duration<long int> >, MagmaKVStore::MagmaKVStore(MagmaKVStoreConfig&)::<lambda(magma::Magma::KVStoreID)>&, short unsigned int> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

#4  std::__invoke_r<std::optional<std::chrono::duration<long int> >, MagmaKVStore::MagmaKVStore(MagmaKVStoreConfig&)::<lambda(magma::Magma::KVStoreID)>&, short unsigned int> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113

#5  std::_Function_handler<std::optional<std::chrono::duration<long int, std::ratio<1, 1> > >(short unsigned int), MagmaKVStore::MagmaKVStore(MagmaKVStoreConfig&)::<lambda(magma::Magma::KVStoreID)> >::_M_invoke(const std::_Any_data &, unsigned short &&) (__functor=..., __args#0=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

#6  0x0000000000a74efd in std::function<std::optional<std::chrono::duration<long, std::ratio<1l, 1l> > > (unsigned short)>::operator()(unsigned short) const (

    __args#0=<optimized out>, this=0x7f91d424e4c8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#7  magma::Magma::Impl::canTimeRetainHistory (kvid=<optimized out>, time=..., this=0x7f91d424e000)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1948

#8  operator() (time=..., __closure=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:306

#9  std::__invoke_impl<bool, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(std::chrono::seconds)>&, std::chrono::duration<long int, std::ratio<1, 1> > > (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

#10 std::__invoke_r<bool, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(std::chrono::seconds)>&, std::chrono::duration<long int, std::ratio<1, 1> > > (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:113

#11 std::_Function_handler<bool(std::chrono::duration<long int, std::ratio<1, 1> >), magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>::<lambda(std::chrono::seconds)> >::_M_invoke(const std::_Any_data &, std::chrono::duration<long, std::ratio<1, 1> > &&) (

    __functor=..., __args#0=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

#12 0x0000000000b67b72 in std::function<bool (std::chrono::duration<long, std::ratio<1l, 1l> >)>::operator()(std::chrono::duration<long, std::ratio<1l, 1l> >) const (

    __args#0=..., this=0x7f91d42cf898) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#13 magma::LSMTree::recomputeHistorySSTableState (this=0x7f91d42cf010, snapshot=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:2136

#14 0x0000000000b691a3 in magma::LSMTree::swapTreeSnapshotAndRecomputeHistory (this=this@entry=0x7f91d42cf010, snapshot=...)

    at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

#15 0x0000000000b85373 in magma::LSMTree::doRecovery (this=this@entry=0x7f91d42cf010, checkpoints=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/checkpoint.cc:239

#16 0x0000000000b794f0 in magma::LSMTree::Open (this=0x7f91d42cf010, versions=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:247

#17 0x0000000000aeea5a in magma::KVStore::Open (this=0x7f91d422f810) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:189

#18 0x0000000000a75f85 in operator() (__closure=0x7f91b2295110) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:330

#19 0x0000000000a761bd in std::__invoke_impl<std::tuple<magma::Status, std::shared_ptr<magma::KVStore> >, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:289

#20 std::__invoke_r<std::tuple<magma::Status, std::shared_ptr<magma::KVStore> >, magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()>&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:115

#21 std::_Function_handler<std::tuple<magma::Status, std::shared_ptr<magma::KVStore> >(), magma::Magma::Impl::createKVStore(magma::Magma::KVStoreID, magma::Magma::KVStoreRevision, magma::KVStoreHandle&)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:292

#22 0x0000000000ac0946 in std::function<std::tuple<magma::Status, std::shared_ptr<magma::KVStore> > ()>::operator()() const (this=0x7f91d57ed560)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#23 magma::KVStoreSet::CreateKVStore(std::function<std::tuple<magma::Status, std::shared_ptr<magma::KVStore> > ()>) (this=this@entry=0x7f91d424e640,

--Type <RET> for more, q to quit, c to continue without paging--

    createKVStore=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore_set.cc:28

#24 0x0000000000a814b8 in magma::Magma::Impl::createKVStore (this=this@entry=0x7f91d424e000, kvID=<optimized out>, kvID@entry=1016, kvsRev=<optimized out>,

    kvstore=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248

#25 0x0000000000afcb44 in magma::Magma::Impl::recovery (this=this@entry=0x7f91d424e000)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_recovery.cc:93

#26 0x0000000000a7d59f in magma::Magma::Impl::Open (this=0x7f91d424e000) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:205

#27 0x0000000000a7d73d in magma::Magma::Open (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:224

#28 0x00000000009cf491 in MagmaMemoryTrackingProxy::Open (this=0x7f9176ccf0c8)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:440

#29 0x00000000009b4582 in MagmaKVStore::MagmaKVStore (this=0x7f91d422a500, configuration=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:750

#30 0x000000000094e9fb in std::make_unique<MagmaKVStore, MagmaKVStoreConfig&> ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/kvstore.cc:225

#31 KVStoreFactory::create (config=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/kvstore.cc:226

#32 0x00000000007dfef7 in KVShard::KVShard (this=0x7f91d4233000, config=..., numShards=<optimized out>, id=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.cc:41

#33 0x0000000000829e09 in std::make_unique<KVShard, Configuration&, unsigned long const&, unsigned long&> ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.h:725

#34 VBucketMap::VBucketMap (this=0x7f91d422a058, bucket=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucketmap.cc:66

#35 0x00000000007cdd6f in KVBucket::KVBucket (this=0x7f91d422a000, theEngine=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.h:682

#36 0x000000000092dd14 in EPBucket::EPBucket (this=0x7f91d422a000, engine=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:248

#37 0x000000000078e388 in std::make_unique<EPBucket, EventuallyPersistentEngine&> ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6794

#38 EventuallyPersistentEngine::makeBucket (this=0x7f9119c67000, config=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6794

#39 0x000000000078ebf2 in EventuallyPersistentEngine::initialize (this=0x7f9119c67000, config=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2093

#40 0x000000000063323c in BucketManager::createEngineInstance (this=0x1165db0 <BucketManager::instance()::instance>, bucket=..., type=<optimized out>, name=...,

    config=..., cid=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:558

#41 0x0000000000634d22 in BucketManager::create (this=0x1165db0 <BucketManager::instance()::instance>, cid=<optimized out>, name=..., config=...,

    type=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:644

#42 0x0000000000635310 in BucketManager::create (this=0x1165db0 <BucketManager::instance()::instance>, cookie=..., name=..., config=..., type=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322

#43 0x00000000006d6417 in operator() (__closure=0x7f92451bb4e0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322

#44 std::__invoke_impl<void, BucketManagementCommandContext::create()::<lambda()>&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60

#45 std::__invoke_r<void, BucketManagementCommandContext::create()::<lambda()>&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110

#46 std::_Function_handler<void(), BucketManagementCommandContext::create()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291

#47 0x00000000006d8991 in non-virtual thunk to LimitedConcurrencyTask::runInner(bool) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/one_shot_limited_concurrency_task.h:61

#48 0x0000000000bd308d in GlobalTask::execute (this=0x7f92383e7b08, threadName=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:104

#49 0x0000000000bd12ba in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f91d57efe20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:314

--Type <RET> for more, q to quit, c to continue without paging--

#50 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (

    p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:363

#51 0x0000000000bd4216 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f91d57efe20)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#52 operator() (__closure=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#53 0x0000000000d6a6ef in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f91d57f00d0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#54 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f9244db7d00, thread=..., task=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#55 0x0000000000d533e7 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f9244db7d00, thread=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

#56 0x0000000000d6d28a in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, __t=<optimized out>, __f=<optimized out>, __t=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

#57 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

#58 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

#59 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

#60 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

#61 0x0000000000bcdf44 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f92451ee600)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#62 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f92451ee600)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#63 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:377

#64 0x00007f9246690f00 in std::execute_native_thread_routine (__p=0x7f9244eb92b0) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

#65 0x00007f9246c93fa3 in start_thread (arg=<optimized out>) at pthread_create.c:486

#66 0x00007f9246353eff in __init_misc (argc=<optimized out>, argv=0x7f91d57fa700, envp=0x3f8) at init-misc.c:33

#67 0x0000000000000000 in ?? ()

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.87918.ini GROUP=P1;drop_recreate,rerun=False,disk_optimized_thread_settings=True,get-cbcollect-info=True,autoCompactionDefined=true,upgrade_version=7.6.0-1452,sirius_url=http://172.23.120.103:4000 -t bucket_collections.collections_drop_recreate_rebalance.CollectionsDropRecreateRebalance.test_data_load_collections_with_hard_failover_recovery,nodes_init=5,nodes_failover=2,bucket_spec=magma_dgm.20_percent_dgm.5_node_2_replica_magma_1024,doc_size=1024,randomize_value=True,recovery_type=delta,sleep_between_collections_crud=240,default_history_retention_for_collections=false,bucket_history_retention_seconds=60,bucket_history_retention_bytes=2147483648,dedupe_update_itrs=10000,GROUP=P0_set2;drop_recreate;P1'