Memcached crashed in magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:96

(gdb) bt

#0  0x0000000000b01f2e in magma::MergeIterator::greaterThan::operator() (this=0x7f80046b7100, a=..., b=...)

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_vector.h:1123

#1  0x0000000000b02364 in operator() (rhs=..., lhs=..., this=0x7f80046b7100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/detail/mutable_heap.hpp:122

#2  operator() (rhs=..., lhs=..., this=0x7f80046b7100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/detail/stable_heap.hpp:211

#3  siftdown (index=0, this=0x7f80046b7100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:290

#4  decrease (index=0, this=0x7f80046b7100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:414

#5  decrease (handle=..., this=0x7f80046b7100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/detail/mutable_heap.hpp:454

#6  decrease (handle=..., this=0x7f80046b7100)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/heap/d_ary_heap.hpp:729

#7  magma::MergeIterator::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/iterator.cc:96

#8  0x0000000000b1888f in magma::KVReader::KVStream::Next() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/common.cc:103

#9  0x0000000000b857b8 in magma::ItemGCFilter::Process(magma::KVReader::KVStream&, std::function<magma::Status (std::pair<magma::Slice, folly::IOBuf>&, magma::Operation)>, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/mvcc.cc:95

#10 0x0000000000b42170 in addKVs (isDocHistoryEnabled=false, gc=..., gcOutputCb=..., kvRdr=..., this=<optimized out>)

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_function.h:239

#11 magma::LSMTree::writeSSTable(magma::SSTableWriter&, magma::Iterator*, long, std::function<bool (magma::Slice const&)>) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:858

#12 0x0000000000b45e5f in magma::LSMTree::writeSSTable(bool, magma::Iterator*, unsigned long, long, bool, std::function<bool (magma::Slice const&)>) ()

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_function.h:573

#13 0x0000000000b46176 in magma::LSMTree::writeMemtable(magma::Memtable*, bool)::{lambda(magma::Iterator*)#1}::operator()(magma::Iterator*) const ()

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/atomic_base.h:503

#14 0x0000000000b47623 in magma::LSMTree::writeMemtable(magma::Memtable*, bool) () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:469

#15 0x0000000000b479aa in magma::LSMTree::doMemtableFlushWork() () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/shared_ptr_base.h:1665

#16 0x0000000000b4840b in magma::LSMTree::tryRetryMemtableFlush() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1850

#17 0x0000000000b4b66b in magma::LSMTree::BeginTxn() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:1737

#18 0x0000000000ab357d in operator() (tree=..., __closure=<synthetic pointer>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:239

#19 magma::KVStore::beginTxn() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:253

#20 0x0000000000ac98dd in magma::KVStore::WriteDocs(magma::WAL*, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>, magma::Magma::HistoryMode, bool) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/write.cc:188

#21 0x0000000000a7bf22 in magma::Magma::Impl::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>, magma::Magma::HistoryMode) () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:469

#22 0x0000000000a7c2de in magma::Magma::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>, magma::Magma::HistoryMode) () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_function.h:573

#23 0x0000000000991d55 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > ---Type <return> to continue, or q <return> to quit---

const&, unsigned int, magma::Magma::HistoryMode, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>) () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_function.h:573

#24 0x000000000098492b in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62

#25 0x000000000096b20b in MagmaKVStore::commit(std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.h:850

#26 0x00000000008eea25 in EPBucket::commit(KVStoreIface&, std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:197

#27 0x00000000008f418f in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:197

#28 0x00000000008f4f07 in EPBucket::flushVBucket(Vbid) () at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_lock.h:112

#29 0x0000000000782bff in Flusher::flushVB (this=0x7f800c3f5a00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:284

#30 0x0000000000782e48 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:204

#31 0x0000000000bacc0f in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:79

#32 0x0000000000904d22 in EpTask::execute(std::basic_string_view<char, std::char_traits<char> >) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_task.cc:43

#33 0x0000000000baa9aa in operator() (__closure=0x7f7fdd7cdd80) at /opt/gcc-13.2.0/include/c++/13.2.0/string_view:146

#34 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (

    p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:363

#35 0x0000000000badd63 in operator() (this=0x7f7fdd7cdd80)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:390

#36 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#37 0x0000000000d485ff in operator() (this=0x7f7fdd7ce030)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#38 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) ()

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#39 0x0000000000d31127 in folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) ()

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

#40 0x0000000000d4b19a in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, __t=<optimized out>, __f=<optimized out>, __t=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

#41 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

#42 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

#43 operator()<> (this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

#44 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

#45 0x0000000000ba6cf0 in operator() (this=0x7f8023268180)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:390

#46 operator() (__closure=0x7f8023268180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#47 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:377

#48 0x00007f8024fa2aa3 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/src/c++11/thread.cc:104

---Type <return> to continue, or q <return> to quit---

#49 0x00007f8025820ea5 in start_thread (arg=0x7f7fdd7d8700) at pthread_create.c:307

#50 0x00007f80248ecb0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

(gdb)

(gdb)

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.124863.ini bucket_storage=magma,rerun=false,bucket_eviction_policy=fullEviction,randomize_value=True,enable_dp=false,GROUP=P0;Majority,get-cbcollect-info=True,autoCompactionDefined=true,get-cbcollect-info=True,infra_log_level=info,log_level=info,bucket_storage=magma,upgrade_version=7.6.0-1767,sirius_url=http://172.23.120.103:4000 -t storage.magma.magma_disk_full.MagmaDiskFull.test_random_seqTree_chmod,nodes_init=4,num_items=5000000,doc_size=2048,sdk_timeout=60,replicas=1,durability=majority,vbuckets=16,GROUP=P0;Majority'