Rebalance failed as memcached crashed during EPVBucket::bgFetchForCompactionExpiry(DocKey const&, Item const&)

172.23.97.37 : Stack Trace of fb73b6ae-84b8-4bb1-53e5f9a4-38858996.dmp

rm -rf /opt/couchbase/var/lib/couchbase/crash/fb73b6ae-84b8-4bb1-53e5f9a4-38858996.core

//opt/couchbase/bin/minidump-2-core /opt/couchbase/var/lib/couchbase/crash/fb73b6ae-84b8-4bb1-53e5f9a4-38858996.dmp > /opt/couchbase/var/lib/couchbase/crash/fb73b6ae-84b8-4bb1-53e5f9a4-38858996.core

gdb --batch /opt/couchbase/bin/memcached -c /opt/couchbase/var/lib/couchbase/crash/fb73b6ae-84b8-4bb1-53e5f9a4-38858996.core -ex bt full -ex quit

Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.

 #0  0x00007f85460ff337 in raise () from /lib64/libc.so.6

 #0  0x00007f85460ff337 in raise () from /lib64/libc.so.6

 #1  0x00007f8546100a28 in abort () from /lib64/libc.so.6

 #2  0x00007f8546c5d195 in __gnu_cxx::__verbose_terminate_handler() () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #3  0x000000000054cfc2 in backtrace_terminate_handler() ()

 #4  0x00007f8546c5af86 in __cxxabiv1::__terminate(void (*)()) () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #5  0x00007f8546c5afd1 in std::terminate() () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #6  0x00007f8546c5b213 in __cxa_throw () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #7  0x00007f8546c85d5e in std::__throw_system_error(int) () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #8  0x00007f854a4116ee in EPVBucket::queueBGFetchItem(DocKey const&, std::unique_ptr<BGFetchItem, std::default_delete<BGFetchItem> >, BgFetcher*) () from /opt/couchbase/bin/../lib/libep.so

 #9  0x00007f854a411bdf in EPVBucket::bgFetchForCompactionExpiry(DocKey const&, Item const&) () from /opt/couchbase/bin/../lib/libep.so

 #10 0x00007f854a4c5e91 in VBucket::deleteExpiredItem(Item const&, long, ExpireBy) () from /opt/couchbase/bin/../lib/libep.so

 #11 0x00007f854a482526 in KVBucket::deleteExpiredItem(Item&, long, ExpireBy) () from /opt/couchbase/bin/../lib/libep.so

 #12 0x00007f854a53af46 in MagmaKVStore::compactionCallBack(MagmaKVStore::MagmaCompactionCB&, magma::Slice const&, magma::Slice const&, magma::Slice const&) () from /opt/couchbase/bin/../lib/libep.so

 #13 0x00007f8549e2010a in magma::DocSeqGCContext::Transform(magma::Slice const&, magma::Slice const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, magma::Slice*) () from /opt/couchbase/bin/../lib/libmagma.so

 #14 0x00007f8549dd554c in magma::ItemGCFilter::Process(std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >&, std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >*) () from /opt/couchbase/bin/../lib/libmagma.so

 #15 0x00007f8549e2fde6 in magma::LSMTree::mergeSSTables(std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, magma::ItemGCFilter&, unsigned long, double, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >*) () from /opt/couchbase/bin/../lib/libmagma.so

 #16 0x00007f8549dce0f1 in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock<std::mutex>&, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, bool, std::function<std::unique_ptr<magma::GCContext, std::default_delete<magma::GCContext> > (bool)>) () from /opt/couchbase/bin/../lib/libmagma.so

 #17 0x00007f8549dcf55e in magma::LSMTree::compactTable(int, std::unique_lock<std::mutex>&, std::shared_ptr<magma::Table>&) () from /opt/couchbase/bin/../lib/libmagma.so

 #18 0x00007f8549e32b44 in magma::LSMTree::doPurgingWork() () from /opt/couchbase/bin/../lib/libmagma.so

 #19 0x00007f8549e78951 in magma::TaskWorker::loop(void*) () from /opt/couchbase/bin/../lib/libmagma.so

 #20 0x00007f8548e61777 in platform_thread_wrap(void*) () from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0

 #21 0x00007f854649ee65 in start_thread () from /lib64/libpthread.so.0

 #22 0x00007f85461c788d in clone () from /lib64/libc.so.6

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/dp_magma_job.ini -p bucket_storage=magma,bucket_eviction_policy=fullEviction,rerun=False -t volumetests.Magma.volume.SystemTestMagma,nodes_init=2,replicas=1,skip_cleanup=True,num_items=1000000000,num_buckets=1,bucket_names=GleamBook,doc_size=4096,bucket_type=membase,compression_mode=off,iterations=1,batch_size=10,sdk_timeout=60,log_level=debug,infra_log_level=debug,rerun=False,skip_cleanup=True,key_size=18,randomize_doc_size=False,randomize_value=True,assert_crashes_on_load=True,maxttl=60,process_concurrency=1 -m rest'