[Magma] magma_0 MagmaKVStore Magma open failed. Status:Corruption: KVStore-512 recovered to offset 57:24719360 but start offset is 59:0 and flush offset is 65:106496 (DISK FULL)

  #0  0x00007faa31a7c337 in raise () from /lib64/libc.so.6

 #0  0x00007faa31a7c337 in raise () from /lib64/libc.so.6

 No symbol table info available.

 #1  0x00007faa31a7da28 in abort () from /lib64/libc.so.6

 No symbol table info available.

 #2  0x00007faa323c763c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

         terminating = false

         t = <optimized out>

 #3  0x0000000000a7c65f in backtrace_terminate_handler() ()

 No symbol table info available.

 #4  0x00007faa323d28f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

 No locals.

 #5  0x00007faa323d2961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

 No locals.

 #6  0x00007faa323d2bf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0xf676e0 <typeinfo for std::runtime_error>, dest=0x4435c0 <_ZNSt13runtime_errorD1Ev@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

         globals = <optimized out>

         header = 0x7fa9d80008c0

 #7  0x00000000004c7874 in MagmaKVStore::makeCompactionContext(Vbid) [clone .cold] ()

 No symbol table info available.

 #8  0x0000000000859585 in MagmaKVStore::compactionCallBack(MagmaKVStore::MagmaCompactionCB&, magma::Slice const&, magma::Slice const&, magma::Slice const&) ()

 No symbol table info available.

 #9  0x000000000094a6bd in magma::DocSeqGCContext::Transform(magma::Slice const&, magma::Slice const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, magma::Slice*) ()

 No symbol table info available.

 #10 0x00000000009cf755 in magma::ItemGCFilter::Process(std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >&, std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >*) ()

 No symbol table info available.

 #11 0x0000000000962bb2 in magma::LSMTree::mergeSSTables(std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, magma::ItemGCFilter&, unsigned long, double, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >*) ()

 No symbol table info available.

 #12 0x00000000009c919d in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock<std::mutex>&, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, bool, std::function<std::unique_ptr<magma::GCContext, std::default_delete<magma::GCContext> > (bool)>) ()

 No symbol table info available.

 #13 0x00000000009cb5b4 in magma::LSMTree::compactLevel(int, std::unique_lock<std::mutex>&, bool, std::function<std::unique_ptr<magma::GCContext, std::default_delete<magma::GCContext> > (bool)>) ()

 No symbol table info available.

 #14 0x00000000009cb855 in magma::LSMTree::compact(std::unique_lock<std::mutex>&) ()

 No symbol table info available.

 #15 0x0000000000957fc8 in std::_Function_handler<void (), magma::LSMTree::queueCompaction()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()

 No symbol table info available.

 #16 0x000000000099653d in magma::TimedTask::Complete(bool) ()

 No symbol table info available.

 #17 0x0000000000997d31 in magma::TaskWorker::loop(void*) ()

 No symbol table info available.

 #18 0x0000000000aab9e9 in platform_thread_wrap(void*) ()

 No symbol table info available.

 #19 0x00007faa33fdfe65 in start_thread () from /lib64/libpthread.so.0

 No symbol table info available.

 #20 0x00007faa31b4488d in clone () from /lib64/libc.so.6

 No symbol table info available.

2021-06-04T12:28:31.391152-07:00 CRITICAL (default) (default) magma_0 MagmaKVStore Magma open failed. Status:Corruption: KVStore-512 recovered to offset 57:24719360 but start offset is 59:0 and flush offset is 65:106496

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.86889.ini bucket_storage=magma,rerun=false,bucket_eviction_policy=fullEviction,randomize_value=True,doc_size=4096,enable_dp=True,doc_size=1024,upgrade_version=7.0.0-4908 -t magma.magma_disk_full.MagmaDiskFull.test_crash_recovery_disk_full,nodes_init=4,num_items=5000000,doc_size=4096,sdk_timeout=60,replicas=1,GROUP=P0'