Diskfull: magma_0 MagmaKVStore Magma open failed. Status:Corruption: KVStore-808 recovered to offset 41:15806464 but start offset is 42:0 and flush offset is 51:26439680

[(default) magma_1]KVStore-753 recovered to offset 42:8577024 but start offset is 43:0 and flush offset is 53:7090176

(default) (default) magma_1 MagmaKVStore Magma open failed. Status:Corruption: KVStore-753 recovered to offset 42:8577024 but start offset is 43:0 and flush offset is 53:7090176

*** Fatal error encountered during exception handling ***

Caught unhandled std::exception-derived exception. what(): MagmaKVStore Magma open failed. Status:Corruption: KVStore-753 recovered to offset 42:8577024 but start offset is 43:0 and flush offset is 53:7090176

Breakpad caught a crash (Couchbase version 7.0.0-4908). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/80c50e17-9ec8-4d46-e6fe87b8-fa9d41ae.dmp before terminating.

Stack backtrace of crashed thread:

#0  /opt/couchbase/bin/memcached() [0x400000+0x1b7533]

#1  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x1c912a]

#2  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x1c9468]

#3  /lib64/libpthread.so.0() [0x7f82eb87e000+0xf630]

#4  /lib64/libc.so.6(gsignal+0x37) [0x7f82eb4b0000+0x36387]

#5  /lib64/libc.so.6(abort+0x148) [0x7f82eb4b0000+0x37a78]

#6  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f82ebfb4000+0x9963c]

#7  /opt/couchbase/bin/memcached() [0x400000+0x1c434f]

#8  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f82ebfb4000+0xa48f6]

#9  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f82ebfb4000+0xa4961]

#10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f82ebfb4000+0xa4bf4]

#11 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0xa4366]

#12 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x210dd3]

#13 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x236a14]

#14 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x281fe0]

#15 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x21de0e]

#16 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x198356]

#17 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x1baf68]

#18 /opt/couchbase/bin/../lib/libep.so() [0x7f82ef8c1000+0x1c6331]

#19 /opt/couchbase/bin/memcached() [0x400000+0x5f2ad]

#20 /opt/couchbase/bin/memcached() [0x400000+0x5f7df]

#21 /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0(_ZN9Couchbase6Thread12thread_entryEv+0x13) [0x7f82ee084000+0x43143]

#22 /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7f82ee084000+0x32be9]

#23 /lib64/libpthread.so.0() [0x7f82eb87e000+0x7ea5]

#24 /lib64/libc.so.6(clone+0x6d) [0x7f82eb4b0000+0xfe8dd]

[(default) magma_0/kvstore-160/rev-000000001/seqIndex]Error occured during memtable flush (IOError: /data/default/magma.0/kvstore-160/rev-000000001/seqIndex/sstable.11.data: Unable to write to the file err=No space left on device offset:1040384 len:520192).

172.23.107.31 : Stack Trace of first crash: d9a6f409-aa40-4220-cc43bc93-558bb76b.dmp

Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.

 #0  0x00007feb61a63387 in raise () from /lib64/libc.so.6

 #0  0x00007feb61a63387 in raise () from /lib64/libc.so.6

 No symbol table info available.

 #1  0x00007feb61a64a78 in abort () from /lib64/libc.so.6

 No symbol table info available.

 #2  0x00007feb625ca63c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

         terminating = false

         t = <optimized out>

 #3  0x00000000005c434f in backtrace_terminate_handler() ()

 No symbol table info available.

 #4  0x00007feb625d58f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

 No locals.

 #5  0x00007feb625d5961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

 No locals.

 #6  0x00007feb625d5bf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0x892d60 <typeinfo for std::runtime_error>, dest=0x419470 <_ZNSt13runtime_errorD1Ev@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

         globals = <optimized out>

         header = 0x7fe8ac0008c0

 #7  0x00007feb65ee37c2 in MagmaKVStore::makeCompactionContext(Vbid) [clone .cold] () from /opt/couchbase/bin/../lib/libep.so

 No symbol table info available.

 #8  0x00007feb6613bd25 in MagmaKVStore::compactionCallBack(MagmaKVStore::MagmaCompactionCB&, magma::Slice const&, magma::Slice const&, magma::Slice const&) () from /opt/couchbase/bin/../lib/libep.so

 No symbol table info available.

 #9  0x00007feb655c9882 in magma::DocSeqGCContext::Transform(magma::Slice const&, magma::Slice const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, magma::Slice*) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #10 0x00007feb65565ea0 in magma::ItemGCFilter::Process(std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >&, std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >*) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #11 0x00007feb655e256f in magma::LSMTree::mergeSSTables(std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, magma::ItemGCFilter&, unsigned long, double, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >*) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #12 0x00007feb6555ee0d in magma::LSMTree::compactLevelForInputs(int, int, std::unique_lock<std::mutex>&, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, bool, std::function<std::unique_ptr<magma::GCContext, std::default_delete<magma::GCContext> > (bool)>) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #13 0x00007feb65561172 in magma::LSMTree::compactLevel(int, std::unique_lock<std::mutex>&, bool, std::function<std::unique_ptr<magma::GCContext, std::default_delete<magma::GCContext> > (bool)>) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #14 0x00007feb6556154d in magma::LSMTree::compact(std::unique_lock<std::mutex>&) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #15 0x00007feb655d756a in std::_Function_handler<void (), magma::LSMTree::queueCompaction()::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #16 0x00007feb65629111 in magma::TimedTask::Complete(bool) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #17 0x00007feb6562a981 in magma::TaskWorker::loop(void*) () from /opt/couchbase/bin/../lib/libmagma.so

 No symbol table info available.

 #18 0x00007feb64633be9 in platform_thread_wrap(void*) () from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0

 No symbol table info available.

 #19 0x00007feb61e02ea5 in start_thread () from /lib64/libpthread.so.0

 No symbol table info available.

 #20 0x00007feb61b2b8dd in clone () from /lib64/libc.so.6

 No symbol table info available.

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.86889.ini bucket_storage=magma,rerun=false,bucket_eviction_policy=fullEviction,randomize_value=True,doc_size=4096,enable_dp=True,doc_size=1024,upgrade_version=7.0.0-4908 -t magma.magma_disk_full.MagmaDiskFull.test_reads_on_disk_full,nodes_init=4,num_items=5000000,doc_size=4096,sdk_timeout=60,replicas=1,GROUP=P0'