[Magma] MagmaKVStore::readLocalDoc(vbid:vb:38 key:_vbstate not found

./sequoia -client 172.23.104.254:2375 -provider file:centos_second_cluster.yml -test tests/rebalance/test_allRebalance_collections.yml -scope tests/rebalance/scope_rebalance_collection.yml -scale 3 -repeat 0 -log_level 0 -version 7.0.0-2422 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=259200 -show_topology=true

2020-06-25T05:13:48.016604-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): MagmaKVStore vbstate vbid:38 not found. Status:NotFound: MagmaKVStore::readLocalDoc(vbid:vb:38 key:_vbstate not found.

memcached.log.000117.txt:2020-06-25T05:13:48.115107-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.0-2422). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/27f72048-9347-4fd3-420bae8b-86b37b03.dmp before terminating.

memcached.log.000117.txt:2020-06-25T05:13:48.115134-07:00 CRITICAL Stack backtrace of crashed thread:

memcached.log.000117.txt:2020-06-25T05:13:48.115352-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x13bebd]

memcached.log.000117.txt:2020-06-25T05:13:48.115372-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x1511ca]

memcached.log.000117.txt:2020-06-25T05:13:48.115379-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x151508]

memcached.log.000117.txt:2020-06-25T05:13:48.115386-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7fda6831b000+0xf5e0]

memcached.log.000117.txt:2020-06-25T05:13:48.115423-07:00 CRITICAL     /lib64/libc.so.6(gsignal+0x37) [0x7fda67f58000+0x351f7]

memcached.log.000117.txt:2020-06-25T05:13:48.115450-07:00 CRITICAL     /lib64/libc.so.6(abort+0x148) [0x7fda67f58000+0x368e8]

memcached.log.000117.txt:2020-06-25T05:13:48.115488-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6(_ZN9__gnu_cxx27__verbose_terminate_handlerEv+0x125) [0x7fda68a50000+0x91195]

memcached.log.000117.txt:2020-06-25T05:13:48.115503-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x14c6c2]

memcached.log.000117.txt:2020-06-25T05:13:48.115524-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fda68a50000+0x8ef86]

memcached.log.000117.txt:2020-06-25T05:13:48.115551-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fda68a50000+0x8efd1]

memcached.log.000117.txt:2020-06-25T05:13:48.115578-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fda68a50000+0x8f213]

memcached.log.000117.txt:2020-06-25T05:13:48.115598-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x21c7fa]

memcached.log.000117.txt:2020-06-25T05:13:48.115606-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x159473]

memcached.log.000117.txt:2020-06-25T05:13:48.115619-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x175ad4]

memcached.log.000117.txt:2020-06-25T05:13:48.115635-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x1c0653]

memcached.log.000117.txt:2020-06-25T05:13:48.115641-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x16372e]

memcached.log.000117.txt:2020-06-25T05:13:48.115648-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0xe80c6]

memcached.log.000117.txt:2020-06-25T05:13:48.115653-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x100839]

memcached.log.000117.txt:2020-06-25T05:13:48.115666-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fda6c19b000+0x11362c]

memcached.log.000117.txt:2020-06-25T05:13:48.115675-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x23be0]

memcached.log.000117.txt:2020-06-25T05:13:48.115690-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x24109]

memcached.log.000117.txt:2020-06-25T05:13:48.115703-07:00 CRITICAL     /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0(_ZN9Couchbase6Thread12thread_entryEv+0xf) [0x7fda6acd5000+0x1b40f]

memcached.log.000117.txt:2020-06-25T05:13:48.115716-07:00 CRITICAL     /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7fda6acd5000+0x10777]

memcached.log.000117.txt:2020-06-25T05:13:48.115723-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7fda6831b000+0x7e25]

memcached.log.000117.txt:2020-06-25T05:13:48.115754-07:00 CRITICAL     /lib64/libc.so.6(clone+0x6d) [0x7fda67f58000+0xf834d]

memcached.log.000118.txt:2020-06-25T05:14:06.067734-07:00 CRITICAL *** Fatal error encountered during exception handling ***

(gdb) bt full

#0  0x00007fdacb08d1f7 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007fdacb08e8e8 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007fdacbbe1195 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x000000000054c6c2 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86

No locals.

#4  0x00007fdacbbdef86 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47

No locals.

#5  0x00007fdacbbdefd1 in std::terminate () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57

No locals.

#6  0x00007fdacbbdf213 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7fda7c039f90, tinfo=0x7fe910 <typeinfo for std::logic_error>, dest=0x4187a0 <_ZNSt11logic_errorD1Ev@plt>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93

        globals = <optimized out>

        header = 0x7fda7c039f10

#7  0x00007fdacf4b77fa in MagmaKVStore::MagmaKVStore (this=0x7fdabca9e600, configuration=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/magma-kvstore/magma-kvstore.cc:517

        __for_range = @0x7fdabdff4bd0: {<std::_Vector_base<unsigned short, std::allocator<unsigned short> >> = {

            _M_impl = {<std::allocator<unsigned short>> = {<__gnu_cxx::new_allocator<unsigned short>> = {<No data fields>}, <No data fields>}, _M_start = 0x7fda6e550980, _M_finish = 0x7fda6e5509cc, 

              _M_end_of_storage = 0x7fda6e550a00}}, <No data fields>}

        currEngine = <optimized out>

        status = {s = {_M_t = {

              _M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {

                    _M_head_impl = 0x7fda6dffc360}, <No data fields>}, <No data fields>}}}}

        commitPointInterval = <optimized out>

        loggerName = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fdabdff4c00 "magma_6"}, _M_string_length = 7, {

            _M_local_buf = "magma_6\000\037\303vt\332\177\000", _M_allocated_capacity = 15304520794333549}}

        kvstoreList = {<std::_Vector_base<unsigned short, std::allocator<unsigned short> >> = {_M_impl = {<std::allocator<unsigned short>> = {<__gnu_cxx::new_allocator<unsigned short>> = {<No data fields>}, <No data fields>}, 

              _M_start = 0x7fda6e550980, _M_finish = 0x7fda6e5509cc, _M_end_of_storage = 0x7fda6e550a00}}, <No data fields>}

#8  0x00007fdacf3f4473 in make_unique<MagmaKVStore, MagmaKVStoreConfig&> () at /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825

No locals.

#9  KVStoreFactory::create (config=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore.cc:197

        backend = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fdabdff4d10 "magma"}, _M_string_length = 5, {

            _M_local_buf = "magma\000\000\000\262OL\317\332\177\000", _M_allocated_capacity = 418447319405}}

#10 0x00007fdacf410ad4 in KVShard::KVShard (this=0x7fda77f02630, numShards=<optimized out>, id=<optimized out>, config=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.cc:56

No locals.

#11 0x00007fdacf45b653 in make_unique<KVShard, unsigned long const&, unsigned long&, Configuration&> () at /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825

No locals.

#12 VBucketMap::VBucketMap (this=0x7fdabcaa6018, config=..., store=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucketmap.cc:30

        shardId = 6

        numShards = 8

#13 0x00007fdacf3fe72e in KVBucket::KVBucket (this=0x7fdabcaa6000, theEngine=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:300

No locals.

#14 0x00007fdacf3830c6 in EPBucket::EPBucket (this=0x7fdabcaa6000, theEngine=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:254

No locals.

#15 0x00007fdacf39b839 in EventuallyPersistentEngine::makeBucket (this=this@entry=0x7fdabc9ff000, config=...) at /usr/local/include/c++/7.3.0/bits/unique_ptr.h:825

        bucketType = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fdabdff5070 "persistent"}, _M_string_length = 10, {

            _M_local_buf = "persistent\000\000\000\000\000", _M_allocated_capacity = 7310594991874794864}}

#16 0x00007fdacf3ae62c in EventuallyPersistentEngine::initialize (this=0x7fdabc9ff000, config=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2215

        switchToEngine = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, EventuallyPersistentEngine*, EPHandleReleaser>> = {<std::_Tuple_impl<1, EPHandleReleaser>> = {<std::_Head_base<1, EPHandleReleaser, true>> = {<EPHandleReleaser> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, EventuallyPersistentEngine*, false>> = {_M_head_impl = 0x7fdabc9ff000}, <No data fields>}, <No data fields>}}}

        env = <optimized out>

        threads = <optimized out>

        pool = <optimized out>

        numShards = <optimized out>

        flowCtlPolicy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fdabdff5110 "aggressive"}, _M_string_length = 10, {

            _M_local_buf = "aggressive\000\000\006\000\000", _M_allocated_capacity = 7598543875852429153}}

#17 0x0000000000423be0 in CreateBucketThread::create (this=this@entry=0x7fdac9affdd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1333

        ii = <optimized out>

---Type <return> to continue, or q <return> to quit---

        first_free = <optimized out>

        found = <optimized out>

        all_bucket_lock = {_M_device = 0x870200 <buckets_lock>, _M_owns = false}

        engine = @0x7fdabc9ff000: <error reading variable>

#18 0x0000000000424109 in CreateBucketThread::run (this=0x7fdac9affdd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/memcached.cc:1376

        guard = <optimized out>

#19 0x00007fdacddf040f in Couchbase::Thread::thread_entry (this=0x7fdac9affdd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:45

No locals.

#20 0x00007fdacdde5777 in run (this=0x7fdac9d92950) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#21 platform_thread_wrap (arg=0x7fdac9d92950) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7fdac9d92950}, <No data fields>}, <No data fields>}}}

#22 0x00007fdacb422e25 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#23 0x00007fdacb15034d in clone () from /lib64/libc.so.6

No symbol table info available.

(gdb)