[Jepsen] Crash in CouchKVStore::writeVBucketState()

gdb /opt/couchbase/bin/memcached -c fc39ae72-f8c5-4942-4d04c6a6-96c10a2f.dmp.core

(gdb) bt full

#0  0x00007fe2b80024f9 in CouchKVStore::writeVBucketState (this=this@entry=0x7fe2a8eabc00, vbucketId=..., vbstate=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1321

        openResult = {status = COUCHSTORE_ERROR_OPEN_FILE, db = {<folly::LockedPtrBase<folly::Synchronized<DbHolder, std::mutex>, std::mutex, folly::LockPolicyExclusive>> = {lock_ = {_M_device = 0x0, _M_owns = false}, parent_ = 0x0}, static AllowsConcurrentAccess = false}, closeOnUnlock = false}

        errorCode = COUCHSTORE_ERROR_OPEN_FILE

        db = <optimized out>

        info = {version = 80, timestamp = 140611701885652, updateSeqNum = 0, purgeSeqNum = 140611462353504, headerPosition = 3148544, filename = 0x0, docCount = 140611473010008, deletedCount = 140611473010016, spaceUsed = 140610616453184, fileSize = 140611462353504}

#1  0x00007fe2b8002e79 in CouchKVStore::snapshotVBucket (this=0x7fe2a8eabc00, vbucketId=..., vbstate=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/couch-kvstore/couch-kvstore.cc:1377

        vbs = <optimized out>

        start = <optimized out>

#2  0x00007fe2b7ee2fef in EPBucket::flushVBucket (this=0x7fe2a8e90000, vbid=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:698

        flushStart = <optimized out>

        vb = {vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe2764a3600, _M_refcount = {_M_pi = 0x7fe2765ff120}}, <No data fields>}, lock = {_M_device = 0x7fe2a8f82318, _M_owns = true}}

        toFlush = {items = {<std::_Vector_base<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >> = {

              _M_impl = {<std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<__gnu_cxx::new_allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x7fe275d86800, _M_finish = 0x7fe275d86818, _M_end_of_storage = 0x7fe275d86820}}, <No data fields>}, 

          ranges = {<std::_Vector_base<CheckpointSnapshotRange, std::allocator<CheckpointSnapshotRange> >> = {_M_impl = {<std::allocator<CheckpointSnapshotRange>> = {<__gnu_cxx::new_allocator<CheckpointSnapshotRange>> = {<No data fields>}, <No data fields>}, _M_start = 0x7fe2a842c290, _M_finish = 0x7fe2a842c2c0, 

                _M_end_of_storage = 0x7fe2a842c2c0}}, <No data fields>}, moreAvailable = false, maxDeletedRevSeqno = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 140611702015580}, 

                _M_engaged = false}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}, checkpointType = CheckpointType::Memory, flushHandle = {_M_t = {

              _M_t = {<std::_Tuple_impl<0, FlushHandle*, std::default_delete<FlushHandle> >> = {<std::_Tuple_impl<1, std::default_delete<FlushHandle> >> = {<std::_Head_base<1, std::default_delete<FlushHandle>, true>> = {<std::default_delete<FlushHandle>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, FlushHandle*, false>> = {

                    _M_head_impl = 0x7fe2a8e66d40}, <No data fields>}, <No data fields>}}}}

        moreAvailable = EPBucket::MoreAvailable::No

        wakeupCheckpointRemover = EPBucket::WakeCkptRemover::No

        range = {<std::_Optional_base<snapshot_range_t>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = {start = 0, end = 0}}, _M_engaged = false}}, <std::_Enable_copy_move<true, true, true, true, std::optional<snapshot_range_t> >> = {<No data fields>}, <No data fields>}

        rwUnderlying = <optimized out>

        prev = 0x0

        persistedVbState = <optimized out>

        vbstate = {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\000\000\000\000\000"}}, highSeqno = 0, purgeSeqno = 0, lastSnapStart = 0, lastSnapEnd = 0, maxCas = 0, hlcCasEpochSeqno = -1, mightContainXattrs = false, supportsNamespaces = true, version = 3, persistedCompletedSeqno = 0, persistedPreparedSeqno = 0, 

          highPreparedSeqno = 0, maxVisibleSeqno = 0, onDiskPrepares = 0, checkpointType = CheckpointType::Memory, transition = {failovers = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fe279ff0b90 ""}, _M_string_length = 0, {

                _M_local_buf = "\000\000\000\000 \000\000\000\000\000\000\000\000\000\000", _M_allocated_capacity = 137438953472}}, replicationTopology = {m_type = nlohmann::detail::value_t::null, m_value = {object = 0x0, array = 0x0, string = 0x0, boolean = false, number_integer = 0, number_unsigned = 0, number_float = 0}}, state = vbucket_state_dead}}

        commitData = {collections = {needsPurge = false, mutated = {

              _M_h = {<std::__detail::_Hashtable_base<CollectionID, CollectionID, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, true, true> >> = {<std::__detail::_Hash_code_base<CollectionID, CollectionID, std::__detail::_Identity, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Identity, true>> = {<std::__detail::_Identity> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<CollectionID>, true>> = {<std::hash<CollectionID>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<CollectionID>, true>> = {<std::equal_to<CollectionID>> = {<std::binary_function<CollectionID, CollectionID, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<No data fields>}, <std::__detail::_Insert<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<std::__detail::_Insert_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<CollectionID, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<CollectionID, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<CollectionID, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<CollectionID, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7fe279ff0ca8, _M_bucket_count = 1, _M_before_begin = {_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2, 

                  _M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}, manifest = @0x7fe2765b5200}, proposedVBState = {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\000\000\000\000\000"}}, highSeqno = 0, purgeSeqno = 0, lastSnapStart = 0, lastSnapEnd = 0, maxCas = 0, hlcCasEpochSeqno = -1, mightContainXattrs = false, 

            supportsNamespaces = true, version = 3, persistedCompletedSeqno = 0, persistedPreparedSeqno = 0, highPreparedSeqno = 0, maxVisibleSeqno = 0, onDiskPrepares = 0, checkpointType = CheckpointType::Memory, transition = {failovers = {static npos = 18446744073709551615, 

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fe2765c4c20 <error: Cannot access memory at address 0x7fe2765c4c20>}, _M_string_length = 30, {_M_local_buf = "\036\000\000\000\000\000\000\000QV\343\267\342\177\000", _M_allocated_capacity = 30}}, replicationTopology = {

                m_type = nlohmann::detail::value_t::null, m_value = {object = 0x0, array = 0x0, string = 0x0, boolean = false, number_integer = 0, number_unsigned = 0, number_float = 0}}, state = vbucket_state_replica}}}

        proposedVBState = @0x7fe279ff0cb8: {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\000\000\000\000\000"}}, highSeqno = 0, purgeSeqno = 0, lastSnapStart = 0, lastSnapEnd = 0, maxCas = 0, hlcCasEpochSeqno = -1, mightContainXattrs = false, supportsNamespaces = true, version = 3, persistedCompletedSeqno = 0, 

          persistedPreparedSeqno = 0, highPreparedSeqno = 0, maxVisibleSeqno = 0, onDiskPrepares = 0, checkpointType = CheckpointType::Memory, transition = {failovers = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

                _M_p = 0x7fe2765c4c20 <error: Cannot access memory at address 0x7fe2765c4c20>}, _M_string_length = 30, {_M_local_buf = "\036\000\000\000\000\000\000\000QV\343\267\342\177\000", _M_allocated_capacity = 30}}, replicationTopology = {m_type = nlohmann::detail::value_t::null, m_value = {object = 0x0, array = 0x0, string = 0x0, boolean = false, 

                number_integer = 0, number_unsigned = 0, number_float = 0}}, state = vbucket_state_replica}}

        maxSeqno = 0

        maxVbStateOpCas = 1599628450251669504

        minSeqno = 18446744073709551615

        flushBatchSize = 0

        mustPersistVBState = true

        hcs = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 0}, _M_engaged = false}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}

        hps = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 0}, _M_engaged = false}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}

        maxVisibleSeqno = {<ThrowExceptionPolicy<unsigned long>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fe279ff0bd0 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000\000\000\000\000", 

                _M_allocated_capacity = 7812730926796664437}}}, val = 0}

        aggStats = {numItems = 2, totalBytes = 413, totalAgeInMicro = 0}

        flushSuccess = <optimized out>

        highSeqno = <optimized out>

#3  0x00007fe2b7f39ecc in Flusher::flushVB (this=this@entry=0x7fe2a8ee48c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:306

        vbid = {vbid = 839}

        res = {moreAvailable = EPBucket::MoreAvailable::No, numFlushed = 140610686095472, wakeupCkptRemover = EPBucket::WakeCkptRemover::No}

#4  0x00007fe2b7f3b0a9 in Flusher::step (this=0x7fe2a8ee48c0, task=0x7fe2a8e9f330) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:207

        more = <optimized out>

        currentState = <optimized out>

#5  0x00007fe2b7f3de03 in GlobalTask::execute (this=0x7fe2a8e9f330) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73

        guard = {previous = 0x0}

#6  0x00007fe2b7e4413f in CB3ExecutorThread::run (this=0x7fe2b04325e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/cb3_executorthread.cc:174

        curTaskDescr = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fe2b02f0bc0 <error: Cannot access memory at address 0x7fe2b02f0bc0>}, _M_string_length = 31, {_M_local_buf = "\037\000\000\000\000\000\000\000Pd\001\000\000\000\000", 

            _M_allocated_capacity = 31}}

        woketime = <optimized out>

        scheduleOverhead = <optimized out>

        again = <optimized out>

        runtime = <optimized out>

        q = <optimized out>

        tick = 72 'H'

        guard = {engine = 0x0}

#7  0x00007fe2b689e3f7 in CouchbaseThread::run (this=0x7fe2b35a70c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#8  platform_thread_wrap (arg=0x7fe2b35a70c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7fe2b35a70c0}, <No data fields>}, <No data fields>}}}

#9  0x00007fe2b3e2d6db in start_thread (arg=0x7fe279ffb700) at pthread_create.c:463

        pd = 0x7fe279ffb700

        now = <optimized out>

        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140610686138112, 26953794092842796, 140610686095808, 0, 140611648385216, 140611495610400, -28374361731711188, -28528407196942548}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}

        not_first_call = <optimized out>

#10 0x00007fe2b3b56a3f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

No locals.