Crash due to GSL assert failing hcs > persistedCompletedSeqno

(gdb) bt full

#0  0x00007f007cabf387 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007f007cac0a78 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007f007d61d15d in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:50

        terminating = false

        t = <optimized out>

#3  0x000000000054d562 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86

No locals.

#4  0x00007f007d61af86 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47

No locals.

#5  0x00007f007d61afd1 in std::terminate () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57

No locals.

#6  0x00007f007d61b213 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7efff800e4a0, tinfo=tinfo@entry=0x7f00812477f0 <typeinfo for gsl::fail_fast>, dest=dest@entry=0x7f0080d464d0 <gsl::fail_fast::~fail_fast()>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93

        globals = <optimized out>

        header = 0x7efff800e420

#7  0x00007f0080d53fe8 in gsl::fail_fast_assert (cond=<optimized out>, message=0x7f0080f94e80 "GSL: Precondition failure at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc: 665") at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.h:473

No locals.

#8  0x00007f0080dde8f3 in EPBucket::flushVBucket (this=0x7f0071491000, vbid=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:665

        vb = {vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f0036cd7c00, _M_refcount = {_M_pi = 0x7f0036d6b700}}, <No data fields>}, lock = {_M_device = 0x7f003648d288, _M_owns = true}}

        toFlush = {items = {<std::_Vector_base<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> >, std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > > >> = {

              _M_impl = {<std::allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<__gnu_cxx::new_allocator<SingleThreadedRCPtr<Item, Item*, std::default_delete<Item> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x7f0048672c58, _M_finish = 0x7f0048672c60, _M_end_of_storage = 0x7f0048672c60}}, <No data fields>}, 

          ranges = {<std::_Vector_base<CheckpointSnapshotRange, std::allocator<CheckpointSnapshotRange> >> = {_M_impl = {<std::allocator<CheckpointSnapshotRange>> = {<__gnu_cxx::new_allocator<CheckpointSnapshotRange>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f003aaa1d70, _M_finish = 0x7f003aaa1da0, 

                _M_end_of_storage = 0x7f003aaa1da0}}, <No data fields>}, moreAvailable = false, maxDeletedRevSeqno = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 2}, 

                _M_engaged = true}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}, checkpointType = Memory, flushHandle = {_M_t = {

              _M_t = {<std::_Tuple_impl<0, FlushHandle*, std::default_delete<FlushHandle> >> = {<std::_Tuple_impl<1, std::default_delete<FlushHandle> >> = {<std::_Head_base<1, std::default_delete<FlushHandle>, true>> = {<std::default_delete<FlushHandle>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, FlushHandle*, false>> = {

                    _M_head_impl = 0x7f00388b80e0}, <No data fields>}, <No data fields>}}}}

        moreAvailable = EPBucket::No

        wakeupCheckpointRemover = EPBucket::No

        range = {<std::_Optional_base<snapshot_range_t>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = {start = 3105, end = 3105}}, _M_engaged = true}}, <std::_Enable_copy_move<true, true, true, true, std::optional<snapshot_range_t> >> = {<No data fields>}, <No data fields>}

        prev = 0x7f003a8d3f60

        persistedVbState = <optimized out>

        vbstate = {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\002\000\000\000\000"}}, highSeqno = 3104, purgeSeqno = 0, lastSnapStart = 3104, lastSnapEnd = 3104, maxCas = 1594089648088481792, hlcCasEpochSeqno = 1, mightContainXattrs = false, supportsNamespaces = true, version = 3, persistedCompletedSeqno = 3519, 

          persistedPreparedSeqno = 3104, highPreparedSeqno = 3104, maxVisibleSeqno = 2611, onDiskPrepares = 990, checkpointType = Memory, transition = {failovers = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

                _M_p = 0x7f003c41dd80 <Address 0x7f003c41dd80 out of bounds>}, _M_string_length = 370, {_M_local_buf = "r\001\000\000\000\000\000\000\067\032\203\177\000\177\000", _M_allocated_capacity = 370}}, replicationTopology = {m_type = nlohmann::detail::array, m_value = {object = 0x7f003a79efc0, array = 0x7f003a79efc0, string = 0x7f003a79efc0, 

                boolean = 192, number_integer = 139638957797312, number_unsigned = 139638957797312, number_float = 6.8990811868727523e-310}}, state = vbucket_state_active}}

        commitData = {collections = {needsPurge = false, mutated = {

              _M_h = {<std::__detail::_Hashtable_base<CollectionID, CollectionID, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, true, true> >> = {<std::__detail::_Hash_code_base<CollectionID, CollectionID, std::__detail::_Identity, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Identity, true>> = {<std::__detail::_Identity> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<CollectionID>, true>> = {<std::hash<CollectionID>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<CollectionID>, true>> = {<std::equal_to<CollectionID>> = {<std::binary_function<CollectionID, CollectionID, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<No data fields>}, <std::__detail::_Insert<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<std::__detail::_Insert_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<CollectionID, CollectionID, std::allocator<CollectionID>, std::__detail::_Identity, std::equal_to<CollectionID>, std::hash<CollectionID>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, true, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<CollectionID, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<CollectionID, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<CollectionID, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<CollectionID, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f0032a44e48, _M_bucket_count = 1, _M_before_begin = {_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2, 

                  _M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}, manifest = @0x7f0036b1f8a0}, proposedVBState = {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\002\000\000\000\000"}}, highSeqno = 3104, purgeSeqno = 0, lastSnapStart = 3105, lastSnapEnd = 3105, maxCas = 1594089702109872128, hlcCasEpochSeqno = 1, 

            mightContainXattrs = false, supportsNamespaces = true, version = 3, persistedCompletedSeqno = 3519, persistedPreparedSeqno = 3104, highPreparedSeqno = 3104, maxVisibleSeqno = 2611, onDiskPrepares = 990, checkpointType = Memory, transition = {failovers = {static npos = 18446744073709551615, 

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f003c41e500 <Address 0x7f003c41e500 out of bounds>}, _M_string_length = 370, {_M_local_buf = "r\001\000\000\000\000\000\000\200;\002s\000\177\000", _M_allocated_capacity = 370}}, replicationTopology = {

                m_type = nlohmann::detail::array, m_value = {object = 0x7f00702feee0, array = 0x7f00702feee0, string = 0x7f00702feee0, boolean = 224, number_integer = 139639858917088, number_unsigned = 139639858917088, number_float = 6.8991257081051633e-310}}, state = vbucket_state_active}}}

        proposedVBState = @0x7f0032a44e58: {static CurrentVersion = 3, maxDeletedSeqno = {counter = {_M_elems = "\002\000\000\000\000"}}, highSeqno = 3104, purgeSeqno = 0, lastSnapStart = 3105, lastSnapEnd = 3105, maxCas = 1594089702109872128, hlcCasEpochSeqno = 1, mightContainXattrs = false, supportsNamespaces = true, version = 3, 

          persistedCompletedSeqno = 3519, persistedPreparedSeqno = 3104, highPreparedSeqno = 3104, maxVisibleSeqno = 2611, onDiskPrepares = 990, checkpointType = Memory, transition = {failovers = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, 

                _M_p = 0x7f003c41e500 <Address 0x7f003c41e500 out of bounds>}, _M_string_length = 370, {_M_local_buf = "r\001\000\000\000\000\000\000\200;\002s\000\177\000", _M_allocated_capacity = 370}}, replicationTopology = {m_type = nlohmann::detail::array, m_value = {object = 0x7f00702feee0, array = 0x7f00702feee0, string = 0x7f00702feee0, 

                boolean = 224, number_integer = 139639858917088, number_unsigned = 139639858917088, number_float = 6.8991257081051633e-310}}, state = vbucket_state_active}}

        maxSeqno = 3105

        maxVbStateOpCas = 0

        minSeqno = 3105

        flushBatchSize = 1

        mustPersistVBState = false

        hcs = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 3103}, _M_engaged = true}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}

        hps = {<std::_Optional_base<unsigned long>> = {_M_payload = {{_M_empty = {<No data fields>}, _M_payload = 0}, _M_engaged = false}}, <std::_Enable_copy_move<true, true, true, true, std::optional<unsigned long> >> = {<No data fields>}, <No data fields>}

        maxVisibleSeqno = {<ThrowExceptionPolicy<unsigned long>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f0032a44ce0 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000\000\000\000\000", 

                _M_allocated_capacity = 7812730926796664437}}}, val = 2611}

        aggStats = {numItems = 1, totalBytes = 134, totalAgeInMicro = 0}

        flushSuccess = <optimized out>

        highSeqno = <optimized out>

#9  0x00007f0080e33c0c in Flusher::flushVB (this=this@entry=0x7f0035fcf340) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:306

        vbid = {vbid = 733}

        res = {moreAvailable = EPBucket::No, numFlushed = 0, wakeupCkptRemover = EPBucket::No}

#10 0x00007f0080e34dd9 in Flusher::step (this=0x7f0035fcf340, task=0x7f0037dc6e70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:207

        more = <optimized out>

#11 0x00007f0080e37d83 in GlobalTask::execute (this=0x7f0037dc6e70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73

        guard = {previous = 0x0}

#12 0x00007f0080e2e5af in ExecutorThread::run (this=0x7f007b5ec5e0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188

        curTaskDescr = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f007b78f000 <Address 0x7f007b78f000 out of bounds>}, _M_string_length = 31, {_M_local_buf = "\037\000\000\000\000\000\000\000^#\000\000\000\000\000", 

            _M_allocated_capacity = 31}}

        woketime = <optimized out>

        scheduleOverhead = <optimized out>

        again = <optimized out>

        runtime = <optimized out>

        q = <optimized out>

        tick = 68 'D'

        guard = {engine = 0x0}

#13 0x00007f007f821777 in run (this=0x7f0073350f30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

No locals.

#14 platform_thread_wrap (arg=0x7f0073350f30) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

        context = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, CouchbaseThread*, std::default_delete<CouchbaseThread> >> = {<std::_Tuple_impl<1, std::default_delete<CouchbaseThread> >> = {<std::_Head_base<1, std::default_delete<CouchbaseThread>, true>> = {<std::default_delete<CouchbaseThread>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, CouchbaseThread*, false>> = {_M_head_impl = 0x7f0073350f30}, <No data fields>}, <No data fields>}}}

#15 0x00007f007ce5eea5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#16 0x00007f007cb878dd in clone () from /lib64/libc.so.6

./testrunner -i /tmp/win10-bucket-ops.ini rerun=False,crash_warning=True,quota_percent=95 -t failover.AutoFailoverTests.AutoFailoverTests.test_autofailover_during_rebalance,timeout=5,num_node_failures=1,nodes_in=0,nodes_out=1,failover_action=restart_server,nodes_init=4,can_abort_rebalance=False,override_spec_params=durability,durability=MAJORITY,bucket_spec=single_bucket.def_scope_fifty_collections,GROUP=auto_failover_durability