[Jepsen] Crash due to Monotonic throwing exception

Caught unhandled std::exception-derived exception. what(): Monotonic<15SnapshotEndInfo> (unlabelled) invariant failed: new value (25{Disk}) breaks invariant on current value (25{Disk})

(gdb) bt

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51

#1  0x00007f937ac9b801 in __GI_abort () at abort.c:79

#2  0x00007f937b8b9165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95

#3  0x000000000054b5f2 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86

#4  0x00007f937b8b6f56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47

#5  0x00007f937b8b6fa1 in std::terminate () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57

#6  0x00007f937b8b71e3 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f9344000d30, tinfo=0x7f75f0 <typeinfo for std::logic_error>, dest=0x4194a0 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93

#7  0x00007f937f055417 in ThrowExceptionPolicy<SnapshotEndInfo>::nonMonotonic (this=this@entry=0x7f933d26d040, curValue=..., newValue=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:53

#8  0x00007f937f05392e in Monotonic<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::operator= (v=..., this=0x7f933d26d040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:145

#9  MonotonicQueue<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::push (value=..., this=0x7f933d26cff0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic_queue.h:64

#10 PassiveDurabilityMonitor::notifySnapshotEndReceived (this=0x7f933d21fa60, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/passive_durability_monitor.cc:235

#11 0x00007f937f1129f5 in VBucket::notifyPassiveDMOfSnapEndReceived (this=<optimized out>, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1040

#12 0x00007f937f019bb3 in PassiveStream::handleSnapshotEnd (this=this@entry=0x7f933cfaef90, vb=..., byseqno=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:1037

#13 0x00007f937f01efc7 in PassiveStream::processMessage (this=this@entry=0x7f933cfaef90, message=message@entry=0x7f933d468db0, messageType=messageType@entry=PassiveStream::Prepare) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:698

#14 0x00007f937f01f178 in PassiveStream::processPrepare (this=this@entry=0x7f933cfaef90, prepare=prepare@entry=0x7f933d468db0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:725

#15 0x00007f937f02077b in PassiveStream::processBufferedMessages (this=0x7f933cfaef90, processed_bytes=@0x7f933f5ebf9c: 0, batchSize=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:466

#16 0x00007f937f00419b in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f933d1e4510, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1144

#17 0x00007f937f004319 in DcpConsumer::processBufferedItems (this=this@entry=0x7f933d1e4510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1183

#18 0x00007f937f008ef8 in DcpConsumerTask::run (this=0x7f933e471cd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:87

#19 0x00007f937f0b56a3 in GlobalTask::execute (this=0x7f933e471cd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73

#20 0x00007f937f0adf4f in ExecutorThread::run (this=0x7f93775e9880) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188

#21 0x00007f937dac4437 in CouchbaseThread::run (this=0x7f937a63d010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58

#22 platform_thread_wrap (arg=0x7f937a63d010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71

#23 0x00007f937b0536db in start_thread (arg=0x7f933f5f6700) at pthread_create.c:463

#24 0x00007f937ad7c88f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

(gdb) bt full

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51

        set = {__val = {0, 140271400810243, 140270477774016, 0, 140271400814768, 140270400091296, 386183668047020927, 2072905128, 1063170792, 281470681751456, 0, 0, 0, 0, 0, 0}}

        pid = <optimized out>

        tid = <optimized out>

        ret = <optimized out>

#1  0x00007f937ac9b801 in __GI_abort () at abort.c:79

        save_stage = 1

        act = {__sigaction_handler = {sa_handler = 0x3d2ef35793c76730, sa_sigaction = 0x3d2ef35793c76730}, sa_mask = {__val = {0, 0, 0, 4613425733568182272, 0, 140271400810243, 140271397273425, 11, 140271400810112, 10, 140270477774160, 140270400093152, 140270400093216, 140270400093248, 140271397274627, 140271400810112}}, sa_flags = 2072731984, 

          sa_restorer = 0x7f9343ff8858}

        sigs = {__val = {32, 0 <repeats 15 times>}}

        __cnt = <optimized out>

        __set = <optimized out>

        __cnt = <optimized out>

        __set = <optimized out>

#2  0x00007f937b8b9165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x000000000054b5f2 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86

No locals.

#4  0x00007f937b8b6f56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47

No locals.

#5  0x00007f937b8b6fa1 in std::terminate () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57

No locals.

#6  0x00007f937b8b71e3 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f9344000d30, tinfo=0x7f75f0 <typeinfo for std::logic_error>, dest=0x4194a0 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93

        globals = <optimized out>

        header = 0x7f9344000cb0

#7  0x00007f937f055417 in ThrowExceptionPolicy<SnapshotEndInfo>::nonMonotonic (this=this@entry=0x7f933d26d040, curValue=..., newValue=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:53

No locals.

#8  0x00007f937f05392e in Monotonic<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::operator= (v=..., this=0x7f933d26d040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:145

No locals.

#9  MonotonicQueue<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::push (value=..., this=0x7f933d26cff0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic_queue.h:64

No locals.

#10 PassiveDurabilityMonitor::notifySnapshotEndReceived (this=0x7f933d21fa60, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/passive_durability_monitor.cc:235

        s = {holder_ = {<folly::LockedPtrBase<folly::Synchronized<std::unique_ptr<PassiveDurabilityMonitor::State, std::default_delete<PassiveDurabilityMonitor::State> >, folly::SharedMutexImpl<false, void, std::atomic, false, false> >, folly::SharedMutexImpl<false, void, std::atomic, false, false>, folly::LockPolicyExclusive>> = {parent_ = 0x7f933d21fa90}, 

            static AllowsConcurrentAccess = false}}

        prevHps = {<ThrowExceptionPolicy<long>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x19 <error: Cannot access memory at address 0x19>}, _M_string_length = 0, {

                _M_local_buf = "\006\000\000\000\020\000\000\000\v\000\000\000\000\000\000", _M_allocated_capacity = 68719476742}}}, val = 1701736192}

#11 0x00007f937f1129f5 in VBucket::notifyPassiveDMOfSnapEndReceived (this=<optimized out>, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1040

No locals.

#12 0x00007f937f019bb3 in PassiveStream::handleSnapshotEnd (this=this@entry=0x7f933cfaef90, vb=..., byseqno=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:1037

No locals.

#13 0x00007f937f01efc7 in PassiveStream::processMessage (this=this@entry=0x7f933cfaef90, message=message@entry=0x7f933d468db0, messageType=messageType@entry=PassiveStream::Prepare) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:698

        taskToString = {0x7f937f20a40b "mutation", 0x7f937f20a414 "deletion", 0x7f937f20a41d "expiration", 0x7f937f20a428 "prepare"}

        vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f933d269c00, _M_refcount = {_M_pi = 0x7f933d3d9ce0}}, <No data fields>}

        consumer = {<std::__shared_ptr<DcpConsumer, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<DcpConsumer, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f933d1e4510, _M_refcount = {_M_pi = 0x7f933d1e4500}}, <No data fields>}

        ret = ENGINE_SUCCESS

        deleteSource = <optimized out>

        switchComplete = true

#14 0x00007f937f01f178 in PassiveStream::processPrepare (this=this@entry=0x7f933cfaef90, prepare=prepare@entry=0x7f933d468db0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:725

        result = <optimized out>

#15 0x00007f937f02077b in PassiveStream::processBufferedMessages (this=0x7f933cfaef90, processed_bytes=@0x7f933f5ebf9c: 0, batchSize=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:466

        ret = ENGINE_SUCCESS

        response = {_M_t = {

            _M_t = {<std::_Tuple_impl<0, DcpResponse*, std::default_delete<DcpResponse> >> = {<std::_Tuple_impl<1, std::default_delete<DcpResponse> >> = {<std::_Head_base<1, std::default_delete<DcpResponse>, true>> = {<std::default_delete<DcpResponse>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, DcpResponse*, false>> = {

                  _M_head_impl = 0x7f933d468db0}, <No data fields>}, <No data fields>}}}

        lh = {_M_device = 0x7f933cfaf110, _M_owns = false}

        count = 0

        message_bytes = 2131026

        total_bytes_processed = 0

        failed = false

        noMem = false

#16 0x00007f937f00419b in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f933d1e4510, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1144

        rval = all_processed

        bytesProcessed = 0

        iterations = 0

#17 0x00007f937f004319 in DcpConsumer::processBufferedItems (this=this@entry=0x7f933d1e4510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1183

        stream = {<std::__shared_ptr<PassiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<PassiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f933cfaef90, _M_refcount = {_M_pi = 0x7f933cfaef80}}, <No data fields>}

        process_ret = all_processed

        vbucket = {vbid = 399}

#18 0x00007f937f008ef8 in DcpConsumerTask::run (this=0x7f933e471cd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:87

        phosphor_internal_category_enabled_76 = {_M_b = {Segmentation fault (core dumped)