Details
-
Bug
-
Resolution: Duplicate
-
Major
-
Cheshire-Cat
-
Ubuntu 1804
Couchbase version 7.0.0-2041
Description
Linearizability failure and crash during kv-engine-cheshire-cat-jepsen-nightly-138 for test lein trampoline run test --nodes-file ./nodes --username root --password couchbase --workload=rebalance --scenario=swap-rebalance --no-autofailover --replicas=2 --disrupt-count=1 --kv-timeout=30 --durability=0:0:0:100 --use-json-docs --doc-padding-size=4096 --enable-tcp-capture --hashdump --enable-memcached-debug-log-level.
Message:
Caught unhandled std::exception-derived exception. what(): Monotonic<15SnapshotEndInfo> (unlabelled) invariant failed: new value (25{Disk}) breaks invariant on current value (25{Disk})
|
(gdb) bt
|
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
|
#1 0x00007f937ac9b801 in __GI_abort () at abort.c:79
|
#2 0x00007f937b8b9165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
#3 0x000000000054b5f2 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
|
#4 0x00007f937b8b6f56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
|
#5 0x00007f937b8b6fa1 in std::terminate () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57
|
#6 0x00007f937b8b71e3 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f9344000d30, tinfo=0x7f75f0 <typeinfo for std::logic_error>, dest=0x4194a0 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93
|
#7 0x00007f937f055417 in ThrowExceptionPolicy<SnapshotEndInfo>::nonMonotonic (this=this@entry=0x7f933d26d040, curValue=..., newValue=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:53
|
#8 0x00007f937f05392e in Monotonic<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::operator= (v=..., this=0x7f933d26d040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:145
|
#9 MonotonicQueue<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::push (value=..., this=0x7f933d26cff0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic_queue.h:64
|
#10 PassiveDurabilityMonitor::notifySnapshotEndReceived (this=0x7f933d21fa60, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/passive_durability_monitor.cc:235
|
#11 0x00007f937f1129f5 in VBucket::notifyPassiveDMOfSnapEndReceived (this=<optimized out>, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1040
|
#12 0x00007f937f019bb3 in PassiveStream::handleSnapshotEnd (this=this@entry=0x7f933cfaef90, vb=..., byseqno=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:1037
|
#13 0x00007f937f01efc7 in PassiveStream::processMessage (this=this@entry=0x7f933cfaef90, message=message@entry=0x7f933d468db0, messageType=messageType@entry=PassiveStream::Prepare) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:698
|
#14 0x00007f937f01f178 in PassiveStream::processPrepare (this=this@entry=0x7f933cfaef90, prepare=prepare@entry=0x7f933d468db0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:725
|
#15 0x00007f937f02077b in PassiveStream::processBufferedMessages (this=0x7f933cfaef90, processed_bytes=@0x7f933f5ebf9c: 0, batchSize=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:466
|
#16 0x00007f937f00419b in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f933d1e4510, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1144
|
#17 0x00007f937f004319 in DcpConsumer::processBufferedItems (this=this@entry=0x7f933d1e4510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1183
|
#18 0x00007f937f008ef8 in DcpConsumerTask::run (this=0x7f933e471cd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:87
|
#19 0x00007f937f0b56a3 in GlobalTask::execute (this=0x7f933e471cd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:73
|
#20 0x00007f937f0adf4f in ExecutorThread::run (this=0x7f93775e9880) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/executorthread.cc:188
|
#21 0x00007f937dac4437 in CouchbaseThread::run (this=0x7f937a63d010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:58
|
#22 platform_thread_wrap (arg=0x7f937a63d010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_pthreads.cc:71
|
#23 0x00007f937b0536db in start_thread (arg=0x7f933f5f6700) at pthread_create.c:463
|
#24 0x00007f937ad7c88f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
(gdb) bt full
|
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
|
set = {__val = {0, 140271400810243, 140270477774016, 0, 140271400814768, 140270400091296, 386183668047020927, 2072905128, 1063170792, 281470681751456, 0, 0, 0, 0, 0, 0}}
|
pid = <optimized out>
|
tid = <optimized out>
|
ret = <optimized out>
|
#1 0x00007f937ac9b801 in __GI_abort () at abort.c:79
|
save_stage = 1
|
act = {__sigaction_handler = {sa_handler = 0x3d2ef35793c76730, sa_sigaction = 0x3d2ef35793c76730}, sa_mask = {__val = {0, 0, 0, 4613425733568182272, 0, 140271400810243, 140271397273425, 11, 140271400810112, 10, 140270477774160, 140270400093152, 140270400093216, 140270400093248, 140271397274627, 140271400810112}}, sa_flags = 2072731984,
|
sa_restorer = 0x7f9343ff8858}
|
sigs = {__val = {32, 0 <repeats 15 times>}}
|
__cnt = <optimized out>
|
__set = <optimized out>
|
__cnt = <optimized out>
|
__set = <optimized out>
|
#2 0x00007f937b8b9165 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x000000000054b5f2 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:86
|
No locals.
|
#4 0x00007f937b8b6f56 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
|
No locals.
|
#5 0x00007f937b8b6fa1 in std::terminate () at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57
|
No locals.
|
#6 0x00007f937b8b71e3 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f9344000d30, tinfo=0x7f75f0 <typeinfo for std::logic_error>, dest=0x4194a0 <std::logic_error::~logic_error()@plt>) at /tmp/deploy/objdir/../gcc-7.3.0/libstdc++-v3/libsupc++/eh_throw.cc:93
|
globals = <optimized out>
|
header = 0x7f9344000cb0
|
#7 0x00007f937f055417 in ThrowExceptionPolicy<SnapshotEndInfo>::nonMonotonic (this=this@entry=0x7f933d26d040, curValue=..., newValue=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:53
|
No locals.
|
#8 0x00007f937f05392e in Monotonic<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::operator= (v=..., this=0x7f933d26d040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic.h:145
|
No locals.
|
#9 MonotonicQueue<SnapshotEndInfo, ThrowExceptionPolicy, cb::greater>::push (value=..., this=0x7f933d26cff0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/monotonic_queue.h:64
|
No locals.
|
#10 PassiveDurabilityMonitor::notifySnapshotEndReceived (this=0x7f933d21fa60, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/passive_durability_monitor.cc:235
|
s = {holder_ = {<folly::LockedPtrBase<folly::Synchronized<std::unique_ptr<PassiveDurabilityMonitor::State, std::default_delete<PassiveDurabilityMonitor::State> >, folly::SharedMutexImpl<false, void, std::atomic, false, false> >, folly::SharedMutexImpl<false, void, std::atomic, false, false>, folly::LockPolicyExclusive>> = {parent_ = 0x7f933d21fa90},
|
static AllowsConcurrentAccess = false}}
|
prevHps = {<ThrowExceptionPolicy<long>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x19 <error: Cannot access memory at address 0x19>}, _M_string_length = 0, {
|
_M_local_buf = "\006\000\000\000\020\000\000\000\v\000\000\000\000\000\000", _M_allocated_capacity = 68719476742}}}, val = 1701736192}
|
#11 0x00007f937f1129f5 in VBucket::notifyPassiveDMOfSnapEndReceived (this=<optimized out>, snapEnd=snapEnd@entry=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1040
|
No locals.
|
#12 0x00007f937f019bb3 in PassiveStream::handleSnapshotEnd (this=this@entry=0x7f933cfaef90, vb=..., byseqno=25) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:1037
|
No locals.
|
#13 0x00007f937f01efc7 in PassiveStream::processMessage (this=this@entry=0x7f933cfaef90, message=message@entry=0x7f933d468db0, messageType=messageType@entry=PassiveStream::Prepare) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:698
|
taskToString = {0x7f937f20a40b "mutation", 0x7f937f20a414 "deletion", 0x7f937f20a41d "expiration", 0x7f937f20a428 "prepare"}
|
vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f933d269c00, _M_refcount = {_M_pi = 0x7f933d3d9ce0}}, <No data fields>}
|
consumer = {<std::__shared_ptr<DcpConsumer, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<DcpConsumer, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f933d1e4510, _M_refcount = {_M_pi = 0x7f933d1e4500}}, <No data fields>}
|
ret = ENGINE_SUCCESS
|
deleteSource = <optimized out>
|
switchComplete = true
|
#14 0x00007f937f01f178 in PassiveStream::processPrepare (this=this@entry=0x7f933cfaef90, prepare=prepare@entry=0x7f933d468db0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:725
|
result = <optimized out>
|
#15 0x00007f937f02077b in PassiveStream::processBufferedMessages (this=0x7f933cfaef90, processed_bytes=@0x7f933f5ebf9c: 0, batchSize=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:466
|
ret = ENGINE_SUCCESS
|
response = {_M_t = {
|
_M_t = {<std::_Tuple_impl<0, DcpResponse*, std::default_delete<DcpResponse> >> = {<std::_Tuple_impl<1, std::default_delete<DcpResponse> >> = {<std::_Head_base<1, std::default_delete<DcpResponse>, true>> = {<std::default_delete<DcpResponse>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, DcpResponse*, false>> = {
|
_M_head_impl = 0x7f933d468db0}, <No data fields>}, <No data fields>}}}
|
lh = {_M_device = 0x7f933cfaf110, _M_owns = false}
|
count = 0
|
message_bytes = 2131026
|
total_bytes_processed = 0
|
failed = false
|
noMem = false
|
#16 0x00007f937f00419b in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f933d1e4510, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1144
|
rval = all_processed
|
bytesProcessed = 0
|
iterations = 0
|
#17 0x00007f937f004319 in DcpConsumer::processBufferedItems (this=this@entry=0x7f933d1e4510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1183
|
stream = {<std::__shared_ptr<PassiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<PassiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f933cfaef90, _M_refcount = {_M_pi = 0x7f933cfaef80}}, <No data fields>}
|
process_ret = all_processed
|
vbucket = {vbid = 399}
|
#18 0x00007f937f008ef8 in DcpConsumerTask::run (this=0x7f933e471cd0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:87
|
phosphor_internal_category_enabled_76 = {_M_b = {Segmentation fault (core dumped)
|
Attachments
Issue Links
- duplicates
-
MB-41024 Crash in DCP consumer when processing SyncWrite Prepare at end of snapshot under memory pressure
-
- Closed
-