Details
-
Bug
-
Resolution: Fixed
-
Critical
-
6.5.2, 6.6.4, 7.0.3, 7.1.0
-
Couchbase version 7.1.0-2024
-
Untriaged
-
-
1
-
Unknown
-
KV 2022-Jan
Description
During the Jepsen nightly run for Neo a crash was observed due memcached throwing a exception std::exception-derived exception. what(): GSL: Precondition failure: 'prepareSeqno >= checkpointManager->getOpenSnapshotStartSeqno()' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1007
cb::engine_errc VBucket::abort( |
const DocKey& key, |
uint64_t prepareSeqno,
|
std::optional<int64_t> abortSeqno,
|
const Collections::VB::CachingReadHandle& cHandle, |
const CookieIface* cookie) { |
...
|
if (prepareSeqno != static_cast<uint64_t>(htRes.pending->getBySeqno())) { |
Expects(getState() != vbucket_state_active);
|
Expects(isReceivingDiskSnapshot());
|
Expects(prepareSeqno >= checkpointManager->getOpenSnapshotStartSeqno()); // <-- Throwing line |
}
|
...
|
}
|
Crash seen on 182.23.107.227:
(gdb) bt full
|
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
|
set = {__val = {0, 139828462741251, 139827357981904, 0, 139828462745776, 0, 540150480307749759, 1514296763, 838767928, 281470681751460, 0, 0, 0, 0, 0, 0}}
|
pid = <optimized out>
|
tid = <optimized out>
|
ret = <optimized out>
|
#1 0x00007f2c599f28b1 in __GI_abort () at abort.c:79
|
save_stage = 1
|
act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0, 0, 0, 0, 0, 139828462741251, 139828459204545, 11, 139828462741120, 10, 139827357962048, 0, 139827794054208, 12887888, 139828459205747, 139828462741120}}, sa_flags = 1514130448, sa_restorer = 0x107b190 <stderr>}
|
sigs = {__val = {32, 0 <repeats 15 times>}}
|
__cnt = <optimized out>
|
__set = <optimized out>
|
__cnt = <optimized out>
|
__set = <optimized out>
|
#2 0x00007f2c5a3f263c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x0000000000b34a9b in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88
|
No locals.
|
#4 0x00007f2c5a3fd8f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48
|
No locals.
|
#5 0x00007f2c5a3fd961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58
|
No locals.
|
#6 0x00007f2c5a3fdbf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f2c18009d40, tinfo=tinfo@entry=0xc4cb78 <typeinfo for gsl::fail_fast>, dest=dest@entry=0x590ae0 <gsl::fail_fast::~fail_fast()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95
|
globals = <optimized out>
|
header = 0x7f2c18009cc0
|
#7 0x0000000000450a7b in gsl::detail::fail_fast_throw (message=0xc6da18 "GSL: Precondition failure: 'prepareSeqno >= checkpointManager->getOpenSnapshotStartSeqno()' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1007")
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769
|
No locals.
|
#8 0x00000000004a74d6 in VBucket::abort (this=0x7f2c09d36600, key=..., prepareSeqno=129, abortSeqno=..., cHandle=..., cookie=0x0) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:245
|
htRes = {pending = {lock = {bucketNum = 10, htLock = {_M_device = 0x7f2c0a466190, _M_owns = true}}, value = 0x7f2c09cf11f0,
|
valueStats = {<std::_Reference_wrapper_base_memfun<HashTable::Statistics, false>> = {<std::_Reference_wrapper_base<HashTable::Statistics>> = {<std::_Weak_result_type<HashTable::Statistics>> = {<std::_Weak_result_type_memfun<HashTable::Statistics, false>> = {<std::_Weak_result_type_impl<HashTable::Statistics>> = {<std::_Maybe_get_result_type<HashTable::Statistics, void>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::_Refwrap_base_arg1<HashTable::Statistics, void>> = {<No data fields>}, <std::_Refwrap_base_arg2<HashTable::Statistics, void>> = {<No data fields>}, <No data fields>}, <No data fields>}, _M_data = 0x7f2c09d36670}, pre = {size = 2131407, metaDataSize = 68,
|
uncompressedSize = 2131302, cid = {value = 0}, datatype = 3 '\003', isValid = true, isResident = true, isDeleted = false, isTempItem = false, isSystemItem = false, isPreparedSyncWrite = true}}, committed = 0x0, ht = @0x7f2c09d36618}
|
notify = {bySeqno = 139828031398912, notifyReplication = 108, notifyFlusher = 41, syncWrite = (unknown: 124), itemCountDifference = 0}
|
#9 0x00000000008e1c38 in PassiveStream::processAbort(AbortSyncWriteConsumer const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/dockey.h:341
|
vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2c09d36600, _M_refcount = {_M_pi = 0x7f2c09dced60}}, <No data fields>}
|
rv = <optimized out>
|
#10 0x00000000008e3bac in PassiveStream::processBufferedMessages(unsigned int&, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:472
|
ret = cb::engine_errc::success
|
response = <synthetic pointer>
|
message_bytes = <synthetic pointer>
|
seqno = {<std::_Optional_base<long, true, true>> = {<std::_Optional_base_impl<long, std::_Optional_base<long, true, true> >> = {<No data fields>}, _M_payload = {<std::_Optional_payload_base<long>> = {_M_payload = {_M_empty = {<No data fields>}, _M_value = 130},
|
_M_engaged = true}, <No data fields>}}, <std::_Enable_copy_move<true, true, true, true, std::optional<long> >> = {<No data fields>}, <No data fields>}
|
lh = {_M_device = 0x7f2c09beba10, _M_owns = false}
|
count = 2
|
total_bytes_processed = 2131468
|
failed = false
|
noMem = false
|
#11 0x00000000008d6672 in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f2c40304380, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1233
|
rval = all_processed
|
bytesProcessed = 0
|
iterations = 0
|
#12 0x00000000008db7a7 in DcpConsumer::processBufferedItems() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100
|
stream = {<std::__shared_ptr<PassiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<PassiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2c09beb890, _M_refcount = {_M_pi = 0x7f2c09beb880}}, <No data fields>}
|
process_ret = all_processed
|
vbucket = {vbid = 236}
|
#13 0x00000000008def85 in DcpConsumerTask::run (this=0x7f2c095c9790) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:83
|
phosphor_internal_category_enabled_72 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_72 = <optimized out>
|
phosphor_internal_tpi_72 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}}
|
phosphor_internal_guard_72 = {tpi = 0x10662e0 <DcpConsumerTask::run()::phosphor_internal_tpi_72>, enabled = true, arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 74747822430500}}}
|
consumer = {<std::__shared_ptr<DcpConsumer, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<DcpConsumer, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2c40304300}}, <No data fields>}
|
sleepFor = 0
|
state = <optimized out>
|
#14 0x0000000000aa3bb2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#15 0x0000000000aa0dc5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f2c31fe9820) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
executedAt = <optimized out>
|
end = <optimized out>
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 74747822427858}}
|
runtime = <optimized out>
|
runAgain = false
|
proxy = <error reading variable>
|
#16 0x0000000000bf7440 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f2c31fe9820) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f2c31fe9820: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f2c40c2dae0, tiny = {__data = "\340\332\302@,\177\000\000\000\000\000\000\000\000\000\000\060\375lX,\177\000\000\217\261\001\000\000\000\000\000@\000\000\000\000\000\000\000\360\230\376\061,\177\000", __align = {<No data fields>}}},
|
call_ = 0xaa1480 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xa9fbd0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
fn = <optimized out>
|
#17 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f2c5867a800, thread=..., task=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 74747822422888}}
|
stats = {expired = false, waitTime = {__r = 30378}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 74747822392510}}, requestId = 0}
|
#18 0x0000000000bdf1fa in folly::CPUThreadPoolExecutor::threadRun (this=0x7f2c5867a800, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = -32 '\340', value = {<folly::ThreadPoolExecutor::Task> = {func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f2c40c2dae0, tiny = {
|
__data = "\340\332\302@,\177\000\000\000\000\000\000\000\000\000\000\060\375lX,\177\000\000\217\261\001\000\000\000\000\000@\000\000\000\000\000\000\000\360\230\376\061,\177\000", __align = {<No data fields>}}},
|
call_ = 0xaa1480 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xa9fbd0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 74747822392510}}, expiration_ = {__r = 0},
|
expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x1b18f, tiny = {__data = "\217\261\001\000\000\000\000\000+\006\243", '\000' <repeats 13 times>, "_.f\\,\177\000\000p\252~1,\177\000\000@Ɔ\\,\177\000", __align = {<No data fields>}}}, call_ = 0x4663d9
|
<folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0}, context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {
|
_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000', queueObserverPayload_ = 139828455001216}}, hasValue = true}}
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcc89eb "CPUThreadPoolExecutor", e_ = 0xcc8a00 ""}}}}
|
#19 0x0000000000bfa3f9 in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#20 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#21 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#22 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#23 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#24 0x0000000000aa0a54 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f2c586091c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
fn = <error reading variable>
|
#25 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f2c586091c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
threadNameOpt = {storage_ = {{emptyState = -128 '\200', value = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f2c31fe9980 "NonIoPool0"}, _M_string_length = 10, {_M_local_buf = "NonIoPool0\000\000\000\000\000",
|
_M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7f2c586091c0)>
|
func = <optimized out>
|
threadNameOpt = <optimized out>
|
#26 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = <error reading variable>
|
#27 0x00007f2c5a426d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
|
__t = <optimized out>
|
#28 0x00007f2c5c22e6db in start_thread (arg=0x7f2c31ffb700) at pthread_create.c:463
|
pd = 0x7f2c31ffb700
|
now = <optimized out>
|
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139827794130688, -829786996715802346, 139827794057920, 0, 139828454714560, 140722526760384, 730817763442176278, 730912838838661398}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
|
not_first_call = <optimized out>
|
#29 0x00007f2c59ad3a3f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
No locals.
|
Test run using Couchstore.
Attachments
Issue Links
- is duplicated by
-
MB-51428 [Jepsen] Crash due to failing Expects while processing an Abort
- Resolved