(gdb) bt full #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 set = {__val = {0, 139892972287555, 139892319326608, 10, 139892495283456, 9778240, 540150480307749759, 1594876611, 1212095120, 281470681751456, 0, 0, 0, 0, 0, 0}} pid = tid = ret = #1 0x00007f3b5ecf0859 in __GI_abort () at abort.c:79 save_stage = 1 act = {__sigaction_handler = {sa_handler = 0xfffdd6, sa_sigaction = 0xfffdd6}, sa_mask = {__val = {16777050, 0, 0, 0, 0, 224, 10, 139892972287424, 139892495329024, 139892319326720, 13142832, 139892495285312, 139892970876947, 10, 139892972287424, 139892495329024}}, sa_flags = 1594697380, sa_restorer = 0xca6ce0 } sigs = {__val = {32, 0 }} #2 0x00007f3b5f0c7951 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 No symbol table info available. #3 0x000000000092440e in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/utilities/terminate_handler.cc:88 No locals. #4 0x00007f3b5f0d347c in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 No symbol table info available. #5 0x00007f3b5f0d34e7 in std::terminate() () from /lib/x86_64-linux-gnu/libstdc++.so.6 No symbol table info available. #6 0x00007f3b5f0d3799 in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6 No symbol table info available. #7 0x000000000046fc00 in gsl::detail::fail_fast_throw ( message=message@entry=0xae8390 "GSL: Precondition failure: 'highestPrepareSeqno == s->highPreparedSeqno.it->getBySeqno()' at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/durability/passive_"...) at ../third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769 No locals. #8 0x0000000000492dfd in PassiveDurabilityMonitor::PassiveDurabilityMonitor (this=0x7f3b483c84a0, vb=..., highPreparedSeqno=77, highCompletedSeqno=72, outstandingPrepares=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/durability/passive_durability_monitor.cc:80 s = { holder_ = { >, folly::SharedMutexImpl >, folly::SharedMutexImpl, folly::LockPolicyExclusive>> = {parent_ = 0x7f3b483c84d0}, static AllowsConcurrentAccess = false}} highestPrepareSeqno = s = { holder_ = { >, folly::SharedMutexImpl >, folly::SharedMutexImpl, folly::LockPolicyExclusive>> = {parent_ = }, static AllowsConcurrentAccess = false}} highestPrepareSeqno = prepare = __for_range = Python Exception value has been optimized out: __for_begin = Python Exception value has been optimized out: __for_end = Python Exception value has been optimized out: lastIt = #9 0x000000000064767b in std::make_unique >, std::allocator > > > > () at /usr/include/c++/9/bits/unique_ptr.h:857 No locals. #10 EPVBucket::loadOutstandingPrepares (this=0x7f3b40175900, vbs=..., outstandingPrepares=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/ep_vb.cc:977 No locals. #11 0x000000000065f917 in EPBucket::loadPreparedSyncWrites (this=, vb=...) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/engines/ep/src/ep_bucket.cc:2068 epVb = start = {__d = {__r = }} kvStore = vbState = 0x7f3b4830ae40 prepares = std::vector of length 9, capacity 9 = { startSeqno = endSeqno = docFilter = DocumentFilter::ALL_ITEMS valFilter = scanCtx = std::unique_ptr = {get() = 0x7f3b4825ae40} scanResult = storageCB = numPrepares = #12 0x0000000000719094 in Warmup::loadPreparedSyncWrites (this=0x7f3b482a9300, shardId=) at ../kv_engine/engines/ep/src/warmup.cc:1578 itr = { >::ahm_iterator, std::hash, std::equal_to, std::allocator, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>, std::pair >, folly::AtomicHashArray, std::hash, std::equal_to, std::allocator, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>::aha_iterator, std::hash, std::equal_to, std::allocator, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>, std::pair > > >, std::pair >, std::forward_iterator_tag>> = {}, ahm_ = 0x7f3b482a94d8, subMap_ = 0, subIt_ = {, std::hash, std::equal_to, std::allocator, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>::aha_iterator, std::hash, std::equal_to, std::allocator, folly::AtomicHashArrayLinearProbeFcn, folly::identity_fn>, std::pair > >, std::pair >, std::forward_iterator_tag>> = {}, aha_ = 0x7f3b4826e000, offset_ = 15}} vb = itemsVisited = 10 success = true preparesLoaded = 10 epStats = vbid = {vbid = 15} __for_range = Python Exception value has been optimized out: __for_begin = Python Exception value has been optimized out: __for_end = #13 0x00000000007254d2 in WarmupLoadPreparedSyncWrites::run (this=0x7f3b483e5d90) at ../kv_engine/engines/ep/src/warmup.cc:309 phosphor_internal_category_enabled_305 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = } phosphor_internal_category_enabled_temp_305 = phosphor_internal_tpi_305 = {category = 0x0, name = 0x0, type = phosphor::TraceEventType::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {phosphor::TraceArgumentType::is_bool, phosphor::TraceArgumentType::is_bool}}} phosphor_internal_guard_305 = {tpi = 0xc8cfc0 , enabled = true, arg1 = 1, arg2 = {}, start = {__d = {__r = 4246184684636}}} #14 0x00000000008b080b in GlobalTask::execute (this=0x7f3b483e5d90) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/executor/globaltask.cc:68 guard = {previous = 0x0} guard = {previous = } #15 FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f3b427d0530) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/executor/folly_executorpool.cc:189 executedAt = {__d = {__r = }} end = {__d = {__r = }} scheduleOverhead = {__r = } start = {__d = {__r = 4246184683259}} runtime = {__r = } runAgain = false proxy = proxy = runAgain = _logger_ = executedAt = {__d = {__r = }} scheduleOverhead = {__r = } start = {__d = {__r = }} end = {__d = {__r = }} runtime = {__r = } #16 folly::detail::function::FunctionTraits::callSmall(folly::detail::function::Data&) (p=...) at tlm/deps/folly.exploded/include/folly/Function.h:387 fn = @0x7f3b427d0530: {__proxy = @0x7f3b403f3cb0} #17 0x0000000000a0b0a0 in folly::detail::function::FunctionTraits::operator()() (this=0x7f3b427d0530) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416 fn = @0x7f3b427d0530: {> = {}, data_ = {big = 0x7f3b403f3cb0, tiny = {__data = "\260, __align = {}}}, call_ = 0x8b06e0 ::callSmall(folly::detail::function::Data&)>, exec_ = 0x8af460 (folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>} fn = #18 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f3b5e83e400, thread=, task=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97 rctx = {prev_ = std::shared_ptr (empty) = {get() = 0x0}} startTime = {__d = {__r = 4246184678893}} stats = {expired = false, waitTime = {__r = 54171}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 4246184624722}}, requestId = 0} #19 0x00000000009f2f5a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f3b5e83e400, thread=) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265 task = {storage_ = {{emptyState = -80 '\260', value = { = {func_ = {> = {}, data_ = {big = 0x7f3b403f3cb0, tiny = { __data = "\260, __align = {}}}, call_ = 0x8b06e0 ::callSmall(folly::detail::function::Data&)>, exec_ = 0x8af460 (folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data*)>}, enqueueTime_ = {__d = { __r = 4246184624722}}, expiration_ = {__r = 0}, expireCallback_ = {> = {}, data_ = {big = 0x0, tiny = { __data = "\000\000\000\000\000\000\000\000`\026\024`;\177\000\000\060\000\000\000\000\000\000\000\001\022\001\000\000\000\000\000\300g\177K;\177\000\000\273,\236\000\000\000\000", __align = {}}}, call_ = 0x460849 ::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0}, context_ = std::shared_ptr (empty) = {get() = 0x0}}, poison = false, priority_ = 0 '\000', queueObserverPayload_ = 0}}, hasValue = true}} guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = , b_ = 0xb36acb "CPUThreadPoolExecutor", e_ = 0xb36ae0 ""}}}} #20 0x0000000000a0e059 in std::__invoke_impl), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__t=, __f=) at /usr/local/include/c++/7.5.0/bits/invoke.h:73 No locals. #21 std::__invoke), folly::ThreadPoolExecutor*&, std::shared_ptr&> (__fn=) at /usr/local/include/c++/7.5.0/bits/invoke.h:95 No locals. #22 std::_Bind))(std::shared_ptr)>::__call(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=) at /usr/local/include/c++/7.5.0/functional:467 No locals. #23 std::_Bind))(std::shared_ptr)>::operator()<, void>() (this=) at /usr/local/include/c++/7.5.0/functional:551 No locals. #24 folly::detail::function::FunctionTraits::callBig))(std::shared_ptr)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401 fn = #25 0x00000000008b0513 in folly::detail::function::FunctionTraits::operator()() (this=0x7f3b5da10dc0) at tlm/deps/folly.exploded/include/folly/Function.h:414 fn = #26 CBRegisteredThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f3b5da10dc0) at /home/couchbase/jenkins/workspace/kv-engine-jepsen-post-commit/kv_engine/executor/folly_executorpool.cc:47 threadNameOpt = {storage_ = {{emptyState = -112 '\220', value = "ReaderPool3"}, hasValue = true}} func = func = {> = {}, data_ = {big = , tiny = {__data = { }, __align = {}}}, call_ = , exec_ = } threadNameOpt = {storage_ = {{emptyState = , Python Exception value has been optimized out: value = }, hasValue = }} #27 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at tlm/deps/folly.exploded/include/folly/Function.h:401 fn = #28 0x00000000008afa57 in folly::detail::function::FunctionTraits::operator()() (this=0x7f3b5e85e920) at tlm/deps/folly.exploded/include/folly/Function.h:416 fn = #29 folly::PriorityThreadFactory::newThread(folly::Function&&)::{lambda()#1}::operator()() (__closure=0x7f3b5e85e910) at tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54 func = priority = func = {> = {}, data_ = {big = , tiny = {__data = { }, __align = {}}}, call_ = , exec_ = } priority = #30 folly::detail::function::FunctionTraits::callBig&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at tlm/deps/folly.exploded/include/folly/Function.h:401 fn = #31 0x00007f3b5f0ffd84 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 No symbol table info available. #32 0x00007f3b600f8609 in start_thread (arg=) at pthread_create.c:477 ret = pd = unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139892495329024, -8482617625064342747, 140726614108766, 140726614108767, 140726614108768, 139892495288384, 8372531234975415077, 8372596505666681637}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = { prev = 0x0, cleanup = 0x0, canceltype = 0}}} not_first_call = 0 #33 0x00007f3b5eded293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 No locals.