Caught unhandled std::exception-derived exception. what(): VBucket::processSet: vb:1011 expected a complete item but the item is a prepare <ud>cid:0x0:00000000000000004724</ud> with seqno:116913. Existing prepare has seqno:116640

./sequoia -client 172.23.104.27:2375 -provider file:centos_pine.yml -test tests/integration/7.2/test_7.2.yml -scope tests/integration/7.2/scope_7.2_magma.yml -scale 3 -repeat 0 -log_level 0 -version 7.2.0-5241 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=604800 -show_topology=true

balakumaran.g@Balakumarans-MacBook-Pro-2 cbcollect_info_ns_1@172.23.99.20_20230310-193218 % grep CRITICAL memcached.log 

2023-03-10T10:15:00.815852-08:00 CRITICAL *** Fatal error encountered during exception handling ***

2023-03-10T10:15:00.815920-08:00 CRITICAL Caught unhandled std::exception-derived exception. what(): VBucket::processSet: vb:1011 expected a complete item but the item is a prepare <ud>cid:0x0:00000000000000004724</ud> with seqno:116913. Existing prepare has seqno:116640

2023-03-10T10:15:59.208824-08:00 CRITICAL Detected previous crash

2023-03-10T10:15:59.208869-08:00 CRITICAL Breakpad caught a crash (Couchbase version 7.2.0-5241). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/77074f66-cf86-4833-bedaaabc-37dfe1da.dmp before terminating.

2023-03-10T10:15:59.208878-08:00 CRITICAL Stack backtrace of crashed thread:

2023-03-10T10:15:59.208878-08:00 CRITICAL    #0  /opt/couchbase/bin/memcached() [0x400000+0x73ebc8]

2023-03-10T10:15:59.208879-08:00 CRITICAL    #1  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x7906ba]

2023-03-10T10:15:59.208881-08:00 CRITICAL    #2  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x7909f8]

2023-03-10T10:15:59.208881-08:00 CRITICAL    #3  /lib64/libpthread.so.0() [0x7f3846956000+0xf630]

2023-03-10T10:15:59.208883-08:00 CRITICAL    #4  /lib64/libc.so.6(gsignal+0x37) [0x7f38441a6000+0x36387]

2023-03-10T10:15:59.208884-08:00 CRITICAL    #5  /lib64/libc.so.6(abort+0x148) [0x7f38441a6000+0x37a78]

2023-03-10T10:15:59.208884-08:00 CRITICAL    #6  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f3844a8e000+0x9963c]

2023-03-10T10:15:59.208885-08:00 CRITICAL    #7  /opt/couchbase/bin/memcached() [0x400000+0x7490eb]

2023-03-10T10:15:59.208886-08:00 CRITICAL    #8  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f3844a8e000+0xa48f6]

2023-03-10T10:15:59.208886-08:00 CRITICAL    #9  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f3844a8e000+0xa4961]

2023-03-10T10:15:59.208887-08:00 CRITICAL    #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f3844a8e000+0xa4bf4]

2023-03-10T10:15:59.208888-08:00 CRITICAL    #11 /opt/couchbase/bin/memcached() [0x400000+0xa7a37]

2023-03-10T10:15:59.208888-08:00 CRITICAL    #12 /opt/couchbase/bin/memcached() [0x400000+0x3396e4]

2023-03-10T10:15:59.208889-08:00 CRITICAL    #13 /opt/couchbase/bin/memcached() [0x400000+0x302dea]

2023-03-10T10:15:59.208890-08:00 CRITICAL    #14 /opt/couchbase/bin/memcached() [0x400000+0x4fe3a6]

2023-03-10T10:15:59.208891-08:00 CRITICAL    #15 /opt/couchbase/bin/memcached() [0x400000+0x4fe9c8]

2023-03-10T10:15:59.208891-08:00 CRITICAL    #16 /opt/couchbase/bin/memcached() [0x400000+0x4fedfb]

2023-03-10T10:15:59.208892-08:00 CRITICAL    #17 /opt/couchbase/bin/memcached() [0x400000+0x4efb02]

2023-03-10T10:15:59.208893-08:00 CRITICAL    #18 /opt/couchbase/bin/memcached() [0x400000+0x4f6107]

2023-03-10T10:15:59.208893-08:00 CRITICAL    #19 /opt/couchbase/bin/memcached() [0x400000+0x4f9635]

2023-03-10T10:15:59.208894-08:00 CRITICAL    #20 /opt/couchbase/bin/memcached() [0x400000+0x6b7749]

2023-03-10T10:15:59.208895-08:00 CRITICAL    #21 /opt/couchbase/bin/memcached() [0x400000+0x6b0e0a]

2023-03-10T10:15:59.208895-08:00 CRITICAL    #22 /opt/couchbase/bin/memcached() [0x400000+0x6b8afe]

2023-03-10T10:15:59.208905-08:00 CRITICAL    #23 /opt/couchbase/bin/memcached() [0x400000+0x816c10]

2023-03-10T10:15:59.208906-08:00 CRITICAL    #24 /opt/couchbase/bin/memcached() [0x400000+0x8016aa]

2023-03-10T10:15:59.208911-08:00 CRITICAL    #25 /opt/couchbase/bin/memcached() [0x400000+0x819bc9]

2023-03-10T10:15:59.208912-08:00 CRITICAL    #26 /opt/couchbase/bin/memcached() [0x400000+0x6b0b04]

2023-03-10T10:15:59.208912-08:00 CRITICAL    #27 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f3844a8e000+0xcdd40]

2023-03-10T10:15:59.208932-08:00 CRITICAL    #28 /lib64/libpthread.so.0() [0x7f3846956000+0x7ea5]

2023-03-10T10:15:59.208933-08:00 CRITICAL    #29 /lib64/libc.so.6(clone+0x6d) [0x7f38441a6000+0xfeb0d]

balakumaran.g@Balakumarans-MacBook-Pro-2 cbcollect_info_ns_1@172.23.99.20_20230310-193218 % 

(gdb) bt

#0  0x00007f38441dc387 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:55

#1  0x00007f38441dda78 in __GI_abort () at abort.c:90

#2  0x00007f3844b2763c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

#3  0x0000000000b490eb in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

#4  0x00007f3844b328f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#5  0x00007f3844b32961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

#6  0x00007f3844b32bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f37c4000940, tinfo=0x1092230 <typeinfo for std::logic_error>, dest=0x4453d0 <_ZNSt11logic_errorD1Ev@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

#7  0x00000000004a7a37 in VBucket::processSet (this=0x7f37d77ebc60, htRes=..., v=@0x7f37d77ebd68: 0x7f35fdc126d0, itm=..., cas=139877815336016, allowExisting=true, hasMetaData=true, queueItmCtx=..., storeIfStatus=cb::Continue, maybeKeyExists=true)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:357

#8  0x00000000007396e4 in VBucket::prepare(Item&, unsigned long, unsigned long*, CookieIface const*, EventuallyPersistentEngine&, CheckConflicts, bool, GenerateBySeqno, GenerateCas, Collections::VB::CachingReadHandle const&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:1946

#9  0x0000000000702dea in KVBucket::prepare(Item&, CookieIface const*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kv_bucket.cc:1801

#10 0x00000000008fe3a6 in PassiveStream::processMessage(MutationConsumerMessage*, PassiveStream::MessageType) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/response.h:478

#11 0x00000000008fe9c8 in PassiveStream::processPrepare(MutationConsumerMessage*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:761

#12 0x00000000008fedfb in PassiveStream::processBufferedMessages(unsigned int&, unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/passive_stream.cc:493

#13 0x00000000008efb02 in DcpConsumer::drainStreamsBufferedItems (this=this@entry=0x7f37a8335080, stream=..., yieldThreshold=10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:1270

#14 0x00000000008f6107 in DcpConsumer::processBufferedItems() () at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

#15 0x00000000008f9635 in DcpConsumerTask::run (this=0x7f37f8344bb0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/consumer.cc:84

#16 0x0000000000ab7749 in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98

#17 0x0000000000ab0e0a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f37d77ec650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

#18 0x0000000000ab8afe in operator() (this=0x7f37d77ec650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#19 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#20 0x0000000000c16c10 in operator() (this=0x7f37d77ec840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

#21 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f3842b51d00, thread=..., 

    task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5241.x86_64.debug, CU 0xa931924, DIE 0xa9b5859>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

#22 0x0000000000c016aa in folly::CPUThreadPoolExecutor::threadRun (this=0x7f3842b51d00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

#23 0x0000000000c19bc9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, 

    __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73

#24 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)

    at /usr/local/include/c++/7.3.0/bits/invoke.h:95

#25 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

#26 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

#27 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

#28 0x0000000000ab0b04 in operator() (this=0x7f3842fddec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#29 operator() (__closure=0x7f3842fddec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#30 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

#31 0x00007f3844b5bd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

#32 0x00007f384695dea5 in start_thread (arg=0x7f37d77fe700) at pthread_create.c:307

#33 0x00007f38442a4b0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

(gdb)