registerCursorBySeqno can place cursor before the requested seqno: Monotonic<m> (ActiveStream(vb:901)::lastReadSeqno) invariant failed: new value (392216) breaks invariant on current value (392216)

./sequoia -client 172.23.104.254:2375 -provider file:centos_third_cluster.yml -test tests/integration/7.2/test_7.2.yml -scope tests/integration/7.2/scope_7.2_magma.yml -scale 3 -repeat 0 -log_level 0 -version 7.2.1-5919 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=1209600 -show_topology=true

[root@localhost logs]# grep CRITICAL memcached.log.0000*

memcached.log.000091.txt:2023-08-20T18:16:00.766567-07:00 CRITICAL *** Fatal error encountered during exception handling ***

memcached.log.000091.txt:2023-08-20T18:16:00.766897-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): Monotonic<m> (ActiveStream(vb:901)::lastReadSeqno) invariant failed: new value (392216) breaks invariant on current value (392216)

memcached.log.000091.txt:2023-08-20T18:16:00.766930-07:00 CRITICAL Exception thrown from:

memcached.log.000091.txt:2023-08-20T18:16:00.767023-07:00 CRITICAL     #0  /opt/couchbase/bin/memcached() [0x400000+0x142bc8]

memcached.log.000091.txt:2023-08-20T18:16:00.767101-07:00 CRITICAL     #1  /opt/couchbase/bin/memcached() [0x400000+0x4db6ac]

memcached.log.000091.txt:2023-08-20T18:16:00.767164-07:00 CRITICAL     #2  /opt/couchbase/bin/memcached() [0x400000+0x4ddf5d]

memcached.log.000091.txt:2023-08-20T18:16:00.767185-07:00 CRITICAL     #3  /opt/couchbase/bin/memcached() [0x400000+0x4e6be1]

memcached.log.000091.txt:2023-08-20T18:16:00.767204-07:00 CRITICAL     #4  /opt/couchbase/bin/memcached() [0x400000+0x4e743c]

memcached.log.000091.txt:2023-08-20T18:16:00.767230-07:00 CRITICAL     #5  /opt/couchbase/bin/memcached() [0x400000+0x4e7564]

memcached.log.000091.txt:2023-08-20T18:16:00.767239-07:00 CRITICAL     #6  /opt/couchbase/bin/memcached() [0x400000+0x4ee299]

memcached.log.000091.txt:2023-08-20T18:16:00.767313-07:00 CRITICAL     #7  /opt/couchbase/bin/memcached() [0x400000+0x6c2099]

memcached.log.000091.txt:2023-08-20T18:16:00.767413-07:00 CRITICAL     #8  /opt/couchbase/bin/memcached() [0x400000+0x6bb77a]

memcached.log.000091.txt:2023-08-20T18:16:00.767659-07:00 CRITICAL     #9  /opt/couchbase/bin/memcached() [0x400000+0x6c344e]

memcached.log.000091.txt:2023-08-20T18:16:00.767816-07:00 CRITICAL     #10 /opt/couchbase/bin/memcached() [0x400000+0x8215b0]

memcached.log.000091.txt:2023-08-20T18:16:00.767886-07:00 CRITICAL     #11 /opt/couchbase/bin/memcached() [0x400000+0x80c04a]

memcached.log.000091.txt:2023-08-20T18:16:00.767931-07:00 CRITICAL     #12 /opt/couchbase/bin/memcached() [0x400000+0x824569]

memcached.log.000091.txt:2023-08-20T18:16:00.767996-07:00 CRITICAL     #13 /opt/couchbase/bin/memcached() [0x400000+0x6bb474]

memcached.log.000091.txt:2023-08-20T18:16:00.768168-07:00 CRITICAL     #14 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56b2b9f000+0xcdd40]

memcached.log.000091.txt:2023-08-20T18:16:00.768187-07:00 CRITICAL     #15 /lib64/libpthread.so.0() [0x7f56b4a66000+0x7ea5]

memcached.log.000091.txt:2023-08-20T18:16:00.768293-07:00 CRITICAL     #16 /lib64/libc.so.6(clone+0x6d) [0x7f56b22b7000+0xfeb0d]

memcached.log.000091.txt:2023-08-20T18:16:03.942104-07:00 CRITICAL Detected previous crash

memcached.log.000091.txt:2023-08-20T18:16:03.942171-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.2.1-5919). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/6436fc7e-70e6-4d04-60e90d91-5868feb6.dmp before terminating.

memcached.log.000091.txt:2023-08-20T18:16:03.942174-07:00 CRITICAL Stack backtrace of crashed thread:

memcached.log.000091.txt:2023-08-20T18:16:03.942176-07:00 CRITICAL    #0  /opt/couchbase/bin/memcached() [0x400000+0x749518]

memcached.log.000091.txt:2023-08-20T18:16:03.942178-07:00 CRITICAL    #1  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x79b05a]

memcached.log.000091.txt:2023-08-20T18:16:03.942179-07:00 CRITICAL    #2  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x79b398]

memcached.log.000091.txt:2023-08-20T18:16:03.942180-07:00 CRITICAL    #3  /lib64/libpthread.so.0() [0x7f56b4a66000+0xf630]

memcached.log.000091.txt:2023-08-20T18:16:03.942181-07:00 CRITICAL    #4  /lib64/libc.so.6(gsignal+0x37) [0x7f56b22b7000+0x36387]

memcached.log.000091.txt:2023-08-20T18:16:03.942184-07:00 CRITICAL    #5  /lib64/libc.so.6(abort+0x148) [0x7f56b22b7000+0x37a78]

memcached.log.000091.txt:2023-08-20T18:16:03.942184-07:00 CRITICAL    #6  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56b2b9f000+0x9963c]

memcached.log.000091.txt:2023-08-20T18:16:03.942185-07:00 CRITICAL    #7  /opt/couchbase/bin/memcached() [0x400000+0x753a8b]

memcached.log.000091.txt:2023-08-20T18:16:03.942186-07:00 CRITICAL    #8  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56b2b9f000+0xa48f6]

memcached.log.000091.txt:2023-08-20T18:16:03.942187-07:00 CRITICAL    #9  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56b2b9f000+0xa4961]

memcached.log.000091.txt:2023-08-20T18:16:03.942188-07:00 CRITICAL    #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56b2b9f000+0xa4bf4]

memcached.log.000091.txt:2023-08-20T18:16:03.942189-07:00 CRITICAL    #11 /opt/couchbase/bin/memcached() [0x400000+0x142ccd]

memcached.log.000091.txt:2023-08-20T18:16:03.942189-07:00 CRITICAL    #12 /opt/couchbase/bin/memcached() [0x400000+0x4db6ac]

memcached.log.000091.txt:2023-08-20T18:16:03.942190-07:00 CRITICAL    #13 /opt/couchbase/bin/memcached() [0x400000+0x4ddf5d]

memcached.log.000091.txt:2023-08-20T18:16:03.942192-07:00 CRITICAL    #14 /opt/couchbase/bin/memcached() [0x400000+0x4e6be1]

memcached.log.000091.txt:2023-08-20T18:16:03.942193-07:00 CRITICAL    #15 /opt/couchbase/bin/memcached() [0x400000+0x4e743c]

memcached.log.000091.txt:2023-08-20T18:16:03.942194-07:00 CRITICAL    #16 /opt/couchbase/bin/memcached() [0x400000+0x4e7564]

memcached.log.000091.txt:2023-08-20T18:16:03.942194-07:00 CRITICAL    #17 /opt/couchbase/bin/memcached() [0x400000+0x4ee299]

memcached.log.000091.txt:2023-08-20T18:16:03.942195-07:00 CRITICAL    #18 /opt/couchbase/bin/memcached() [0x400000+0x6c2099]

memcached.log.000091.txt:2023-08-20T18:16:03.942196-07:00 CRITICAL    #19 /opt/couchbase/bin/memcached() [0x400000+0x6bb77a]

memcached.log.000091.txt:2023-08-20T18:16:03.942197-07:00 CRITICAL    #20 /opt/couchbase/bin/memcached() [0x400000+0x6c344e]

memcached.log.000091.txt:2023-08-20T18:16:03.942198-07:00 CRITICAL    #21 /opt/couchbase/bin/memcached() [0x400000+0x8215b0]

memcached.log.000091.txt:2023-08-20T18:16:03.942198-07:00 CRITICAL    #22 /opt/couchbase/bin/memcached() [0x400000+0x80c04a]

memcached.log.000091.txt:2023-08-20T18:16:03.942199-07:00 CRITICAL    #23 /opt/couchbase/bin/memcached() [0x400000+0x824569]

memcached.log.000091.txt:2023-08-20T18:16:03.942200-07:00 CRITICAL    #24 /opt/couchbase/bin/memcached() [0x400000+0x6bb474]

memcached.log.000091.txt:2023-08-20T18:16:03.942209-07:00 CRITICAL    #25 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f56b2b9f000+0xcdd40]

memcached.log.000091.txt:2023-08-20T18:16:03.942209-07:00 CRITICAL    #26 /lib64/libpthread.so.0() [0x7f56b4a66000+0x7ea5]

memcached.log.000091.txt:2023-08-20T18:16:03.942209-07:00 CRITICAL    #27 /lib64/libc.so.6(clone+0x6d) [0x7f56b22b7000+0xfeb0d]

[root@localhost logs]# 

(gdb) bt 

#0  0x00007f56b22ed387 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:55

#1  0x00007f56b22eea78 in __GI_abort () at abort.c:90

#2  0x00007f56b2c3863c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

#3  0x0000000000b53a8b in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

#4  0x00007f56b2c438f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#5  0x00007f56b2c43961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

#6  0x00007f56b2c43bf4 in __cxxabiv1::__cxa_throw (obj=obj@entry=0x7f5604000980, tinfo=tinfo@entry=0x109a3a8 <typeinfo for boost::exception_detail::error_info_injector<std::logic_error>>, 

    dest=dest@entry=0xbb69f0 <boost::exception_detail::error_info_injector<std::logic_error>::~error_info_injector()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95

#7  0x0000000000542ccd in cb::throwWithTrace<std::logic_error> (exception=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/exception/info.hpp:129

#8  0x00000000008db6ac in ThrowExceptionPolicy<unsigned long>::nonMonotonic(unsigned long const&, unsigned long const&) [clone .isra.0] () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322

#9  0x00000000008ddf5d in store (memoryOrder=std::memory_order_seq_cst, desired=<optimized out>, this=0x7f55bf847ec0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:1525

#10 ActiveStream::snapshot(ActiveStream::OutstandingItemsResult const&, std::deque<std::unique_ptr<DcpResponse, std::default_delete<DcpResponse> >, std::allocator<std::unique_ptr<DcpResponse, std::default_delete<DcpResponse> > > >&, unsigned long, std::optional<unsigned long>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:1484

#11 0x00000000008e6be1 in ActiveStream::processItems(ActiveStream::OutstandingItemsResult&, std::lock_guard<std::mutex> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:1336

#12 0x00000000008e743c in ActiveStream::nextCheckpointItemTask(std::lock_guard<std::mutex> const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:1009

#13 0x00000000008e7564 in ActiveStream::nextCheckpointItemTask() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:993

#14 0x00000000008ee299 in ActiveStreamCheckpointProcessorTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream_checkpoint_processor_task.cc:52

#15 0x0000000000ac2099 in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98

#16 0x0000000000abb77a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f5613fed650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

#17 0x0000000000ac344e in operator() (this=0x7f5613fed650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#18 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#19 0x0000000000c215b0 in operator() (this=0x7f5613fed840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

#20 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f56b0d46d00, thread=..., 

    task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.1-5919.x86_64.debug, CU 0xaa3a58a, DIE 0xaabe4bf>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

#21 0x0000000000c0c04a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f56b0d46d00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

#22 0x0000000000c24569 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, 

    __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73

#23 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)

    at /usr/local/include/c++/7.3.0/bits/invoke.h:95

#24 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

#25 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

#26 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

#27 0x0000000000abb474 in operator() (this=0x7f56b11f2240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#28 operator() (__closure=0x7f56b11f2240) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#29 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

#30 0x00007f56b2c6cd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

#31 0x00007f56b4a6dea5 in start_thread (arg=0x7f5613fff700) at pthread_create.c:307

#32 0x00007f56b23b5b0d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

(gdb)