[System Test] :- Caught unhandled std::exception-derived exception. what(): GSL: Precondition failure: 'getNumCursorsInCheckpoint() == 0' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.cc:102

./sequoia -client 172.23.104.27:2375 -provider file:debian_pine.yml -test tests/integration/7.6/test_7.6.yml -scope tests/integration/7.6/scope_7.6_magma.yml -scale 3 -repeat 0 -log_level 0 -version 7.6.0-1503 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=1209600 -show_topology=true

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:29.640488-07:00 CRITICAL *** Fatal error encountered during exception handling ***

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:29.640540-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): GSL: Precondition failure: 'getNumCursorsInCheckpoint() == 0' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.cc:102

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692744-07:00 CRITICAL Detected previous crash

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692903-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.6.0-1503). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/2bfdff89-fda5-42bd-89eeb4bf-32e35118.dmp before terminating. Writing dump succeeded: yes

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692923-07:00 CRITICAL Stack backtrace of crashed thread:

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692925-07:00 CRITICAL    #0  /opt/couchbase/bin/memcached() [0x400000+0x8768f5]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692927-07:00 CRITICAL    #1  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x385) [0x400000+0x894745]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692929-07:00 CRITICAL    #2  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0x9f) [0x400000+0x894a8f]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692930-07:00 CRITICAL    #3  /lib/x86_64-linux-gnu/libc.so.6() [0x7f9f0121f000+0x3bfd0]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692932-07:00 CRITICAL    #4  /lib/x86_64-linux-gnu/libc.so.6() [0x7f9f0121f000+0x8ad3c]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692934-07:00 CRITICAL    #5  /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x12) [0x7f9f0121f000+0x3bf32]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692935-07:00 CRITICAL    #6  /lib/x86_64-linux-gnu/libc.so.6(abort+0xd3) [0x7f9f0121f000+0x26472]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692936-07:00 CRITICAL    #7  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f9f0162b000+0xa27fc]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692938-07:00 CRITICAL    #8  /opt/couchbase/bin/memcached() [0x400000+0x886f26]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692939-07:00 CRITICAL    #9  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f9f0162b000+0xadab6]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692940-07:00 CRITICAL    #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f9f0162b000+0xacb99]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692942-07:00 CRITICAL    #11 /opt/couchbase/bin/../lib/libstdc++.so.6(__gxx_personality_v0+0x271) [0x7f9f0162b000+0xad4d1]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692943-07:00 CRITICAL    #12 /opt/couchbase/bin/../lib/libgcc_s.so.1() [0x7f9f022bd000+0x106df]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692945-07:00 CRITICAL    #13 /opt/couchbase/bin/../lib/libgcc_s.so.1(_Unwind_RaiseException+0x331) [0x7f9f022bd000+0x10d61]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692946-07:00 CRITICAL    #14 /opt/couchbase/bin/../lib/libstdc++.so.6(__cxa_throw+0x37) [0x7f9f0162b000+0xadda7]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692948-07:00 CRITICAL    #15 /opt/couchbase/bin/memcached() [0x400000+0x970665]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692949-07:00 CRITICAL    #16 /opt/couchbase/bin/memcached() [0x400000+0x620b2]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692950-07:00 CRITICAL    #17 /opt/couchbase/bin/memcached() [0x400000+0x10162a]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692951-07:00 CRITICAL    #18 /opt/couchbase/bin/memcached() [0x400000+0x4d18a4]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692953-07:00 CRITICAL    #19 /opt/couchbase/bin/memcached() [0x400000+0x7d30bd]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692954-07:00 CRITICAL    #20 /opt/couchbase/bin/memcached() [0x400000+0x7d12ea]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692955-07:00 CRITICAL    #21 /opt/couchbase/bin/memcached() [0x400000+0x7d4256]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692957-07:00 CRITICAL    #22 /opt/couchbase/bin/memcached() [0x400000+0x96c65f]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692958-07:00 CRITICAL    #23 /opt/couchbase/bin/memcached() [0x400000+0x955357]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692960-07:00 CRITICAL    #24 /opt/couchbase/bin/memcached() [0x400000+0x96f1fa]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692961-07:00 CRITICAL    #25 /opt/couchbase/bin/memcached() [0x400000+0x7cd4f4]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692963-07:00 CRITICAL    #26 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f9f0162b000+0xd6f00]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692964-07:00 CRITICAL    #27 /lib/x86_64-linux-gnu/libc.so.6() [0x7f9f0121f000+0x89044]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000003.txt:2023-09-14T10:18:51.692965-07:00 CRITICAL    #28 /lib/x86_64-linux-gnu/libc.so.6() [0x7f9f0121f000+0x1095fc]

(gdb) 

#0  __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6, no_tid=no_tid@entry=0) at ./nptl/pthread_kill.c:44

#1  0x00007f9f012a9d9f in __pthread_kill_internal (signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:78

#2  0x00007f9f0125af32 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26

#3  0x00007f9f01245472 in __GI_abort () at ./stdlib/abort.c:79

#4  0x00007f9f016cd7fc in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

#5  0x0000000000c86f26 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

#6  0x00007f9f016d8ab6 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#7  0x00007f9f016d7b99 in __cxa_call_terminate (ue_header=ue_header@entry=0x7f9e4804a1d0) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_call.cc:54

#8  0x00007f9f016d84d1 in __cxxabiv1::__gxx_personality_v0 (version=<optimized out>, actions=6, exception_class=5138137972254386944, ue_header=0x7f9e4804a1d0, context=<optimized out>)

    at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_personality.cc:685

#9  0x00007f9f022cd6df in _Unwind_RaiseException_Phase2 (exc=exc@entry=0x7f9e4804a1d0, context=context@entry=0x7f9e617ef6b0, frames_p=frames_p@entry=0x7f9e617ef7a0) at /tmp/deploy/objdir/../gcc-10.2.0/libgcc/unwind.inc:64

#10 0x00007f9f022cdd61 in _Unwind_RaiseException (exc=exc@entry=0x7f9e4804a1d0) at /tmp/deploy/objdir/../gcc-10.2.0/libgcc/unwind.inc:136

#11 0x00007f9f016d8da7 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0xe77718 <typeinfo for gsl::fail_fast>, dest=0x639520 <gsl::fail_fast::~fail_fast()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:90

#12 0x0000000000d70665 in __cxxabiv1::__cxa_throw (thrownException=0x7f9e4804a1f0, type=0xe77718 <typeinfo for gsl::fail_fast>, destructor=<optimized out>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp:106

#13 0x00000000004620b2 in gsl::detail::fail_fast_throw (message=0xe35f40 "GSL: Precondition failure: 'getNumCursorsInCheckpoint() == 0' at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.cc:102")

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/gsl-lite.hpp:1769

#14 0x000000000050162a in Checkpoint::~Checkpoint (this=0x7f9e1b637940, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint.cc:102

#15 0x00000000008d18a4 in std::default_delete<Checkpoint>::operator() (__ptr=0x7f9e1b637940, this=0x7f9e13f71510) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:79

#16 std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >::~unique_ptr (this=0x7f9e13f71510, __in_chrg=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:361

#17 boost::container::allocator_traits<boost::container::new_allocator<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false> > >::priv_destroy<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> > > (p=0x7f9e13f71510) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/container/allocator_traits.hpp:403

#18 boost::container::allocator_traits<boost::container::new_allocator<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false> > >::destroy<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> > > (p=0x7f9e13f71510, a=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/container/allocator_traits.hpp:331

#19 boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false>::destructor<boost::container::new_allocator<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false> > > (a=..., this=0x7f9e13f71500)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/container/detail/node_alloc_holder.hpp:142

#20 boost::container::dtl::allocator_node_destroyer<boost::container::new_allocator<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false> > >::operator() (

    p=<optimized out>, this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/container/detail/destroyers.hpp:412

#21 boost::intrusive::list_impl<boost::intrusive::bhtraits<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, unsigned long, true, void>::erase_and_dispose<boost::container::dtl::allocator_node_destroyer<boost::container::new_allocator<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false> > > > (disposer=..., i=..., this=0x7f9e617efac0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/intrusive/list.hpp:677

#22 boost::intrusive::list_impl<boost::intrusive::bhtraits<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false>, boost::intrusive::list_node_traits<void*>, (boost::intrusive::link_mode_type)0, boost::intrusive::dft_tag, 1u>, unsigned long, true, void>::erase_and_dispose<boost::container::dtl::allocator_node_destroyer<boost::container::new_allocator<boost::container::base_node<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, boost::container::dtl::list_hook<void*>, false> > > > (disposer=..., i=..., this=0x7f9e617efac0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/intrusive/list.hpp:684

#23 boost::container::list<std::unique_ptr<Checkpoint, std::default_delete<Checkpoint> >, void>::erase (p=..., this=0x7f9e617efac0)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/boost.exploded/include/boost/container/list.hpp:972

#24 CheckpointDestroyerTask::run (this=0x7f9e11995a90) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/checkpoint_remover.cc:52

#25 0x0000000000bd30bd in GlobalTask::execute (this=0x7f9e11995a90, threadName=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:104

#26 0x0000000000bd12ea in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f9e617efe00) at /opt/gcc-10.2.0/include/c++/10.2.0/string_view:133

#27 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:363

#28 0x0000000000bd4256 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f9e617efe00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:390

#29 operator() (__closure=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#30 0x0000000000d6c65f in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f9e617f00b0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#31 folly::ThreadPoolExecutor::runTask (this=this@entry=0x7f9efcebfd00, thread=..., task=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#32 0x0000000000d55357 in folly::CPUThreadPoolExecutor::threadRun (this=0x7f9efcebfd00, thread=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

#33 0x0000000000d6f1fa in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, 

    __t=<optimized out>, __f=<optimized out>, __t=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:73

#34 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

#35 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

#36 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

#37 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

#38 0x0000000000bcd4f4 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f9efcd2c800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:390

#39 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f9efcd2c800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#40 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:377

#41 0x00007f9f01701f00 in std::execute_native_thread_routine (__p=0x7f9efcf75630) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

#42 0x00007f9f012a8044 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442

#43 0x00007f9f013285fc in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

(gdb)