[System Test] :- Minidumps seen on longevity cluster

./sequoia -client 172.23.104.27:2375 -provider file:debian_pine.yml -test tests/integration/7.6/test_7.6.yml -scope tests/integration/7.6/scope_7.6_magma.yml -scale 3 -repeat 0 -log_level 0 -version 7.6.0-1812 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=1209600 -show_topology=true

memcached.log.000054.txt:2023-11-17T17:41:13.607112-08:00 CRITICAL Detected previous crash

memcached.log.000054.txt:2023-11-17T17:41:13.607160-08:00 CRITICAL Breakpad caught a crash (Couchbase version 7.6.0-1812). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/be37ce38-c6c1-431e-855ee6a8-c13efbe6.dmp before terminating. Writing dump succeeded: yes

memcached.log.000054.txt:2023-11-17T17:41:13.607162-08:00 CRITICAL Stack backtrace of crashed thread:

memcached.log.000054.txt:2023-11-17T17:41:13.607163-08:00 CRITICAL    #0  /opt/couchbase/bin/memcached() [0x400000+0x852211]

memcached.log.000054.txt:2023-11-17T17:41:13.607164-08:00 CRITICAL    #1  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x385) [0x400000+0x86fc75]

memcached.log.000054.txt:2023-11-17T17:41:13.607164-08:00 CRITICAL    #2  /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0x9f) [0x400000+0x86ffbf]

memcached.log.000054.txt:2023-11-17T17:41:13.607166-08:00 CRITICAL    #3  /lib/x86_64-linux-gnu/libc.so.6() [0x7fc63541e000+0x3c510]

memcached.log.000054.txt:2023-11-17T17:41:13.607188-08:00 CRITICAL    #4  /lib/x86_64-linux-gnu/libc.so.6() [0x7fc63541e000+0x8a0fc]

memcached.log.000054.txt:2023-11-17T17:41:13.607189-08:00 CRITICAL    #5  /lib/x86_64-linux-gnu/libc.so.6(gsignal+0x12) [0x7fc63541e000+0x3c472]

memcached.log.000054.txt:2023-11-17T17:41:13.607190-08:00 CRITICAL    #6  /lib/x86_64-linux-gnu/libc.so.6(abort+0xd3) [0x7fc63541e000+0x264b2]

memcached.log.000054.txt:2023-11-17T17:41:13.607190-08:00 CRITICAL    #7  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xa89ab]

memcached.log.000054.txt:2023-11-17T17:41:13.607191-08:00 CRITICAL    #8  /opt/couchbase/bin/memcached() [0x400000+0x861e66]

memcached.log.000054.txt:2023-11-17T17:41:13.607192-08:00 CRITICAL    #9  /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xb82fa]

memcached.log.000054.txt:2023-11-17T17:41:13.607193-08:00 CRITICAL    #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xb8365]

memcached.log.000054.txt:2023-11-17T17:41:13.607196-08:00 CRITICAL    #11 /opt/couchbase/bin/../lib/libstdc++.so.6(__cxa_rethrow+0x48) [0x7fc632600000+0xb8608]

memcached.log.000054.txt:2023-11-17T17:41:13.607197-08:00 CRITICAL    #12 /opt/couchbase/bin/memcached() [0x400000+0x945c3e]

memcached.log.000054.txt:2023-11-17T17:41:13.607198-08:00 CRITICAL    #13 /opt/couchbase/bin/memcached() [0x400000+0x13d605]

memcached.log.000054.txt:2023-11-17T17:41:13.607200-08:00 CRITICAL    #14 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xb82fa]

memcached.log.000054.txt:2023-11-17T17:41:13.607202-08:00 CRITICAL    #15 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xb8365]

memcached.log.000054.txt:2023-11-17T17:41:13.607203-08:00 CRITICAL    #16 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xb9063]

memcached.log.000054.txt:2023-11-17T17:41:13.607205-08:00 CRITICAL    #17 /opt/couchbase/bin/memcached() [0x400000+0x20cd6e]

memcached.log.000054.txt:2023-11-17T17:41:13.607206-08:00 CRITICAL    #18 /opt/couchbase/bin/memcached() [0x400000+0x227432]

memcached.log.000054.txt:2023-11-17T17:41:13.607209-08:00 CRITICAL    #19 /opt/couchbase/bin/memcached() [0x400000+0x227519]

memcached.log.000054.txt:2023-11-17T17:41:13.607210-08:00 CRITICAL    #20 /opt/couchbase/bin/memcached() [0x400000+0x20750f]

memcached.log.000054.txt:2023-11-17T17:41:13.607212-08:00 CRITICAL    #21 /opt/couchbase/bin/memcached() [0x400000+0x21cd6c]

memcached.log.000054.txt:2023-11-17T17:41:13.607234-08:00 CRITICAL    #22 /opt/couchbase/bin/memcached() [0x400000+0x1cf7e4]

memcached.log.000054.txt:2023-11-17T17:41:13.607236-08:00 CRITICAL    #23 /opt/couchbase/bin/memcached() [0x400000+0x21e9ea]

memcached.log.000054.txt:2023-11-17T17:41:13.607237-08:00 CRITICAL    #24 /opt/couchbase/bin/../lib/libevent_core-2.1.so.7() [0x7fc633c00000+0xf84e]

memcached.log.000054.txt:2023-11-17T17:41:13.607238-08:00 CRITICAL    #25 /opt/couchbase/bin/../lib/libevent_core-2.1.so.7() [0x7fc633c00000+0x18b99]

memcached.log.000054.txt:2023-11-17T17:41:13.607239-08:00 CRITICAL    #26 /opt/couchbase/bin/../lib/libevent_core-2.1.so.7(event_base_loop+0x357) [0x7fc633c00000+0x19287]

memcached.log.000054.txt:2023-11-17T17:41:13.607240-08:00 CRITICAL    #27 /opt/couchbase/bin/memcached() [0x400000+0x97bcc6]

memcached.log.000054.txt:2023-11-17T17:41:13.607242-08:00 CRITICAL    #28 /opt/couchbase/bin/memcached() [0x400000+0x97c1ce]

memcached.log.000054.txt:2023-11-17T17:41:13.607243-08:00 CRITICAL    #29 /opt/couchbase/bin/memcached() [0x400000+0x97e698]

memcached.log.000054.txt:2023-11-17T17:41:13.607244-08:00 CRITICAL    #30 /opt/couchbase/bin/memcached() [0x400000+0x1cf0d9]

memcached.log.000054.txt:2023-11-17T17:41:13.607245-08:00 CRITICAL    #31 /opt/couchbase/bin/memcached() [0x400000+0x8ce91c]

memcached.log.000054.txt:2023-11-17T17:41:13.607246-08:00 CRITICAL    #32 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fc632600000+0xe4aa3]

memcached.log.000054.txt:2023-11-17T17:41:13.607247-08:00 CRITICAL    #33 /lib/x86_64-linux-gnu/libc.so.6() [0x7fc63541e000+0x883ec]

memcached.log.000054.txt:2023-11-17T17:41:13.607263-08:00 CRITICAL    #34 /lib/x86_64-linux-gnu/libc.so.6() [0x7fc63541e000+0x108a4c]

(gdb) bt

#0  __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6, no_tid=no_tid@entry=0) at ./nptl/pthread_kill.c:44

#1  0x00007fc6354a815f in __pthread_kill_internal (signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:78

#2  0x00007fc63545a472 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26

#3  0x00007fc6354444b2 in __GI_abort () at ./stdlib/abort.c:79

#4  0x00007fc6326a89ab in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

#5  0x0000000000c61e66 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

#6  0x00007fc6326b82fa in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#7  0x00007fc6326b8365 in std::terminate () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

#8  0x00007fc6326b8608 in __cxxabiv1::__cxa_rethrow () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/eh_throw.cc:136

#9  0x0000000000d45c3e in __cxxabiv1::__cxa_rethrow () at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp:119

#10 0x000000000053d605 in log_handled_exception () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:39

#11 backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:79

#12 0x00007fc6326b82fa in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

#13 0x00007fc6326b8365 in std::terminate () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58

#14 0x00007fc6326b9063 in __cxxabiv1::__cxa_pure_virtual () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/libsupc++/pure.cc:50

#15 0x000000000060cd6e in Connection::to_json[abi:cxx11]() const (this=0x7fc62d5fcd80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:226

#16 0x0000000000627432 in Cookie::~Cookie (this=0x7fc62d22f000, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:775

#17 0x0000000000627519 in Cookie::~Cookie (this=0x7fc62d22f000, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/cookie.cc:785

#18 0x000000000060750f in std::default_delete<Cookie>::operator() (__ptr=<optimized out>, this=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:99

#19 std::unique_ptr<Cookie, std::default_delete<Cookie> >::~unique_ptr (this=0x7fc54f03ce00, __in_chrg=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:404

#20 std::_Destroy<std::unique_ptr<Cookie, std::default_delete<Cookie> > > (__pointer=0x7fc54f03ce00) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_construct.h:151

#21 std::_Destroy_aux<false>::__destroy<std::unique_ptr<Cookie, std::default_delete<Cookie> >*> (__last=<optimized out>, __first=0x7fc54f03ce00) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_construct.h:163

#22 std::_Destroy<std::unique_ptr<Cookie, std::default_delete<Cookie> >*> (__last=<optimized out>, __first=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_construct.h:196

#23 std::_Destroy<std::unique_ptr<Cookie, std::default_delete<Cookie> >*, std::unique_ptr<Cookie, std::default_delete<Cookie> > > (__last=<optimized out>, __first=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/alloc_traits.h:947

#24 std::deque<std::unique_ptr<Cookie, std::default_delete<Cookie> >, std::allocator<std::unique_ptr<Cookie, std::default_delete<Cookie> > > >::_M_destroy_data_aux (this=<optimized out>, __last=..., __first=...)

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/deque.tcc:875

#25 std::deque<std::unique_ptr<Cookie, std::default_delete<Cookie> >, std::allocator<std::unique_ptr<Cookie, std::default_delete<Cookie> > > >::_M_destroy_data (this=0x7fc62d5fcdc8, __first=..., __last=...)

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_deque.h:2091

#26 std::deque<std::unique_ptr<Cookie, std::default_delete<Cookie> >, std::allocator<std::unique_ptr<Cookie, std::default_delete<Cookie> > > >::~deque (this=0x7fc62d5fcdc8, __in_chrg=<optimized out>)

    at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_deque.h:1028

#27 Connection::~Connection (this=this@entry=0x7fc62d5fcd80, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection.cc:1263

#28 0x000000000061cd6c in LibeventConnection::~LibeventConnection (this=0x7fc62d5fcd80, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection_libevent.cc:76

#29 LibeventConnection::~LibeventConnection (this=0x7fc62d5fcd80, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection_libevent.cc:76

#30 0x00000000005cf7e4 in std::default_delete<Connection>::operator() (__ptr=<optimized out>, this=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:99

#31 std::unique_ptr<Connection, std::default_delete<Connection> >::~unique_ptr (this=0x7fc62fd00290, __in_chrg=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/unique_ptr.h:404

#32 std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >::~pair (this=0x7fc62fd00288, __in_chrg=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/stl_pair.h:187

#33 std::__new_allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, false> >::destroy<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > > > (__p=0x7fc62fd00288, this=<synthetic pointer>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/new_allocator.h:194

#34 std::allocator_traits<std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, false> > >::destroy<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > > > (__p=0x7fc62fd00288, __a=<synthetic pointer>...) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/alloc_traits.h:557

#35 std::_Node_handle_common<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, false> > >::_M_reset (this=<synthetic pointer>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/node_handle.h:166

#36 std::_Node_handle_common<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, false> > >::~_Node_handle_common (this=<synthetic pointer>, __in_chrg=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/node_handle.h:89

#37 std::_Node_handle<Connection*, std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, std::allocator<std::__detail::_Hash_node<std::pair<Connection* const, std::unique_ptr<Connection, std::default_delete<Connection> > >, false> > >::~_Node_handle (this=<synthetic pointer>, __in_chrg=<optimized out>) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/node_handle.h:243

#38 FrontEndThread::destroy_connection (this=<optimized out>, connection=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:332

#39 0x000000000061e9ea in LibeventConnection::read_callback (this=0x7fc62d5fcd80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/connection_libevent.cc:136

#40 0x00007fc633c0f84e in bufferevent_run_deferred_callbacks_unlocked (cb=<optimized out>, arg=0x7fc62d5fb980) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/bufferevent.c:208

#41 0x00007fc633c18b99 in event_process_active_single_queue (base=base@entry=0x7fc62f865200, activeq=0x7fc6303ec0e0, max_to_process=max_to_process@entry=2147483647, endtime=endtime@entry=0x0)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1726

#42 0x00007fc633c19287 in event_process_active (base=0x7fc62f865200) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1789

#43 event_base_loop (base=0x7fc62f865200, flags=flags@entry=1) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:2012

#44 0x0000000000d7bcc6 in (anonymous namespace)::EventBaseBackend::eb_event_base_loop (flags=1, this=<optimized out>) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:74

#45 folly::EventBase::loopBody (this=this@entry=0x7fc62fd58c80, flags=flags@entry=0, ignoreKeepAlive=ignoreKeepAlive@entry=false) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:381

#46 0x0000000000d7c1ce in folly::EventBase::loop (this=this@entry=0x7fc62fd58c80) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:305

#47 0x0000000000d7e698 in folly::EventBase::loopForever (this=this@entry=0x7fc62fd58c80) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/io/async/EventBase.cpp:524

#48 0x00000000005cf0d9 in worker_libevent (arg=0x7fc62fd58c70) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/thread.cc:272

#49 0x0000000000cce91c in std::function<void ()>::operator()() const (this=0x7fc62fd07838) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_function.h:591

#50 operator() (__closure=0x7fc62fd07818) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/thread.cc:117

#51 std::__invoke_impl<void, create_thread(std::function<void()>, std::string)::<lambda()> > (__f=...) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/invoke.h:61

#52 std::__invoke<create_thread(std::function<void()>, std::string)::<lambda()> > (__fn=...) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/invoke.h:96

#53 std::thread::_Invoker<std::tuple<create_thread(std::function<void()>, std::string)::<lambda()> > >::_M_invoke<0> (this=0x7fc62fd07818) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_thread.h:292

#54 std::thread::_Invoker<std::tuple<create_thread(std::function<void()>, std::string)::<lambda()> > >::operator() (this=0x7fc62fd07818) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_thread.h:299

--Type <RET> for more, q to quit, c to continue without paging--

#55 std::thread::_State_impl<std::thread::_Invoker<std::tuple<create_thread(std::function<void()>, std::string)::<lambda()> > > >::_M_run(void) (this=0x7fc62fd07810) at /opt/gcc-13.2.0/include/c++/13.2.0/bits/std_thread.h:244

#56 0x00007fc6326e4aa3 in std::execute_native_thread_routine (__p=0x7fc62fd07810) at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/src/c++11/thread.cc:104

#57 0x00007fc6354a63ec in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:444

#58 0x00007fc635526a4c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81