[System Test] :- CRITICAL Caught unhandled std::exception-derived exception. what(): ThrowExceptionUnderflowPolicy current:0 arg:-23

./sequoia -client 172.23.104.168:2375 -provider file:centos_second_cluster.yml -test tests/integration/7.2/test_7.2.yml -scope tests/integration/7.2/scope_7.2_magma.yml -scale 1 -repeat 0 -log_level 0 -version 7.6.0-1627 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=1209600 -show_topology=true

2023-10-11T08:55:43.697070-07:00 CRITICAL *** Fatal error encountered during exception handling ***

2023-10-11T08:55:43.697101-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): ThrowExceptionUnderflowPolicy current:0 arg:-23

2023-10-11T08:55:43.697104-07:00 CRITICAL Exception thrown from:

2023-10-11T08:55:43.697159-07:00 CRITICAL     #0  /opt/couchbase/bin/memcached() [0x400000+0x13a6b7]

2023-10-11T08:55:43.697202-07:00 CRITICAL     #1  /opt/couchbase/bin/memcached() [0x400000+0x599c38]

2023-10-11T08:55:43.697235-07:00 CRITICAL     #2  /opt/couchbase/bin/memcached() [0x400000+0x59a41f]

2023-10-11T08:55:43.697256-07:00 CRITICAL     #3  /opt/couchbase/bin/memcached() [0x400000+0x57975d]

2023-10-11T08:55:43.697279-07:00 CRITICAL     #4  /opt/couchbase/bin/memcached() [0x400000+0x589fb7]

2023-10-11T08:55:43.697309-07:00 CRITICAL     #5  /opt/couchbase/bin/memcached() [0x400000+0x6c47d2]

2023-10-11T08:55:43.697347-07:00 CRITICAL     #6  /opt/couchbase/bin/memcached(_ZN5magma5Magma4Impl9WriteDocsEtRKSt6vectorINS0_14WriteOperationESaIS3_EEjSt8functionIFvRKS3_bNS_5SliceEEES8_IFSt4pairINS_6StatusEPS6_EvEES8_IFbSA_RS3_EENS0_11HistoryModeE+0x32c) [0x400000+0x6778bc]

2023-10-11T08:55:43.697378-07:00 CRITICAL     #7  /opt/couchbase/bin/memcached(_ZN5magma5Magma9WriteDocsEtRKSt6vectorINS0_14WriteOperationESaIS2_EEjSt8functionIFvRKS2_bNS_5SliceEEES7_IFSt4pairINS_6StatusEPS5_EvEES7_IFbS9_RS2_EENS0_11HistoryModeE+0x13e) [0x400000+0x677bfe]

2023-10-11T08:55:43.697409-07:00 CRITICAL     #8  /opt/couchbase/bin/memcached() [0x400000+0x58e235]

2023-10-11T08:55:43.697438-07:00 CRITICAL     #9  /opt/couchbase/bin/memcached() [0x400000+0x580a8b]

2023-10-11T08:55:43.697461-07:00 CRITICAL     #10 /opt/couchbase/bin/memcached() [0x400000+0x5665bb]

2023-10-11T08:55:43.697483-07:00 CRITICAL     #11 /opt/couchbase/bin/memcached() [0x400000+0x4ec755]

2023-10-11T08:55:43.697504-07:00 CRITICAL     #12 /opt/couchbase/bin/memcached() [0x400000+0x4f1def]

2023-10-11T08:55:43.697524-07:00 CRITICAL     #13 /opt/couchbase/bin/memcached() [0x400000+0x4f2b57]

2023-10-11T08:55:43.697539-07:00 CRITICAL     #14 /opt/couchbase/bin/memcached() [0x400000+0x37509f]

2023-10-11T08:55:43.697556-07:00 CRITICAL     #15 /opt/couchbase/bin/memcached() [0x400000+0x3752e8]

2023-10-11T08:55:43.697592-07:00 CRITICAL     #16 /opt/couchbase/bin/memcached() [0x400000+0x7a817a]

2023-10-11T08:55:43.697615-07:00 CRITICAL     #17 /opt/couchbase/bin/memcached() [0x400000+0x7a61aa]

2023-10-11T08:55:43.697637-07:00 CRITICAL     #18 /opt/couchbase/bin/memcached() [0x400000+0x7a94c3]

2023-10-11T08:55:43.697682-07:00 CRITICAL     #19 /opt/couchbase/bin/memcached() [0x400000+0x94c1bf]

2023-10-11T08:55:43.697720-07:00 CRITICAL     #20 /opt/couchbase/bin/memcached() [0x400000+0x934ce7]

2023-10-11T08:55:43.697751-07:00 CRITICAL     #21 /opt/couchbase/bin/memcached() [0x400000+0x94ed5a]

2023-10-11T08:55:43.697781-07:00 CRITICAL     #22 /opt/couchbase/bin/memcached() [0x400000+0x7a2590]

2023-10-11T08:55:43.697849-07:00 CRITICAL     #23 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7f390e02f000+0xe4aa3]

2023-10-11T08:55:43.697859-07:00 CRITICAL     #24 /lib64/libpthread.so.0() [0x7f390e98a000+0x7e25]

Dwarf Error: wrong version in compilation unit header (is 5, should be 2, 3, or 4) [in module /usr/lib/debug/opt/couchbase/bin/memcached-7.6.0-1627.x86_64.debug]

(gdb) bt

#0  0x00007f390d99f1f7 in __GI_____strtod_l_internal (nptr=<optimized out>, endptr=<optimized out>, group=<optimized out>, loc=<optimized out>) at strtod_l.c:1290

#1  0x3935383537353635 in ?? ()

#2  0x3336323631363036 in ?? ()

#3  0x3736363635363436 in ?? ()

#4  0x3137303739363836 in ?? ()

#5  0x3537343733373237 in ?? ()

#6  0x3937383737373637 in ?? ()

#7  0x3338323831383038 in ?? ()

#8  0x3738363835383438 in ?? ()

#9  0x3139303939383838 in ?? ()

#10 0x3539343933393239 in ?? ()

#11 0x3939383937393639 in ?? ()

#12 0x00007f378170e000 in ?? ()

#13 0x00007f37cdfd7870 in ?? ()

#14 0x00007f377f3623c0 in ?? ()

#15 0x00007f37cdfd85c0 in ?? ()

#16 0x00007f37cdfd7560 in ?? ()

#17 0x00007f37cdfd74f0 in ?? ()

#18 0xffffffffffffb771 in ?? ()

#19 0xffffffffffffffe9 in ?? ()

#20 0x000000000099a41f in cb::NonNegativeCounter<unsigned long, cb::ThrowExceptionUnderflowPolicy>::store (desired=<optimized out>, this=0x3734363435343434)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/synchronization/SharedMutex.h:575

#21 cb::NonNegativeCounter<unsigned long, cb::ThrowExceptionUnderflowPolicy>::NonNegativeCounter (initial=<optimized out>, this=0x3734363435343434) at /opt/gcc-13.2.0/include/c++/13.2.0/x86_64-pc-linux-gnu/bits/bitset:81

#22 Collections::VB::PersistedStats::PersistedStats (diskSize=3545793286775257140, highSeqno=168036, itemCount=<optimized out>, this=0x3734363435343434) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/atomic.h:27

#23 Collections::VB::Flush::saveCollectionStats(std::function<void (CollectionID, Collections::VB::PersistedStats const&)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/flush.cc:53

#24 0x000000000097975d in operator() (__closure=0x7f377e353260) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1613

#25 std::__invoke_impl<std::pair<magma::Status, const std::vector<magma::Magma::WriteOperation>*>, MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode)::<lambda()>&> (__f=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/storage_common/storage_common/kvstore_iface.h:61

#26 std::__invoke_r<std::pair<magma::Status, const std::vector<magma::Magma::WriteOperation>*>, MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode)::<lambda()>&> (__fn=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/storage_common/storage_common/kvstore_iface.h:116

#27 std::_Function_handler<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> (), MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/kv_magma_common/magma-kvstore_rollback_purge_seqno_ctx.h:291

#28 0x0000000000989fb7 in std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>::operator()() const (this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/HdrHistogram_c/src/gsl-lite.hpp:247

#29 operator() (__closure=0x7f37cdfd7a70, __closure@entry=<error reading variable: dwarf2_find_location_expression: Corrupted DWARF expression.>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-memory-tracking-proxy.cc:533

#30 std::__invoke_impl<std::pair<magma::Status, const std::vector<magma::Magma::WriteOperation>*>, MagmaMemoryTrackingProxy::WriteDocs(magma::Magma::KVStoreID, const std::vector<magma::Magma::WriteOperation>&, magma::Magma::KVStoreRevision, magma::Magma::HistoryMode, magma::Magma::WriteDocsCallback, magma::Magma::PostWriteDocsCallback, magma::Magma::DocTransformCallback)::<lambda()>&> (__f=..., __f@entry=<error reading variable: dwarf2_find_location_expression: Corrupted DWARF expression.>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/hdrhistogram/cb_arena_malloc.h:61

#31 std::__invoke_r<std::pair<magma::Status, const std::vector<magma::Magma::WriteOperation>*>, MagmaMemoryTrackingProxy::WriteDocs(magma::Magma::KVStoreID, const std::vector<magma::Magma::WriteOperation>&, magma::Magma::KVStoreRevision, magma::Magma::HistoryMode, magma::Magma::WriteDocsCallback, magma::Magma::PostWriteDocsCallback, magma::Magma::DocTransformCallback)::<lambda()>&> (__fn=..., __fn@entry=<error reading variable: dwarf2_find_location_expression: Corrupted DWARF expression.>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/include/hdrhistogram/cb_arena_malloc.h:116

#32 std::_Function_handler<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> (), MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, magma::Magma::HistoryMode, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/HdrHistogram_c/src/gsl-lite.hpp:291

#33 0x0000000000ac47d2 in std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>::operator()() const (

    this=<error reading variable: dwarf2_find_location_expression: Corrupted DWARF expression.>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/write.cc:309

#34 magma::KVStore::WriteDocs(magma::WAL*, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>, magma::Magma::HistoryMode, bool) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/write.cc:309

#35 0x0000000000a778bc in magma::Magma::Impl::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/stl_algobase.h:247

#36 0x0000000000a77bfe in magma::Magma::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>, magma::Magma::HistoryMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/stl_algobase.h:247

#37 0x000000000098e235 in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, magma::Magma::HistoryMode, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<std::pair<magma::Status, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const*> ()>, std::function<bool (magma::Magma::WriteOperation const&, magma::Magma::WriteOperation&)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/HdrHistogram_c/src/gsl-lite.hpp:247

#38 0x0000000000980a8b in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&, magma::Magma::HistoryMode) () at /opt/gcc-13.2.0/include/c++/13.2.0/x86_64-pc-linux-gnu/bits/atomic:62

#39 0x00000000009665bb in MagmaKVStore::commit(std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/item.h:850

#40 0x00000000008ec755 in EPBucket::commit(KVStoreIface&, std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:968

---Type <return> to continue, or q <return> to quit---

#41 0x00000000008f1def in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:840

#42 0x00000000008f2b57 in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:392

#43 0x000000000077509f in Flusher::flushVB (this=0x7f3781c0d500) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:284

#44 0x00000000007752e8 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:204

#45 0x0000000000ba817a in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:105

#46 0x0000000000ba61aa in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f37cdfd8d40) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:314

#47 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/char_traits.h:363

#48 0x0000000000ba94c3 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f37cdfd8d40, this@entry=<error reading variable: dwarf2_find_location_expression: Corrupted DWARF expression.>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#49 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

#50 0x0000000000d4c1bf in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f37cdfd8ff0) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#51 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) ()

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:98

#52 0x0000000000d34ce7 in folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) ()

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:306

#53 0x0000000000d4ed5a in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__f=<optimized out>, 

    __t=<optimized out>, __f=<optimized out>, __t=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:100

#54 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95

#55 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/functional:416

#56 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>)

    at /opt/gcc-10.2.0/include/c++/10.2.0/functional:499

#57 folly::detail::function::FunctionTraits<void ()>::callSmall<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:363

#58 0x0000000000ba2590 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=<error reading variable: dwarf2_find_location_expression: Corrupted DWARF expression.>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#59 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x7f390827a740) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

#60 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/third_party/gsl-lite/include/gsl/char_traits.h:377

#61 0x00007f390e113aa3 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-13.2.0/libstdc++-v3/src/c++11/thread.cc:104

#62 0x00007f390e991e25 in start_thread (arg=0x0) at pthread_create.c:282

#63 0x00007f390da6234d in putspent (p=0x0, stream=0x7f37cdfe3700) at putspent.c:64

#64 0x0000000000000000 in ?? ()

(gdb) quit