[Magma] - Memcached crashes seen for graceful failover + delta recovery + rebalance + collections CRUD on magma backend

./sequoia -client 172.23.97.101:2375 -provider file:centos_component.yml -test tests/rebalance/test_allRebalance_collections.yml -scope tests/rebalance/scope_rebalance_collection.yml -scale 10 -repeat 0 -log_level 0 -version 7.0.0-3009 -skip_setup=false -skip_test=false -skip_teardown=true -skip_cleanup=false -continue=false -collect_on_error=false -stop_on_error=false -duration=0 -show_topology=true

[2020-09-04T05:37:51-07:00, sequoiatools/couchbase-cli:7.0:bea0bb] failover -c 172.23.104.16:8091 --server-failover 172.23.104.17:8091 -u Administrator -p password

[2020-09-04T05:42:07-07:00, sequoiatools/couchbase-cli:7.0:844cb9] recovery -c 172.23.104.16:8091 --server-recovery 172.23.104.17:8091 --recovery-type delta -u Administrator -p password

[2020-09-04T05:42:14-07:00, sequoiatools/couchbase-cli:7.0:ee59f8] rebalance -c 172.23.104.16:8091 -u Administrator -p password

[root@cnt7-qe1 logs]# grep CRITICAL memcached.log.0000*

memcached.log.000011.txt:2020-09-04T05:36:16.874849-07:00 CRITICAL *** Fatal error encountered during exception handling ***

memcached.log.000011.txt:2020-09-04T05:36:16.874906-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): Tree Snapshot:

memcached.log.000011.txt:2020-09-04T05:36:17.118079-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.0-3009). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/90aeb100-0407-471f-570193ad-a44a8b9e.dmp before terminating.

memcached.log.000011.txt:2020-09-04T05:36:17.118102-07:00 CRITICAL Stack backtrace of crashed thread:

memcached.log.000011.txt:2020-09-04T05:36:17.118410-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x13e2fd]

memcached.log.000011.txt:2020-09-04T05:36:17.118428-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x1543ea]

memcached.log.000011.txt:2020-09-04T05:36:17.118438-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x154728]

memcached.log.000011.txt:2020-09-04T05:36:17.118448-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7fe95cc98000+0xf5e0]

memcached.log.000011.txt:2020-09-04T05:36:17.118480-07:00 CRITICAL     /lib64/libc.so.6(gsignal+0x37) [0x7fe95c8d5000+0x351f7]

memcached.log.000011.txt:2020-09-04T05:36:17.118507-07:00 CRITICAL     /lib64/libc.so.6(abort+0x148) [0x7fe95c8d5000+0x368e8]

memcached.log.000011.txt:2020-09-04T05:36:17.118936-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6(_ZN9__gnu_cxx27__verbose_terminate_handlerEv+0x125) [0x7fe95d3cd000+0x91195]

memcached.log.000011.txt:2020-09-04T05:36:17.118954-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x14edb2]

memcached.log.000011.txt:2020-09-04T05:36:17.118981-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fe95d3cd000+0x8ef86]

memcached.log.000011.txt:2020-09-04T05:36:17.119007-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fe95d3cd000+0x8efd1]

memcached.log.000011.txt:2020-09-04T05:36:17.119033-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fe95d3cd000+0x8f213]

memcached.log.000011.txt:2020-09-04T05:36:17.119042-07:00 CRITICAL     /opt/couchbase/bin/../lib/libmagma.so() [0x7fe960586000+0x783da]

memcached.log.000011.txt:2020-09-04T05:36:17.119050-07:00 CRITICAL     /opt/couchbase/bin/../lib/libmagma.so() [0x7fe960586000+0x67bc4]

memcached.log.000011.txt:2020-09-04T05:36:17.119057-07:00 CRITICAL     /opt/couchbase/bin/../lib/libmagma.so() [0x7fe960586000+0x6a136]

memcached.log.000011.txt:2020-09-04T05:36:17.119068-07:00 CRITICAL     /opt/couchbase/bin/../lib/libmagma.so() [0x7fe960586000+0xb4ccc]

memcached.log.000011.txt:2020-09-04T05:36:17.119078-07:00 CRITICAL     /opt/couchbase/bin/../lib/libmagma.so(_ZN5magma5Magma4Impl14CompactKVStoreEtRKNS_5SliceES4_St8functionIFSt10unique_ptrINS0_18CompactionCallbackESt14default_deleteIS7_EEvEE+0x236) [0x7fe960586000+0x886c6]

memcached.log.000011.txt:2020-09-04T05:36:17.119091-07:00 CRITICAL     /opt/couchbase/bin/../lib/libmagma.so(_ZN5magma5Magma14CompactKVStoreEtRKNS_5SliceES3_St8functionIFSt10unique_ptrINS0_18CompactionCallbackESt14default_deleteIS6_EEvEE+0x45) [0x7fe960586000+0x88915]

memcached.log.000011.txt:2020-09-04T05:36:17.119111-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x24bb40]

memcached.log.000011.txt:2020-09-04T05:36:17.119120-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x24cc92]

memcached.log.000011.txt:2020-09-04T05:36:17.119130-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x113fff]

memcached.log.000011.txt:2020-09-04T05:36:17.119138-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x114304]

memcached.log.000011.txt:2020-09-04T05:36:17.119148-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x1c6728]

memcached.log.000011.txt:2020-09-04T05:36:17.119157-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x1725f3]

memcached.log.000011.txt:2020-09-04T05:36:17.119166-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7fe960b76000+0x78faf]

memcached.log.000011.txt:2020-09-04T05:36:17.119177-07:00 CRITICAL     /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7fe95f652000+0x10777]

memcached.log.000011.txt:2020-09-04T05:36:17.119185-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7fe95cc98000+0x7e25]

memcached.log.000011.txt:2020-09-04T05:36:17.119222-07:00 CRITICAL     /lib64/libc.so.6(clone+0x6d) [0x7fe95c8d5000+0xf834d]

[root@cnt7-qe1 logs]#