Ephemeral HTCleaner may remove in-flight SyncWrite from the HashTable

Node 172.23.107.63 - Core dump seen: 1

172.23.107.63 : Stack Trace of first crash: c1d8239e-ad57-4f6d-74ec5b87-4acca00c.dmp

Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.

 #0  0x00007efc2c8a8207 in raise () from /lib64/libc.so.6

 #0  0x00007efc2c8a8207 in raise () from /lib64/libc.so.6

 #1  0x00007efc2c8a98f8 in abort () from /lib64/libc.so.6

 #2  0x00007efc2d405195 in __gnu_cxx::__verbose_terminate_handler() () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #3  0x000000000054ebb2 in backtrace_terminate_handler() ()

 #4  0x00007efc2d402f86 in __cxxabiv1::__terminate(void (*)()) () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #5  0x00007efc2d402fd1 in std::terminate() () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #6  0x00007efc2d403213 in __cxa_throw () from /opt/couchbase/bin/../lib/libstdc++.so.6

 #7  0x00007efc30c0cc60 in void ActiveDurabilityMonitor::throwException<std::logic_error>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const () from /opt/couchbase/bin/../lib/libep.so

 #8  0x00007efc30c07546 in ActiveDurabilityMonitor::commit(DurabilityMonitor::ActiveSyncWrite const&) () from /opt/couchbase/bin/../lib/libep.so

 #9  0x00007efc30c07683 in ActiveDurabilityMonitor::processCompletedSyncWriteQueue() () from /opt/couchbase/bin/../lib/libep.so

 #10 0x00007efc30cd88c0 in VBucket::processResolvedSyncWrites() () from /opt/couchbase/bin/../lib/libep.so

 #11 0x00007efc30c0e68d in DurabilityCompletionTask::run() () from /opt/couchbase/bin/../lib/libep.so

 #12 0x00007efc30c73583 in GlobalTask::execute() () from /opt/couchbase/bin/../lib/libep.so

 #13 0x00007efc30b867df in CB3ExecutorThread::run() () from /opt/couchbase/bin/../lib/libep.so

 #14 0x00007efc2f609777 in platform_thread_wrap(void*) () from /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0

 #15 0x00007efc2cc46dd5 in start_thread () from /lib64/libpthread.so.0

 #16 0x00007efc2c96fead in clone () from /lib64/libc.so.6

[root@sa1601-cnt76 ~]# grep CRITICAL /opt/couchbase/var/lib/couchbase/logs/memcached.log.00000*

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.039657-07:00 CRITICAL *** Fatal error encountered during exception handling ***

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.039712-07:00 CRITICAL Caught unhandled std::exception-derived exception. what(): ActiveDurabilityMonitor::commit vb:672 failed with status:1

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191278-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.0-2908). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/c1d8239e-ad57-4f6d-74ec5b87-4acca00c.dmp before terminating.

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191292-07:00 CRITICAL Stack backtrace of crashed thread:

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191470-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x13e11d]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191482-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x1541ea]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191491-07:00 CRITICAL     /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x154528]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191500-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7efc2cc3f000+0xf5d0]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191523-07:00 CRITICAL     /lib64/libc.so.6(gsignal+0x37) [0x7efc2c872000+0x36207]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191544-07:00 CRITICAL     /lib64/libc.so.6(abort+0x148) [0x7efc2c872000+0x378f8]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191575-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6(_ZN9__gnu_cxx27__verbose_terminate_handlerEv+0x125) [0x7efc2d374000+0x91195]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191585-07:00 CRITICAL     /opt/couchbase/bin/memcached() [0x400000+0x14ebb2]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191601-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7efc2d374000+0x8ef86]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191616-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7efc2d374000+0x8efd1]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191632-07:00 CRITICAL     /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7efc2d374000+0x8f213]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191643-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0xf0c60]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191652-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0xeb546]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191658-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0xeb683]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191668-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0x1bc8c0]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191675-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0xf268d]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191683-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0x157583]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191690-07:00 CRITICAL     /opt/couchbase/bin/../lib/libep.so() [0x7efc30b1c000+0x6a7df]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191696-07:00 CRITICAL     /opt/couchbase/bin/../lib/libplatform_so.so.0.1.0() [0x7efc2f5f9000+0x10777]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191704-07:00 CRITICAL     /lib64/libpthread.so.0() [0x7efc2cc3f000+0x7dd5]

/opt/couchbase/var/lib/couchbase/logs/memcached.log.000005.txt:2020-08-22T18:42:52.191736-07:00 CRITICAL     /lib64/libc.so.6(clone+0x6d) [0x7efc2c872000+0xfdead]

num_items=5000,doc_size=512,transaction_timeout=30,nodes_init=2,replicas=1,bucket_size=100,GROUP=P0_Set1,durability=MAJORITY,get-cbcollect-info=False,infra_log_level=critical,log_level=error,upgrade_version=7.0.0-2908 -t Atomicity.basic_ops.basic_ops.test_basic_commit,nodes_init=2,replicas=1,transaction_commit=True,op_type=create;update,bucket_type=ephemeral,num_buckets=2,default_bucket=False,GROUP=P0_Set1