[Ephemeral] ThrowExceptionUnderflowPolicy on BasicLinkedList::numDeletedItems in frontend connection run-loop

2020-11-06T16:02:55.173707+01:00 WARNING 75: exception occurred in runloop during packet execution. Closing connection: ThrowExceptionUnderflowPolicy current:0 arg:1. Cookies: [{"aiostat":"would block","connection":"[ {\"ip\":\"192.168.1.231\",\"port\":51585} - {\"ip\":\"192.168.1.231\",\"port\":12000} (<ud>admin</ud>) ]","engine_storage":"0x1644f36353180000","ewouldblock":true,"packet":{"bodylen":75,"cas":0,"datatype":"raw","extlen":0,"frameid":{"durability":{"level":"Majority","timeout":9000}},"key":"<ud>._txn:atr-264-#ff9</ud>","keylen":18,"magic":"AltClientRequest","opaque":3053781504,"opcode":"SUBDOC_MULTI_MUTATION","vbucket":8},"refcount":0},{"aiostat":"success","connection":"[ {\"ip\":\"192.168.1.231\",\"port\":51585} - {\"ip\":\"192.168.1.231\",\"port\":12000} (<ud>admin</ud>) ]","engine_storage":"0x0000000000000000","ewouldblock":false,"packet":{"bodylen":43,"cas":0,"datatype":"raw","extlen":8,"frameid":{"durability":{"level":"Majority","timeout":9000}},"key":"<ud>.tx64_key876</ud>","keylen":12,"magic":"AltClientRequest","opaque":3087335936,"opcode":"ADD","vbucket":0},"refcount":0}]

* thread #15, name = 'mc:worker_5', stop reason = signal SIGABRT

    frame #0: 0x00007fff724132c6 libsystem_kernel.dylib` __pthread_kill  + 10

libsystem_kernel.dylib`__pthread_kill:

->  0x7fff724132c6 <+10>: jae    0x7fff724132d0            ; <+20>

    0x7fff724132c8 <+12>: movq   %rax, %rdi

    0x7fff724132cb <+15>: jmp    0x7fff7240d457            ; cerror_nocancel

    0x7fff724132d0 <+20>: retq   

Target 0: (memcached) stopped.

(lldb) bt

* thread #15, name = 'mc:worker_5', stop reason = signal SIGABRT

  * frame #0: 0x00007fff724132c6 libsystem_kernel.dylib` __pthread_kill  + 10

    frame #1: 0x00007fff724cebf1 libsystem_pthread.dylib` pthread_kill  + 284

    frame #2: 0x00007fff7237d6a6 libsystem_c.dylib` abort  + 127

    frame #3: 0x00007fff7234620d libsystem_c.dylib` __assert_rtn  + 324

    frame #4: 0x00007fff6f5059a4 libc++.1.dylib` std::__1::mutex::unlock()  + 46

    frame #5: 0x00000001079a59da libep.dylib` std::__1::unique_lock<std::__1::mutex>::~unique_lock(this=0x000070000fe28898)  + 42 at __mutex_base:153

    frame #6: 0x00000001079a5305 libep.dylib` std::__1::unique_lock<std::__1::mutex>::~unique_lock(this=0x000070000fe28898)  + 21 at __mutex_base:151

    frame #7: 0x00000001079a72a9 libep.dylib` folly::LockedPtrBase<folly::Synchronized<std::__1::vector<cb::tracing::Span, std::__1::allocator<cb::tracing::Span> >, std::__1::mutex>, std::__1::mutex, folly::LockPolicyExclusive>::LockedPtrBase(this=0x000070000fe28898, parent=0x000000010d3c2290)  + 377 at Synchronized.h:1287

    frame #8: 0x00000001079a7128 libep.dylib` folly::LockedPtr<folly::Synchronized<std::__1::vector<cb::tracing::Span, std::__1::allocator<cb::tracing::Span> >, std::__1::mutex>, folly::LockPolicyExclusive>::LockedPtr(this=0x000070000fe28898, parent=0x000000010d3c2290)  + 40 at Synchronized.h:1402

    frame #9: 0x00000001079a70ed libep.dylib` folly::LockedPtr<folly::Synchronized<std::__1::vector<cb::tracing::Span, std::__1::allocator<cb::tracing::Span> >, std::__1::mutex>, folly::LockPolicyExclusive>::LockedPtr(this=0x000070000fe28898, parent=0x000000010d3c2290)  + 29 at Synchronized.h:1402

    frame #10: 0x00000001079a312c libep.dylib` folly::SynchronizedBase<folly::Synchronized<std::__1::vector<cb::tracing::Span, std::__1::allocator<cb::tracing::Span> >, std::__1::mutex>, (folly::detail::MutexLevel)0>::lock(this=0x000000010d3c2290)  + 28 at Synchronized.h:384

    frame #11: 0x00000001079a2bac libep.dylib` auto folly::SynchronizedBase<folly::Synchronized<std::__1::vector<cb::tracing::Span, std::__1::allocator<cb::tracing::Span> >, std::__1::mutex>, (folly::detail::MutexLevel)0>::withLock<cb::tracing::Tracer::begin(this=0x000000010d3c2290, function=0x000070000fe288d8)::$_0>(cb::tracing::Tracer::begin(cb::tracing::Code, std::__1::chrono::time_point<std::__1::chrono::steady_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> > >)::$_0&&)  + 44 at Synchronized.h:443

    frame #12: 0x00000001079a2b74 libep.dylib` cb::tracing::Tracer::begin(this=0x000000010d3c2290, tracecode=SyncWriteAckLocal, startTime=std::__1::chrono::steady_clock::time_point @ 0x000070000fe288f8)  + 52 at tracer.cc:35

    frame #13: 0x000000010732a59a libep.dylib` TracerStopwatch::~TracerStopwatch(this=0x000070000fe28a00)  + 138 at trace_helpers.h:87

    frame #14: 0x0000000107320945 libep.dylib` TracerStopwatch::~TracerStopwatch(this=0x000070000fe28a00)  + 21 at trace_helpers.h:83

    frame #15: 0x000000010732063a libep.dylib` ActiveDurabilityMonitor::State::advanceAndAckForPosition(this=0x000000010f76bb00, pos=0x000000010f7dbc18, node="n_0@192.168.1.231", shouldAck=true)  + 474 at active_durability_monitor.cc:657

* thread #21, name = 'mc:worker_11', stop reason = EXC_BAD_ACCESS (code=1, address=0x1d0)

  * frame #0: 0x000000010a8c5803 libevent_core-2.1.7.dylib` bufferevent_trigger  + 19

    frame #1: 0x0000000103cf6a25 memcached` Connection::triggerCallback(this=0x000000010df3d300)  + 53 at connection.cc:931

    frame #2: 0x0000000103fc1485 memcached` thread_libevent_process(fd=51, (null)=2, arg=0x000000010bdff748)  + 1253 at thread.cc:286

    frame #3: 0x000000010a8d2897 libevent_core-2.1.7.dylib` event_process_active_single_queue  + 1399

    frame #4: 0x000000010a8cf23e libevent_core-2.1.7.dylib` event_base_loop  + 1790

    frame #5: 0x0000000103fbf285 memcached` worker_libevent(arg=0x000000010bdff748)  + 117 at thread.cc:187

    frame #6: 0x000000010a2c6431 libplatform_so.0.1.0.dylib` CouchbaseThread::run(this=0x000000010bc2d250)  + 113 at cb_pthreads.cc:58

    frame #7: 0x000000010a2c595e libplatform_so.0.1.0.dylib` platform_thread_wrap(arg=0x000000010bc2d250)  + 46 at cb_pthreads.cc:71

    frame #8: 0x00007fff724cc2eb libsystem_pthread.dylib` _pthread_body  + 126

    frame #9: 0x00007fff724cf249 libsystem_pthread.dylib` _pthread_start  + 66

    frame #10: 0x00007fff724cb40d libsystem_pthread.dylib` thread_start  + 13

(lldb) fr s 1

frame #1: 0x0000000103cf6a25 memcached` Connection::triggerCallback(this=0x000000010df3d300)  + 53 at connection.cc:931

   928 	

   929 	void Connection::triggerCallback() {

   930 	    const auto opt = BEV_TRIG_IGNORE_WATERMARKS | BEV_TRIG_DEFER_CALLBACKS;

-> 931 	    bufferevent_trigger(bev.get(), EV_READ, opt);

   932 	}

   933 	

   934 	bool Connection::dcpUseWriteBuffer(size_t size) const {

(lldb) p bev

(cb::libevent::unique_bufferevent_ptr) $0 = {

  __ptr_ = {

    std::__1::__compressed_pair_elem<bufferevent *, 0, false> = {

      __value_ = 0x0000000000000000

    }

  }

}