Memcached core due assert failure in TapProducer

Chiyoung, seeing this on 2.0 as well. (I think you said, we should not see it on 2.0). Filing it for you to investigatew. This is during regular regression runs.

Not sure what is the severity of this bug. Def we would see data loss when this happens.

#0 0x00000033fb030265 in raise () from /lib64/libc.so.6
(gdb) t a a bt

Thread 28 (Thread 0x2b06a0591220 (LWP 21389)):
#0 0x00000033fb0d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b06a0113be8 in epoll_dispatch (base=0xef88000, tv=<value optimized out>) at epoll.c:404
#2 0x00002b06a0101d5a in event_base_loop (base=0xef88000, flags=<value optimized out>) at event.c:1558
#3 0x0000000000408dba in main (argc=<value optimized out>, argv=<value optimized out>) at daemon/memcached.c:7574

Thread 27 (Thread 21397):
#0 0x00000033fb0c678b in read () from /lib64/libc.so.6
#1 0x00000033fb06cd57 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00000033fb06d71e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00000033fb062804 in _IO_getline_info_internal () from /lib64/libc.so.6
#4 0x00000033fb0616a9 in fgets () from /lib64/libc.so.6
#5 0x00002b06a0592930 in check_stdin_thread (arg=<value optimized out>) at extensions/daemon/stdin_check.c:37
#6 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#7 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 26 (Thread 21398):
#0 0x00000033fb0d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b06a0113be8 in epoll_dispatch (base=0xef88500, tv=<value optimized out>) at epoll.c:404
#2 0x00002b06a0101d5a in event_base_loop (base=0xef88500, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xef60500) at daemon/thread.c:301
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 25 (Thread 21399):
#0 0x00000033fb0d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b06a0113be8 in epoll_dispatch (base=0xef88280, tv=<value optimized out>) at epoll.c:404
#2 0x00002b06a0101d5a in event_base_loop (base=0xef88280, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xef605f8) at daemon/thread.c:301
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 24 (Thread 21400):
#0 0x00000033fb0d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b06a0113be8 in epoll_dispatch (base=0xef88c80, tv=<value optimized out>) at epoll.c:404
#2 0x00002b06a0101d5a in event_base_loop (base=0xef88c80, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xef606f0) at daemon/thread.c:301
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 23 (Thread 21401):
#0 0x00000033fb0d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b06a0113be8 in epoll_dispatch (base=0xef88a00, tv=<value optimized out>) at epoll.c:404
#2 0x00002b06a0101d5a in event_base_loop (base=0xef88a00, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xef607e8) at daemon/thread.c:301
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 22 (Thread 21402):
#0 0x00000033fb0d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b06a0113be8 in epoll_dispatch (base=0xef88780, tv=<value optimized out>) at epoll.c:404
#2 0x00002b06a0101d5a in event_base_loop (base=0xef88780, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xef608e0) at daemon/thread.c:301
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 21 (Thread 21404):
#0 0x00000033fb09a541 in nanosleep () from /lib64/libc.so.6
#1 0x00000033fb0cded4 in usleep () from /lib64/libc.so.6
#2 0x00002aaaaad53678 in updateStatsThread (arg=<value optimized out>) at memory_tracker.cc:31
#3 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#4 0x00000033fb0d44bd in clone () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--

Thread 20 (Thread 21405):
#0 0x00000033fbc0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0980a in wait (this=0xefa2380) at syncobject.hh:31
#2 Dispatcher::run (this=0xefa2380) at dispatcher.cc:89
#3 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa23d4) at dispatcher.cc:28
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 19 (Thread 21406):
#0 0x00000033fb0cda97 in fdatasync () from /lib64/libc.so.6
#1 0x00002aaaaad8df49 in full_fsync (id=<value optimized out>, flags=2) at embedded/sqlite3.c:25510
#2 unixSync (id=<value optimized out>, flags=2) at embedded/sqlite3.c:25558
#3 0x00002aaaaad896b7 in vfsepstatSync (pFile=0xf021150, flags=2) at sqlite-vfs.c:160
#4 0x00002aaaaada8f78 in walCheckpoint (pWal=0xef8cb08, sync_flags=2, nBuf=1024, zBuf=0xeff0888 "\r") at embedded/sqlite3.c:13413
#5 sqlite3WalCheckpoint (pWal=0xef8cb08, sync_flags=2, nBuf=1024, zBuf=0xeff0888 "\r") at embedded/sqlite3.c:43431
#6 0x00002aaaaadd8167 in sqlite3BtreeCheckpoint (db=0xf004388, zDb=<value optimized out>) at embedded/sqlite3.c:40675
#7 sqlite3Checkpoint (db=0xf004388, zDb=<value optimized out>) at embedded/sqlite3.c:39117
#8 sqlite3_wal_checkpoint (db=0xf004388, zDb=<value optimized out>) at embedded/sqlite3.c:39082
#9 0x00002aaaaadd8321 in sqlite3WalDefaultHook (pClientData=<value optimized out>, db=0xf004388, zDb=0xf018988 "kv_3", nFrame=-1) at embedded/sqlite3.c:104546
#10 0x00002aaaaae13c37 in sqlite3Step (pStmt=0x2aaab1f98c08) at embedded/sqlite3.c:57877
#11 sqlite3_step (pStmt=0x2aaab1f98c08) at embedded/sqlite3.c:58011
#12 0x00002aaaaad7fbf9 in PreparedStatement::execute (this=0x46520ac0) at sqlite-pst.cc:73
#13 0x00002aaaaad7e78b in execute (this=0xef6a480) at sqlite-kvstore.hh:168
#14 StrategicSqlite3::commit (this=0xef6a480) at sqlite-kvstore.hh:74
#15 0x00002aaaaad0dcb9 in TransactionContext::commit (this=0x102d8418) at ep.cc:2205
#16 0x00002aaaaad1a225 in EventuallyPersistentStore::flushSome (this=0x102d8280, q=0x102d83b0, rejectQueue=0x2c443ea0) at ep.cc:1528
#17 0x00002aaaaad4c746 in Flusher::doFlush (this=0x103f55f0) at flusher.cc:242
#18 0x00002aaaaad4d2b5 in Flusher::step (this=0x36, d=..., tid=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 32, in to_string
return '%s (count %d) %s' % (self.typename,
TypeError: int argument required
) at flusher.cc:156
#19 0x00002aaaaad4d9ee in FlusherStepper::callback (this=0x11482800, d=..., t=<value optimized out>) at flusher.cc:23
#20 0x00002aaaaad0af7f in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#21 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa2540) at dispatcher.cc:123
#22 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0x36) at dispatcher.cc:28
#23 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#24 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 18 (Thread 21407):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0xf00a980, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0xf00a980, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa2a80) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa2ad4) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 17 (Thread 21408):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad36909 in wait (this=0xeff4000) at syncobject.hh:42
#2 wait (this=0xeff4000) at syncobject.hh:58
#3 wait (this=0xeff4000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0xeff4000) at ep_engine.cc:3918
#5 0x00002aaaaad36be3 in EvpNotifyPendingConns (arg=0xeff4000) at ep_engine.cc:1088
#6 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#7 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 16 (Thread 21431):
#0 0x00000033fbc0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0980a in wait (this=0xefa28c0) at syncobject.hh:31
#2 Dispatcher::run (this=0xefa28c0) at dispatcher.cc:89
--Type <return> to continue, or q <return> to quit--
#3 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa2914) at dispatcher.cc:28
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 15 (Thread 21432):
#0 0x00000033fb0cda97 in fdatasync () from /lib64/libc.so.6
#1 0x00002aaaaad8df7b in full_fsync (id=<value optimized out>, flags=<value optimized out>) at embedded/sqlite3.c:25510
#2 unixSync (id=<value optimized out>, flags=<value optimized out>) at embedded/sqlite3.c:25573
#3 0x00002aaaaad896b7 in vfsepstatSync (pFile=0x2aaaaedccb08, flags=2) at sqlite-vfs.c:160
#4 0x00002aaaaade3a17 in vdbeCommit (db=0x12d42008, p=<value optimized out>) at embedded/sqlite3.c:13413
#5 0x00002aaaaade525d in sqlite3VdbeHalt (p=0x3aa62908) at embedded/sqlite3.c:56514
#6 0x00002aaaaae34e89 in sqlite3VdbeExec (p=0x3aa62908) at embedded/sqlite3.c:62196
#7 0x00002aaaaae137fa in sqlite3Step (pStmt=0x3aa62908) at embedded/sqlite3.c:57947
#8 sqlite3_step (pStmt=0x3aa62908) at embedded/sqlite3.c:58011
#9 0x00002aaaaad7fbf9 in PreparedStatement::execute (this=0x48d24ac0) at sqlite-pst.cc:73
#10 0x00002aaaaad7e78b in execute (this=0x12b3a720) at sqlite-kvstore.hh:168
#11 StrategicSqlite3::commit (this=0x12b3a720) at sqlite-kvstore.hh:74
#12 0x00002aaaaad0dcb9 in TransactionContext::commit (this=0x14018b98) at ep.cc:2205
#13 0x00002aaaaad1a225 in EventuallyPersistentStore::flushSome (this=0x14018a00, q=0x14018b30, rejectQueue=0x116b1090) at ep.cc:1528
#14 0x00002aaaaad4c746 in Flusher::doFlush (this=0x14115c20) at flusher.cc:242
#15 0x00002aaaaad4d2b5 in Flusher::step (this=0xe2, d=..., tid=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 32, in to_string
return '%s (count %d) %s' % (self.typename,
TypeError: int argument required
) at flusher.cc:156
#16 0x00002aaaaad4d9ee in FlusherStepper::callback (this=0x15169200, d=..., t=<value optimized out>) at flusher.cc:23
#17 0x00002aaaaad0af7f in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#18 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa2c40) at dispatcher.cc:123
#19 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xe2) at dispatcher.cc:28
#20 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#21 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 14 (Thread 21433):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0x117fe100, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x117fe100, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa3180) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa31d4) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 13 (Thread 21434):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad36909 in wait (this=0x12c34000) at syncobject.hh:42
#2 wait (this=0x12c34000) at syncobject.hh:58
#3 wait (this=0x12c34000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0x12c34000) at ep_engine.cc:3918
#5 0x00002aaaaad36be3 in EvpNotifyPendingConns (arg=0x12c34000) at ep_engine.cc:1088
#6 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#7 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 12 (Thread 21459):
#0 0x00000033fb0cda97 in fdatasync () from /lib64/libc.so.6
#1 0x00002aaaaad8df7b in full_fsync (id=<value optimized out>, flags=<value optimized out>) at embedded/sqlite3.c:25510
#2 unixSync (id=<value optimized out>, flags=<value optimized out>) at embedded/sqlite3.c:25573
#3 0x00002aaaaad896b7 in vfsepstatSync (pFile=0x3cbd4908, flags=2) at sqlite-vfs.c:160
#4 0x00002aaaaade3a17 in vdbeCommit (db=0x1684a008, p=<value optimized out>) at embedded/sqlite3.c:13413
#5 0x00002aaaaade525d in sqlite3VdbeHalt (p=0x2c6a8608) at embedded/sqlite3.c:56514
#6 0x00002aaaaae34e89 in sqlite3VdbeExec (p=0x2c6a8608) at embedded/sqlite3.c:62196
#7 0x00002aaaaae137fa in sqlite3Step (pStmt=0x2c6a8608) at embedded/sqlite3.c:57947
#8 sqlite3_step (pStmt=0x2c6a8608) at embedded/sqlite3.c:58011
#9 0x00002aaaaad7fbf9 in PreparedStatement::execute (this=0x4ab27c50) at sqlite-pst.cc:73
#10 0x00002aaaaad7e78b in execute (this=0x16635620) at sqlite-kvstore.hh:168
--Type <return> to continue, or q <return> to quit--
#11 StrategicSqlite3::commit (this=0x16635620) at sqlite-kvstore.hh:74
#12 0x00002aaaaad7e3cb in StrategicSqlite3::delVBucket (this=0x16635620, vbucket=<value optimized out>, vb_version=<value optimized out>) at sqlite-kvstore.cc:265
#13 0x00002aaaaad1a56f in EventuallyPersistentStore::completeVBucketDeletion (this=0x17b29180, vbid=2, vbver=0) at ep.cc:798
#14 0x00002aaaaad26452 in FastVBucketDeletionCallback::callback (this=0x2aaab7babf00) at ep.cc:177
#15 0x00002aaaaad0af7f in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#16 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa3500) at dispatcher.cc:123
#17 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xaa) at dispatcher.cc:28
#18 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#19 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 11 (Thread 21460):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0x1298b280, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x1298b280, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa3a40) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa3a94) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 10 (Thread 21461):
#0 0x00000033fbc0d524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00000033fbc08e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x00000033fbc08cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00002aaaaad367ab in acquire (this=0x15cf0000) at mutex.hh:70
#4 lock (this=0x15cf0000) at locks.hh:48
#5 LockHolder (this=0x15cf0000) at locks.hh:26
#6 wait (this=0x15cf0000) at tapconnmap.hh:169
#7 EventuallyPersistentEngine::notifyPendingConnections (this=0x15cf0000) at ep_engine.cc:3918
#8 0x00002aaaaad36be3 in EvpNotifyPendingConns (arg=0x15cf0000) at ep_engine.cc:1088
#9 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#10 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 9 (Thread 21485):
#0 0x00000033fbc0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0980a in wait (this=0xefa3880) at syncobject.hh:31
#2 Dispatcher::run (this=0xefa3880) at dispatcher.cc:89
#3 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa38d4) at dispatcher.cc:28
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 8 (Thread 21486):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0x18e77e00, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x18e77e00, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa2e00) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa2e54) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 7 (Thread 21487):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0x18e77480, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x18e77480, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa3dc0) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0xefa3e14) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 6 (Thread 21488):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad36909 in wait (this=0x12c33000) at syncobject.hh:42
#2 wait (this=0x12c33000) at syncobject.hh:58
#3 wait (this=0x12c33000) at tapconnmap.hh:171
--Type <return> to continue, or q <return> to quit--
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0x12c33000) at ep_engine.cc:3918
#5 0x00002aaaaad36be3 in EvpNotifyPendingConns (arg=0x12c33000) at ep_engine.cc:1088
#6 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#7 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 5 (Thread 21526):
#0 0x00000033fbc0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0980a in wait (this=0x1ef4a540) at syncobject.hh:31
#2 Dispatcher::run (this=0x1ef4a540) at dispatcher.cc:89
#3 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0x1ef4a594) at dispatcher.cc:28
#4 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 4 (Thread 21527):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0x18e76200, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x18e76200, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0x1ef4a700) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0x1ef4a754) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 3 (Thread 21528):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad076e0 in wait (this=0x18e77800, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x18e77800, d=...) at dispatcher.cc:286
#3 0x00002aaaaad099d6 in Dispatcher::run (this=0x1ef4a380) at dispatcher.cc:123
#4 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0x1ef4a3d4) at dispatcher.cc:28
#5 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 2 (Thread 21529):
#0 0x00000033fbc0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad36909 in wait (this=0x12c32000) at syncobject.hh:42
#2 wait (this=0x12c32000) at syncobject.hh:58
#3 wait (this=0x12c32000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0x12c32000) at ep_engine.cc:3918
#5 0x00002aaaaad36be3 in EvpNotifyPendingConns (arg=0x12c32000) at ep_engine.cc:1088
#6 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#7 0x00000033fb0d44bd in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x40aab940 (LWP 21458)):
#0 0x00000033fb030265 in raise () from /lib64/libc.so.6
#1 0x00000033fb031d10 in abort () from /lib64/libc.so.6
#2 0x00000033fb0296e6 in __assert_fail () from /lib64/libc.so.6
#3 0x00002aaaaad64f46 in TapProducer::completeBGFetchJob (this=0x12986800, itm=0x17adf200, implicitEnqueue=false) at tapconnection.cc:1019
#4 0x00002aaaaacf8449 in TapConnMap::performTapOp<Item*> (this=0x15cf05e8, name=<value optimized out>, tapop=..., arg=0x17adf200) at tapconnmap.hh:115
#5 0x00002aaaaad71f97 in TapBGFetchCallback::callback (this=0x2aaabb5474f0, d=..., t=std::tr1::shared_ptr (count 1084930368) 0x40aaaf40) at tapconnection.cc:950
#6 0x00002aaaaad0af7f in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#7 0x00002aaaaad099d6 in Dispatcher::run (this=0xefa3340) at dispatcher.cc:123
#8 0x00002aaaaad0a27b in launch_dispatcher_thread (arg=0x538d) at dispatcher.cc:28
#9 0x00000033fbc0673d in start_thread () from /lib64/libpthread.so.0
#10 0x00000033fb0d44bd in clone () from /lib64/libc.so.6