Details
-
Bug
-
Resolution: Cannot Reproduce
-
Major
-
1.8.1-release-candidate
-
Security Level: Public
-
None
-
181-933-rel
Centos 64 bit.
5 buckets.
This happened when trying to repro CBSE-165
Description
The crash happens on the node that is being failed over.
Core was generated by `/opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler'.
Program terminated with signal 6, Aborted.
#0 0x0000003866c30285 in raise () from /lib64/libc.so.6
(gdb) t a a bt
Thread 28 (Thread 0x2b51184fb220 (LWP 5380)):
#0 0x0000003866cd3648 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b5118083be8 in epoll_dispatch (base=0xc9ee000, tv=<value optimized out>) at epoll.c:404
#2 0x00002b5118071d5a in event_base_loop (base=0xc9ee000, flags=<value optimized out>) at event.c:1558
#3 0x0000000000408dba in main (argc=<value optimized out>, argv=<value optimized out>) at daemon/memcached.c:7574
Thread 27 (Thread 5389):
#0 0x0000003866cc545b in read () from /lib64/libc.so.6
#1 0x0000003866c6b677 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x0000003866c6c03e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x0000003866c61124 in _IO_getline_info_internal () from /lib64/libc.so.6
#4 0x0000003866c5ffc9 in fgets () from /lib64/libc.so.6
#5 0x00002b51184fc930 in check_stdin_thread (arg=<value optimized out>) at extensions/daemon/stdin_check.c:37
#6 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#7 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 26 (Thread 5390):
#0 EventuallyPersistentEngine::walkTapQueue (this=0x18564000, cookie=0x2aaaef9dbb88, itm=0x419e3fc8, es=0x419e3fc0, nes=0x419e3fdc, ttl=0x419e3fdf "\377\210\273\235\357\252*", flags=0x419e3fda, seqno=0x419e3fd4, vbucket=0x419e3fd8)
at ep_engine.cc:2242
#1 0x00002aaaaad394af in EvpTapIterator (handle=0x18564000, cookie=0x2aaaef9dbb88, itm=0x419e3fc8, es=0x419e3fc0, nes=0x419e3fdc, ttl=0x419e3fdf "\377\210\273\235\357\252*", flags=0x419e3fda, seqno=0x419e3fd4, vbucket=0x419e3fd8)
at ep_engine.cc:997
#2 0x00002aaaaaab06f2 in bucket_tap_iterator_shim (handle=0x2aaaaacb47e0, cookie=0x2aaaef9dbb88, itm=0x419e3fc8, engine_specific=0x419e3fc0, nengine_specific=0x419e3fdc, ttl=0x419e3fdf "\377\210\273\235\357\252*", flags=0x419e3fda,
seqno=0x419e3fd4, vbucket=0x419e3fd8) at bucket_engine.c:1876
#3 0x00000000004108b1 in ship_tap_log (c=0x2aaaef9dbb88) at daemon/memcached.c:2383
#4 conn_ship_log (c=0x2aaaef9dbb88) at daemon/memcached.c:5197
#5 0x000000000040637e in event_handler (fd=292, which=<value optimized out>, arg=0x2aaaef9dbb88) at daemon/memcached.c:5610
#6 0x00002b5118072109 in event_base_loop (base=0xc9ee500, flags=<value optimized out>) at event.c:1308
#7 0x0000000000413ad4 in worker_libevent (arg=0xc9c6500) at daemon/thread.c:301
#8 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#9 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 25 (Thread 5392):
#0 0x0000003866cb9e87 in sched_yield () from /lib64/libc.so.6
#1 0x00002aaaaad21277 in gimme (this=0x2aaac715e2f0, id=33) at atomic.hh:259
#2 RCPtr (this=0x2aaac715e2f0, id=33) at atomic.hh:336
#3 VBucketMap::getBucket (this=0x2aaac715e2f0, id=33) at vbucket.hh:495
#4 0x00002aaaaad0e003 in EventuallyPersistentStore::queueDirty (this=0x2aaac715e280, key="417c09e7169b1e1e", vbid=<value optimized out>, op=queue_op_set, rowid=-1, tapBackfill=false) at ep.cc:1998
#5 0x00002aaaaad162b9 in EventuallyPersistentStore::set (this=0x2aaac715e280, itm=..., cookie=<value optimized out>, force=<value optimized out>) at ep.cc:600
#6 0x00002aaaaad2df08 in EventuallyPersistentEngine::tapNotify(const void *, void *, uint16_t, uint8_t, uint16_t, ._107, uint32_t, const void *, size_t, uint32_t, uint32_t, uint64_t, const void *, size_t, uint16_t) (this=0x18564000,
cookie=0x2aaaf1971088, engine_specific=<value optimized out>, nengine=<value optimized out>, tap_flags=<value optimized out>, tap_event=TAP_MUTATION, tap_seqno=6567, key=0x2aaaf4d08028, nkey=16, flags=0, exptime=0,
data=0x2aaaf4d08038, ndata=255, vbucket=33) at ep_engine.cc:2513
#7 0x00002aaaaad2e792 in EvpTapNotify(ENGINE_HANDLE *, const void *, void *, uint16_t, uint8_t, uint16_t, ._107, uint32_t, const void *, size_t, uint32_t, uint32_t, uint64_t, const void *, size_t, uint16_t) (handle=0x18564000,
cookie=0x2aaaf1971088, engine_specific=0x2aaaf4d08020, nengine=0, ttl=254 '\376', tap_flags=0, tap_event=TAP_MUTATION, tap_seqno=6567, key=0x2aaaf4d08028, nkey=16, flags=0, exptime=0, cas=4141065, data=0x2aaaf4d08038, ndata=255,
vbucket=33) at ep_engine.cc:986
#8 0x00002aaaaaab08ae in bucket_tap_notify (handle=<value optimized out>, cookie=0x2aaaf1971088, engine_specific=0x2aaaf4d08020, nengine=0, ttl=254 '\376', tap_flags=0, tap_event=TAP_MUTATION, tap_seqno=6567, key=0x2aaaf4d08028,
nkey=16, flags=0, exptime=0, cas=4141065, data=0x2aaaf4d08038, ndata=255, vbucket=33) at bucket_engine.c:1847
#9 0x0000000000407739 in process_bin_tap_packet (event=<value optimized out>, c=0x2aaaf1971088) at daemon/memcached.c:2736
#10 0x00000000004122dc in complete_nread (c=0x42e5a790) at daemon/memcached.c:3437
#11 conn_nread (c=0x42e5a790) at daemon/memcached.c:5347
#12 0x000000000040637e in event_handler (fd=290, which=<value optimized out>, arg=0x2aaaf1971088) at daemon/memcached.c:5610
#13 0x00002b5118072109 in event_base_loop (base=0xc9eec80, flags=<value optimized out>) at event.c:1308
#14 0x0000000000413ad4 in worker_libevent (arg=0xc9c66f0) at daemon/thread.c:301
#15 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#16 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 24 (Thread 5393):
#0 0x0000003866cd3648 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b5118083be8 in epoll_dispatch (base=0xc9eea00, tv=<value optimized out>) at epoll.c:404
#2 0x00002b5118071d5a in event_base_loop (base=0xc9eea00, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xc9c67e8) at daemon/thread.c:301
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 23 (Thread 5394):
#0 0x0000003866cd3648 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b5118083be8 in epoll_dispatch (base=0xc9ee780, tv=<value optimized out>) at epoll.c:404
#2 0x00002b5118071d5a in event_base_loop (base=0xc9ee780, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ad4 in worker_libevent (arg=0xc9c68e0) at daemon/thread.c:301
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 22 (Thread 5396):
#0 0x0000003866c99221 in nanosleep () from /lib64/libc.so.6
#1 0x0000003866cccba4 in usleep () from /lib64/libc.so.6
#2 0x00002aaaaad52678 in updateStatsThread (arg=<value optimized out>) at memory_tracker.cc:31
#3 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#4 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 21 (Thread 5397):
#0 0x000000386740af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0880a in wait (this=0xca08380) at syncobject.hh:31
#2 Dispatcher::run (this=0xca08380) at dispatcher.cc:89
#3 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca083d4) at dispatcher.cc:28
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 20 (Thread 5398):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0xca70300, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0xca70300, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0xca08540) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca08594) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 19 (Thread 5399):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0xca70980, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0xca70980, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0xca08a80) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca08ad4) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 18 (Thread 5400):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad35909 in wait (this=0xca5a000) at syncobject.hh:42
#2 wait (this=0xca5a000) at syncobject.hh:58
#3 wait (this=0xca5a000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0xca5a000) at ep_engine.cc:3918
#5 0x00002aaaaad35be3 in EvpNotifyPendingConns (arg=0xca5a000) at ep_engine.cc:1088
#6 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#7 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 17 (Thread 5410):
#0 0x000000386740af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0880a in wait (this=0xca088c0) at syncobject.hh:31
#2 Dispatcher::run (this=0xca088c0) at dispatcher.cc:89
#3 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca08914) at dispatcher.cc:28
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 16 (Thread 5411):
--Type <return> to continue, or q <return> to quit--
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0xca71000, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0xca71000, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0xca08c40) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca08c94) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 15 (Thread 5412):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0xf6dda00, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0xf6dda00, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0xca09180) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca091d4) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 14 (Thread 5413):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad35909 in wait (this=0xca5c000) at syncobject.hh:42
#2 wait (this=0xca5c000) at syncobject.hh:58
#3 wait (this=0xca5c000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0xca5c000) at ep_engine.cc:3918
#5 0x00002aaaaad35be3 in EvpNotifyPendingConns (arg=0xca5c000) at ep_engine.cc:1088
#6 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#7 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 13 (Thread 5423):
#0 0x000000386740af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0880a in wait (this=0xca09500) at syncobject.hh:31
#2 Dispatcher::run (this=0xca09500) at dispatcher.cc:89
#3 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca09554) at dispatcher.cc:28
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 12 (Thread 5424):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0x126b4900, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x126b4900, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0xca096c0) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca09714) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 11 (Thread 5425):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0x126b4580, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x126b4580, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0xca09c00) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0xca09c54) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 10 (Thread 5426):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad35909 in wait (this=0xf285000) at syncobject.hh:42
#2 wait (this=0xf285000) at syncobject.hh:58
#3 wait (this=0xf285000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0xf285000) at ep_engine.cc:3918
#5 0x00002aaaaad35be3 in EvpNotifyPendingConns (arg=0xf285000) at ep_engine.cc:1088
#6 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#7 0x0000003866cd325d in clone () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--
Thread 9 (Thread 5435):
#0 0x000000386740af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0880a in wait (this=0x166fa000) at syncobject.hh:31
#2 Dispatcher::run (this=0x166fa000) at dispatcher.cc:89
#3 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0x166fa054) at dispatcher.cc:28
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 8 (Thread 5436):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0x152f8200, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x152f8200, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0x166fa1c0) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0x166fa214) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 7 (Thread 5437):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0x126b4f00, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x126b4f00, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0x166fa700) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0x166fa754) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 6 (Thread 5438):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad35909 in wait (this=0xca5d000) at syncobject.hh:42
#2 wait (this=0xca5d000) at syncobject.hh:58
#3 wait (this=0xca5d000) at tapconnmap.hh:171
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0xca5d000) at ep_engine.cc:3918
#5 0x00002aaaaad35be3 in EvpNotifyPendingConns (arg=0xca5d000) at ep_engine.cc:1088
#6 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#7 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 5 (Thread 5948):
#0 0x000000386740af59 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad0880a in wait (this=0x166fac40) at syncobject.hh:31
#2 Dispatcher::run (this=0x166fac40) at dispatcher.cc:89
#3 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0x166fac94) at dispatcher.cc:28
#4 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#5 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 4 (Thread 5949):
#0 0x0000003866ccc767 in fdatasync () from /lib64/libc.so.6
#1 0x00002aaaaad8cf7b in full_fsync (id=<value optimized out>, flags=<value optimized out>) at embedded/sqlite3.c:25510
#2 unixSync (id=<value optimized out>, flags=<value optimized out>) at embedded/sqlite3.c:25573
#3 0x00002aaaaad886b7 in vfsepstatSync (pFile=0xf183808, flags=2) at sqlite-vfs.c:160
#4 0x00002aaaaade2a17 in vdbeCommit (db=0x2aaad7088388, p=<value optimized out>) at embedded/sqlite3.c:13413
#5 0x00002aaaaade425d in sqlite3VdbeHalt (p=0x21984d88) at embedded/sqlite3.c:56514
#6 0x00002aaaaae33e89 in sqlite3VdbeExec (p=0x21984d88) at embedded/sqlite3.c:62196
#7 0x00002aaaaae127fa in sqlite3Step (pStmt=0x21984d88) at embedded/sqlite3.c:57947
#8 sqlite3_step (pStmt=0x21984d88) at embedded/sqlite3.c:58011
#9 0x00002aaaaad7ebf9 in PreparedStatement::execute (this=0x5006fc50) at sqlite-pst.cc:73
#10 0x00002aaaaad7d78b in execute (this=0x1fb9fa40) at sqlite-kvstore.hh:168
#11 StrategicSqlite3::commit (this=0x1fb9fa40) at sqlite-kvstore.hh:74
#12 0x00002aaaaad7d3cb in StrategicSqlite3::delVBucket (this=0x1fb9fa40, vbucket=<value optimized out>, vb_version=<value optimized out>) at sqlite-kvstore.cc:265
#13 0x00002aaaaad1956f in EventuallyPersistentStore::completeVBucketDeletion (this=0x2aaac715e280, vbid=2, vbver=0) at ep.cc:798
#14 0x00002aaaaad25452 in FastVBucketDeletionCallback::callback (this=0x2aaaf4af1fe0) at ep.cc:177
#15 0x00002aaaaad09f7f in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
--Type <return> to continue, or q <return> to quit--
#16 0x00002aaaaad089d6 in Dispatcher::run (this=0x166fae00) at dispatcher.cc:123
#17 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0x125) at dispatcher.cc:28
#18 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#19 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 3 (Thread 5950):
#0 0x000000386740b1c0 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad066e0 in wait (this=0x126b5a00, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x126b5a00, d=...) at dispatcher.cc:286
#3 0x00002aaaaad089d6 in Dispatcher::run (this=0x166fbdc0) at dispatcher.cc:123
#4 0x00002aaaaad0927b in launch_dispatcher_thread (arg=0x166fbe14) at dispatcher.cc:28
#5 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#6 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 2 (Thread 5951):
#0 0x000000386740d594 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003867408e8a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003867408d4c in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00000000004143f6 in notify_io_complete (cookie=0x1af538c8, status=ENGINE_SUCCESS) at daemon/thread.c:485
#4 0x00002aaaaad757eb in notifyIOComplete<std::list<void const*, std::allocator<void const*> > > (this=0x185645e8) at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_function.h:519
#5 TapConnMap::notifyIOThreadMain (this=0x185645e8) at tapconnmap.cc:467
#6 0x00002aaaaad35787 in EventuallyPersistentEngine::notifyPendingConnections (this=0x18564000) at ep_engine.cc:3911
#7 0x00002aaaaad35be3 in EvpNotifyPendingConns (arg=0x18564000) at ep_engine.cc:1088
#8 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#9 0x0000003866cd325d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x4245a940 (LWP 5391)):
#0 0x0000003866c30285 in raise () from /lib64/libc.so.6
#1 0x0000003866c31d30 in abort () from /lib64/libc.so.6
#2 0x0000003866c29706 in __assert_fail () from /lib64/libc.so.6
#3 0x00002aaaaad008b3 in CheckpointManager::queueDirty (this=0x25c9c080, item=..., vbucket=...) at checkpoint.cc:718
#4 0x00002aaaaad0e13c in EventuallyPersistentStore::queueDirty (this=0x2aaac715e280, key="45a64e86a9dc2b03", vbid=<value optimized out>, op=queue_op_set, rowid=<value optimized out>, tapBackfill=false) at ep.cc:2005
#5 0x00002aaaaad162b9 in EventuallyPersistentStore::set (this=0x2aaac715e280, itm=..., cookie=<value optimized out>, force=<value optimized out>) at ep.cc:600
#6 0x00002aaaaad3452f in EventuallyPersistentEngine::store(const void *, item *, uint64_t *, ._34, uint16_t) (this=0x18564000, cookie=0x2aaaef9db088, itm=0x2aaaf17a23c0, cas=0x2aaaef9db2e0, operation=OPERATION_SET, vbucket=501)
at ep_engine.cc:1849
#7 0x00002aaaaad35273 in EvpStore(ENGINE_HANDLE *, const void *, item *, uint64_t *, ._34, uint16_t) (handle=0x18564000, cookie=0x2aaaef9db088, item=0x2aaaf17a23c0, cas=0x2aaaef9db2e0, operation=OPERATION_SET, vbucket=501)
at ep_engine.cc:206
#8 0x00002aaaaaab0d10 in bucket_store (handle=<value optimized out>, cookie=0x2aaaef9db088, itm=0x2aaaf17a23c0, cas=0x2aaaef9db2e0, operation=OPERATION_SET, vbucket=501) at bucket_engine.c:1715
#9 0x00000000004126fc in complete_nread (c=0x2aaaef9db088) at daemon/memcached.c:1589
#10 conn_nread (c=0x2aaaef9db088) at daemon/memcached.c:5347
#11 0x000000000040637e in event_handler (fd=306, which=<value optimized out>, arg=0x2aaaef9db088) at daemon/memcached.c:5610
#12 0x00002b5118072109 in event_base_loop (base=0xc9ee280, flags=<value optimized out>) at event.c:1308
#13 0x0000000000413ad4 in worker_libevent (arg=0xc9c65f8) at daemon/thread.c:301
#14 0x000000386740677d in start_thread () from /lib64/libpthread.so.0
#15 0x0000003866cd325d in clone () from /lib64/libc.so.6