Deadlock and issues related to bucket_engine crash

Latest deadlock with 4 buckets

Thread 25 (Thread 0x42150940 (LWP 19484)):
#0 0x00000038286c678b in read () from /lib64/libc.so.6
#1 0x000000382866cd57 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x000000382866d71e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x0000003828668fdb in getc () from /lib64/libc.so.6
#4 0x00002ac5fbea0875 in check_stdin_thread (arg=0x4034c0) at extensions/daemon/stdin_check.c:19
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 24 (Thread 0x410eb940 (LWP 19485)):
#0 0x000000382869a541 in nanosleep () from /lib64/libc.so.6
#1 0x000000382869a364 in sleep () from /lib64/libc.so.6
#2 0x0000000000415638 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
#3 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#4 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 23 (Thread 0x42b51940 (LWP 19486)):
#0 0x00000038286d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002ac5fba21be8 in epoll_dispatch (base=0x1e0237b0, tv=<value optimized out>) at epoll.c:404
#2 0x00002ac5fba0fd5a in event_base_loop (base=0x1e0237b0, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ae4 in worker_libevent (arg=0x1e023280) at daemon/thread.c:301
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 22 (Thread 0x43552940 (LWP 19487)):
#0 0x00000038286d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002ac5fba21be8 in epoll_dispatch (base=0x1e024050, tv=<value optimized out>) at epoll.c:404
#2 0x00002ac5fba0fd5a in event_base_loop (base=0x1e024050, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ae4 in worker_libevent (arg=0x1e023378) at daemon/thread.c:301
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 21 (Thread 0x43f53940 (LWP 19488)):
#0 0x000000382920d524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003829208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003829208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000404dde in release_cookie (cookie=0x1dfb71e0) at daemon/memcached.c:6523
#4 0x00002aaaaaaaf1a5 in bucket_engine_release_cookie (cookie=0x1dfb71e0) at bucket_engine.c:2425
#5 0x00002aaaaad71544 in TapConnMap::newProducer (this=0x19e265e8, cookie=0x1dfa9da0, name="eq_tapq:replication_ns_1@10.1.3.116", flags=85, backfillAge=0,
tapKeepAlive=300) at tapconnmap.cc:306
#6 0x00002aaaaad29ecb in EventuallyPersistentEngine::createTapQueue (this=0x19e26000, cookie=0x1dfa9da0, client=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 469, in to_string
return self.val['_M_dataplus']['_M_p'].string (encoding, length = len)
RuntimeError: Cannot access memory at address 0xffffffffffffffe8
, flags=<value optimized out>, userdata=<value optimized out>, nuserdata=0) at ep_engine.cc:2284
#7 0x00002aaaaad2a2e0 in EvpGetTapIterator (handle=0x19e26000, cookie=0x1dfa9da0, client=0x2fcee68c, nclient=27, flags=85, userdata=0x2fcee6a7, nuserdata=1776)
at ep_engine.cc:989
#8 0x00002aaaaaaae27f in bucket_get_tap_iterator (handle=<value optimized out>, cookie=0x1dfa9da0, client=0x2fcee68c, nclient=27, flags=85,
userdata=0x2fcee6a7, nuserdata=1776) at bucket_engine.c:1903
#9 0x00000000004061fc in process_bin_tap_connect (c=0x1dfa9da0) at daemon/memcached.c:2890
#10 conn_setup_tap_stream (c=0x1dfa9da0) at daemon/memcached.c:5719
#11 0x0000000000406ec6 in event_handler (fd=199, which=2, arg=0x1dfa9da0) at daemon/memcached.c:5762
#12 0x00002ac5fba10109 in event_base_loop (base=0x1e0248f0, flags=<value optimized out>) at event.c:1308
#13 0x0000000000413ae4 in worker_libevent (arg=0x1e023470) at daemon/thread.c:301
#14 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#15 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 20 (Thread 0x44954940 (LWP 19489)):
--Type <return> to continue, or q <return> to quit--#0 0x000000382920d524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003829208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003829208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000404dde in release_cookie (cookie=0x1dfc7b30) at daemon/memcached.c:6523
#4 0x00002aaaaaaaf1a5 in bucket_engine_release_cookie (cookie=0x1dfc7b30) at bucket_engine.c:2425
#5 0x00002aaaaad71544 in TapConnMap::newProducer (this=0x19e265e8, cookie=0x1dfb71e0, name="eq_tapq:replication_ns_1@10.1.3.116", flags=85, backfillAge=0,
tapKeepAlive=300) at tapconnmap.cc:306
#6 0x00002aaaaad29ecb in EventuallyPersistentEngine::createTapQueue (this=0x19e26000, cookie=0x1dfb71e0, client=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 469, in to_string
return self.val['_M_dataplus']['_M_p'].string (encoding, length = len)
RuntimeError: Cannot access memory at address 0xffffffffffffffe8
, flags=<value optimized out>, userdata=<value optimized out>, nuserdata=0) at ep_engine.cc:2284
#7 0x00002aaaaad2a2e0 in EvpGetTapIterator (handle=0x19e26000, cookie=0x1dfb71e0, client=0x2aaab541c62c, nclient=27, flags=85, userdata=0x2aaab541c647,
nuserdata=1788) at ep_engine.cc:989
#8 0x00002aaaaaaae27f in bucket_get_tap_iterator (handle=<value optimized out>, cookie=0x1dfb71e0, client=0x2aaab541c62c, nclient=27, flags=85,
userdata=0x2aaab541c647, nuserdata=1788) at bucket_engine.c:1903
#9 0x00000000004061fc in process_bin_tap_connect (c=0x1dfb71e0) at daemon/memcached.c:2890
#10 conn_setup_tap_stream (c=0x1dfb71e0) at daemon/memcached.c:5719
#11 0x0000000000406ec6 in event_handler (fd=190, which=2, arg=0x1dfb71e0) at daemon/memcached.c:5762
#12 0x00002ac5fba10109 in event_base_loop (base=0x1e025190, flags=<value optimized out>) at event.c:1308
#13 0x0000000000413ae4 in worker_libevent (arg=0x1e023568) at daemon/thread.c:301
#14 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#15 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 19 (Thread 0x45355940 (LWP 19490)):
#0 0x00000038286d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002ac5fba21be8 in epoll_dispatch (base=0x1e025b30, tv=<value optimized out>) at epoll.c:404
#2 0x00002ac5fba0fd5a in event_base_loop (base=0x1e025b30, flags=<value optimized out>) at event.c:1558
#3 0x0000000000413ae4 in worker_libevent (arg=0x1e023660) at daemon/thread.c:301
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 18 (Thread 0x45d56940 (LWP 19492)):
#0 0x000000382869a541 in nanosleep () from /lib64/libc.so.6
#1 0x00000038286cded4 in usleep () from /lib64/libc.so.6
#2 0x00002aaaaad4fe48 in updateStatsThread (arg=<value optimized out>) at memory_tracker.cc:31
#3 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#4 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x46757940 (LWP 19493)):
#0 0x000000382920aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad06d0a in wait (this=0x19ea0380) at syncobject.hh:31
#2 Dispatcher::run (this=0x19ea0380) at dispatcher.cc:89
#3 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea03d4) at dispatcher.cc:28
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x47158940 (LWP 19494)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4000, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4000, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea0000) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0054) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 15 (Thread 0x47b59940 (LWP 19495)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4080, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4080, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea01c0) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0214) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 14 (Thread 0x4855a940 (LWP 19496)):
#0 0x000000382920d524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003829208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003829208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--#3 0x0000000000413494 in notify_io_complete (cookie=0x1df7b6c0, status=ENGINE_SUCCESS) at daemon/thread.c:485
#4 0x00002aaaaad73970 in notifyIOComplete<std::list<void const*, std::allocator<void const*> > > (this=0x19e245e8)
at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_function.h:519
#5 TapConnMap::notifyIOThreadMain (this=0x19e245e8) at tapconnmap.cc:495
#6 0x00002aaaaad25be7 in EventuallyPersistentEngine::notifyPendingConnections (this=0x19e24000) at ep_engine.cc:3798
#7 0x00002aaaaad25e13 in EvpNotifyPendingConns (arg=0x19e24000) at ep_engine.cc:1046
#8 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#9 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 13 (Thread 0x48f5b940 (LWP 19497)):
#0 0x000000382920aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad06d0a in wait (this=0x19ea08c0) at syncobject.hh:31
#2 Dispatcher::run (this=0x19ea08c0) at dispatcher.cc:89
#3 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0914) at dispatcher.cc:28
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x4995c940 (LWP 19498)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4200, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4200, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea0540) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0594) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x4a35d940 (LWP 19499)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4280, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4280, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea0700) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0754) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x4ad5e940 (LWP 19500)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad25cb1 in wait (this=0x19e25000) at syncobject.hh:42
#2 wait (this=0x19e25000) at syncobject.hh:58
#3 wait (this=0x19e25000) at tapconnmap.hh:167
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0x19e25000) at ep_engine.cc:3805
#5 0x00002aaaaad25e13 in EvpNotifyPendingConns (arg=0x19e25000) at ep_engine.cc:1046
#6 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#7 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x4b75f940 (LWP 19501)):
#0 0x000000382920aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad06d0a in wait (this=0x19ea0e00) at syncobject.hh:31
#2 Dispatcher::run (this=0x19ea0e00) at dispatcher.cc:89
#3 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0e54) at dispatcher.cc:28
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x4c160940 (LWP 19502)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4880, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4880, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea0a80) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0ad4) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x4cb61940 (LWP 19503)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4900, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4900, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea0c40) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea0c94) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x4d562940 (LWP 19504)):
#0 0x000000382920d524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003829208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003829208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000413494 in notify_io_complete (cookie=0x1dfb71e0, status=ENGINE_SUCCESS) at daemon/thread.c:485
#4 0x00002aaaaad73970 in notifyIOComplete<std::list<void const*, std::allocator<void const*> > > (this=0x19e265e8)
at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_function.h:519
#5 TapConnMap::notifyIOThreadMain (this=0x19e265e8) at tapconnmap.cc:495
#6 0x00002aaaaad25be7 in EventuallyPersistentEngine::notifyPendingConnections (this=0x19e26000) at ep_engine.cc:3798
#7 0x00002aaaaad25e13 in EvpNotifyPendingConns (arg=0x19e26000) at ep_engine.cc:1046
#8 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#9 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 5 (Thread 0x4df63940 (LWP 19505)):
#0 0x000000382920aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad06d0a in wait (this=0x19ea1340) at syncobject.hh:31
#2 Dispatcher::run (this=0x19ea1340) at dispatcher.cc:89
#3 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea1394) at dispatcher.cc:28
#4 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x4e964940 (LWP 19506)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4f00, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4f00, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea0fc0) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea1014) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x4f365940 (LWP 19507)):
#0 0x000000382920b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad04be0 in wait (this=0x19ea4f80, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x19ea4f80, d=...) at dispatcher.cc:286
#3 0x00002aaaaad06ed6 in Dispatcher::run (this=0x19ea1180) at dispatcher.cc:123
#4 0x00002aaaaad0777b in launch_dispatcher_thread (arg=0x19ea11d4) at dispatcher.cc:28
#5 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x4fd66940 (LWP 19508)):
#0 0x000000382920d524 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003829208e1a in _L_lock_1034 () from /lib64/libpthread.so.0
#2 0x0000003829208cdc in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000413494 in notify_io_complete (cookie=0x1dfa3380, status=ENGINE_SUCCESS) at daemon/thread.c:485
#4 0x00002aaaaad73970 in notifyIOComplete<std::list<void const*, std::allocator<void const*> > > (this=0x19e275e8)
at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_function.h:519
#5 TapConnMap::notifyIOThreadMain (this=0x19e275e8) at tapconnmap.cc:495
#6 0x00002aaaaad25be7 in EventuallyPersistentEngine::notifyPendingConnections (this=0x19e27000) at ep_engine.cc:3798
#7 0x00002aaaaad25e13 in EvpNotifyPendingConns (arg=0x19e27000) at ep_engine.cc:1046
#8 0x000000382920673d in start_thread () from /lib64/libpthread.so.0
#9 0x00000038286d44bd in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x2ac5fbe9f1c0 (LWP 19475)):
#0 0x00000038286d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002ac5fba21be8 in epoll_dispatch (base=0x1e018420, tv=<value optimized out>) at epoll.c:404
#2 0x00002ac5fba0fd5a in event_base_loop (base=0x1e018420, flags=<value optimized out>) at event.c:1558
#3 0x000000000040a314 in main (argc=<value optimized out>, argv=<value optimized out>) at daemon/memcached.c:7721