2.0 dp4 gets stuck when warmping up after memcached crashes

10.1.3.103 node crashed when i was adding .107
the node was in warm up state for few hours and still stuck. i am not able to get stats from memcached either

Thread 24 (Thread 0x7fc422b9c700 (LWP 19262)):
#0 0x00007fc4233a24bd in read () from /lib/libc.so.6
#1 0x00007fc42333d348 in _IO_file_underflow () from /lib/libc.so.6
#2 0x00007fc42333eeee in _IO_default_uflow () from /lib/libc.so.6
#3 0x00007fc423335c7b in getc () from /lib/libc.so.6
#4 0x00007fc422b9d9c9 in check_stdin_thread (arg=0x403880) at extensions/daemon/stdin_check.c:19
#5 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#6 0x00007fc4233b070d in clone () from /lib/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 23 (Thread 0x7fc42239b700 (LWP 19263)):
#0 0x00007fc42337439d in nanosleep () from /lib/libc.so.6
#1 0x00007fc423374210 in sleep () from /lib/libc.so.6
#2 0x0000000000415df8 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
#3 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#4 0x00007fc4233b070d in clone () from /lib/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 22 (Thread 0x7fc42198f700 (LWP 19264)):
#0 0x00007fc4233b0d03 in epoll_wait () from /lib/libc.so.6
#1 0x00007fc4241eb5a6 in epoll_dispatch (base=0x50ae500, tv=<value optimized out>) at epoll.c:404
#2 0x00007fc4241da171 in event_base_loop (base=0x50ae500, flags=<value optimized out>)
--Type <return> to continue, or q <return> to quit--
at event.c:1558
#3 0x0000000000414544 in worker_libevent (arg=0xc4c500) at daemon/thread.c:304
#4 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#5 0x00007fc4233b070d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 21 (Thread 0x7fc42118e700 (LWP 19265)):
#0 0x00007fc4233b0d03 in epoll_wait () from /lib/libc.so.6
#1 0x00007fc4241eb5a6 in epoll_dispatch (base=0x50ae280, tv=<value optimized out>) at epoll.c:404
#2 0x00007fc4241da171 in event_base_loop (base=0x50ae280, flags=<value optimized out>)
at event.c:1558
#3 0x0000000000414544 in worker_libevent (arg=0xc4c600) at daemon/thread.c:304
#4 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#5 0x00007fc4233b070d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 20 (Thread 0x7fc42098d700 (LWP 19266)):
#0 0x00007fc4233b0d03 in epoll_wait () from /lib/libc.so.6
#1 0x00007fc4241eb5a6 in epoll_dispatch (base=0x50aec80, tv=<value optimized out>) at epoll.c:404
#2 0x00007fc4241da171 in event_base_loop (base=0x50aec80, flags=<value optimized out>)
at event.c:1558
#3 0x0000000000414544 in worker_libevent (arg=0xc4c700) at daemon/thread.c:304
#4 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#5 0x00007fc4233b070d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 19 (Thread 0x7fc42018c700 (LWP 19267)):
#0 0x00007fc4233b0d03 in epoll_wait () from /lib/libc.so.6
#1 0x00007fc4241eb5a6 in epoll_dispatch (base=0x50aea00, tv=<value optimized out>) at epoll.c:404
#2 0x00007fc4241da171 in event_base_loop (base=0x50aea00, flags=<value optimized out>)
at event.c:1558
#3 0x0000000000414544 in worker_libevent (arg=0xc4c800) at daemon/thread.c:304
#4 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#5 0x00007fc4233b070d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 18 (Thread 0x7fc41f98b700 (LWP 19268)):
#0 0x00007fc4233b0d03 in epoll_wait () from /lib/libc.so.6
#1 0x00007fc4241eb5a6 in epoll_dispatch (base=0x50ae780, tv=<value optimized out>) at epoll.c:404
#2 0x00007fc4241da171 in event_base_loop (base=0x50ae780, flags=<value optimized out>)
at event.c:1558
#3 0x0000000000414544 in worker_libevent (arg=0xc4c900) at daemon/thread.c:304
#4 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#5 0x00007fc4233b070d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()

--Type <return> to continue, or q <return> to quit--
Thread 17 (Thread 0x7fc41cbca700 (LWP 19274)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a000) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 16 (Thread 0x7fc41c3c9700 (LWP 19279)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a0c0) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 15 (Thread 0x7fc41bbc8700 (LWP 19288)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a240) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 14 (Thread 0x7fc41b3c7700 (LWP 19294)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a3c0) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 13 (Thread 0x7fc41abc6700 (LWP 19299)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a180) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 12 (Thread 0x7fc41a3c5700 (LWP 19304)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a300) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 11 (Thread 0x7fc419bc4700 (LWP 19309)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a600) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 10 (Thread 0x7fc4193c3700 (LWP 19314)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a840) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 9 (Thread 0x7fc418bc2700 (LWP 19320)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a540) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 8 (Thread 0x7fc4183c1700 (LWP 19325)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a780) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()
--Type <return> to continue, or q <return> to quit--

Thread 7 (Thread 0x7fc417bc0700 (LWP 19330)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a480) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 6 (Thread 0x7fc4173bf700 (LWP 19335)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc42199331f in engine_shutdown_thread (arg=0x516a6c0) at bucket_engine.c:1469
#2 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#3 0x00007fc4233b070d in clone () from /lib/libc.so.6
#4 0x0000000000000000 in ?? ()

Thread 5 (Thread 0x7fc41ebce700 (LWP 19342)):
#0 0x00007fc42365885c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc41ee0c707 in SyncObject::wait (this=0x89c8fc0) at syncobject.hh:31
#2 Dispatcher::run (this=0x89c8fc0) at dispatcher.cc:85
#3 0x00007fc41ee0d2e3 in launch_dispatcher_thread (arg=0x89c9014) at dispatcher.cc:28
#4 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#5 0x00007fc4233b070d in clone () from /lib/libc.so.6
#6 0x0000000000000000 in ?? ()
--Type <return> to continue, or q <return> to quit--

Thread 4 (Thread 0x7fc41e3cd700 (LWP 19343)):
#0 0x00007fc4233a3f93 in poll () from /lib/libc.so.6
#1 0x00007fc41ee8fad2 in MemcachedEngine::waitForReadable (this=0x567c800)
at mc-kvstore/mc-engine.cc:1015
#2 0x00007fc41ee90318 in MemcachedEngine::wait (this=0x567c800, cb=DWARF-2 expression error: DW_OP_reg operations must be used either alone or in conjuction with DW_OP_piece.
)
at mc-kvstore/mc-engine.cc:1048
#3 MemcachedEngine::tapKeys (this=0x567c800, cb=DWARF-2 expression error: DW_OP_reg operations must be used either alone or in conjuction with DW_OP_piece.
) at mc-kvstore/mc-engine.cc:1303

#4 0x00007fc41ee95f3f in MCKVStore::dumpKeys (this=0x901b640, vbids=<value optimized out>, cb=DWARF-2 expression error: DW_OP_reg operations must be used either alone or in conjuction with DW_OP_piece.
)
at mc-kvstore/mc-kvstore.cc:169

#5 0x00007fc41ee15773 in EventuallyPersistentStore::warmup (this=0x85f2000,
st=<value optimized out>, keysOnly=<value optimized out>) at ep.cc:2679

#6 0x00007fc41ee4961c in Flusher::step (this=0x51ad440, d=<value optimized out>, tid=DWARF-2 expression error: DW_OP_reg operations must be used either alone or in conjuction with DW_OP_piece.
)
at flusher.cc:194
#7 0x00007fc41ee49f20 in FlusherStepper::callback (this=<value optimized out>, d=...,
t=<value optimized out>) at flusher.cc:23
#8 0x00007fc41ee0d88f in Task::run (this=<value optimized out>, d=..., t=<value optimized out>)
at dispatcher.hh:139
#9 0x00007fc41ee0c482 in Dispatcher::run (this=0x89c9880) at dispatcher.cc:119
#10 0x00007fc41ee0d2e3 in launch_dispatcher_thread (arg=0x7fc41e3cc760) at dispatcher.cc:28
#11 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#12 0x00007fc4233b070d in clone () from /lib/libc.so.6
#13 0x0000000000000000 in ?? ()
--Type <return> to continue, or q <return> to quit--

Thread 3 (Thread 0x7fc41dbcc700 (LWP 19344)):
#0 0x00007fc423658bc9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/libpthread.so.0
#1 0x00007fc41ee08fc0 in SyncObject::wait (this=0xc52900, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0xc52900, d=...) at dispatcher.cc:244
#3 0x00007fc41ee0c482 in Dispatcher::run (this=0x89c9dc0) at dispatcher.cc:119
#4 0x00007fc41ee0d2e3 in launch_dispatcher_thread (arg=0x89c9e14) at dispatcher.cc:28
#5 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#6 0x00007fc4233b070d in clone () from /lib/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 2 (Thread 0x7fc416bbe700 (LWP 19346)):
#0 0x00007fc42337439d in nanosleep () from /lib/libc.so.6
#1 0x00007fc4233a9844 in usleep () from /lib/libc.so.6
#2 0x00007fc41ee47e3a in Flusher::wait (this=0x51ad440) at flusher.cc:59
#3 0x00007fc41ee1d036 in EventuallyPersistentStore::stopFlusher (this=0x85f2000,
__in_chrg=<value optimized out>) at ep.cc:754
#4 ~EventuallyPersistentStore (this=0x85f2000, __in_chrg=<value optimized out>) at ep.cc:720
#5 0x00007fc41ee459ac in EventuallyPersistentEngine::~EventuallyPersistentEngine() ()
from /opt/couchbase/lib/memcached/ep.so
#6 0x00007fc41ee37abc in EvpDestroy (handle=0x62f4d00, force=<value optimized out>)
at ep_engine.cc:85
#7 0x00007fc421993191 in engine_shutdown_thread (arg=0x516a900) at bucket_engine.c:1431
--Type <return> to continue, or q <return> to quit--
#8 0x00007fc4236539ca in start_thread () from /lib/libpthread.so.0
#9 0x00007fc4233b070d in clone () from /lib/libc.so.6
#10 0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7fc42481a720 (LWP 19261)):
#0 0x00007fc4233b0d03 in epoll_wait () from /lib/libc.so.6
#1 0x00007fc4241eb5a6 in epoll_dispatch (base=0x50ae000, tv=<value optimized out>) at epoll.c:404
#2 0x00007fc4241da171 in event_base_loop (base=0x50ae000, flags=<value optimized out>)
at event.c:1558
#3 0x0000000000409b3f in main (argc=<value optimized out>, argv=<value optimized out>)
at daemon/memcached.c:7781