Segmentation fault in ep-engine shutdown

I was doing some stress tests for disabled async io threads overnight and got 3 memcached core dumps. One of them has this in backtraces and others look similar:

#0 0xf7751430 in __kernel_vsyscall ()
(gdb) bt
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf755d941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0xf7560d72 in *__GI_abort () at abort.c:92
#3 0xf37267dc in Mutex::acquire() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf3739b91 in TapResumeCallback::callback(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf36e2a42 in Task::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#6 0xf36e1154 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#7 0xf36e296a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#8 0xf7695c39 in start_thread (arg=0xf1d49b70) at pthread_create.c:304
#9 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
(gdb) thread app all bt

Thread 7 (Thread 0xf354cb70 (LWP 22469)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf769dfeb in do_pwrite64 (offset=<optimized out>, count=<optimized out>, buf=<optimized out>, fd=<optimized out>) at ../sysdeps/unix/sysv/linux/pwrite64.c:54
#2 __libc_pwrite64 (fd=fd@entry=49, buf=buf@entry=0x24da842d, count=count@entry=765, offset=56431572) at ../sysdeps/unix/sysv/linux/pwrite64.c:77
#3 0xf611e1d0 in couch_pwrite (handle=0x31, buf=0x24da842d, nbyte=765, offset=56431572) at src/os.c:56
#4 0xf376ed4a in cfs_pwrite () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf611c72c in flush_buffer (buf=0x24da8410) at src/iobuffer.c:106
#6 0xf611c901 in buffered_pread (handle=0x20b5e518, buf=0xf354b9f8, nbyte=8, offset=49725540) at src/iobuffer.c:271
#7 0xf611afd8 in read_skipping_prefixes (pos=pos@entry=0xf354b9e8, len=len@entry=8, dst=dst@entry=0xf354b9f8, db=<error reading variable: Unhandled dwarf expression opcode 0xfa>,
db=<error reading variable: Unhandled dwarf expression opcode 0xfa>) at src/couch_file_read.c:27
#8 0xf611b084 in pread_bin_internal (db=db@entry=0xf3f8d28, pos=49725540, ret_ptr=ret_ptr@entry=0xf354ba38, header=header@entry=0) at src/couch_file_read.c:53
#9 0xf611b1f6 in pread_compressed (db=0xf3f8d28, pos=49725540, ret_ptr=ret_ptr@entry=0xf354ba9c) at src/couch_file_read.c:92
#10 0xf61172b7 in modify_node (rq=rq@entry=0xf354bd1c, nptr=0xc386b31c, start=start@entry=8084, end=end@entry=8090, dst=dst@entry=0xc386b028) at src/btree_modify.c:321
#11 0xf6117420 in modify_node (rq=rq@entry=0xf354bd1c, nptr=0xc386b008, start=<optimized out>, start@entry=8080, end=end@entry=8152, dst=dst@entry=0x8c03a988) at src/btree_modify.c:468
#12 0xf6117420 in modify_node (rq=rq@entry=0xf354bd1c, nptr=0x8c03a968, start=<optimized out>, start@entry=6924, end=end@entry=8330, dst=dst@entry=0x1b2f0060) at src/btree_modify.c:468
#13 0xf6117420 in modify_node (rq=rq@entry=0xf354bd1c, nptr=nptr@entry=0x17549e10, start=<optimized out>, start@entry=0, end=8738, dst=0x1b2f0000) at src/btree_modify.c:468
#14 0xf6117a4b in modify_btree (rq=rq@entry=0xf354bd1c, root=0x17549e10, errcode=errcode@entry=0xf354bcc8) at src/btree_modify.c:620
#15 0xf611ac88 in update_indexes (numdocs=4369, idvals=0x24e2b330, ids=0x24e1a220, seqvals=0x24e22aa8, seqs=0x24e11998, db=0xf3f8d28) at src/couch_save.c:198
#16 couchstore_save_documents (db=0xf3f8d28, docs=0x186cb200, infos=0x19b22ed0, numdocs=4369, options=1) at src/couch_save.c:354
#17 0xf37603aa in CouchKVStore::saveDocs(unsigned short, unsigned long long, _doc*, _docinfo*, int) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#18 0xf3760b22 in CouchKVStore::commit2couchstore() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#19 0xf3760d0e in CouchKVStore::commit() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#20 0xf36e4d21 in TransactionContext::commit() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#21 0xf36eebf0 in EventuallyPersistentStore::flushVBQueue(std::queue<SingleThreadedRCPtr<QueuedItem>, std::deque<SingleThreadedRCPtr<QueuedItem>, std::allocator<SingleThreadedRCPtr<QueuedItem> > > >&, unsigned short, int) ()
from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#22 0xf36ef865 in EventuallyPersistentStore::flushOutgoingQueue(std::map<unsigned short, std::queue<SingleThreadedRCPtr<QueuedItem>, std::deque<SingleThreadedRCPtr<QueuedItem>, std::allocator<SingleThreadedRCPtr<QueuedItem> > > >, std::less<unsigned short>, std::allocator<std::pair<unsigned short const, std::queue<SingleThreadedRCPtr<QueuedItem>, std::deque<SingleThreadedRCPtr<QueuedItem>, std::allocator<SingleThreadedRCPtr<QueuedItem> > > > > > >*, unsigned int) ()
from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#23 0xf371ebca in Flusher::doFlush() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#24 0xf371f338 in Flusher::step(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#25 0xf36e2a42 in Task::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#26 0xf36e1154 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#27 0xf36e296a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#28 0xf7695c39 in start_thread (arg=0xf354cb70) at pthread_create.c:304
#29 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 6 (Thread 0xf692db70 (LWP 22455)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf769a703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf7746874 in logger_thead_main (arg=0x94f5e98) at extensions/loggers/file_logger.c:368
#3 0xf7695c39 in start_thread (arg=0xf692db70) at pthread_create.c:304
#4 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 5 (Thread 0xf254ab70 (LWP 22471)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf769a703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf36ddedc in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf36e1154 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf36e296a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7695c39 in start_thread (arg=0xf254ab70) at pthread_create.c:304
#6 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 4 (Thread 0xf2d4bb70 (LWP 22470)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf769a703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf36ddedc in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf36e1154 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf36e296a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7695c39 in start_thread (arg=0xf2d4bb70) at pthread_create.c:304
#6 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
--Type <return> to continue, or q <return> to quit--

Thread 3 (Thread 0xf75316c0 (LWP 22449)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf75cdde6 in nanosleep () at ../sysdeps/unix/syscall-template.S:82
#2 0xf75fcf2c in usleep (useconds=1000) at ../sysdeps/unix/sysv/linux/usleep.c:33
#3 0xf371db14 in Flusher::wait() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf36e41c6 in EventuallyPersistentStore::stopFlusher() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf36e8af4 in EventuallyPersistentStore::~EventuallyPersistentStore() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#6 0xf37146b4 in EventuallyPersistentEngine::~EventuallyPersistentEngine() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#7 0xf3702a76 in EvpDestroy () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#8 0xf61235fa in bucket_shutdown_engine (key=0xc01b8c8, nkey=7, val=0xbfedff8, nval=0, args=0x0) at bucket_engine.c:1290
#9 0xf6129099 in genhash_iter (h=0xbfd7200, iterfunc=iterfunc@entry=0xf61235a0 <bucket_shutdown_engine>, arg=arg@entry=0x0) at genhash.c:275
#10 0xf6127f22 in bucket_destroy (handle=0xf612c940, force=<optimized out>) at bucket_engine.c:1327
#11 bucket_destroy (handle=0xf612c940, force=false) at bucket_engine.c:1307
#12 0x0804be30 in main (argc=21, argv=0xfff637a4) at daemon/memcached.c:7924

Thread 2 (Thread 0xf7530b70 (LWP 22454)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf75f3f3b in read () at ../sysdeps/unix/syscall-template.S:82
#2 0xf759d25b in _IO_new_file_underflow (fp=0xf768c440) at fileops.c:606
#3 0xf759eafb in _IO_default_uflow (fp=0xf768c440) at genops.c:440
#4 0xf759ff28 in *_GI__uflow (fp=0xf768c440) at genops.c:394
#5 0xf759304c in _IO_getline_info (fp=0xf768c440, buf=0xf7530330 "shutdown\n", n=79, delim=10, extract_delim=1, eof=0x0) at iogetline.c:74
#6 0xf7592f91 in _IO_getline (fp=0xf768c440, buf=0xf7530330 "shutdown\n", n=79, delim=10, extract_delim=1) at iogetline.c:42
#7 0xf7591fda in _IO_fgets (buf=buf@entry=0xf7530330 "shutdown\n", n=n@entry=80, fp=0xf768c440) at iofgets.c:58
#8 0xf774988c in check_stdin_thread (arg=0x804cf30) at extensions/daemon/stdin_check.c:37
#9 0xf7695c39 in start_thread (arg=0xf7530b70) at pthread_create.c:304
#10 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 1 (Thread 0xf1d49b70 (LWP 22472)):
#0 0xf7751430 in __kernel_vsyscall ()
#1 0xf755d941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0xf7560d72 in *__GI_abort () at abort.c:92
#3 0xf37267dc in Mutex::acquire() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf3739b91 in TapResumeCallback::callback(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf36e2a42 in Task::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#6 0xf36e1154 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#7 0xf36e296a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#8 0xf7695c39 in start_thread (arg=0xf1d49b70) at pthread_create.c:304
#9 0xf760323e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

So apparently something caused ns_server to shutdown memcached (most likely max_restart_intensity_reached condition) and ep-engine could not handle it correctly.