memcached dumps core during rebalance

Everything latest, memcached consistently dumps core during rebalance.

I've bisected it to commit 2b3c69b428ae500a76074ed8c2cbc3a518a79220 of ep-engine.

Here's backtraces:

Program terminated with signal 6, Aborted.
#0 0xffffe430 in __kernel_vsyscall ()
(gdb) thread apply all bt

Thread 10 (Thread 73):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x555e8c39 in __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/lowlevellock.S:142
#2 0x555e403b in _L_lock_748 () from /lib/i686/cmov/libpthread.so.0
#3 0x555e3e61 in __pthread_mutex_lock (mutex=0x8eb0034) at pthread_mutex_lock.c:61
#4 0x565a4644 in acquire (this=0x56593790, m=...) at mutex.hh:50
#5 lock (this=0x56593790, m=...) at locks.hh:39
#6 LockHolder::LockHolder (this=0x56593790, m=...) at locks.hh:25
#7 0x565aabd5 in EventuallyPersistentStore::setVBucketState(uint16_t, <anonymous enum>, struct

{...} *) (this=0x8eafdc0, vbid=120,
to=dead, core=0x55972680) at ep.cc:293
#8 0x565c1f1e in EventuallyPersistentEngine::doWalkTapQueue(void const*, void*, void, unsigned short, unsigned char*, unsigned short*, unsigned int*, unsigned short*, TapConnection**) () from ./bin/ep_engine/ep.so
#9 0x565b8aa4 in walkTapQueue (handle=0x8ea53d0, cookie=0x9be2698, itm=0x56593a0c, es=0x56593a08, nes=0x56593a06,
ttl=0x56593a05 "\377\004", flags=0x56593a02, seqno=0x565939fc, vbucket=0x565939fa) at ep_engine.cc:1074
#10 EvpTapIterator (handle=0x8ea53d0, cookie=0x9be2698, itm=0x56593a0c, es=0x56593a08, nes=0x56593a06, ttl=0x56593a05 "\377\004",
flags=0x56593a02, seqno=0x565939fc, vbucket=0x565939fa) at ep_engine.cc:569
#11 0x5596c9bc in bucket_tap_iterator_shim (handle=0x559725a0, cookie=0x9be2698, item=0x56593a0c, engine_specific=0x56593a08,
nengine_specific=0x56593a06, ttl=0x56593a05 "\377\004", flags=0x56593a02, seqno=0x565939fc, vbucket=0x565939fa)
at bucket_engine.c:956
#12 0x080505b6 in ship_tap_log (c=0x9be2698) at memcached.c:2301
#13 0x08057e5a in conn_ship_log (c=0x9be2698) at memcached.c:4756
#14 0x0805dbf4 in libevent_tap_process (fd=26, which=2, arg=0x806aa80) at thread.c:439
#15 0x5557967c in event_process_active (base=0x8ec4930) at event.c:395
#16 0x55579a43 in event_base_loop (base=0x8ec4930, flags=0) at event.c:547
#17 0x0805d228 in worker_libevent (arg=0x806aa80) at thread.c:252
#18 0x555e1955 in start_thread (arg=0x56594b70) at pthread_create.c:300
#19 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 9 (Thread 76):
#0 0xffffe430 in __kernel_vsyscall ()
--Type <return> to continue, or q <return> to quit--
#1 0x555e6482 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:179
#2 0x565b5c43 in wait (arg=0x8ea53d0) at syncobject.hh:42
#3 wait (arg=0x8ea53d0) at syncobject.hh:58
#4 notifyTapIoThread (arg=0x8ea53d0) at ep_engine.cc:2083
#5 EvpNotifyTapIo (arg=0x8ea53d0) at ep_engine.cc:631
#6 0x555e1955 in start_thread (arg=0x56b85b70) at pthread_create.c:300
#7 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 8 (Thread 71):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x556c16f6 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0x5558a249 in epoll_dispatch (base=0x8ec2ca0, arg=0x8ec3058, tv=0x0) at epoll.c:198
#3 0x555799f9 in event_base_loop (base=0x8ec2ca0, flags=0) at event.c:538
#4 0x0805d228 in worker_libevent (arg=0x8ec1110) at thread.c:252
#5 0x555e1955 in start_thread (arg=0x56192b70) at pthread_create.c:300
#6 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 7 (Thread 67):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x556b1f5b in read () at ../sysdeps/unix/syscall-template.S:82
#2 0x5565c11b in _IO_new_file_underflow (fp=0x55737420) at fileops.c:606
#3 0x5565d9bb in _IO_default_uflow (fp=0x55737420) at genops.c:440
#4 0x5565ede8 in __uflow (fp=0x55737420) at genops.c:394
#5 0x556546dc in _IO_getc (fp=0x55737420) at getc.c:41
#6 0x5558e6ed in check_stdin_thread (arg=0x805b275) at stdin_check.c:18
#7 0x555e1955 in start_thread (arg=0x55968b70) at pthread_create.c:300
#8 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 6 (Thread 68):
#0 0xffffe430 in __kernel_vsyscall ()
--Type <return> to continue, or q <return> to quit--
#1 0x5568c516 in nanosleep () at ../sysdeps/unix/syscall-template.S:82
#2 0x5568c340 in __sleep (seconds=0) at ../sysdeps/unix/sysv/linux/sleep.c:138
#3 0x0806091d in check_isasl_db_thread (arg=0x806a9a8) at isasl.c:215
#4 0x555e1955 in start_thread (arg=0x55b8fb70) at pthread_create.c:300
#5 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 5 (Thread 70):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x556c16f6 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
#2 0x5558a249 in epoll_dispatch (base=0x8ec1f78, arg=0x8ec2330, tv=0x0) at epoll.c:198
#3 0x555799f9 in event_base_loop (base=0x8ec1f78, flags=0) at event.c:538
#4 0x0805d228 in worker_libevent (arg=0x8ec1080) at thread.c:252
#5 0x555e1955 in start_thread (arg=0x55f91b70) at pthread_create.c:300
#6 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 4 (Thread 69):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x555e8c39 in __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/lowlevellock.S:142
#2 0x555e403b in _L_lock_748 () from /lib/i686/cmov/libpthread.so.0
#3 0x555e3e61 in __pthread_mutex_lock (mutex=0x806aae0) at pthread_mutex_lock.c:61
#4 0x08058f2a in conn_add_tap_client (c=0x9be5b68) at memcached.c:5111
#5 0x08059439 in event_handler (fd=43, which=2, arg=0x9be5b68) at memcached.c:5185
#6 0x5557967c in event_process_active (base=0x8ec1250) at event.c:395
#7 0x55579a43 in event_base_loop (base=0x8ec1250, flags=0) at event.c:547
#8 0x0805d228 in worker_libevent (arg=0x8ec0ff0) at thread.c:252
#9 0x555e1955 in start_thread (arg=0x55d90b70) at pthread_create.c:300
#10 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 3 (Thread 53):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x556c16f6 in epoll_wait () at ../sysdeps/unix/syscall-template.S:82
--Type <return> to continue, or q <return> to quit--
#2 0x5558a249 in epoll_dispatch (base=0x8eb0b40, arg=0x8eb09d8, tv=0xfffc7478) at epoll.c:198
#3 0x555799f9 in event_base_loop (base=0x8eb0b40, flags=0) at event.c:538
#4 0x0805c6f4 in main (argc=17, argv=0xfffc8a14) at memcached.c:6837

Thread 2 (Thread 75):
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x555e909b in read () at ../sysdeps/unix/syscall-template.S:82
#2 0x565def64 in seekAndRead (id=0x8ea60c8, offset=3072, pBuf=0x56983a98, cnt=8) at embedded/sqlite3.c:24956
#3 0x565defd4 in unixRead (id=0x8ea60c8, pBuf=0x56983a98, amt=8, offset=3072) at embedded/sqlite3.c:24988
#4 0x565d72c6 in sqlite3OsRead (id=0x8ea60c8, pBuf=0x56983a98, amt=8, offset=3072) at embedded/sqlite3.c:12548
#5 0x565e557b in syncJournal (pPager=0x8ea5f90) at embedded/sqlite3.c:34944
#6 0x565e7554 in sqlite3PagerCommitPhaseOne (pPager=0x8ea5f90, zMaster=0x0, noSync=0) at embedded/sqlite3.c:36871
#7 0x565ec2f4 in sqlite3BtreeCommitPhaseOne (p=0x8ea5ad8, zMaster=0x0) at embedded/sqlite3.c:41557
#8 0x565f9069 in vdbeCommit (db=0x8ea5730, p=0x8eabd50) at embedded/sqlite3.c:49928
#9 0x565f98ec in sqlite3VdbeHalt (p=0x8eabd50) at embedded/sqlite3.c:50342
#10 0x565fdc09 in sqlite3VdbeExec (p=0x8eabd50) at embedded/sqlite3.c:54122
#11 0x565fb974 in sqlite3Step (p=0x8eabd50) at embedded/sqlite3.c:51732
#12 0x565fbb8d in sqlite3_step (pStmt=0x8eabd50) at embedded/sqlite3.c:51792
#13 0x565cb276 in PreparedStatement::execute (this=0x8f23dc8) at sqlite-pst.cc:47
#14 0x565c8f4f in StrategicSqlite3::setVBState (this=0x8ec3fa8, vbucket=<value optimized out>, state_str=...)
at sqlite-kvstore.cc:170
#15 0x565a5a9b in EventuallyPersistentStore::completeSetVBState(uint16_t, const std::string &, struct {...}

*) (this=0x8eafdc0,
vbid=<value optimized out>, key=..., core=0x55972680) at ep.cc:279
#16 0x565ae1a0 in SetVBStateCallback::callback (this=0x9bf1240, d=..., t=...) at ep.cc:85
#17 0x565a372c in run (this=0x8eadc30) at dispatcher.hh:87
#18 Dispatcher::run (this=0x8eadc30) at dispatcher.cc:81
#19 0x565a4135 in launch_dispatcher_thread (arg=0x8eadc30) at dispatcher.cc:26
#20 0x555e1955 in start_thread (arg=0x56984b70) at pthread_create.c:300
#21 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 1 (Thread 72):
--Type <return> to continue, or q <return> to quit--
#0 0xffffe430 in __kernel_vsyscall ()
#1 0x5561f751 in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0x55622b82 in abort () at abort.c:92
#3 0x556188b8 in __assert_fail (assertion=0x8067ddf "me == c->thread", file=0x8067d3b "thread.c", line=318,
function=0x8067d23 "thread_libevent_process") at assert.c:81
#4 0x0805d4ba in thread_libevent_process (fd=21, which=2, arg=0x8ec11a0) at thread.c:318
#5 0x5557967c in event_process_active (base=0x8ec39c8) at event.c:395
#6 0x55579a43 in event_base_loop (base=0x8ec39c8, flags=0) at event.c:547
#7 0x0805d228 in worker_libevent (arg=0x8ec11a0) at thread.c:252
#8 0x555e1955 in start_thread (arg=0x56393b70) at pthread_create.c:300
#9 0x556c0e7e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130
(gdb)