Crash in memcached's release_cookie

(I was having some crashes regularly for months already, but figured it wasn't worth filing as QEs most likely found some of them as well. Pre-beta things are quite different)

Just found core dump. Doing nothing. 16 cluster_run nodes. And 5 empty buckets.

Here's what backtrace shows me:

Thread 26 (Thread 0xeccf9b70 (LWP 25048)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xeccf9b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 25 (Thread 0xeb4f6b70 (LWP 25164)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xeb4f6b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 24 (Thread 0xefcffb70 (LWP 24864)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xefcffb70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 23 (Thread 0xe8cf1b70 (LWP 25289)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xe8cf1b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 22 (Thread 0xf36fab70 (LWP 24868)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e3687f in EventuallyPersistentEngine::notifyPendingConnections() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e369a2 in EvpNotifyPendingConns () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf7644c39 in start_thread (arg=0xf36fab70) at pthread_create.c:304
#5 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 21 (Thread 0xe9cf3b70 (LWP 25167)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xe9cf3b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 20 (Thread 0xe84f0b70 (LWP 25290)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xe84f0b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 19 (Thread 0xf1500b70 (LWP 24950)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xf1500b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 18 (Thread 0xed4fab70 (LWP 25047)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xed4fab70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 17 (Thread 0xebcf7b70 (LWP 25050)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e3687f in EventuallyPersistentEngine::notifyPendingConnections() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e369a2 in EvpNotifyPendingConns () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf7644c39 in start_thread (arg=0xebcf7b70) at pthread_create.c:304
#5 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 16 (Thread 0xee4fcb70 (LWP 24954)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e3687f in EventuallyPersistentEngine::notifyPendingConnections() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e369a2 in EvpNotifyPendingConns () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf7644c39 in start_thread (arg=0xee4fcb70) at pthread_create.c:304
#5 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 15 (Thread 0xf2502b70 (LWP 24866)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xf2502b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 14 (Thread 0xe94f2b70 (LWP 25168)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e3687f in EventuallyPersistentEngine::notifyPendingConnections() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e369a2 in EvpNotifyPendingConns () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf7644c39 in start_thread (arg=0xe94f2b70) at pthread_create.c:304
#5 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 13 (Thread 0xe7cefb70 (LWP 25291)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xe7cefb70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 12 (Thread 0xeacf5b70 (LWP 25165)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xeacf5b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 11 (Thread 0xedcfbb70 (LWP 25046)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xedcfbb70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 10 (Thread 0xea4f4b70 (LWP 25166)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xea4f4b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 9 (Thread 0xeecfdb70 (LWP 24953)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xeecfdb70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 8 (Thread 0xf74e06c0 (LWP 23970)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf75a2fbb in write () at ../sysdeps/unix/syscall-template.S:82
#2 0xf754bff4 in _IO_new_file_write (f=0xf763b580, data=0xff82fad0, n=90) at fileops.c:1276
#3 0xf754bc7f in new_do_write (fp=0xf763b580, data=0xff82fad0 "Wed Sep 12 23:21:10.772212 3: Schedule cleanup of \"eq_tapq:replication_n_14@10.17.22.233\"\n", to_do=90) at fileops.c:530
#4 0xf754bf46 in _IO_new_file_xsputn (f=0xf763b580, data=0xff82fad0, n=90) at fileops.c:1370
#5 0xf75415b0 in *_GI_IO_fputs (str=str@entry=0xff82fad0 "Wed Sep 12 23:21:10.772212 3: Schedule cleanup of \"eq_tapq:replication_n_14@10.17.22.233\"\n", fp=0xf763b580) at iofputs.c:42
#6 0xf76f5521 in logger_log (severity=EXTENSION_LOG_WARNING, client_cookie=0x0, fmt=0xf2eca190 "Schedule cleanup of \"%s\"") at extensions/loggers/file_logger.c:268
#7 0xf2e7388e in TapConnMap::shutdownAllTapConnections() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#8 0xf2e3a1be in EvpDestroy () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#9 0xf609d5fa in bucket_shutdown_engine (key=0xf0000de8, nkey=4, val=0xf0000b78, nval=0, args=0x0) at bucket_engine.c:1290
#10 0xf60a30b9 in genhash_iter (h=0xb9d5d80, iterfunc=iterfunc@entry=0xf609d5a0 <bucket_shutdown_engine>, arg=arg@entry=0x0) at genhash.c:275
#11 0xf60a1f42 in bucket_destroy (handle=0xf60a6960, force=<optimized out>) at bucket_engine.c:1327
#12 bucket_destroy (handle=0xf60a6960, force=false) at bucket_engine.c:1307
#13 0x0804be30 in main (argc=21, argv=0xff831b34) at daemon/memcached.c:7923

Thread 7 (Thread 0xec4f8b70 (LWP 25049)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xec4f8b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 6 (Thread 0xf1d01b70 (LWP 24867)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xf1d01b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 5 (Thread 0xf2d03b70 (LWP 24865)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xf2d03b70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 4 (Thread 0xef4feb70 (LWP 24952)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xef4feb70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 3 (Thread 0xf0cffb70 (LWP 24951)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf2e16e8c in IdleTask::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#3 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#4 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#5 0xf7644c39 in start_thread (arg=0xf0cffb70) at pthread_create.c:304
#6 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 2 (Thread 0xf68dcb70 (LWP 23985)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf7649703 in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/i386/i686/../i486/pthread_cond_timedwait.S:236
#2 0xf76f5864 in logger_thead_main (arg=0x8ef4e40) at extensions/loggers/file_logger.c:361
#3 0xf7644c39 in start_thread (arg=0xf68dcb70) at pthread_create.c:304
#4 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

Thread 1 (Thread 0xe74eeb70 (LWP 25292)):
#0 0xf7700430 in __kernel_vsyscall ()
#1 0xf750c941 in *__GI_raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#2 0xf750fd72 in *__GI_abort () at abort.c:92
#3 0x0804da3c in release_cookie (cookie=0xb806b68) at daemon/memcached.c:6693
#4 0xf60a0252 in bucket_engine_release_cookie (cookie=0xb806b68) at bucket_engine.c:2565
#5 0xf2e36234 in EventuallyPersistentEngine::releaseCookie(void const*) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#6 0xf2e64e6b in TapConnection::releaseReference(bool) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#7 0xf2e77adb in TapConnectionReaperCallback::callback(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#8 0xf2e1b9f2 in Task::run(Dispatcher&, SingleThreadedRCPtr<Task>&) () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#9 0xf2e1a104 in Dispatcher::run() () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#10 0xf2e1b91a in launch_dispatcher_thread () from /root/src/altoros/moxi/repo20/install/lib/memcached/ep.so
#11 0xf7644c39 in start_thread (arg=0xe74eeb70) at pthread_create.c:304
#12 0xf75b223e in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:130

and extra stuff:

(gdb) p thr
$1 = (LIBEVENT_THREAD *) 0xb9ec624
(gdb) p *thr
$2 = {thread_id = 4100967280, base = 0xb9ed828, notify_event = {ev_active_next =

{tqe_next = 0x0, tqe_prev = 0xb9edb38}

, ev_next =

{tqe_next = 0xb9ca8d0, tqe_prev = 0xb9ed8fc}

, ev_timeout_pos = {ev_next_with_common_timeout =

{tqe_next = 0xffffffff, tqe_prev = 0x0}, min_heap_idx = -1}, ev_fd = 24, ev_base = 0xb9ed828, ev = {ev_io = {ev_io_next = {tqe_next = 0x0, tqe_prev = 0xb9edbd0}, ev_timeout = {tv_sec = 0, tv_usec = 0}}, ev_signal = {ev_signal_next = {tqe_next = 0x0, tqe_prev = 0xb9edbd0}, ev_ncalls = 0, ev_pncalls = 0x0}}, ev_events = 18, ev_res = 2, ev_flags = 128, ev_pri = 0 '\000', ev_closure = 2 '\002', ev_timeout = {tv_sec = 0, tv_usec = 0}, ev_callback = 0x805b280 <thread_libevent_process>, ev_arg = 0xb9ec624}, notify = {24, 25}, new_conn_queue = 0xb9edbe0, suffix_cache = 0xb9edc38, mutex = {data = {lock = 0, __count = 0, __owner = 0, __kind = 376, __nusers = 80, {spins = 61, __list = {_next = 0x3d}}}, __size = '\000' <repeats 12 times>, "x\001\000\000P\000\000\000=\000\000", __align = 0}, is_locked = 61, pending_io = 0x1, index = 1885431122, type = 543649385, last_checked = 544235892}
(gdb) l
6688 conn *c = (conn *)cookie;
6689 assert(c);
6690
6691 LIBEVENT_THREAD *thr = c->thread;
6692 assert(thr);
6693 LOCK_THREAD(thr);
6694 -c>refcount;
6695
6696 /* Releasing the refererence to the object may cause it to change
6697 * state. (NOTE: the release call shall never be called from the
(gdb) p *c
$3 = {sfd = 254, nevents = 0, sasl_conn = 0xf0205fa8, state = 0x805aa40 <conn_ship_log>, substate = bin_reading_packet, registered_in_libevent = true, event = {ev_active_next = {tqe_next = 0xb7fdf20, tqe_prev = 0xb9edb38}, ev_next = {tqe_next = 0xb7fdf20, tqe_prev = 0xb9ed8fc}, ev_timeout_pos = {ev_next_with_common_timeout = {tqe_next = 0xffffffff, tqe_prev = 0x0}

, min_heap_idx = -1}, ev_fd = 254, ev_base = 0xb9ed828, _ev = {ev_io = {ev_io_next =

{tqe_next = 0x0, tqe_prev = 0xf0205f28}, ev_timeout = {tv_sec = 0, tv_usec = 0}}, ev_signal = {ev_signal_next = {tqe_next = 0x0, tqe_prev = 0xf0205f28}

, ev_ncalls = 0, ev_pncalls = 0x0}}, ev_events = 18, ev_res = 0, ev_flags = 128, ev_pri = 0 '\000', ev_closure = 2 '\002', ev_timeout =

{tv_sec = 0, tv_usec = 0}

, ev_callback = 0x804f700 <event_handler>, ev_arg = 0xb806b68}, ev_flags = 18, which = 4, rbuf = 0xb806d48 "\201F", rcurr = 0xb806df0 "", rsize = 2048, rbytes = 0, wbuf = 0xb807550 "\200\n", wcurr = 0xb807550 "\200\n", wsize = 2048, wbytes = 1992, write_and_go = 0x805aa40 <conn_ship_log>, write_and_free = 0x0, ritem = 0xb806df0 "", rlbytes = 0, item = 0x0, store_op = 0, sbytes = 0, iov = 0xb8080d8, iovsize = 400, iovused = 0, msglist = 0xb808d60, msgsize = 10, msgused = 1, msgcurr = 0, msgbytes = 0, ilist = 0xb807d58, isize = 200, icurr = 0xb807d58, ileft = 0, suffixlist = 0xb808080, suffixsize = 20, suffixcurr = 0xb808080, suffixleft = 0, protocol = binary_prot, transport = tcp_transport, request_id = 0, request_addr =

{ss_family = 0, __ss_align = 0, __ss_padding = '\000' <repeats 119 times>}

, request_addr_size = 0, hdrbuf = 0x0, hdrsize = 0, noreply = false, refcount = 2 '\002', dynamic_buffer =

{buffer = 0x0, size = 0, offset = 0}

, engine_storage = 0xf0205f38, ascii_cmd = 0x0, binary_header = {request =

{magic = 129 '\201', opcode = 70 'F', keylen = 0, extlen = 0 '\000', datatype = 0 '\000', vbucket = 0, bodylen = 0, opaque = 654311424, cas = 0}

, bytes = "\201F", '\000' <repeats 13 times>, "'\000\000\000\000\000\000\000"}, cas = 0, cmd = 70, opaque = 654311424, keylen = 0, list_state = 0, next = 0x0, thread = 0xb9ec624, aiostat = ENGINE_SUCCESS, ewouldblock = true, tap_iterator = 0xf609ea80 <bucket_tap_iterator_shim>, parent_port = 11999}