memcached core dump in a 8 node cluster under load (Flusher::step () )

#0 0x00002aaaabcc2265 in raise () from /lib64/libc.so.6
(gdb) t a a bt

Thread 31 (process 8236):
#0 0x00002aaaabd66018 in epoll_wait () from /lib64/libc.so.6
Cannot access memory at address 0x7fff8797aef8

Thread 30 (process 8245):
#0 0x00002aaaabd57f3b in read () from /lib64/libc.so.6
#1 0x00002aaaabcfec07 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00002aaaabcff5ce in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00002aaaabcfae8b in getc () from /lib64/libc.so.6
#4 0x00002aaaac4fa875 in ?? ()
from /opt/couchbase/lib/memcached/stdin_term_handler.so
#5 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#6 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 29 (process 8246):
#0 0x00002aaaabd2c0b1 in nanosleep () from /lib64/libc.so.6
#1 0x00002aaaabd2bed4 in sleep () from /lib64/libc.so.6
#2 0x0000000000414288 in parse_config ()
#3 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#4 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 28 (process 8249):
#0 0x00002aaaabd66018 in epoll_wait () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--
#1 0x00002aaaaaeedc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00002aaaaaedca4c in event_base_loop ()
from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in parse_config ()
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 27 (process 8250):
#0 0x00002aaaabd66018 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaaeedc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00002aaaaaedca4c in event_base_loop ()
from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in parse_config ()
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 26 (process 8251):
#0 0x00002aaaabd66018 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaaeedc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00002aaaaaedca4c in event_base_loop ()
from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in parse_config ()
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 25 (process 8252):
#0 0x00002aaaabd66018 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaaeedc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00002aaaaaedca4c in event_base_loop ()
from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in parse_config ()
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 24 (process 8253):
#0 0x00002aaaabd66018 in epoll_wait () from /lib64/libc.so.6
#1 0x00002aaaaaeedc28 in ?? () from /opt/couchbase/lib/libevent-2.0.so.5
#2 0x00002aaaaaedca4c in event_base_loop ()
from /opt/couchbase/lib/libevent-2.0.so.5
#3 0x0000000000412b14 in parse_config ()
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 23 (process 8300):
#0 0x00002aaaaba81d09 in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#1 0x00002aaaac96bcb2 in Dispatcher::run ()
from /opt/couchbase/lib/memcached/ep.so
#2 0x00002aaaac96c6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#4 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 22 (process 8301):
#0 0x00002aaaabd5ce46 in poll () from /lib64/libc.so.6
#1 0x00002aaaac9f59c3 in MemcachedEngine::waitForWritable ()
from /opt/couchbase/lib/memcached/ep.so
#2 0x00002aaaac9f4a99 in MemcachedEngine::sendCommand ()
from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaac9f6916 in MemcachedEngine::setmq ()
from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaac97a4a1 in EventuallyPersistentStore::flushOneDelOrSet ()
from /opt/couchbase/lib/memcached/ep.so
#5 0x00002aaaac97aa0c in EventuallyPersistentStore::flushOne ()
from /opt/couchbase/lib/memcached/ep.so
#6 0x00002aaaac97ab3e in EventuallyPersistentStore::flushSome ()
from /opt/couchbase/lib/memcached/ep.so
#7 0x00002aaaac9af7f7 in Flusher::doFlush ()
from /opt/couchbase/lib/memcached/ep.so
#8 0x00002aaaac9afe6c in Flusher::completeFlush ()
--Type <return> to continue, or q <return> to quit--
from /opt/couchbase/lib/memcached/ep.so
#9 0x00002aaaac9aff9a in Flusher::step ()
from /opt/couchbase/lib/memcached/ep.so
#10 0x00002aaaac9b09ee in FlusherStepper::callback ()
from /opt/couchbase/lib/memcached/ep.so
#11 0x00002aaaac96cd2f in Task::run () from /opt/couchbase/lib/memcached/ep.so
#12 0x00002aaaac96be81 in Dispatcher::run ()
from /opt/couchbase/lib/memcached/ep.so
#13 0x00002aaaac96c6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#14 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#15 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 21 (process 8302):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac96a038 in IdleTask::run ()
from /opt/couchbase/lib/memcached/ep.so
#2 0x00002aaaac96be81 in Dispatcher::run ()
from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaac96c6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

--Type <return> to continue, or q <return> to quit--
Thread 20 (process 8604):
#0 0x00002aaaabd2c0b1 in nanosleep () from /lib64/libc.so.6
#1 0x00002aaaabd5f684 in usleep () from /lib64/libc.so.6
#2 0x00002aaaac9aef6a in Flusher::wait ()
from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaac97ed10 in EventuallyPersistentStore::~EventuallyPersistentStore
() from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaac9ab2f7 in EventuallyPersistentEngine::~EventuallyPersistentEngine () from /opt/couchbase/lib/memcached/ep.so
#5 0x00002aaaac9996c9 in ?? () from /opt/couchbase/lib/memcached/ep.so
#6 0x00002aaaac6ff815 in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#7 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#8 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 19 (process 8605):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

--Type <return> to continue, or q <return> to quit--
Thread 18 (process 8611):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 17 (process 8616):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 16 (process 8621):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--

Thread 15 (process 8626):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 14 (process 8631):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 13 (process 8636):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 12 (process 8642):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 11 (process 8647):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 10 (process 8656):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
--Type <return> to continue, or q <return> to quit--
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 9 (process 8661):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 8 (process 8666):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 7 (process 8671):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
--Type <return> to continue, or q <return> to quit--
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 6 (process 8677):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 5 (process 8682):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac6ff99f in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#2 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#3 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 4 (process 8683):
#0 0x00002aaaaba81d09 in pthread_cond_wait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#1 0x00002aaaac96bcb2 in Dispatcher::run ()
from /opt/couchbase/lib/memcached/ep.so
#2 0x00002aaaac96c6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#4 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 3 (process 8685):
#0 0x00002aaaaba81f70 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
from /lib64/libpthread.so.0
#1 0x00002aaaac96a038 in IdleTask::run ()
from /opt/couchbase/lib/memcached/ep.so
#2 0x00002aaaac96be81 in Dispatcher::run ()
from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaac96c6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#5 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 2 (process 8687):
#0 0x00002aaaabd2c0b1 in nanosleep () from /lib64/libc.so.6
#1 0x00002aaaabd5f684 in usleep () from /lib64/libc.so.6
#2 0x00002aaaac9aef6a in Flusher::wait ()
from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaac97ed10 in EventuallyPersistentStore::~EventuallyPersistentStore
--Type <return> to continue, or q <return> to quit--
() from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaac9ab2f7 in EventuallyPersistentEngine::~EventuallyPersistentEngine () from /opt/couchbase/lib/memcached/ep.so
#5 0x00002aaaac9996c9 in ?? () from /opt/couchbase/lib/memcached/ep.so
#6 0x00002aaaac6ff815 in ?? ()
from /opt/couchbase/lib/memcached/bucket_engine.so
#7 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#8 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6

Thread 1 (process 8684):
#0 0x00002aaaabcc2265 in raise () from /lib64/libc.so.6
#1 0x00002aaaabcc3d10 in abort () from /lib64/libc.so.6
#2 0x00002aaaabcbb6e6 in __assert_fail () from /lib64/libc.so.6
#3 0x00002aaaac9b06c1 in Flusher::step ()
from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaac9b09ee in FlusherStepper::callback ()
from /opt/couchbase/lib/memcached/ep.so
#5 0x00002aaaac96cd2f in Task::run () from /opt/couchbase/lib/memcached/ep.so
#6 0x00002aaaac96be81 in Dispatcher::run ()
from /opt/couchbase/lib/memcached/ep.so
#7 0x00002aaaac96c6d3 in ?? () from /opt/couchbase/lib/memcached/ep.so
#8 0x00002aaaaba7d617 in start_thread () from /lib64/libpthread.so.0
#9 0x00002aaaabd65c2d in clone () from /lib64/libc.so.6