segmentation fault at #0 HashTable::visit (this=0x183dc608, visitor=...) at stored-value.hh:167

gdb) t a a bt

Thread 17 (Thread 0x2b37e8283ca0 (LWP 26004)):
#0 0x00000031712d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b37e7e02c28 in epoll_dispatch (base=0x14094000, tv=<value optimized out>) at epoll.c:404
#2 0x00002b37e7df1a4c in event_base_loop (base=0x14094000, flags=0) at event.c:1558
#3 0x0000000000409df3 in main (argc=<value optimized out>, argv=<value optimized out>) at daemon/memcached.c:7588

Thread 16 (Thread 26013):
#0 0x00000031712c678b in read () from /lib64/libc.so.6
#1 0x000000317126cd57 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x000000317126d71e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x0000003171268fdb in getc () from /lib64/libc.so.6
#4 0x00002b37e8285875 in check_stdin_thread (arg=0x403500) at extensions/daemon/stdin_check.c:19
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 15 (Thread 26014):
#0 0x000000317129a541 in nanosleep () from /lib64/libc.so.6
#1 0x000000317129a364 in sleep () from /lib64/libc.so.6
#2 0x0000000000415968 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
#3 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#4 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 14 (Thread 26015):
#0 0x00000031712d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b37e7e02c28 in epoll_dispatch (base=0x14094500, tv=<value optimized out>) at epoll.c:404
#2 0x00002b37e7df1a4c in event_base_loop (base=0x14094500, flags=0) at event.c:1558
#3 0x00000000004139a4 in worker_libevent (arg=0x1406e500) at daemon/thread.c:305
#4 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 13 (Thread 26016):
#0 0x00000031712d48a8 in epoll_wait () from /lib64/libc.so.6
--Type <return> to continue, or q <return> to quit--
#1 0x00002b37e7e02c28 in epoll_dispatch (base=0x14094280, tv=<value optimized out>) at epoll.c:404
#2 0x00002b37e7df1a4c in event_base_loop (base=0x14094280, flags=0) at event.c:1558
#3 0x00000000004139a4 in worker_libevent (arg=0x1406e600) at daemon/thread.c:305
#4 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 12 (Thread 26017):
#0 0x00000031712d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b37e7e02c28 in epoll_dispatch (base=0x14094c80, tv=<value optimized out>) at epoll.c:404
#2 0x00002b37e7df1a4c in event_base_loop (base=0x14094c80, flags=0) at event.c:1558
#3 0x00000000004139a4 in worker_libevent (arg=0x1406e700) at daemon/thread.c:305
#4 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 11 (Thread 26018):
#0 0x00000031722044eb in clock_gettime () from /lib64/librt.so.1
#1 0x00002b37e7dee2c3 in gettime (base=0x14094a00, tp=0x14094b88) at event.c:356
#2 0x00002b37e7df22f8 in update_time_cache (base=0x14094a00, flags=0) at event.c:414
#3 event_base_loop (base=0x14094a00, flags=0) at event.c:1567
#4 0x00000000004139a4 in worker_libevent (arg=0x1406e800) at daemon/thread.c:305
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 10 (Thread 26019):
#0 0x00000031712d48a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b37e7e02c28 in epoll_dispatch (base=0x14094780, tv=<value optimized out>) at epoll.c:404
#2 0x00002b37e7df1a4c in event_base_loop (base=0x14094780, flags=0) at event.c:1558
#3 0x00000000004139a4 in worker_libevent (arg=0x1406e900) at daemon/thread.c:305
#4 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 9 (Thread 26301):
#0 0x0000003171e0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
--Type <return> to continue, or q <return> to quit--
#1 0x00002aaaaad07788 in wait (this=0x140b16c0) at syncobject.hh:31
#2 Dispatcher::run (this=0x140b16c0) at dispatcher.cc:89
#3 0x00002aaaaad081fb in launch_dispatcher_thread (arg=0x140b1714) at dispatcher.cc:28
#4 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 8 (Thread 26302):
#0 0x0000003171e0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad05660 in wait (this=0x185fdd80, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x185fdd80, d=...) at dispatcher.cc:286
#3 0x00002aaaaad07956 in Dispatcher::run (this=0x140b1c00) at dispatcher.cc:123
#4 0x00002aaaaad081fb in launch_dispatcher_thread (arg=0x140b1c54) at dispatcher.cc:28
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 7 (Thread 26304):
#0 0x0000003171e0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad2bcbd in wait (this=0x2350a000) at syncobject.hh:42
#2 wait (this=0x2350a000) at tapconnmap.hh:199
#3 EventuallyPersistentEngine::notifyPendingConnections (this=0x2350a000) at ep_engine.cc:3708
#4 0x00002aaaaad2bf71 in EvpNotifyPendingConns (arg=0x2350a000) at ep_engine.cc:971
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 6 (Thread 26305):
#0 0x0000003171e0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad07788 in wait (this=0x140b1180) at syncobject.hh:31
#2 Dispatcher::run (this=0x140b1180) at dispatcher.cc:89
#3 0x00002aaaaad081fb in launch_dispatcher_thread (arg=0x140b11d4) at dispatcher.cc:28
#4 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#5 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 5 (Thread 26306):

--Type <return> to continue, or q <return> to quit--
#0 perftools_pthread_once (ctl=0x2b37e827c5a8, init_routine=0x2b37e804ed40 <InitModule()>)
at src/maybe_threads.cc:124
#1 0x00002b37e804ea87 in MallocExtension::instance () at src/malloc_extension.cc:207
#2 0x00002b37e804eac9 in MallocExtension_GetAllocatedSize (p=0x2b37e827c5a8) at src/malloc_extension.cc:365
#3 0x00002aaaaad502a4 in DeleteHook (ptr=0x3af74ea0) at memory_tracker.cc:45
#4 0x00002b37e804c568 in MallocHook::InvokeDeleteHookSlow (p=0x3af74ea0) at src/malloc_hook.cc:529
#5 0x00002b37e8051189 in InvokeDeleteHook (ptr=0x3af74ea0) at src/malloc_hook-inl.h:176
#6 tc_free (ptr=0x3af74ea0) at src/tcmalloc.cc:1498
#7 0x00002aaaaad895ea in sqlite3_free (p=0x3af74ea8) at embedded/sqlite3.c:17539
#8 0x00002aaaaade0325 in releaseMemArray (p=0x24255508) at embedded/sqlite3.c:55417
#9 closeAllCursors (p=0x24255508) at embedded/sqlite3.c:55939
#10 sqlite3VdbeHalt (p=0x24255508) at embedded/sqlite3.c:56442
#11 0x00002aaaaae2f3aa in sqlite3VdbeExec (p=0x24255508) at embedded/sqlite3.c:60326
#12 0x00002aaaaae0edca in sqlite3Step (pStmt=0x24255508) at embedded/sqlite3.c:57947
#13 sqlite3_step (pStmt=0x24255508) at embedded/sqlite3.c:58011
#14 0x00002aaaaad7bc49 in PreparedStatement::execute (this=0x3ab84b60) at sqlite-pst.cc:73
#15 0x00002aaaaad7ad64 in operator() (__first=<value optimized out>, __last=<value optimized out>, __f=...)
at sqlite-kvstore.cc:236
#16 std::for_each<std::Rb_tree_const_iterator<std::pair<std::pair<unsigned short, unsigned short> const, vbucket_state> >, map_setter<std::pair<unsigned short, unsigned short>, vbucket_state> > (_first=<value optimized out>,
__last=<value optimized out>, __f=...)
at /usr/lib/gcc/x86_64-redhat-linux/4.1.2/../../../../include/c++/4.1.2/bits/stl_algo.h:159
#17 0x00002aaaaad7af0a in StrategicSqlite3::storeMap<std::pair<unsigned short, unsigned short>, vbucket_state> (
this=0x3ba49290, clearSt=0x3ab85920, insSt=0x3ab84b60, m=std::map with 956 elements =

)
at sqlite-kvstore.cc:256
#18 0x00002aaaaad79e0e in StrategicSqlite3::snapshotVBuckets (this=0x2b37e827c5a8,
m=Traceback (most recent call last):
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 282, in to_string
return '%s with %d elements' % (self.typename,
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 212, in _len_
return int (self.size)
RuntimeError: Cannot access memory at address 0x29
Traceback (most recent call last):
--Type <return> to continue, or q <return> to quit--
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 267, in next
n = self.rbiter.next()
File "/usr/share/gdb/python/libstdcxx/v6/printers.py", line 215, in next
if self.count == self.size:
RuntimeError: Cannot access memory at address 0x29
) at sqlite-kvstore.cc:207
#19 0x00002aaaaad14ad8 in EventuallyPersistentStore::snapshotVBuckets (this=0x1ecbf680, priority=...) at ep.cc:719
#20 0x00002aaaaad239c1 in SnapshotVBucketsCallback::callback (this=<value optimized out>) at ep.cc:151
#21 0x00002aaaaad08eaf in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#22 0x00002aaaaad07956 in Dispatcher::run (this=0x140b0fc0) at dispatcher.cc:123
#23 0x00002aaaaad081fb in launch_dispatcher_thread (arg=0x2b37e827c5a8) at dispatcher.cc:28
#24 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#25 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 4 (Thread 26307):
#0 0x0000003171e0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad05660 in wait (this=0x185f4100, d=...) at syncobject.hh:42
#2 IdleTask::run (this=0x185f4100, d=...) at dispatcher.cc:286
#3 0x00002aaaaad07956 in Dispatcher::run (this=0x140b0a80) at dispatcher.cc:123
#4 0x00002aaaaad081fb in launch_dispatcher_thread (arg=0x140b0ad4) at dispatcher.cc:28
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 3 (Thread 26308):
#0 0x0000003171e0b150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad2bcbd in wait (this=0x19eca000) at syncobject.hh:42
#2 wait (this=0x19eca000) at tapconnmap.hh:199
#3 EventuallyPersistentEngine::notifyPendingConnections (this=0x19eca000) at ep_engine.cc:3708
#4 0x00002aaaaad2bf71 in EvpNotifyPendingConns (arg=0x19eca000) at ep_engine.cc:971
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 2 (Thread 26457):
--Type <return> to continue, or q <return> to quit--
#0 0x0000003171e0aee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaaaadb24 in engine_shutdown_thread (arg=0x18d5ff80) at bucket_engine.c:1373
#2 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#3 0x00000031712d44bd in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x479dd940 (LWP 26303)):
#0 HashTable::visit (this=0x183dc608, visitor=...) at stored-value.hh:167
#1 0x00002aaaaad0cdda in VBCBAdaptor::callback (this=0x28241cb0, d=<value optimized out>, t=<value optimized out>)
at ep.cc:2240
#2 0x00002aaaaad08eaf in Task::run (this=<value optimized out>, d=..., t=<value optimized out>) at dispatcher.hh:139
#3 0x00002aaaaad07956 in Dispatcher::run (this=0x140b1340) at dispatcher.cc:123
#4 0x00002aaaaad081fb in launch_dispatcher_thread (arg=0x31715524c0) at dispatcher.cc:28
#5 0x0000003171e0673d in start_thread () from /lib64/libpthread.so.0
#6 0x00000031712d44bd in clone () from /lib64/libc.so.6
(gdb)