Details
-
Bug
-
Resolution: Duplicate
-
Major
-
None
-
Security Level: Public
-
None
-
2
Description
observed two crashes on one of the automated view runs which rebalances in and out after loading data into the bucket
warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fffac1fd000
Core was generated by `/opt/couchbase/bin/memcached -X /opt/couchbase/lib/memcached/stdin_term_handler'.
Program terminated with signal 11, Segmentation fault.
#0 0x00002aaaaadbd9ea in _gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > > std::unguarded_partition<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, RCPtr<QueuedItem>, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, RCPtr<QueuedItem>, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
Thread 12 (Thread 0x2b2f8c511240 (LWP 21497)):
#0 0x00002b2f8bd7d8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b2f8af8aba6 in epoll_dispatch (base=0x12a12000, tv=<value optimized out>) at epoll.c:404
#2 0x00002b2f8af79c11 in event_base_loop (base=0x12a12000, flags=0) at event.c:1558
#3 0x00000000004095b2 in main (argc=<value optimized out>, argv=<value optimized out>) at daemon/memcached.c:7905
Thread 11 (Thread 21498):
#0 0x00002b2f8bd6f78b in read () from /lib64/libc.so.6
#1 0x00002b2f8bd15d57 in _IO_new_file_underflow () from /lib64/libc.so.6
#2 0x00002b2f8bd1671e in _IO_default_uflow_internal () from /lib64/libc.so.6
#3 0x00002b2f8bd11fdb in getc () from /lib64/libc.so.6
#4 0x00002b2f8c512879 in check_stdin_thread (arg=0x4033f0) at extensions/daemon/stdin_check.c:19
#5 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#6 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 10 (Thread 21499):
#0 0x00002b2f8bd43541 in nanosleep () from /lib64/libc.so.6
#1 0x00002b2f8bd43364 in sleep () from /lib64/libc.so.6
#2 0x0000000000415cd8 in check_isasl_db_thread (arg=<value optimized out>) at daemon/isasl.c:233
#3 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#4 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 9 (Thread 21500):
#0 0x00002b2f8bd7d8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b2f8af8aba6 in epoll_dispatch (base=0x12a12500, tv=<value optimized out>) at epoll.c:404
#2 0x00002b2f8af79c11 in event_base_loop (base=0x12a12500, flags=0) at event.c:1558
#3 0x0000000000414074 in worker_libevent (arg=0xe5b2500) at daemon/thread.c:305
#4 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 8 (Thread 21501):
#0 0x00002b2f8bd7d8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b2f8af8aba6 in epoll_dispatch (base=0x12a12280, tv=<value optimized out>) at epoll.c:404
#2 0x00002b2f8af79c11 in event_base_loop (base=0x12a12280, flags=0) at event.c:1558
#3 0x0000000000414074 in worker_libevent (arg=0xe5b2600) at daemon/thread.c:305
#4 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 7 (Thread 21502):
#0 0x00002b2f8bd7d8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b2f8af8aba6 in epoll_dispatch (base=0x12a12c80, tv=<value optimized out>) at epoll.c:404
#2 0x00002b2f8af79c11 in event_base_loop (base=0x12a12c80, flags=0) at event.c:1558
#3 0x0000000000414074 in worker_libevent (arg=0xe5b2700) at daemon/thread.c:305
#4 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 6 (Thread 21503):
#0 0x00002b2f8bd7d8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b2f8af8aba6 in epoll_dispatch (base=0x12a12a00, tv=<value optimized out>) at epoll.c:404
#2 0x00002b2f8af79c11 in event_base_loop (base=0x12a12a00, flags=0) at event.c:1558
#3 0x0000000000414074 in worker_libevent (arg=0xe5b2800) at daemon/thread.c:305
#4 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 5 (Thread 21504):
#0 0x00002b2f8bd7d8a8 in epoll_wait () from /lib64/libc.so.6
#1 0x00002b2f8af8aba6 in epoll_dispatch (base=0x12a12780, tv=<value optimized out>) at epoll.c:404
#2 0x00002b2f8af79c11 in event_base_loop (base=0x12a12780, flags=0) at event.c:1558
#3 0x0000000000414074 in worker_libevent (arg=0xe5b2900) at daemon/thread.c:305
#4 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 4 (Thread 22027):
#0 0x00002b2f8ba98ee9 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad2d417 in wait (this=0x12a5ddc0) at syncobject.hh:36
#2 Dispatcher::run (this=0x12a5ddc0) at dispatcher.cc:89
#3 0x00002aaaaad2da4b in launch_dispatcher_thread (arg=<value optimized out>) at dispatcher.cc:28
#4 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#5 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 3 (Thread 22029):
#0 0x00002b2f8ba99150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad2abd0 in wait (this=0x12aff700, d=...) at syncobject.hh:47
#2 IdleTask::run (this=0x12aff700, d=...) at dispatcher.cc:294
#3 0x00002aaaaad2d1c0 in Dispatcher::run (this=0x12a5c1c0) at dispatcher.cc:123
#4 0x00002aaaaad2da4b in launch_dispatcher_thread (arg=<value optimized out>) at dispatcher.cc:28
#5 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#6 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 2 (Thread 22030):
#0 0x00002b2f8ba99150 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00002aaaaad4a99d in wait (this=0x13120c00) at syncobject.hh:47
#2 wait (this=0x13120c00) at syncobject.hh:63
#3 wait (this=0x13120c00) at tapconnmap.hh:203
#4 EventuallyPersistentEngine::notifyPendingConnections (this=0x13120c00) at ep_engine.cc:3302
#5 0x00002aaaaad4aa91 in EvpNotifyPendingConns (arg=0x13120c00) at ep_engine.cc:1015
#6 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#7 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x41da2940 (LWP 22028)):
#0 0x00002aaaaadbd9ea in _gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > > std::unguarded_partition<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, RCPtr<QueuedItem>, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, RCPtr<QueuedItem>, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#1 0x00002aaaaadbdc9a in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#2 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#3 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#4 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#5 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#6 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#7 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#8 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#9 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#10 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#11 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#12 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#13 0x00002aaaaadbdcbb in void std::_introsort_loop<gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey>(gnu_cxx::normal_iterator<RCPtr<QueuedItem>, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, __gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, long, CompareQueuedItemsByVBAndKey) ()
from /opt/couchbase/lib/memcached/ep.so
#14 0x00002aaaaadc3dbf in sort<_gnu_cxx::_normal_iterator<RCPtr<QueuedItem>*, std::vector<RCPtr<QueuedItem>, std::allocator<RCPtr<QueuedItem> > > >, CompareQueuedItemsByVBAndKey> (this=<value optimized out>, items=<value optimized out>)
at /usr/lib/gcc/x86_64-redhat-linux6E/4.4.6/../../../../include/c++/4.4.6/bits/stl_algo.h:5258
#15 CouchKVStore::optimizeWrites (this=<value optimized out>, items=<value optimized out>) at couch-kvstore/couch-kvstore.cc:603
#16 0x00002aaaaad2fe84 in EventuallyPersistentStore::pushToOutgoingQueue (this=0x12ce3080) at ep.cc:1887
#17 0x00002aaaaad3ca3b in EventuallyPersistentStore::beginFlush (this=0x12ce3080) at ep.cc:1867
#18 0x00002aaaaad64cff in Flusher::doFlush (this=0x1361d830) at flusher.cc:232
#19 0x00002aaaaad65805 in Flusher::step (this=0x1361d830, d=..., tid=std::tr1::shared_ptr (count 5) 0x12a7a380) at flusher.cc:158
#20 0x00002aaaaad65e80 in FlusherStepper::callback (this=<value optimized out>, d=<value optimized out>, t=<value optimized out>)
at flusher.cc:23
#21 0x00002aaaaad2e30f in Task::run(Dispatcher&, std::tr1::shared_ptr<Task>) () from /opt/couchbase/lib/memcached/ep.so
#22 0x00002aaaaad2d1c0 in Dispatcher::run (this=0x12a5d180) at dispatcher.cc:123
#23 0x00002aaaaad2da4b in launch_dispatcher_thread (arg=<value optimized out>) at dispatcher.cc:28
#24 0x00002b2f8ba9473d in start_thread () from /lib64/libpthread.so.0
#25 0x00002b2f8bd7d4bd in clone () from /lib64/libc.so.6