root@deb-2702:/tmp/backup_crash/21_08_2014_05_45# gdb /opt/couchbase/bin/memcached core.memcached.18715 GNU gdb (GDB) 7.4.1-debian Copyright (C) 2012 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-linux-gnu". For bug reporting instructions, please see: ... Reading symbols from /opt/couchbase/bin/memcached...done. [New LWP 18727] [New LWP 18732] [New LWP 18730] [New LWP 18715] [New LWP 18719] [New LWP 18720] [New LWP 18722] [New LWP 18725] [New LWP 18729] [New LWP 18731] [New LWP 18733] [New LWP 18728] [New LWP 18716] [New LWP 18717] [New LWP 18718] [New LWP 18721] [New LWP 18723] [New LWP 18724] [New LWP 18726] warning: Can't read pathname for load map: Input/output error. [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'. Program terminated with signal 11, Segmentation fault. #0 HashTable::insert (this=0x8, itm=..., policy=VALUE_ONLY, eject=false, partial=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/stored-value.cc:203 203 /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/stored-value.cc: No such file or directory. (gdb) t a a bt Thread 19 (Thread 0x7f6a879a5700 (LWP 18726)): #0 0x00007f6a8faf3df5 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a8faf37a2 in readdir64 () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f6a8b2c4d60 in CouchbaseDirectoryUtilities::findFilesContaining (dir=..., name=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/dirutils.cc:174 #3 0x00007f6a8b7b7a04 in discoverDbFiles (dir=..., v=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:151 #4 0x00007f6a8b7bb3dc in CouchKVStore::loadDB (this=this@entry=0x8bc2080, cb=..., cl=..., sr=..., keysOnly=keysOnly@entry=true, vbid=2, startSeqno=startSeqno@entry=0, options=options@entry=4) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1282 #5 0x00007f6a8b7bb7d4 in CouchKVStore::dumpKeys (this=this@entry=0x8bc2080, vbids=..., cb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1160 #6 0x00007f6a8b7a62fa in Warmup::keyDumpforShard (this=0x5ec9b20, shardId=2) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.cc:565 #7 0x00007f6a8b7a8e71 in WarmupKeyDump::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.h:295 #8 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec8620) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #9 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8620) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #10 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ebd0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #11 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #12 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #13 0x0000000000000000 in ?? () Thread 18 (Thread 0x7f6a889a7700 (LWP 18724)): #0 0x00007f6a9098c64b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f6a91c7faf5 in cb_cond_timedwait (cond=0x5ec4300, mutex=0x5ec42c8, ms=725) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00007f6a8b7865e1 in wait (tv=..., this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/syncobject.h:74 #3 TaskQueue::_doSleep (this=this@entry=0x5ec42c0, t=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/taskqueue.cc:77 #4 0x00007f6a8b78a47b in TaskQueue::_fetchNextTask (this=this@entry=0x5ec42c0, t=..., toSleep=toSleep@entry=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/taskqueue.cc:98 #5 0x00007f6a8b78a79f in TaskQueue::fetchNextTask (this=0x5ec42c0, thread=..., toSleep=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/taskqueue.cc:142 #6 0x00007f6a8b756285 in ExecutorPool::_nextTask (this=this@entry=0x6047660, t=..., tick=tick@entry=22 '\026') at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorpool.cc:214 #7 0x00007f6a8b75633f in ExecutorPool::nextTask (this=0x6047660, t=..., tick=22 '\026') at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorpool.cc:229 #8 0x00007f6a8b769993 in ExecutorThread::run (this=this@entry=0x5ec8460) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:78 #9 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8460) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #10 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ebc0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #11 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #12 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #13 0x0000000000000000 in ?? () Thread 17 (Thread 0x7f6a891a8700 (LWP 18723)): #0 0x00007f6a8faf76dd in nanosleep () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a8fb21424 in usleep () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f6a8b7682f5 in updateStatsThread (arg=0x15be040) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/memory_tracker.cc:36 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e1f0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 16 (Thread 0x7f6a8ca1c700 (LWP 18721)): #0 0x00007f6a8fb27863 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a912179e3 in epoll_dispatch (base=0x5e88a00, tv=) at epoll.c:404 #2 0x00007f6a91200e69 in event_base_loop (base=0x5e88a00, flags=) at event.c:1558 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e130) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- Thread 15 (Thread 0x7f6a8e21f700 (LWP 18718)): #0 0x00007f6a8fb27863 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a912179e3 in epoll_dispatch (base=0x5e88280, tv=) at epoll.c:404 #2 0x00007f6a91200e69 in event_base_loop (base=0x5e88280, flags=) at event.c:1558 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e100) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 14 (Thread 0x7f6a8ec2d700 (LWP 18717)): #0 0x00007f6a9098c64b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f6a91c7faf5 in cb_cond_timedwait (cond=cond@entry=0x7f6a8ee30920, mutex=mutex@entry=0x7f6a8ee308e0, ms=19000) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00007f6a8ec2fb7b in logger_thead_main (arg=0x15be900) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/memcached/extensions/loggers/file_logger.c:342 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e070) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 13 (Thread 0x7f6a8f631700 (LWP 18716)): #0 0x00007f6a8fb1b23d in read () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a8fabe138 in _IO_file_underflow () from /lib/x86_64-linux-gnu/libc.so.6 #2 0x00007f6a8fabf81e in _IO_default_uflow () from /lib/x86_64-linux-gnu/libc.so.6 #3 0x00007f6a8fab465a in _IO_getline_info () from /lib/x86_64-linux-gnu/libc.so.6 #4 0x00007f6a8fab3539 in fgets () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x00007f6a8f6329f1 in check_stdin_thread (arg=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/memcached/extensions/daemon/stdin_check.c:38 #6 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e060) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #7 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #8 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #9 0x0000000000000000 in ?? () Thread 12 (Thread 0x7f6a869a3700 (LWP 18728)): #0 0x00007f6a8fb20f3d in fdatasync () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a8b4e8f07 in couch_sync (errinfo=0x8408ba0, handle=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/couchstore/src/os.c:133 #2 0x00007f6a8b7c2d45 in cfs_sync (errinfo=0x8408ba0, h=0x6052800) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-fs-stats.cc:128 #3 0x00007f6a8b4cffba in couchstore_commit (db=0x8408b80) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/couchstore/src/couch_db.cc:203 #4 0x00007f6a8b7b5a5d in CouchKVStore::setVBucketState (this=this@entry=0x632e700, vbucketId=vbucketId@entry=851, vbstate=..., vb_change_type=vb_change_type@entry=1, kvcb=kvcb@entry=0x7f6a869a2ca0, notify=notify@entry=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1094 #5 0x00007f6a8b7b5f11 in CouchKVStore::snapshotVBucket (this=0x632e700, vbucketId=851, vbstate=..., cb=0x7f6a869a2ca0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:972 #6 0x00007f6a8b727f42 in EventuallyPersistentStore::snapshotVBuckets (this=0x5eaa6c0, priority=..., shardId=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep.cc:971 #7 0x00007f6a8b783f7c in VBSnapshotTask::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/tasks.cc:48 #8 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec87e0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #9 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec87e0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #10 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ec00) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #11 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #12 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #13 0x0000000000000000 in ?? () Thread 11 (Thread 0x7f6a8419e700 (LWP 18733)): #0 0x00007f6a902e3418 in std::locale::locale() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #1 0x00007f6a8b716c2c in basic_ios (this=0x7f6a8419d588) at /usr/include/c++/4.7/bits/basic_ios.h:454 #2 std::basic_stringstream, std::allocator >::basic_stringstream (this=this@entry=0x7f6a8419d520, __m=24, __in_chrg=, __vtt_parm=) at /usr/include/c++/4.7/sstream:519 ---Type to continue, or q to quit--- #3 0x00007f6a8b71dda1 in STATWRITER_NAMESPACE::add_casted_stat (k=0x7f6a8419d790 "vb_184:num_checkpoint_items", v=@0x7f6a8419d6c8: 1, add_stat=0x40cff0 , cookie=0x5e1a000) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/statwriter.h:44 #4 0x00007f6a8b71de76 in STATWRITER_NAMESPACE::add_casted_stat (k=k@entry=0x7f6a8419d790 "vb_184:num_checkpoint_items", v=..., add_stat=add_stat@entry=0x40cff0 , cookie=cookie@entry=0x5e1a000) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/statwriter.h:52 #5 0x00007f6a8b7175c5 in CheckpointManager::addStats (this=0xa883498, add_stat=0x40cff0 , cookie=0x5e1a000) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/checkpoint.cc:1616 #6 0x00007f6a8b73d266 in StatCheckpointVisitor::addCheckpointStat (cookie=0x5e1a000, add_stat=0x40cff0 , eps=0x5ead840, vb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:3599 #7 0x00007f6a8b74e8af in StatCheckpointVisitor::visitBucket (this=, vb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:3582 #8 0x00007f6a8b723ce4 in EventuallyPersistentStore::visit (this=0x5ead840, visitor=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep.cc:3047 #9 0x00007f6a8b750075 in StatCheckpointTask::run (this=0x6590160) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep_engine.cc:3619 #10 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec8c40) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #11 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8c40) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #12 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ec50) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #13 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #14 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #15 0x0000000000000000 in ?? () Thread 10 (Thread 0x7f6a851a0700 (LWP 18731)): #0 0x00007f6a9098ecec in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f6a9098a339 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f6a9098a15b in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f6a91c7f956 in cb_mutex_enter (mutex=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:85 #4 0x00007f6a8b768f2d in Mutex::acquire (this=0x6319110) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/mutex.cc:31 #5 0x00007f6a8b7c6b4c in lock (this=0x7f6a8519f800) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/locks.h:70 #6 LockHolder (m=..., this=0x7f6a8519f800, tryLock=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/locks.h:47 #7 CouchNotifier::notify_update (this=0x6319000, vbs=..., file_version=1, header_offset=20480, cb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-notifier.cc:853 #8 0x00007f6a8b7b5bb8 in CouchKVStore::setVBucketState (this=this@entry=0x632c000, vbucketId=vbucketId@entry=840, vbstate=..., vb_change_type=vb_change_type@entry=1, kvcb=kvcb@entry=0x7f6a8519fca0, notify=notify@entry=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1110 #9 0x00007f6a8b7b5f11 in CouchKVStore::snapshotVBucket (this=0x632c000, vbucketId=840, vbstate=..., cb=0x7f6a8519fca0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:972 #10 0x00007f6a8b727f42 in EventuallyPersistentStore::snapshotVBuckets (this=0x5eaa6c0, priority=..., shardId=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep.cc:971 #11 0x00007f6a8b783f7c in VBSnapshotTask::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/tasks.cc:48 #12 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec8a80) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #13 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8a80) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #14 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ec30) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #15 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #16 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #17 0x0000000000000000 in ?? () Thread 9 (Thread 0x7f6a861a2700 (LWP 18729)): #0 0x00007f6a9098ecec in __lll_lock_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f6a9098a339 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0 #2 0x00007f6a9098a15b in pthread_mutex_lock () from /lib/x86_64-linux-gnu/libpthread.so.0 #3 0x00007f6a91c7f956 in cb_mutex_enter (mutex=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:85 #4 0x00007f6a8b768f2d in Mutex::acquire (this=0x6319110) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/mutex.cc:31 #5 0x00007f6a8b7c6b4c in lock (this=0x7f6a861a1800) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/locks.h:70 #6 LockHolder (m=..., this=0x7f6a861a1800, tryLock=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/locks.h:47 #7 CouchNotifier::notify_update (this=0x6319000, vbs=..., file_version=1, header_offset=20480, cb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-notifier.cc:853 #8 0x00007f6a8b7b5bb8 in CouchKVStore::setVBucketState (this=this@entry=0x632da00, vbucketId=vbucketId@entry=850, vbstate=..., vb_change_type=vb_change_type@entry=1, kvcb=kvcb@entry=0x7f6a861a1ca0, notify=notify@entry=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1110 #9 0x00007f6a8b7b5f11 in CouchKVStore::snapshotVBucket (this=0x632da00, vbucketId=850, vbstate=..., cb=0x7f6a861a1ca0) ---Type to continue, or q to quit--- at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:972 #10 0x00007f6a8b727f42 in EventuallyPersistentStore::snapshotVBuckets (this=0x5eaa6c0, priority=..., shardId=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep.cc:971 #11 0x00007f6a8b783f7c in VBSnapshotTask::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/tasks.cc:48 #12 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec88c0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #13 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec88c0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #14 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ec10) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #15 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #16 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #17 0x0000000000000000 in ?? () Thread 8 (Thread 0x7f6a881a6700 (LWP 18725)): #0 __shared_count (__r=..., this=0x7f6a881a5d48) at /usr/include/c++/4.7/bits/shared_ptr_base.h:811 #1 __shared_ptr (this=0x7f6a881a5d40) at /usr/include/c++/4.7/bits/shared_ptr_base.h:811 #2 shared_ptr (this=0x7f6a881a5d40) at /usr/include/c++/4.7/bits/shared_ptr.h:103 #3 CouchKVStore::dumpKeys (this=this@entry=0x8bc0680, vbids=..., cb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1160 #4 0x00007f6a8b7a62fa in Warmup::keyDumpforShard (this=0x5ec9b20, shardId=0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.cc:565 #5 0x00007f6a8b7a8e71 in WarmupKeyDump::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.h:295 #6 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec8540) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #7 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8540) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #8 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ebe0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #9 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #10 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #11 0x0000000000000000 in ?? () Thread 7 (Thread 0x7f6a8c21b700 (LWP 18722)): #0 0x00007f6a8fb27863 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a912179e3 in epoll_dispatch (base=0x5e88c80, tv=) at epoll.c:404 #2 0x00007f6a91200e69 in event_base_loop (base=0x5e88c80, flags=) at event.c:1558 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e140) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 6 (Thread 0x7f6a8d21d700 (LWP 18720)): #0 0x00007f6a8fb27863 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a912179e3 in epoll_dispatch (base=0x5e88780, tv=) at epoll.c:404 #2 0x00007f6a91200e69 in event_base_loop (base=0x5e88780, flags=) at event.c:1558 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e120) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 5 (Thread 0x7f6a8da1e700 (LWP 18719)): #0 0x00007f6a8fb27863 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a912179e3 in epoll_dispatch (base=0x5e88500, tv=) at epoll.c:404 #2 0x00007f6a91200e69 in event_base_loop (base=0x5e88500, flags=) at event.c:1558 #3 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156e110) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #4 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #5 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x0000000000000000 in ?? () Thread 4 (Thread 0x7f6a9249a720 (LWP 18715)): #0 0x00007f6a8fb27863 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a912179e3 in epoll_dispatch (base=0x5e88000, tv=) at epoll.c:404 ---Type to continue, or q to quit--- #2 0x00007f6a91200e69 in event_base_loop (base=0x5e88000, flags=flags@entry=3) at event.c:1558 #3 0x0000000000406611 in main (argc=, argv=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/memcached/daemon/memcached.c:8755 Thread 3 (Thread 0x7f6a859a1700 (LWP 18730)): #0 0x00007f6a8fb1c5c3 in poll () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x00007f6a8b7c63e9 in waitForReadable (this=, tryOnce=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-notifier.cc:726 #2 CouchNotifier::waitOnce (this=this@entry=0x6319000) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-notifier.cc:776 #3 0x00007f6a8b7c6c59 in CouchNotifier::notify_update (this=0x6319000, vbs=..., file_version=1, header_offset=20480, cb=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-notifier.cc:875 #4 0x00007f6a8b7b5bb8 in CouchKVStore::setVBucketState (this=this@entry=0x632cd00, vbucketId=vbucketId@entry=841, vbstate=..., vb_change_type=vb_change_type@entry=1, kvcb=kvcb@entry=0x7f6a859a0ca0, notify=notify@entry=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1110 #5 0x00007f6a8b7b5f11 in CouchKVStore::snapshotVBucket (this=0x632cd00, vbucketId=841, vbstate=..., cb=0x7f6a859a0ca0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:972 #6 0x00007f6a8b727f42 in EventuallyPersistentStore::snapshotVBuckets (this=0x5eaa6c0, priority=..., shardId=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/ep.cc:971 #7 0x00007f6a8b783f7c in VBSnapshotTask::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/tasks.cc:48 #8 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec89a0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #9 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec89a0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #10 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ec20) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #11 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #12 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #13 0x0000000000000000 in ?? () Thread 2 (Thread 0x7f6a8499f700 (LWP 18732)): #0 0x00007f6a9098c64b in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x00007f6a91c7faf5 in cb_cond_timedwait (cond=0x5ec4460, mutex=0x5ec4428, ms=2000) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:156 #2 0x00007f6a8b7865e1 in wait (tv=..., this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/syncobject.h:74 #3 TaskQueue::_doSleep (this=this@entry=0x5ec4420, t=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/taskqueue.cc:77 #4 0x00007f6a8b78a47b in TaskQueue::_fetchNextTask (this=this@entry=0x5ec4420, t=..., toSleep=toSleep@entry=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/taskqueue.cc:98 #5 0x00007f6a8b78a79f in TaskQueue::fetchNextTask (this=0x5ec4420, thread=..., toSleep=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/taskqueue.cc:142 #6 0x00007f6a8b756285 in ExecutorPool::_nextTask (this=this@entry=0x6047660, t=..., tick=tick@entry=5 '\005') at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorpool.cc:214 #7 0x00007f6a8b75633f in ExecutorPool::nextTask (this=0x6047660, t=..., tick=5 '\005') at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorpool.cc:229 #8 0x00007f6a8b769993 in ExecutorThread::run (this=this@entry=0x5ec8b60) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:78 #9 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8b60) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #10 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ec40) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #11 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #12 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #13 0x0000000000000000 in ?? () Thread 1 (Thread 0x7f6a871a4700 (LWP 18727)): #0 HashTable::insert (this=0x8, itm=..., policy=VALUE_ONLY, eject=false, partial=true) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/stored-value.cc:203 #1 0x00007f6a8b7a5899 in LoadStorageKVPairCallback::callback (this=0xc259d00, val=...) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.cc:218 #2 0x00007f6a8b7b3b36 in CouchKVStore::recordDbDump (db=0x8403e80, docinfo=0xb459db0, ctx=0x7f6a871a3c40) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1733 #3 0x00007f6a8b4d0b17 in lookup_callback (rq=, k=0x7f6a871a3ae0, v=0x7f6a871a3af0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/couchstore/src/couch_db.cc:731 #4 0x00007f6a8b4cf218 in btree_lookup_inner (rq=0x7f6a871a3ba0, diskpos=, current=0, end=1) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/couchstore/src/btree_read.cc:103 #5 0x00007f6a8b4d102f in couchstore_changes_since (db=0x8403e80, since=, options=, callback=, ctx=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/couchstore/src/couch_db.cc:774 #6 0x00007f6a8b7bb4dc in CouchKVStore::loadDB (this=this@entry=0x8bc2d80, cb=..., cl=..., sr=..., keysOnly=keysOnly@entry=true, vbid=95, startSeqno=startSeqno@entry=0, options=options@entry=4) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1314 #7 0x00007f6a8b7bb7d4 in CouchKVStore::dumpKeys (this=this@entry=0x8bc2d80, vbids=..., cb=...) ---Type to continue, or q to quit--- at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/couch-kvstore/couch-kvstore.cc:1160 #8 0x00007f6a8b7a62fa in Warmup::keyDumpforShard (this=0x5ec9b20, shardId=3) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.cc:565 #9 0x00007f6a8b7a8e71 in WarmupKeyDump::run (this=) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/warmup.h:295 #10 0x00007f6a8b769b8f in ExecutorThread::run (this=this@entry=0x5ec8700) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:110 #11 0x00007f6a8b76a149 in launch_executor_thread (arg=0x5ec8700) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/ep-engine/src/executorthread.cc:34 #12 0x00007f6a91c7f7fa in platform_thread_wrap (arg=0x156ebf0) at /home/buildbot/buildbot_slave/debian-7-x64-300-builder/build/build/platform/src/cb_pthreads.c:19 #13 0x00007f6a90987b50 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #14 0x00007f6a8fb2720d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #15 0x0000000000000000 in ?? () (gdb)