GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /opt/couchbase/bin/memcached...Reading symbols from /usr/lib/debug//opt/couchbase/bin/memcached...done.
done.
[New LWP 32364]
[New LWP 32322]
[New LWP 32335]
[New LWP 32336]
[New LWP 32337]
[New LWP 32360]
[New LWP 32361]
[New LWP 32362]
[New LWP 32363]
[New LWP 32365]
[New LWP 32366]
[New LWP 32367]
[New LWP 32368]
[New LWP 32369]
[New LWP 32370]
[New LWP 32371]
[New LWP 32372]
[New LWP 32373]
[New LWP 32374]
[New LWP 32375]
[New LWP 32376]
[New LWP 32377]
[New LWP 32378]
[New LWP 419]
[New LWP 420]
[New LWP 421]
[New LWP 422]
[New LWP 423]
[New LWP 424]
[New LWP 425]
[New LWP 426]
[New LWP 427]
[New LWP 428]
[New LWP 429]
To enable execution of this file add
	add-auto-load-safe-path /lib/x86_64-linux-gnu/libthread_db-1.0.so
line to your configuration file "/root/.gdbinit".
To completely disable this security protection add
	set auto-load safe-path /
line to your configuration file "/root/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual.  E.g., run from the shell:
	info "(gdb)Auto-loading safe path"
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
#0  0x00007f92ba377cc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56

Thread 34 (LWP 429):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92b9648588) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92bbd0289d in cb_mutex_enter (mutex=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:192
#4  0x0000000000421d6f in notify_io_complete (cookie=0x7f91dd0b3f00, status=ENGINE_SUCCESS) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:430
#5  0x00007f92b6cb24a0 in EventuallyPersistentEngine::notifyIOComplete (this=0x7f92b9742c00, cookie=0x7f91dd0b3f00, status=ENGINE_SUCCESS)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.h:560
#6  0x00007f92b6cb568a in ConnMap::notifyAllPausedConnections (this=0x7f92a8147800) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/connmap.cc:254
#7  0x00007f92b6cb579a in ConnNotifier::notifyConnections (this=0x7f92a814c1e0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/connmap.cc:125
#8  0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969fe00) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#9  0x00007f92bbd0263a in run (this=0x7f92a814c0c0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#10 platform_thread_wrap (arg=0x7f92a814c0c0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#11 0x00007f92bbae7182 in start_thread (arg=0x7f92a2bf6700) at pthread_create.c:312
#12 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 33 (LWP 428):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007f92b6d4f9c8 in __gthread_cond_timedwait (__abs_timeout=0x7f92a33f6c70, __mutex=<optimized out>, __cond=0x7f92a81241e8)
    at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:871
#2  __wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:160
#3  wait_until<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:100
#4  wait_for<long, std::ratio<1l, 1000000000l> > (__rtime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:132
#5  wait_for (nanoSecs=<optimized out>, lock=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/syncobject.h:49
#6  TaskQueue::_doSleep (this=this@entry=0x7f92a81241c0, t=..., lock=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:95
#7  0x00007f92b6d519d0 in _fetchNextTask (toSleep=true, t=..., this=0x7f92a81241c0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:118
#8  TaskQueue::fetchNextTask (this=0x7f92a81241c0, thread=..., toSleep=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:161
#9  0x00007f92b6d17506 in ExecutorPool::_nextTask (this=0x7f92a812fb00, t=..., tick=27 '\033', tick@entry=18 '\022')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorpool.cc:235
#10 0x00007f92b6d175e1 in ExecutorPool::nextTask (this=0x7f92a812fb00, t=..., tick=tick@entry=18 '\022')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorpool.cc:250
#11 0x00007f92b6d1fe20 in ExecutorThread::run (this=0x7f92b969fd20) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:81
#12 0x00007f92bbd0263a in run (this=0x7f92a7492ca0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#13 platform_thread_wrap (arg=0x7f92a7492ca0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#14 0x00007f92bbae7182 in start_thread (arg=0x7f92a33f7700) at pthread_create.c:312
#15 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 32 (LWP 427):
#0  0x00007f92bbaeecc3 in pread64 () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92b6a76bfa in pread (__offset=77594624, __nbytes=8192, __buf=0x7f919e776831, __fd=125) at /usr/include/x86_64-linux-gnu/bits/unistd.h:99
#2  couch_pread (errinfo=0x7f917c9ff820, handle=<optimized out>, buf=0x7f919e776831, nbyte=8192, offset=77594624)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/os.c:43
#3  0x00007f92b6d7d117 in cfs_pread (errinfo=0x7f917c9ff820, h=0x7f918b8909c0, buf=0x7f919e776831, sz=8192, off=77594624)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/couch-kvstore/couch-fs-stats.cc:105
#4  0x00007f92b6a60bcc in load_buffer_from (nbyte=4096, offset=77594624, buf=0x7f919e776800, errinfo=0x7f917c9ff820)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/iobuffer.cc:167
#5  buffered_pread (errinfo=0x7f917c9ff820, handle=0x7f91a73a9f20, buf=0x7f928387ae8b, nbyte=4095, offset=77594625)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/iobuffer.cc:315
#6  0x00007f92b6a5c7e8 in read_skipping_prefixes (dst=0x7f928387ae8b, len=27942, pos=<synthetic pointer>, file=0x7f917c9ff800)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/couch_file_read.cc:86
#7  pread_bin_internal (file=0x7f917c9ff800, pos=77594625, ret_ptr=0x7f92a3bf7350, max_header_size=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/couch_file_read.cc:135
#8  0x00007f92b6a5e3bc in compact_seq_fetchcb (rq=0x7f92a3bf7630, k=0x7f92a3bf73f0, v=0x7f92a3bf7400) at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/db_compact.cc:207
#9  0x00007f92b6a59d09 in btree_lookup_inner (rq=rq@entry=0x7f92a3bf7630, diskpos=<optimized out>, current=current@entry=0, end=end@entry=1)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/btree_read.cc:101
#10 0x00007f92b6a59b57 in btree_lookup_inner (rq=rq@entry=0x7f92a3bf7630, diskpos=<optimized out>, current=current@entry=0, end=end@entry=1)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/btree_read.cc:69
#11 0x00007f92b6a59b57 in btree_lookup_inner (rq=0x7f92a3bf7630, diskpos=<optimized out>, current=0, end=1)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/btree_read.cc:69
#12 0x00007f92b6a5e6e1 in compact_seq_tree (ctx=0x7f92a3bf7670, target=0x7f918e41eb00, source=0x7f917c9ff800)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/db_compact.cc:264
#13 couchstore_compact_db_ex (source=0x7f917c9ff800, target_filename=0x7f91e83c1258 <error: Cannot access memory at address 0x7f91e83c1258>, flags=<optimized out>, 
    hook=0x7f92b6d691c0 <time_purge_hook(Db*, DocInfo*, void*)>, dhook=0x7f92b6d68f20 <edit_docinfo_hook(DocInfo**, sized_buf const*)>, hook_ctx=0x7f91df938b00, ops=0x7f92a81b6b78)
    at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/db_compact.cc:74
#14 0x00007f92b6d75436 in CouchKVStore::compactDB (this=0x7f92a81b6800, hook_ctx=0x7f91df938b00, kvcb=...)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/couch-kvstore/couch-kvstore.cc:809
#15 0x00007f92b6ce43bc in EventuallyPersistentStore::doCompact (this=0x7f92a8169a00, ctx=0x7f91df938b00, cookie=0x7f9295c78b00)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:1651
#16 0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969fc40) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#17 0x00007f92bbd0263a in run (this=0x7f92a8159880) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#18 platform_thread_wrap (arg=0x7f92a8159880) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#19 0x00007f92bbae7182 in start_thread (arg=0x7f92a3bf8700) at pthread_create.c:312
#20 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 31 (LWP 426):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92a1a1eca8) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92b6ca7574 in __gthread_mutex_lock (__mutex=0x7f92a1a1eca8) at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:748
#4  lock (this=0x7f92a1a1eca8) at /usr/include/c++/4.8/mutex:134
#5  lock (this=0x7f92a43f8910) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:73
#6  LockHolder (tryLock=false, m=..., this=0x7f92a43f8910) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:50
#7  CheckpointManager::getAllItemsForCursor (this=0x7f92a1a1ec98, name=..., items=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/checkpoint.cc:1036
#8  0x00007f92b6cef4fc in EventuallyPersistentStore::flushVBucket (this=0x7f92a8169a00, vbid=vbid@entry=593) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:3366
#9  0x00007f92b6d24135 in Flusher::flushVB (this=0x7f92a8123680) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/flusher.cc:293
#10 0x00007f92b6d2496d in Flusher::step (this=0x7f92a8123680, task=0x7f92a810f080) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/flusher.cc:183
#11 0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969fb60) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#12 0x00007f92bbd0263a in run (this=0x7f92a7491460) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#13 platform_thread_wrap (arg=0x7f92a7491460) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#14 0x00007f92bbae7182 in start_thread (arg=0x7f92a43f9700) at pthread_create.c:312
#15 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 30 (LWP 425):
#0  0x00007f92ba432fdd in fdatasync () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92b6a76cf3 in couch_sync (errinfo=0x7f9196100c20, handle=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/os.c:137
#2  0x00007f92b6d7cc55 in cfs_sync (errinfo=0x7f9196100c20, h=0x7f91a8dc6260) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/couch-kvstore/couch-fs-stats.cc:138
#3  0x00007f92b6a5a742 in precommit (db=0x7f9196100c00) at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/couch_db.cc:230
#4  couchstore_commit (db=0x7f9196100c00) at /home/couchbase/jenkins/workspace/watson-unix/couchstore/src/couch_db.cc:240
#5  0x00007f92b6d72242 in CouchKVStore::saveDocs (this=this@entry=0x7f929cb84800, vbid=vbid@entry=244, rev=rev@entry=1, docs=docs@entry=0x7f91629ff198, 
    docinfos=docinfos@entry=0x7f91629ff5d0, docCount=docCount@entry=1, kvctx=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/couch-kvstore/couch-kvstore.cc:1858
#6  0x00007f92b6d72c0b in CouchKVStore::commit2couchstore (this=0x7f929cb84800, cb=0x7f92a4bf98f0)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/couch-kvstore/couch-kvstore.cc:1753
#7  0x00007f92b6d7322d in CouchKVStore::commit (this=0x7f929cb84800, cb=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/couch-kvstore/couch-kvstore.cc:988
#8  0x00007f92b6ce3126 in EventuallyPersistentStore::commit (this=this@entry=0x7f929cebf000, shardId=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:3486
#9  0x00007f92b6cefab9 in EventuallyPersistentStore::flushVBucket (this=0x7f929cebf000, vbid=vbid@entry=244) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:3434
#10 0x00007f92b6d24135 in Flusher::flushVB (this=0x7f92a75dd780) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/flusher.cc:293
#11 0x00007f92b6d2496d in Flusher::step (this=0x7f92a75dd780, task=0x7f929c1090e0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/flusher.cc:183
#12 0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969fa80) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#13 0x00007f92bbd0263a in run (this=0x7f92a7490040) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#14 platform_thread_wrap (arg=0x7f92a7490040) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#15 0x00007f92bbae7182 in start_thread (arg=0x7f92a4bfa700) at pthread_create.c:312
#16 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 29 (LWP 424):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92a1a1eca8) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92b6ce54d2 in __gthread_mutex_lock (__mutex=0x7f92a1a1eca8) at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:748
#4  lock (this=0x7f92a1a1eca8) at /usr/include/c++/4.8/mutex:134
#5  lock (this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:73
#6  LockHolder (tryLock=false, m=..., this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:50
#7  getHighSeqno (this=0x7f92a1a1ec98) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/checkpoint.h:683
#8  getHighSeqno (this=0x7f92a1a1ec00) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/vbucket.h:209
#9  EventuallyPersistentStore::VBucketStateVisitor::visitBucket (this=this@entry=0x7f92a53facd0, vb=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:1178
#10 0x00007f92b6ce58c3 in visit (visitor=..., this=0x7f92a8169a00) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:3864
#11 EventuallyPersistentStore::snapshotVBuckets (this=0x7f92a8169a00, priority=..., shardId=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:1204
#12 0x00007f92b6d4c43c in VBSnapshotTask::run (this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/tasks.cc:69
#13 0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969f9a0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#14 0x00007f92bbd0263a in run (this=0x7f92a7489c20) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#15 platform_thread_wrap (arg=0x7f92a7489c20) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#16 0x00007f92bbae7182 in start_thread (arg=0x7f92a53fb700) at pthread_create.c:312
#17 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 28 (LWP 423):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92b9648450) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92bbd0289d in cb_mutex_enter (mutex=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:192
#4  0x0000000000421d6f in notify_io_complete (cookie=0x7f9295dda100, status=ENGINE_SUCCESS) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:430
#5  0x00007f92b6ceb5e0 in notifyIOComplete (status=<optimized out>, cookie=0x7f9295dda100, this=0x7f92b9742c00)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.h:560
#6  EventuallyPersistentStore::completeBGFetchMulti (this=0x7f92a8169a00, vbId=vbId@entry=259, fetchedItems=..., startTime=startTime@entry=1462814650399992370)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:2022
#7  0x00007f92b6ca20a6 in BgFetcher::doFetch (this=this@entry=0x7f92a8127f00, vbId=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/bgfetcher.cc:77
#8  0x00007f92b6ca2661 in BgFetcher::run (this=0x7f92a8127f00, task=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/bgfetcher.cc:133
#9  0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969f8c0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#10 0x00007f92bbd0263a in run (this=0x7f92a7488820) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#11 platform_thread_wrap (arg=0x7f92a7488820) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#12 0x00007f92bbae7182 in start_thread (arg=0x7f92a5bfc700) at pthread_create.c:312
#13 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 27 (LWP 422):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007f92b6d4f9c8 in __gthread_cond_timedwait (__abs_timeout=0x7f92a63fcc70, __mutex=<optimized out>, __cond=0x7f92a81240a8)
    at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:871
#2  __wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:160
#3  wait_until<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:100
#4  wait_for<long, std::ratio<1l, 1000000000l> > (__rtime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:132
#5  wait_for (nanoSecs=<optimized out>, lock=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/syncobject.h:49
#6  TaskQueue::_doSleep (this=this@entry=0x7f92a8124080, t=..., lock=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:95
#7  0x00007f92b6d519d0 in _fetchNextTask (toSleep=true, t=..., this=0x7f92a8124080) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:118
#8  TaskQueue::fetchNextTask (this=0x7f92a8124080, thread=..., toSleep=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:161
#9  0x00007f92b6d17506 in ExecutorPool::_nextTask (this=0x7f92a812fb00, t=..., tick=118 'v', tick@entry=52 '4')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorpool.cc:235
#10 0x00007f92b6d175e1 in ExecutorPool::nextTask (this=0x7f92a812fb00, t=..., tick=tick@entry=52 '4')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorpool.cc:250
#11 0x00007f92b6d1fe20 in ExecutorThread::run (this=0x7f92b969f7e0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:81
#12 0x00007f92bbd0263a in run (this=0x7f92a7487400) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#13 platform_thread_wrap (arg=0x7f92a7487400) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#14 0x00007f92bbae7182 in start_thread (arg=0x7f92a63fd700) at pthread_create.c:312
#15 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 26 (LWP 421):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92b9648450) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92bbd0289d in cb_mutex_enter (mutex=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:192
#4  0x0000000000421d6f in notify_io_complete (cookie=0x7f918ca86700, status=ENGINE_SUCCESS) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:430
#5  0x00007f92b6ceb5e0 in notifyIOComplete (status=<optimized out>, cookie=0x7f918ca86700, this=0x7f92b9742c00)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.h:560
#6  EventuallyPersistentStore::completeBGFetchMulti (this=0x7f92a8169a00, vbId=vbId@entry=186, fetchedItems=..., startTime=startTime@entry=1462814650400212887)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:2022
#7  0x00007f92b6ca20a6 in BgFetcher::doFetch (this=this@entry=0x7f92a8127e40, vbId=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/bgfetcher.cc:77
#8  0x00007f92b6ca2661 in BgFetcher::run (this=0x7f92a8127e40, task=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/bgfetcher.cc:133
#9  0x00007f92b6d1fc80 in ExecutorThread::run (this=0x7f92b969f700) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:115
#10 0x00007f92bbd0263a in run (this=0x7f92a7485fc0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#11 platform_thread_wrap (arg=0x7f92a7485fc0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#12 0x00007f92bbae7182 in start_thread (arg=0x7f92a6bfe700) at pthread_create.c:312
#13 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 25 (LWP 420):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007f92b6d4f9c8 in __gthread_cond_timedwait (__abs_timeout=0x7f92a73fec70, __mutex=<optimized out>, __cond=0x7f92a81240a8)
    at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:871
#2  __wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:160
#3  wait_until<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:100
#4  wait_for<long, std::ratio<1l, 1000000000l> > (__rtime=..., __lock=..., this=<optimized out>) at /usr/include/c++/4.8/condition_variable:132
#5  wait_for (nanoSecs=<optimized out>, lock=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/syncobject.h:49
#6  TaskQueue::_doSleep (this=this@entry=0x7f92a8124080, t=..., lock=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:95
#7  0x00007f92b6d519d0 in _fetchNextTask (toSleep=true, t=..., this=0x7f92a8124080) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:118
#8  TaskQueue::fetchNextTask (this=0x7f92a8124080, thread=..., toSleep=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/taskqueue.cc:161
#9  0x00007f92b6d17506 in ExecutorPool::_nextTask (this=0x7f92a812fb00, t=..., tick=119 'w', tick@entry=64 '@')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorpool.cc:235
#10 0x00007f92b6d175e1 in ExecutorPool::nextTask (this=0x7f92a812fb00, t=..., tick=tick@entry=64 '@')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorpool.cc:250
#11 0x00007f92b6d1fe20 in ExecutorThread::run (this=0x7f92b969f620) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/executorthread.cc:81
#12 0x00007f92bbd0263a in run (this=0x7f92a7483ba0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#13 platform_thread_wrap (arg=0x7f92a7483ba0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#14 0x00007f92bbae7182 in start_thread (arg=0x7f92a73ff700) at pthread_create.c:312
#15 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 24 (LWP 419):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007f92b6d2b690 in __gthread_cond_timedwait (__abs_timeout=0x7f92a7ffee20, __mutex=0x7f92a80e3d20, __cond=<optimized out>)
    at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:871
#2  __wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=<synthetic pointer>, this=0x7f92a80e3d48)
    at /usr/include/c++/4.8/condition_variable:160
#3  wait_until<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=<synthetic pointer>, this=0x7f92a80e3d48)
    at /usr/include/c++/4.8/condition_variable:100
#4  wait_until<std::chrono::_V2::system_clock, std::chrono::duration<long int, std::ratio<1l, 1000000000l> >, MemoryTracker::statsThreadMainLoop(void*)::__lambda0> (__atime=..., 
    __lock=<synthetic pointer>, this=0x7f92a80e3d48, __p=...) at /usr/include/c++/4.8/condition_variable:123
#5  wait_for<long int, std::ratio<1l, 1000l>, MemoryTracker::statsThreadMainLoop(void*)::__lambda0> (__rtime=..., __lock=<synthetic pointer>, this=0x7f92a80e3d48, __p=...)
    at /usr/include/c++/4.8/condition_variable:139
#6  MemoryTracker::statsThreadMainLoop (arg=0x7f92a80e3ce0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/memory_tracker.cc:38
#7  0x00007f92bbd0263a in run (this=0x7f92a811d960) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#8  platform_thread_wrap (arg=0x7f92a811d960) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#9  0x00007f92bbae7182 in start_thread (arg=0x7f92a7fff700) at pthread_create.c:312
#10 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 23 (LWP 32378):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96af280) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96af280) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bda0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bda0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92a93ed700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 22 (LWP 32377):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96af100) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96af100) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bd80) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bd80) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92a9bee700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 21 (LWP 32376):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96aef80) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96aef80) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bd60) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bd60) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92aa3ef700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 20 (LWP 32375):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96aee00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96aee00) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bd40) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bd40) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92aabf0700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 19 (LWP 32374):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96aec80) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96aec80) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bd20) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bd20) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92ab3f1700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 18 (LWP 32373):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96aeb00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96aeb00) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bd00) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bd00) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92abbf2700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 17 (LWP 32372):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96ae980) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96ae980) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bce0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bce0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92ac3f3700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 16 (LWP 32371):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96ae800) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96ae800) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bcc0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bcc0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92acbf4700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 15 (LWP 32370):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007f92ba9ca4bc in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#2  0x0000000000438e70 in Executor::run (this=0x7f92b96ae680) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/executor.cc:43
#3  0x00007f92bbd0659b in Couchbase::Thread::thread_entry (this=0x7f92b96ae680) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/thread.cc:46
#4  0x00007f92bbd0263a in run (this=0x7f92b967bca0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bca0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92ad3f5700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 14 (LWP 32369):
#0  0x00007f92ba43bb13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92bb483333 in epoll_dispatch (base=0x7f92b9615900, tv=<optimized out>)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:407
#2  0x00007f92bb470172 in event_base_loop (base=0x7f92b9615900, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1625
#3  0x0000000000420cc4 in worker_libevent (arg=0x7f92b9648af8) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#4  0x00007f92bbd0263a in run (this=0x7f92b967bc60) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bc60) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92adbf6700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 13 (LWP 32368):
#0  0x00007f92ba43bb13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92bb483333 in epoll_dispatch (base=0x7f92b9615680, tv=<optimized out>)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:407
#2  0x00007f92bb470172 in event_base_loop (base=0x7f92b9615680, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1625
#3  0x0000000000420cc4 in worker_libevent (arg=0x7f92b96489c0) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#4  0x00007f92bbd0263a in run (this=0x7f92b967bc40) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bc40) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92ae3f7700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 12 (LWP 32367):
#0  0x00007f92ba43bb13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92bb483333 in epoll_dispatch (base=0x7f92b9615400, tv=<optimized out>)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:407
#2  0x00007f92bb470172 in event_base_loop (base=0x7f92b9615400, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1625
#3  0x0000000000420cc4 in worker_libevent (arg=0x7f92b9648888) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#4  0x00007f92bbd0263a in run (this=0x7f92b967bc20) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bc20) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92aebf8700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 11 (LWP 32366):
#0  0x00007f92ba43bb13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92bb483333 in epoll_dispatch (base=0x7f92b9615180, tv=<optimized out>)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:407
#2  0x00007f92bb470172 in event_base_loop (base=0x7f92b9615180, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1625
#3  0x0000000000420cc4 in worker_libevent (arg=0x7f92b9648750) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#4  0x00007f92bbd0263a in run (this=0x7f92b967bc00) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967bc00) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92af3f9700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 10 (LWP 32365):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92a1a1eca8) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92b6d01d0d in __gthread_mutex_lock (__mutex=0x7f92a1a1eca8) at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:748
#4  lock (this=0x7f92a1a1eca8) at /usr/include/c++/4.8/mutex:134
#5  lock (this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:73
#6  LockHolder (tryLock=false, m=..., this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:50
#7  getHighSeqno (this=0x7f92a1a1ec98) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/checkpoint.h:683
#8  getHighSeqno (this=0x7f92a1a1ec00) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/vbucket.h:209
#9  EventuallyPersistentEngine::addSeqnoVbStats_UNLOCKED (this=this@entry=0x7f92b9742c00, cookie=cookie@entry=0x7f919b646400, add_stat=add_stat@entry=
    0x444200 <append_stats(char const*, uint16_t, char const*, uint32_t, void const*)>, vb=...) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:4411
#10 0x00007f92b6d021e1 in EventuallyPersistentEngine::doSeqnoStats (this=this@entry=0x7f92b9742c00, cookie=cookie@entry=0x7f919b646400, 
    add_stat=add_stat@entry=0x444200 <append_stats(char const*, uint16_t, char const*, uint32_t, void const*)>, 
    stat_key=stat_key@entry=0x7f91e0d81758 <error: Cannot access memory at address 0x7f91e0d81758>, nkey=nkey@entry=13)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:4481
#11 0x00007f92b6d0e47c in EventuallyPersistentEngine::getStats (this=0x7f92b9742c00, this@entry=<error reading variable: Cannot access memory at address 0x7f92afbf9608>, 
    cookie=0x7f919b646400, stat_key=<optimized out>, nkey=13, add_stat=0x444200 <append_stats(char const*, uint16_t, char const*, uint32_t, void const*)>)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:4597

Thread 9 (LWP 32363):
#0  0x00007f92bbae9475 in __GI___pthread_mutex_lock (mutex=0x7f92a0b10968) at ../nptl/pthread_mutex_lock.c:79
#1  0x00007f92b6cf3770 in __gthread_mutex_lock (__mutex=0x7f92a0b10968) at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:748
#2  lock (this=<optimized out>) at /usr/include/c++/4.8/mutex:134
#3  lock (this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:73
#4  LockHolder (tryLock=false, m=..., this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:50
#5  getLockedBucket (bucket=0x7f92b0bf7a30, h=-912603622, this=0x7f92a0a5a708) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/stored-value.h:1392
#6  HashTable::getLockedBucket (this=0x7f92a0a5a708, s=..., bucket=bucket@entry=0x7f92b0bf7a30) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/stored-value.h:1421
#7  0x00007f92b6cea411 in EventuallyPersistentStore::getInternal (this=0x7f92a8169a00, key=..., vbucket=vbucket@entry=31, cookie=cookie@entry=0x7f9295f2b200, 
    allowedState=allowedState@entry=vbucket_state_active, options=options@entry=(QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP | HIDE_LOCKED_CAS))
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:2103
#8  0x00007f92b6d01422 in get (options=(QUEUE_BG_FETCH | HONOR_STATES | TRACK_REFERENCE | DELETE_TEMP | HIDE_LOCKED_CAS), cookie=0x7f9295f2b200, vbucket=31, key=..., 
    this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.h:277
#9  get (options=TRACK_STATISTICS, vbucket=31, nkey=<optimized out>, key=0x7f91baefd018, itm=0x7f92b0bf7b80, cookie=0x7f9295f2b200, this=0x7f92b9742c00)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.h:305
#10 EvpGet (handle=0x7f92b9742c00, cookie=0x7f9295f2b200, itm=0x7f92b0bf7b80, key=0x7f91baefd018, nkey=<optimized out>, vbucket=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:213
#11 0x0000000000440211 in bucket_get (vbucket=<optimized out>, nkey=18, key=0x7f91baefd018, item_=0x7f92b0bf7b80, c=0x7f9295f2b200)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.h:347
#12 process_bin_get (c=0x7f9295f2b200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:558
#13 0x0000000000443a02 in process_bin_packet (c=0x7f9295f2b200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4605
#14 mcbp_complete_nread (c=c@entry=0x7f9295f2b200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4714
#15 0x000000000044c56c in conn_nread (c=0x7f9295f2b200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:310
#16 0x0000000000432508 in execute (connection=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43
#17 McbpConnection::runStateMachinery (this=0x7f9295f2b200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1004
#18 0x000000000043254b in McbpConnection::runEventLoop (this=0x7f9295f2b200, which=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1274
#19 0x0000000000434ab0 in run_event_loop (c=0x7f9295f2b200, which=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connections.cc:147
#20 0x0000000000411a11 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7f9295f2b200)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.cc:841
#21 0x00007f92bb4706b8 in event_persist_closure (ev=<optimized out>, base=0x7f92b9614a00)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319
#22 event_process_active_single_queue (activeq=0x7f92b961a130, base=0x7f92b9614a00)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363
#23 event_process_active (base=<optimized out>) at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438
#24 event_base_loop (base=0x7f92b9614a00, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639
#25 0x0000000000420cc4 in worker_libevent (arg=0x7f92b96483a8) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#26 0x00007f92bbd0263a in run (this=0x7f92b967bba0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#27 platform_thread_wrap (arg=0x7f92b967bba0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#28 0x00007f92bbae7182 in start_thread (arg=0x7f92b0bfc700) at pthread_create.c:312
#29 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 8 (LWP 32362):
#0  0x00007f92bb695ed4 in je_tcache_dalloc_small (binind=3, ptr=0x7f91e3d3e710, tcache=0x7f92b97be000, tsd=<optimized out>) at include/jemalloc/internal/tcache.h:375
#1  je_arena_dalloc (tcache=0x7f92b97be000, ptr=0x7f91e3d3e710, tsd=<optimized out>) at include/jemalloc/internal/arena.h:1271
#2  je_idalloctm (is_metadata=false, tcache=0x7f92b97be000, ptr=0x7f91e3d3e710, tsd=<optimized out>) at include/jemalloc/internal/jemalloc_internal.h:1009
#3  je_iqalloc (tcache=0x7f92b97be000, ptr=0x7f91e3d3e710, tsd=<optimized out>) at include/jemalloc/internal/jemalloc_internal.h:1033
#4  ifree (tsd=<optimized out>, tcache=0x7f92b97be000, ptr=0x7f91e3d3e710) at src/jemalloc.c:1745
#5  je_free (ptr=0x7f91e3d3e710) at src/jemalloc.c:1839
#6  0x00007f92b6cfa2ff in ~basic_string (this=0x7f9202cc26d0, __in_chrg=<optimized out>) at /usr/include/c++/4.8/bits/basic_string.h:539
#7  ~Item (this=0x7f9202cc26a0, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/item.h:419
#8  itemRelease (this=<optimized out>, cookie=<optimized out>, itm=0x7f9202cc26a0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.h:292
#9  EvpItemRelease (handle=<optimized out>, cookie=<optimized out>, itm=0x7f9202cc26a0) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:200
#10 0x000000000044d160 in reset_cmd_handler (c=0x7f929583e200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:122
#11 conn_new_cmd (c=0x7f929583e200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:271
#12 0x0000000000432508 in execute (connection=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43
#13 McbpConnection::runStateMachinery (this=0x7f929583e200) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1004
#14 0x000000000043254b in McbpConnection::runEventLoop (this=0x7f929583e200, which=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1274
#15 0x0000000000434ab0 in run_event_loop (c=0x7f929583e200, which=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connections.cc:147
#16 0x0000000000411a11 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7f929583e200)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.cc:841
#17 0x00007f92bb4706b8 in event_persist_closure (ev=<optimized out>, base=0x7f92b9614780)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319
#18 event_process_active_single_queue (activeq=0x7f92b961a110, base=0x7f92b9614780)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363
#19 event_process_active (base=<optimized out>) at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438
#20 event_base_loop (base=0x7f92b9614780, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639
#21 0x0000000000420cc4 in worker_libevent (arg=0x7f92b9648270) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#22 0x00007f92bbd0263a in run (this=0x7f92b967bb80) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#23 platform_thread_wrap (arg=0x7f92b967bb80) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#24 0x00007f92bbae7182 in start_thread (arg=0x7f92b13fd700) at pthread_create.c:312
#25 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 7 (LWP 32361):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007f92bbae9657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread-2.19.so
#2  0x00007f92bbae9480 in __GI___pthread_mutex_lock (mutex=0x7f92996df980) at ../nptl/pthread_mutex_lock.c:79
#3  0x00007f92b6cced97 in __gthread_mutex_lock (__mutex=0x7f92996df980) at /usr/include/x86_64-linux-gnu/c++/4.8/bits/gthr-default.h:748
#4  lock (this=0x7f92996df980) at /usr/include/c++/4.8/mutex:134
#5  lock (this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:73
#6  LockHolder (tryLock=false, m=..., this=<synthetic pointer>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/locks.h:50
#7  popFront (frontValue=<synthetic pointer>, this=0x7f92996df980) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/dcp/dcp-types.h:75
#8  DcpProducer::getNextItem (this=this@entry=0x7f92996df800) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/dcp/producer.cc:857
#9  0x00007f92b6ccf62c in DcpProducer::step (this=0x7f92996df800, producers=0x689b60 <ship_mcbp_dcp_log(McbpConnection*)::producers>)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/dcp/producer.cc:360
#10 0x00007f92b6cfa83b in EvpDcpStep (handle=<optimized out>, cookie=0x7f929844c700, producers=0x689b60 <ship_mcbp_dcp_log(McbpConnection*)::producers>)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:1460
#11 0x00000000004432d4 in ship_mcbp_dcp_log (c=0x7f929844c700) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:1914
#12 0x000000000044cde5 in conn_ship_log (c=0x7f929844c700) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:189
#13 0x0000000000432508 in execute (connection=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43
#14 McbpConnection::runStateMachinery (this=0x7f929844c700) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1004
#15 0x000000000043254b in McbpConnection::runEventLoop (this=0x7f929844c700, which=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1274
#16 0x0000000000434ab0 in run_event_loop (c=0x7f929844c700, which=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connections.cc:147
#17 0x0000000000411a11 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7f929844c700)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.cc:841
#18 0x00007f92bb4706b8 in event_persist_closure (ev=<optimized out>, base=0x7f92b9614500)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319
#19 event_process_active_single_queue (activeq=0x7f92b961a0f0, base=0x7f92b9614500)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363
#20 event_process_active (base=<optimized out>) at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438
#21 event_base_loop (base=0x7f92b9614500, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639
#22 0x0000000000420cc4 in worker_libevent (arg=0x7f92b9648138) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#23 0x00007f92bbd0263a in run (this=0x7f92b967bb60) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#24 platform_thread_wrap (arg=0x7f92b967bb60) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#25 0x00007f92bbae7182 in start_thread (arg=0x7f92b1bfe700) at pthread_create.c:312
#26 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 6 (LWP 32360):
#0  0x00007f92b6d88796 in getSize (this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/item.h:176
#1  getValMemSize (this=0x7f91fc726770) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/item.h:509
#2  size (this=0x7f91fc726770) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/item.h:601
#3  ObjectRegistry::onDeleteItem (pItem=pItem@entry=0x7f91fc726770) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/objectregistry.cc:156
#4  0x00007f92b6cfa2ed in ~Item (this=0x7f91fc726770, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/item.h:420
#5  itemRelease (this=<optimized out>, cookie=<optimized out>, itm=0x7f91fc726770) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.h:292
#6  EvpItemRelease (handle=<optimized out>, cookie=<optimized out>, itm=0x7f91fc726770) at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:200
#7  0x000000000044d160 in reset_cmd_handler (c=0x7f92959ae300) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:122
#8  conn_new_cmd (c=0x7f92959ae300) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:271
#9  0x0000000000432508 in execute (connection=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43
#10 McbpConnection::runStateMachinery (this=0x7f92959ae300) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1004
#11 0x000000000043254b in McbpConnection::runEventLoop (this=0x7f92959ae300, which=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1274
#12 0x0000000000434ab0 in run_event_loop (c=0x7f92959ae300, which=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connections.cc:147
#13 0x0000000000411a11 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7f92959ae300)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.cc:841
#14 0x00007f92bb4706b8 in event_persist_closure (ev=<optimized out>, base=0x7f92b9614280)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319
#15 event_process_active_single_queue (activeq=0x7f92b961a0d0, base=0x7f92b9614280)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363
#16 event_process_active (base=<optimized out>) at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438
#17 event_base_loop (base=0x7f92b9614280, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639
#18 0x0000000000420cc4 in worker_libevent (arg=0x7f92b9648000) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#19 0x00007f92bbd0263a in run (this=0x7f92b967bb40) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#20 platform_thread_wrap (arg=0x7f92b967bb40) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#21 0x00007f92bbae7182 in start_thread (arg=0x7f92b23ff700) at pthread_create.c:312
#22 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 5 (LWP 32337):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007f92bbd02e39 in cb_cond_timedwait (cond=0x7f92bcb62570 <audit+240>, mutex=0x7f92bcb625a0 <audit+288>, ms=86400000)
    at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:280
#2  0x00007f92bc9581ce in consume_events (arg=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/auditd/src/auditd.cc:59
#3  0x00007f92bbd0263a in run (this=0x7f92b96d91a0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#4  platform_thread_wrap (arg=0x7f92b96d91a0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#5  0x00007f92bbae7182 in start_thread (arg=0x7f92b7bfe700) at pthread_create.c:312
#6  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 4 (LWP 32336):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007f92bbd02e39 in cb_cond_timedwait (cond=0x7f92b8bfbc40 <cond>, mutex=0x7f92b8bfbc80 <mutex>, ms=19000)
    at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:280
#2  0x00007f92b89f8c75 in logger_thread_main (arg=0x7f92b9625d00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/extensions/loggers/file_logger.cc:540
#3  0x00007f92bbd0263a in run (this=0x7f92b967b940) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#4  platform_thread_wrap (arg=0x7f92b967b940) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#5  0x00007f92bbae7182 in start_thread (arg=0x7f92b83ff700) at pthread_create.c:312
#6  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 3 (LWP 32335):
#0  0x00007f92ba42e12d in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92b93fdc28 in poll (__timeout=60000, __nfds=1, __fds=0x7f92b93fbdd0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  get_command (buffsize=80, buffer=0x7f92b93fbde0 "\017") at /home/couchbase/jenkins/workspace/watson-unix/memcached/extensions/daemon/stdin_check.c:51
#3  check_stdin_thread (arg=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/extensions/daemon/stdin_check.c:93
#4  0x00007f92bbd0263a in run (this=0x7f92b967b840) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#5  platform_thread_wrap (arg=0x7f92b967b840) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#6  0x00007f92bbae7182 in start_thread (arg=0x7f92b93fc700) at pthread_create.c:312
#7  0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 2 (LWP 32322):
#0  0x00007f92ba43bb13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:81
#1  0x00007f92bb483333 in epoll_dispatch (base=0x7f92b9614000, tv=<optimized out>)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/epoll.c:407
#2  0x00007f92bb470172 in event_base_loop (base=0x7f92b9614000, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1625
#3  0x0000000000415419 in memcached_main (argc=<optimized out>, argv=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.cc:2658
#4  0x00007f92ba362ec5 in __libc_start_main (main=0x40d300 <main(int, char**)>, argc=3, argv=0x7ffd48ebc2f8, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, 
    stack_end=0x7ffd48ebc2e8) at libc-start.c:287
#5  0x000000000040f44f in _start ()

Thread 1 (LWP 32364):
#0  0x00007f92ba377cc9 in __GI_raise (sig=sig@entry=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:56
#1  0x00007f92ba37b0d8 in __GI_abort () at abort.c:89
#2  0x00007f92ba97c535 in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.19
#3  0x0000000000427f4f in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/watson-unix/memcached/utilities/terminate_handler.cc:63
#4  0x74706f2f20202020 in ?? ()
#5  0x61626863756f632f in ?? ()
#6  0x6d2f6e69622f6573 in ?? ()
#7  0x6465686361636d65 in ?? ()
#8  0x313478305b202928 in ?? ()
#9  0x20200a5d31316131 in ?? ()
#10 0x632f74706f2f2020 in ?? ()
#11 0x657361626863756f in ?? ()
#12 0x2f2e2e2f6e69622f in ?? ()
#13 0x6562696c2f62696c in ?? ()
#14 0x726f635f746e6576 in ?? ()
#15 0x6f732e302e322d65 in ?? ()
#16 0x746e65766528352e in ?? ()
#17 0x6f6c5f657361625f in ?? ()
#18 0x38373678302b706f in ?? ()
#19 0x39663778305b2029 in ?? ()
#20 0x6236303734626232 in ?? ()
#21 0x2f202020200a5d38 in ?? ()
#22 0x63756f632f74706f in ?? ()
#23 0x69622f6573616268 in ?? ()
#24 0x6361636d656d2f6e in ?? ()
#25 0x305b202928646568 in ?? ()
#26 0x5d34636330323478 in ?? ()
#27 0x706f2f202020200a in ?? ()
#28 0x626863756f632f74 in ?? ()
#29 0x2f6e69622f657361 in ?? ()
#30 0x6c2f62696c2f2e2e in ?? ()
#31 0x6f6674616c706269 in ?? ()
#32 0x2e302e6f732e6d72 in ?? ()
#33 0x305b202928302e31 in ?? ()
#34 0x6462623239663778 in ?? ()
#35 0x200a5d6133363230 in ?? ()
#36 0x2f62696c2f202020 in ?? ()
#37 0x6c2d34365f363878 in ?? ()
#38 0x756e672d78756e69 in ?? ()
#39 0x7268747062696c2f in ?? ()
#40 0x302e6f732e646165 in ?? ()
#41 0x663778305b202928 in ?? ()
#42 0x3137656162623239 in ?? ()
#43 0x202020200a5d3238 in ?? ()
#44 0x3638782f62696c2f in ?? ()
#45 0x756e696c2d34365f in ?? ()
#46 0x696c2f756e672d78 in ?? ()
#47 0x28362e6f732e6362 in ?? ()
#48 0x78302b656e6f6c63 in ?? ()
#49 0x3778305b20296436 in ?? ()
#50 0x6233346162323966 in ?? ()
#51 0x0000000a5d643734 in ?? ()
#52 0x00007f92bbd031db in cb_rw_reader_enter (rw=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:391
#53 0x00007f92b03fa870 in ?? ()
#54 0x0000000000000004 in ?? ()
#55 0x00007f92b6dac1a0 in StoredValue::state_temp_init () from /opt/couchbase/lib/ep.so
#56 0x00007f919fc57ee0 in ?? ()
#57 0x00007f92b6cf48fd in HashTable::unlocked_set (this=0x7f92b6da063f, v=@0x100000000: <error reading variable>, val=..., cas=<optimized out>, allowExisting=<optimized out>, 
    hasMetaData=<optimized out>, policy=FULL_EVICTION, nru=0 '\000', maybeKeyExists=false, isReplication=true)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/stored-value.h:1109
#58 0x00007f92b6cecdc2 in EventuallyPersistentStore::setWithMeta (this=0x7f92a8169a00, itm=..., cas=cas@entry=0, seqno=seqno@entry=0x0, cookie=0x7f91dd0b3f00, force=force@entry=true, 
    allowExisting=allowExisting@entry=true, nru=nru@entry=2 '\002', genBySeqno=genBySeqno@entry=false, emd=0x7f917ff70400, isReplication=isReplication@entry=true)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep.cc:2386
#59 0x00007f92b6cd9a94 in PassiveStream::processMutation (this=this@entry=0x7f9294ac6cc0, mutation=mutation@entry=0x7f9164ffdae0)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/dcp/stream.cc:1554
#60 0x00007f92b6cda448 in PassiveStream::messageReceived (this=0x7f9294ac6cc0, resp=resp@entry=0x7f9164ffdae0)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/dcp/stream.cc:1399
#61 0x00007f92b6cc5a43 in DcpConsumer::mutation (this=0x7f91545fd580, opaque=<optimized out>, key=<optimized out>, nkey=20, value=<optimized out>, nvalue=340, 
    cas=1462814623569018880, vbucket=593, flags=0, datatype=0 '\000', locktime=0, bySeqno=97060, revSeqno=8571, exptime=0, nru=2 '\002', meta=0x7f915218fbf6, nmeta=5)
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/dcp/consumer.cc:324
#62 0x00007f92b6cfa5ca in EvpDcpMutation (handle=<optimized out>, cookie=0x7f91dd0b3f00, opaque=70, key=0x7f915218fa8e, nkey=<optimized out>, value=0x7f915218faa2, nvalue=340, 
    cas=1462814623569018880, vbucket=593, flags=0, datatype=0 '\000', bySeqno=97060, revSeqno=8571, expiration=0, lockTime=0, meta=0x7f915218fbf6, nmeta=5, nru=2 '\002')
    at /home/couchbase/jenkins/workspace/watson-unix/ep-engine/src/ep_engine.cc:1615
#63 0x000000000043c4ea in dcp_mutation_executor (c=c@entry=0x7f91dd0b3f00, packet=packet@entry=0x7f915218fa57)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:2400
#64 0x0000000000443a02 in process_bin_packet (c=0x7f91dd0b3f00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4605
#65 mcbp_complete_nread (c=c@entry=0x7f91dd0b3f00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/mcbp_executors.cc:4714
#66 0x000000000044c56c in conn_nread (c=0x7f91dd0b3f00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.cc:310
#67 0x0000000000432508 in execute (connection=..., this=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/statemachine_mcbp.h:43
#68 McbpConnection::runStateMachinery (this=0x7f91dd0b3f00) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1004
#69 0x000000000043254b in McbpConnection::runEventLoop (this=0x7f91dd0b3f00, which=<optimized out>)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connection_mcbp.cc:1274
#70 0x0000000000434ab0 in run_event_loop (c=0x7f91dd0b3f00, which=<optimized out>) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/connections.cc:147
#71 0x0000000000411a11 in event_handler (fd=<optimized out>, which=<optimized out>, arg=0x7f91dd0b3f00)
    at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/memcached.cc:841
#72 0x00007f92bb4706b8 in event_persist_closure (ev=<optimized out>, base=0x7f92b9614c80)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1319
#73 event_process_active_single_queue (activeq=0x7f92b961a150, base=0x7f92b9614c80)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1363
#74 event_process_active (base=<optimized out>) at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1438
#75 event_base_loop (base=0x7f92b9614c80, flags=0)
    at /home/couchbase/serverjenkins/workspace/cbdeps-platform-build/deps/packages/build/libevent/libevent-prefix/src/libevent/event.c:1639
#76 0x0000000000420cc4 in worker_libevent (arg=0x7f92b96484e0) at /home/couchbase/jenkins/workspace/watson-unix/memcached/daemon/thread.cc:231
#77 0x00007f92bbd0263a in run (this=0x7f92b967bbc0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:54
#78 platform_thread_wrap (arg=0x7f92b967bbc0) at /home/couchbase/jenkins/workspace/watson-unix/platform/src/cb_pthreads.cc:66
#79 0x00007f92bbae7182 in start_thread (arg=0x7f92b03fb700) at pthread_create.c:312
#80 0x00007f92ba43b47d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
(gdb) quit