GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-120.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /opt/couchbase/bin/memcached...Reading symbols from /usr/lib/debug/opt/couchbase/bin/memcached.debug...done.
done.
[New LWP 23701]
[New LWP 18163]
[New LWP 18164]
[New LWP 18165]
[New LWP 18166]
[New LWP 18167]
[New LWP 18171]
[New LWP 18172]
[New LWP 18193]
[New LWP 18194]
[New LWP 18195]
[New LWP 18196]
[New LWP 18197]
[New LWP 18198]
[New LWP 18199]
[New LWP 18200]
[New LWP 18201]
[New LWP 18202]
[New LWP 18203]
[New LWP 18204]
[New LWP 18205]
[New LWP 18206]
[New LWP 18207]
[New LWP 18208]
[New LWP 18209]
[New LWP 18210]
[New LWP 18215]
[New LWP 18216]
[New LWP 18217]
[New LWP 18218]
[New LWP 18650]
[New LWP 18651]
[New LWP 18652]
[New LWP 18653]
[New LWP 18654]
[New LWP 18655]
[New LWP 18656]
[New LWP 18657]
[New LWP 18658]
[New LWP 18659]
[New LWP 18660]
[New LWP 18661]
[New LWP 18662]
[New LWP 18663]
[New LWP 18664]
[New LWP 18665]
[New LWP 18666]
[New LWP 18667]
[New LWP 18668]
[New LWP 18669]
[New LWP 18670]
[New LWP 18675]
[New LWP 18676]
[New LWP 18679]
[New LWP 22670]
[New LWP 23172]
[New LWP 23699]
[New LWP 23700]
[New LWP 23702]
To enable execution of this file add
	add-auto-load-safe-path /usr/lib64/libthread_db-1.0.so
line to your configuration file "/root/.gdbinit".
To completely disable this security protection add
	set auto-load safe-path /
line to your configuration file "/root/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual.  E.g., run from the shell:
	info "(gdb)Auto-loading safe path"
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
#0  0x00007fe431ab5387 in __bsd_signal (sig=18163, handler=0x5c95) at ../sysdeps/posix/signal.c:50
50	}

Thread 59 (LWP 23702):
#0  0x00007ffdfeb3a6c2 in ?? ()
#1  0x00007fe376bc2250 in ?? ()
#2  0x00007fe390076000 in ?? ()
#3  0x0000000000000000 in ?? ()

Thread 58 (LWP 23700):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fe43242f8bc in __gthread_cond_wait (__mutex=<optimized out>, __cond=__cond@entry=0x7fe3754f6c88) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2  std::condition_variable::wait (this=this@entry=0x7fe3754f6c88, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x000000000092b1af in magma::KVStore::acquireFlushBarrier (this=this@entry=0x7fe3754f6a10) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1485
#4  0x00000000009315a0 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:356
#5  0x0000000000931f48 in magma::KVStore::FlushMemTables (this=<optimized out>, wal=<optimized out>, flushMode=<optimized out>, blockMode=<optimized out>)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:336
#6  0x00000000008e023a in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1106
#7  0x00000000008ff3ee in magma::Magma::Impl::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:50
#8  0x00000000008ff542 in magma::Magma::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#9  0x0000000000856fcf in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#10 0x0000000000840d9c in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62
#11 0x000000000083a5ef in MagmaKVStore::commit(std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:651
#12 0x00000000007e8045 in EPBucket::commit(KVStoreIface&, std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:929
#13 0x00000000007eefd5 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:801
#14 0x00000000007ef59f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:378
#15 0x00000000006c80b0 in Flusher::flushVB (this=0x7fe3900bc400) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:285
#16 0x00000000006c8a00 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:200
#17 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
#18 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fe3f77ec840)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
#19 0x0000000000b649a0 in operator() (this=0x7fe3f77ec840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
#20 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fe4309e4400, thread=..., 
    task=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached.debug, CU 0x65a24fb, DIE 0x65e7bd8>)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
#21 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fe4309e4400, thread=...)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
#22 0x0000000000b67959 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (
    __t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
#23 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)
    at /usr/local/include/c++/7.3.0/bits/invoke.h:95
#24 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
#25 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
#26 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
#27 0x0000000000a19ce4 in operator() (this=0x7fe4306d6900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#28 operator() (__closure=0x7fe4306d6900) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#29 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
#30 0x00007fe432434d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
#31 0x00007fe434258ea5 in start_thread (arg=0x7fe3f77fe700) at pthread_create.c:307
#32 0x00007fe431b7d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81
#33 0x0000000000000000 in ?? ()

Thread 57 (LWP 23699):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fe43242f8bc in __gthread_cond_wait (__mutex=<optimized out>, __cond=__cond@entry=0x7fe377aaf288) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2  std::condition_variable::wait (this=this@entry=0x7fe377aaf288, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x000000000092b1af in magma::KVStore::acquireFlushBarrier (this=this@entry=0x7fe377aaf010) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1485
#4  0x00000000009315a0 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:356
#5  0x0000000000931f48 in magma::KVStore::FlushMemTables (this=<optimized out>, wal=<optimized out>, flushMode=<optimized out>, blockMode=<optimized out>)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:336
#6  0x00000000008e023a in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1106
#7  0x00000000008ff3ee in magma::Magma::Impl::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:50
#8  0x00000000008ff542 in magma::Magma::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#9  0x0000000000856fcf in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#10 0x0000000000840d9c in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62
#11 0x000000000083a5ef in MagmaKVStore::commit(std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:651
#12 0x00000000007e8045 in EPBucket::commit(KVStoreIface&, std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:929
#13 0x00000000007eefd5 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:801
#14 0x00000000007ef59f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:378
#15 0x00000000006c80b0 in Flusher::flushVB (this=0x7fe3900bd200) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:285
#16 0x00000000006c8a00 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:200
#17 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
#18 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fe40cfc5840)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
#19 0x0000000000b649a0 in operator() (this=0x7fe40cfc5840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
#20 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fe4309e4400, thread=..., 
    task=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached.debug, CU 0x65a24fb, DIE 0x65e7bd8>)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
#21 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fe4309e4400, thread=...)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
#22 0x0000000000b67959 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (
    __t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
#23 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)
    at /usr/local/include/c++/7.3.0/bits/invoke.h:95
#24 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
#25 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
#26 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
#27 0x0000000000a19ce4 in operator() (this=0x7fe4306d6180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#28 operator() (__closure=0x7fe4306d6180) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#29 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
#30 0x00007fe432434d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
#31 0x00007fe434258ea5 in start_thread (arg=0x7fe40cfd7700) at pthread_create.c:307
#32 0x00007fe431b7d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81
#33 0x0000000000000000 in ?? ()

Thread 56 (LWP 23172):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00007fe43425ae9b in _L_lock_883 () from /lib64/libpthread.so.0
#2  0x00007fe43425ad68 in __GI___pthread_mutex_lock (mutex=mutex@entry=0x7fe3a00084f8) at ../nptl/pthread_mutex_lock.c:78
#3  0x00007fe43379a629 in malloc_mutex_lock_final (mutex=0x7fe3a00084b8) at include/jemalloc/internal/mutex.h:155
#4  je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7fe3a00084b8) at src/mutex.c:85
#5  0x00007fe433757a5d in malloc_mutex_lock (mutex=<optimized out>, tsdn=0x7fe3a37ed220) at include/jemalloc/internal/mutex.h:221
#6  je_arena_bin_choose_lock (binshard=<synthetic pointer>, binind=8, arena=0x7fe3a00008c0, tsdn=0x7fe3a37ed220) at src/arena.c:1375
#7  je_arena_tcache_fill_small (tsdn=0x7fe3a37ed220, arena=0x7fe3a00008c0, tcache=0x7fe35d9b4000, tbin=0x7fe35d9b40d0, binind=8, prof_accumbytes=<optimized out>) at src/arena.c:1392
#8  0x00007fe4337c6f45 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7fe3a37ed220, arena=<optimized out>, tcache=tcache@entry=0x7fe35d9b4000, tbin=tbin@entry=0x7fe35d9b40d0, binind=<optimized out>, 
    tcache_success=tcache_success@entry=0x7fe3a37eb360) at src/tcache.c:94
#9  0x00007fe43374ba0b in tcache_alloc_small (slow_path=false, zero=<optimized out>, binind=<optimized out>, size=128, tcache=0x7fe35d9b4000, arena=<optimized out>, tsd=<optimized out>) at include/jemalloc/internal/tcache_inlines.h:60
#10 arena_malloc (slow_path=false, tcache=0x7fe35d9b4000, zero=<optimized out>, ind=<optimized out>, size=128, arena=<optimized out>, tsdn=<optimized out>) at include/jemalloc/internal/arena_inlines_b.h:165
#11 iallocztm (slow_path=false, arena=<optimized out>, is_internal=false, tcache=0x7fe35d9b4000, zero=<optimized out>, ind=<optimized out>, size=128, tsdn=<optimized out>) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:53
#12 imalloc_no_sample (ind=<optimized out>, usize=128, size=128, tsd=0x7fe3a37ed220, dopts=<synthetic pointer>, sopts=<synthetic pointer>) at src/jemalloc.c:1949
#13 imalloc_body (tsd=0x7fe3a37ed220, dopts=<synthetic pointer>, sopts=<synthetic pointer>) at src/jemalloc.c:2123
#14 imalloc (dopts=<synthetic pointer>, sopts=<synthetic pointer>) at src/jemalloc.c:2258
#15 je_mallocx (size=size@entry=128, flags=<optimized out>) at src/jemalloc.c:3137
#16 0x0000000000b1216d in cb::_JEArenaMalloc<cb::JEArenaCoreLocalTracker>::malloc (size=size@entry=128) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/je_arena_malloc.cc:266
#17 0x00000000005af02f in malloc (size=128) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39
#18 cb_malloc (size=128) at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/cb_malloc_arena.cc:39
#19 0x0000000000b07209 in operator new(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/platform/src/global_new_replacement.cc:63
#20 0x000000000090e012 in allocate (__n=<optimized out>, this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_uninitialized.h:1013
#21 allocate (__n=<optimized out>, __a=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:460
#22 _M_allocate (this=<optimized out>, __n=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:346
#23 std::vector<std::shared_ptr<magma::KVStore>, std::allocator<std::shared_ptr<magma::KVStore> > >::_M_realloc_insert<std::shared_ptr<magma::KVStore>&> (this=this@entry=0x7fe3a37eb4d0, __position=...)
    at /opt/gcc-10.2.0/include/c++/10.2.0/bits/vector.tcc:440
#24 0x000000000090cd0d in emplace_back<std::shared_ptr<magma::KVStore>&> (this=0x7fe3a37eb4d0) at /opt/gcc-10.2.0/include/c++/10.2.0/ext/atomicity.h:84
#25 magma::KVStoreSet::GetAllKVStores() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore_set.cc:53
#26 0x00000000008d87d1 in magma::Magma::Impl::GetKVStoreList() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:339
#27 0x00000000008da43e in magma::Magma::Impl::GetResourceUsage() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1041
#28 0x00000000008e044f in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1077
#29 0x00000000008ff3ee in magma::Magma::Impl::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:50
#30 0x00000000008ff542 in magma::Magma::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#31 0x0000000000856fcf in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#32 0x0000000000840d9c in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62
#33 0x000000000083a5ef in MagmaKVStore::commit(std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:651
#34 0x00000000007e8045 in EPBucket::commit(KVStoreIface&, std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:929
#35 0x00000000007eefd5 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:801
#36 0x00000000007ef59f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:378
#37 0x00000000006c80b0 in Flusher::flushVB (this=0x7fe3900bb600) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:285
#38 0x00000000006c8a00 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:200
#39 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
#40 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fe3a37ec840)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
#41 0x0000000000b649a0 in operator() (this=0x7fe3a37ec840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
#42 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fe4309e4400, thread=..., 
    task=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached.debug, CU 0x65a24fb, DIE 0x65e7bd8>)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
#43 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fe4309e4400, thread=...)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
#44 0x0000000000b67959 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (
    __t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
#45 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)
    at /usr/local/include/c++/7.3.0/bits/invoke.h:95
#46 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
#47 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
#48 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
#49 0x0000000000a19ce4 in operator() (this=0x7fe4309dca00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#50 operator() (__closure=0x7fe4309dca00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#51 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
#52 0x00007fe432434d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
#53 0x00007fe434258ea5 in start_thread (arg=0x7fe3a37fe700) at pthread_create.c:307
#54 0x00007fe431b7d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81
#55 0x0000000000000000 in ?? ()

Thread 55 (LWP 22670):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00007fe43242f8bc in __gthread_cond_wait (__mutex=<optimized out>, __cond=__cond@entry=0x7fe375b05688) at /tmp/deploy/objdir/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2  std::condition_variable::wait (this=this@entry=0x7fe375b05688, __lock=...) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/condition_variable.cc:53
#3  0x000000000092b1af in magma::KVStore::acquireFlushBarrier (this=this@entry=0x7fe375b05410) at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:1485
#4  0x00000000009315a0 in magma::KVStore::flushMemTables(magma::WAL*, magma::WALOffset, magma::FlushMode, magma::BlockingMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:356
#5  0x0000000000931f48 in magma::KVStore::FlushMemTables (this=<optimized out>, wal=<optimized out>, flushMode=<optimized out>, blockMode=<optimized out>)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/kvstore.cc:336
#6  0x00000000008e023a in magma::Magma::Impl::tryWriteCacheFlush(magma::BlockingMode, magma::FlushMode) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db.cc:1106
#7  0x00000000008ff3ee in magma::Magma::Impl::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/db_write.cc:50
#8  0x00000000008ff542 in magma::Magma::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#9  0x0000000000856fcf in MagmaMemoryTrackingProxy::WriteDocs(unsigned short, std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> > const&, unsigned int, std::function<void (magma::Magma::WriteOperation const&, bool, magma::Slice)>, std::function<magma::Status (std::vector<magma::Magma::WriteOperation, std::allocator<magma::Magma::WriteOperation> >&)>) () at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
#10 0x0000000000840d9c in MagmaKVStore::saveDocs(MagmaKVStoreTransactionContext&, VB::Commit&, kvstats_ctx&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/include/memcached/vbucket.h:62
#11 0x000000000083a5ef in MagmaKVStore::commit(std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:651
#12 0x00000000007e8045 in EPBucket::commit(KVStoreIface&, std::unique_ptr<TransactionContext, std::default_delete<TransactionContext> >, VB::Commit&) ()
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:929
#13 0x00000000007eefd5 in EPBucket::flushVBucket_UNLOCKED(LockedVBucketPtr) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:801
#14 0x00000000007ef59f in EPBucket::flushVBucket(Vbid) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:378
#15 0x00000000006c80b0 in Flusher::flushVB (this=0x7fe3900bcb00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:285
#16 0x00000000006c8a00 in Flusher::step(GlobalTask*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/flusher.cc:200
#17 0x0000000000a1cf32 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
#18 0x0000000000a1a055 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fe3effed840)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
#19 0x0000000000b649a0 in operator() (this=0x7fe3effed840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
#20 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=0x7fe4309e4400, thread=..., 
    task=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached.debug, CU 0x65a24fb, DIE 0x65e7bd8>)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
#21 0x0000000000b4c75a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fe4309e4400, thread=...)
    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
#22 0x0000000000b67959 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (
    __t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
#23 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>)
    at /usr/local/include/c++/7.3.0/bits/invoke.h:95
#24 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
#25 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
#26 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
#27 0x0000000000a19ce4 in operator() (this=0x7fe4304d5800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#28 operator() (__closure=0x7fe4304d5800) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
#29 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
#30 0x00007fe432434d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
#31 0x00007fe434258ea5 in start_thread (arg=0x7fe3effff700) at pthread_create.c:307
#32 0x00007fe431b7d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81
#33 0x0000000000000000 in ?? ()

Thread 54 (LWP 18679):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00007fe43375de51 in background_thread_sleep (tsdn=<optimized out>, interval=10000000000, info=<optimized out>) at src/background_thread.c:255
#2  background_work_sleep_once (ind=<optimized out>, info=<optimized out>, tsdn=<optimized out>) at src/background_thread.c:307
#3  background_work (ind=<optimized out>, tsd=0x7fe3853ee220) at src/background_thread.c:497
#4  background_thread_entry (ind_arg=<optimized out>) at src/background_thread.c:522
#5  0x00007fe434258ea5 in start_thread (arg=0x7fe3853ff700) at pthread_create.c:307
#6  0x00007fe431b7d8dd in ioperm () at ../sysdeps/unix/syscall-template.S:81
#7  0x0000000000000000 in ?? ()

Thread 53 (LWP 18676):
#0  0x00007fe43425ffc3 in pread64 () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000948888 in magma::BasicFile::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_linux.cc:145
#2  0x00000000009a793a in magma::FileWithStats::Read(long, unsigned long, char*, unsigned long*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/file/file_impl_stats.cc:39
#3  0x000000000094ca26 in magma::BufferedFileReader::Read(magma::File*, long, unsigned long, magma::Slice&, unsigned long&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/rw_file.cc:273
#4  0x0000000000957c44 in magma::ReadAndTransform(magma::File*, long, unsigned long, std::function<magma::Status (char const*, unsigned long, std::unique_ptr<char [], std::default_delete<char []> >&, unsigned long&)>, std::unique_ptr<char [], std::default_delete<char []> >&, unsigned long&, unsigned long&, magma::BufferedFileReader*) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/util/block_cache.cc:39
No core file now.