KV-Engine crashes if AccessScanner task fails to write access.log due to disk being full

(gdb) bt full

#0  0x00007fed1dd40337 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007fed1dd41a28 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00007fed1e68b63c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95

        terminating = false

        t = <optimized out>

#3  0x0000000000b48a1b in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88

No locals.

#4  0x00007fed1e6968f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48

No locals.

#5  0x00007fed1e6959d9 in __cxa_call_terminate (ue_header=ue_header@entry=0x7fece4000f50) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_call.cc:54

        xh = 0x7fece4000f00

#6  0x00007fed1e696311 in __gxx_personality_v0 () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_personality.cc:685

        found_type = <optimized out>

        info = {Start = 140655783217168, LPStart = 140656388238664, ttype_base = 0, TType = 0x7fecf9fe87b8 "\227\026\344", action_table = 0x0, ttype_encoding = 151 '\227',

          call_site_encoding = 22 '\026'}

        language_specific_data = <optimized out>

        action_record = <optimized out>

        p = <optimized out>

        landing_pad = <optimized out>

        ip = <optimized out>

        handler_switch_value = <optimized out>

        thrown_ptr = 0x0

        foreign_exception = <optimized out>

        ip_before_insn = 0

        xh = 0x7fece4000f00

#7  0x00007fed1e0e7ebf in _Unwind_RaiseException_Phase2 () at /tmp/deploy/objdir/../gcc-10.2.0/libgcc/unwind.inc:64

        fs = {regs = {reg = {{loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0,

                  exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551584, offset = -32, exp = 0xffffffffffffffe0 <Address 0xffffffffffffffe0 out of bounds>},

                how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {

                  reg = 18446744073709551592, offset = -24, exp = 0xffffffffffffffe8 <Address 0xffffffffffffffe8 out of bounds>}, how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0,

                  exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {

                  reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551600, offset = -16,

                  exp = 0xfffffffffffffff0 <Address 0xfffffffffffffff0 out of bounds>}, how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {

                  reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}, {loc = {reg = 18446744073709551608, offset = -8,

                  exp = 0xfffffffffffffff8 <Address 0xfffffffffffffff8 out of bounds>}, how = REG_SAVED_OFFSET}, {loc = {reg = 0, offset = 0, exp = 0x0}, how = REG_UNSAVED}},

            prev = 0x0, cfa_offset = 48, cfa_reg = 7, cfa_exp = 0x0, cfa_how = CFA_REG_OFFSET}, pc = 0x8555b2 <MutationLog::~MutationLog()+226>,

          personality = 0x444220 <__gxx_personality_v0@plt>, data_align = -8, code_align = 1, retaddr_column = 16, fde_encoding = 27 '\033', lsda_encoding = 27 '\033',

          saw_z = 1 '\001', signal_frame = 0 '\000', eh_ptr = 0x0}

        match_handler = 4

        code = _URC_NO_REASON

        frames = 3

#8  0x00007fed1e0e8866 in _Unwind_Resume () at /tmp/deploy/objdir/../gcc-10.2.0/libgcc/unwind.inc:241

        this_context = {reg = {0x7fecf9fe8bc8, 0x7fecf9fe8bd0, 0x0, 0x7fecf9fe8bd8, 0x0, 0x0, 0x7fecf9fe8c00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7fecf9fe8be0, 0x7fecf9fe8be8, 0x7fecf9fe8bf0,

            0x7fecf9fe8bf8, 0x7fecf9fe8c08, 0x0}, cfa = 0x7fecf9fe8c10, ra = 0x4d3cae <MutationLog::updateInitialBlock()+232>, lsda = 0x0, bases = {tbase = 0x0, dbase = 0x0,

            func = 0x7fed1e0e8740 <_Unwind_Resume>}, flags = 4611686018427387904, version = 0, args_size = 0, by_value = '\000' <repeats 17 times>}

        cur_context = {reg = {0x7fecf9fe8bc8, 0x7fecf9fe8bd0, 0x0, 0x7fecf9fe8c70, 0x0, 0x0, 0x7fecf9fe8c78, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7fecf9fe8c80, 0x7fecf9fe8c60, 0x7fecf9fe8bf0,

---Type <return> to continue, or q <return> to quit---

            0x7fecf9fe8bf8, 0x7fecf9fe8c88, 0x0}, cfa = 0x7fecf9fe8c90, ra = 0x85551f <MutationLog::~MutationLog()+79>, lsda = 0xe41697, bases = {tbase = 0x0, dbase = 0x0,

            func = 0x8554d0 <MutationLog::~MutationLog()>}, flags = 4611686018427387904, version = 0, args_size = 0, by_value = '\000' <repeats 17 times>}

        code = <optimized out>

        frames = 0

#9  0x00000000004d3cae in deallocate (__t=<optimized out>, __p=<optimized out>, this=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/mutation_log.cc:442

No locals.

#10 deallocate (__n=<optimized out>, __p=<optimized out>, __a=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/alloc_traits.h:492

No locals.

#11 _M_deallocate (__n=<optimized out>, __p=<optimized out>, this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:354

No locals.

#12 ~_Vector_base (this=<optimized out>, __in_chrg=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:335

No locals.

#13 ~vector (this=<optimized out>, __in_chrg=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/stl_vector.h:683

No locals.

#14 MutationLog::updateInitialBlock (this=this@entry=0x7fec657dece0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/mutation_log.cc:437

        byteswritten = <optimized out>

#15 0x0000000000855431 in MutationLog::close() [clone .part.0] () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/mutation_log.cc:590

No locals.

#16 0x000000000085551f in close (this=0x7fec657dece0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/mutation_log.h:169

No locals.

#17 MutationLog::~MutationLog (this=0x7fec657dece0, __in_chrg=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/mutation_log.cc:283

        doLog = <optimized out>

#18 0x00000000007d07d3 in operator() (__ptr=0x7fec657dece0, this=0x7feca1056c20) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322

No locals.

#19 ~unique_ptr (this=0x7feca1056c20, __in_chrg=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:361

        __ptr = @0x7feca1056c20: <error reading variable>

#20 ItemAccessVisitor::ItemAccessVisitor (this=0x7feca1056b00, _store=..., conf=..., _stats=..., sh=<optimized out>, sfin=..., aS=..., items_to_scan=<optimized out>)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:48

No locals.

#21 0x00000000007cd37a in make_unique<ItemAccessVisitor, KVBucket&, Configuration&, EPStats&, unsigned long const&, std::atomic<bool>&, AccessScanner&, unsigned long&> ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:368

No locals.

#22 AccessScanner::createAndScheduleTask(unsigned long) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:369

        maxExpectedDuration = <optimized out>

#23 0x00000000007cdc7b in AccessScanner::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/access_scanner.cc:325

        i = 0

        deleteAccessLogFiles = <optimized out>

        aggregator = {<VBucketVisitor> = {_vptr.VBucketVisitor = 0x1079f20 <vtable for VBucketStatAggregator+16>, vBucketFilter = {acceptable = {_M_t = {

                  _M_impl = {<std::allocator<std::_Rb_tree_node<Vbid> >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<Vbid> >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<Vbid> >> = {_M_key_compare = {<std::binary_function<Vbid, Vbid, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {

                        _M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7fecf9fe8f90, _M_right = 0x7fecf9fe8f90}, _M_node_count = 0}, <No data fields>}}}}}, visitorMap = {_M_t = {

              _M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<vbucket_state_t const, std::vector<VBucketStatVisitor*, std::allocator<VBucketStatVisitor*> > > > >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<vbucket_state_t const, std::vector<VBucketStatVisitor*, std::allocator<VBucketStatVisitor*> > > > >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<vbucket_state_t> >> = {

                  _M_key_compare = {<std::binary_function<vbucket_state_t, vbucket_state_t, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {

                    _M_color = std::_S_red, _M_parent = 0x7fecf8583a00, _M_left = 0x7fecf8583a00, _M_right = 0x7fecf85d8980}, _M_node_count = 2}, <No data fields>}}}}

        activeCountVisitor = {<VBucketStatVisitor> = {<VBucketVisitor> = {_vptr.VBucketVisitor = 0x1079ec0 <vtable for VBucketCountVisitor+16>, vBucketFilter = {acceptable = {_M_t = {

---Type <return> to continue, or q <return> to quit---

                    _M_impl = {<std::allocator<std::_Rb_tree_node<Vbid> >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<Vbid> >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<Vbid> >> = {_M_key_compare = {<std::binary_function<Vbid, Vbid, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {

                          _M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7fecf9fe9000, _M_right = 0x7fecf9fe9000}, _M_node_count = 0}, <No data fields>}}}}},

            desired_state = vbucket_state_active}, numItems = 1983959, numTempItems = 0, nonResident = 1891524, numVbucket = 341, htMemory = 1587984, htItemMemory = 147066524,

          htUncompressedItemMemory = 147066524, htCacheSize = 147066524, numEjects = 1987541, numExpiredItems = 0, metaDataMemory = 7489674, metaDataDisk = 111245477,

          checkpointMemory = 161027069, checkpointMemoryQueue = 155443406, checkpointMemoryUnreferenced = 0, checkpointMemOverheadAllocatorBytes = 14924975,

          checkpointMemOverheadAllocatorBytesQueue = 2450568, checkpointMemOverheadAllocatorBytesIndex = 7963232, checkpointMemOverheadAllocatorBytesIndexKey = 3032823,

          checkpointMemOverhead = 7062015, checkpointMemOverheadQueue = 2450568, checkpointMemOverheadIndex = 3133095, checkpointMemFreedByItemExpel = 148977555,

          checkpointMemFreedByRemoval = 3331082990, bloomFilterMemory = 0, opsCreate = 1983959, opsDelete = 0, opsGet = 0, opsReject = 89951238, opsUpdate = 0, queueSize = 92435,

          queueMemory = 11092200, queueFill = 2086624, queueDrain = 1994189, pendingWrites = 153721844, chkPersistRemaining = 0, queueAge = 7303904767419, rollbackItemCount = 0,

          numHpVBReqs = 0, totalAbsHLCDrift = {total = 0, updates = 0}, totalHLCDriftExceptionCounters = {ahead = 0, behind = 0}, syncWriteAcceptedCount = 0,

          syncWriteCommittedCount = 0, syncWriteAbortedCount = 0, maxHistoryDiskSize = 0}

        replicaCountVisitor = {<VBucketStatVisitor> = {<VBucketVisitor> = {_vptr.VBucketVisitor = 0x1079ec0 <vtable for VBucketCountVisitor+16>, vBucketFilter = {acceptable = {_M_t = {

                    _M_impl = {<std::allocator<std::_Rb_tree_node<Vbid> >> = {<__gnu_cxx::new_allocator<std::_Rb_tree_node<Vbid> >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<Vbid> >> = {_M_key_compare = {<std::binary_function<Vbid, Vbid, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {

                          _M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7fecf9fe91b0, _M_right = 0x7fecf9fe91b0}, _M_node_count = 0}, <No data fields>}}}}},

            desired_state = vbucket_state_replica}, numItems = 1982174, numTempItems = 0, nonResident = 1889753, numVbucket = 341, htMemory = 1253104, htItemMemory = 147044090,

          htUncompressedItemMemory = 147044090, htCacheSize = 147044090, numEjects = 1982174, numExpiredItems = 0, metaDataMemory = 7488380, metaDataDisk = 111145205,

          checkpointMemory = 161230684, checkpointMemoryQueue = 155557541, checkpointMemoryUnreferenced = 0, checkpointMemOverheadAllocatorBytes = 15276865,

          checkpointMemOverheadAllocatorBytesQueue = 2463216, checkpointMemOverheadAllocatorBytesIndex = 8152192, checkpointMemOverheadAllocatorBytesIndexKey = 3107409,

          checkpointMemOverhead = 7227191, checkpointMemOverheadQueue = 2463216, checkpointMemOverheadIndex = 3209927, checkpointMemFreedByItemExpel = 97866245,

          checkpointMemFreedByRemoval = 3376880159, bloomFilterMemory = 0, opsCreate = 1982174, opsDelete = 0, opsGet = 0, opsReject = 89998630, opsUpdate = 0, queueSize = 92421,

          queueMemory = 11090520, queueFill = 2085166, queueDrain = 1992745, pendingWrites = 153698402, chkPersistRemaining = 0, queueAge = 7302893244678, rollbackItemCount = 0,

          numHpVBReqs = 0, totalAbsHLCDrift = {total = 179386964010, updates = 2074595}, totalHLCDriftExceptionCounters = {ahead = 0, behind = 56}, syncWriteAcceptedCount = 0,

          syncWriteCommittedCount = 0, syncWriteAbortedCount = 0, maxHistoryDiskSize = 0}

        phosphor_internal_category_enabled_282 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_282 = <optimized out>

        phosphor_internal_tpi_282 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {

              phosphor::is_bool, phosphor::is_bool}}}

        phosphor_internal_guard_282 = {tpi = 0x107a920 <AccessScanner::run()::phosphor_internal_tpi_282>, enabled = true, arg1 = {<No data fields>}, arg2 = {<No data fields>},

          start = {__d = {__r = 7500518794445971}}}

        inverse = true

#24 0x0000000000ab7079 in GlobalTask::execute(std::basic_string_view<char, std::char_traits<char> >) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98

        guard = {previous = 0x0}

        start = <optimized out>

        runAgain = <optimized out>

#25 0x0000000000ab073a in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fecf9fe9650)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309

        runAgain = <optimized out>

        proxy = @0x7fed1ca29cf0: <error reading variable>

#26 0x0000000000ab842e in operator() (this=0x7fecf9fe9650) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        fn = @0x7fecf9fe9650: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fed1ca29cf0, tiny = {

              __data = "\360\234\242\034\355\177\000\000@\227\376\371\354\177\000\000\000\000\000\000\000\000\000\000@\233\246\034\355\177\000\000@\231\376\371\354\177\000\000\305\003\331\035\355\177\000", __align = {<No data fields>}}},

          call_ = 0xab0c70 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xaaf160 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, fol---Type <return> to continue, or q <return> to quit---

ly::detail::function::Data)>}

#27 CancellableCPUExecutor::add(GlobalTask*, folly::Function<void ()>)::{lambda()#1}::operator()() const ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42

        task = {storage_ = {{emptyState = -48 '\320', value = {task = 0x7fed102309d0, func = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {

                    big = 0x7fed1ca29cf0, tiny = {

                      __data = "\360\234\242\034\355\177\000\000@\227\376\371\354\177\000\000\000\000\000\000\000\000\000\000@\233\246\034\355\177\000\000@\231\376\371\354\177\000\000\305\003\331\035\355\177\000", __align = {<No data fields>}}},

                  call_ = 0xab0c70 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

                  exec_ = 0xaaf160 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}}}, hasValue = true}}

        this = <optimized out>

#28 0x0000000000c16540 in operator() (this=0x7fecf9fe9840) at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

        fn = @0x7fecf9fe9840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fed1ca57c00, tiny = {

              __data = "\000|\245\034\355\177\000\000\320\347\257 \355\177\000\000\060\000\000\000\000\000\000\000\231\027\000\000\000\000\000\000H\000\000\000\000\000\000\000\020\231\376\371\354\177\000", __align = {<No data fields>}}},

          call_ = 0xab87b0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

          exec_ = 0xab7d60 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}

#29 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7fed1ca57d00, thread=...,

    task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.2.0-5235.x86_64.debug, CU 0xa927aa4, DIE 0xa9ab9d9>)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 7500518792538456}}

        stats = {expired = false, waitTime = {__r = 18498122}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 7500518774040334}}, requestId = 0}

#30 0x0000000000c00fda in folly::CPUThreadPoolExecutor::threadRun (this=0x7fed1ca57d00, thread=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>},

                    data_ = {big = 0x7fed1ca57c00, tiny = {

                        __data = "\000|\245\034\355\177\000\000\320\347\257 \355\177\000\000\060\000\000\000\000\000\000\000\231\027\000\000\000\000\000\000H\000\000\000\000\000\000\000\020\231\376\371\354\177\000", __align = {<No data fields>}}},

                    call_ = 0xab87b0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,

                    exec_ = 0xab7d60 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}, enqueueTime_ = {__d = {__r = 7500518774040334}}, expiration_ = {__r = 0},

                  expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x1799, tiny = {

                        __data = "\231\027\000\000\000\000\000\000Kt\246", '\000' <repeats 13 times>, "_.\217 \355\177\000\000\240\231\376\371\354\177\000\000@\306\257 \355\177\000",

                        __align = {<No data fields>}}}

, call_ = 0x4675df <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000', queueObserverPayload_ = 140656364369664}},

            hasValue = true}}

        guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcdbdd3 "CPUThreadPoolExecutor", e_ = 0xcdbde8 ""}}}}

#31 0x0000000000c194f9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73

No locals.

---Type <return> to continue, or q <return> to quit---

#32 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&>

    (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95

No locals.

#33 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

No locals.

#34 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

No locals.

#35 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

        fn = <optimized out>

#36 0x0000000000ab0434 in operator() (this=0x7fed1cac7d80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        fn = @0x7fed1cac7d80: <error reading variable>

#37 operator() (__closure=0x7fed1cac7d80) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49

        threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fecf9fe99a0 "AuxIoPool0"},

                _M_string_length = 10, {_M_local_buf = "AuxIoPool0\000\000\000\000\000", _M_allocated_capacity = 8029725099529106753}}}, hasValue = true}}

        func = <error reading variable func (Cannot access memory at address 0x7fed1cac7d80)>

#38 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

        fn = @0x7fed1cac7d80: <error reading variable>

#39 0x00007fed1e6bfd40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80

No locals.

#40 0x00007fed204c1e65 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#41 0x00007fed1de0888d in clone () from /lib64/libc.so.6

No symbol table info available.

(gdb)

(gdb)

Subcomponent: analytics_5_percent_dgm_magma_failover

Job: analytics_5_percent_dgm_magma_failover

conf : /TAF/conf/cbas/magma/py-cbas-collection-failover.conf

test_cbas_with_kv_graceful_failover_rebalance_out,data_load_stage=during,run_kv_queries=True,run_cbas_queries=True,no_of_dv=2,ds_per_dv=2,no_of_synonyms=2,no_of_indexes=1,no_of_threads=20,GROUP=sanity,doc_size=1536,randomize_value=True,parallel_load_percent=10