[NexusKVStore] Abort due to differing expiry callbacks during compaction

(gdb) bt full

#0  0x00007f5690ed7337 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007f5690ed8a28 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x0000000000497396 in cb::handleError (logger=..., logLevel=<optimized out>, msg=..., method=cb::Abort)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:41

No locals.

#3  0x00000000008f6bfc in NexusKVStore::compactDB(std::unique_lock<std::mutex>&, std::shared_ptr<CompactionContext>) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:1085

        ss = <incomplete type>

        msg = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7f560150b9e0 <Address 0x7f560150b9e0 out of bounds>}, _M_string_length = 111, {

            _M_local_buf = "o\000\000\000\000\000\000\000\060 ~\002V\177\000", _M_allocated_capacity = 111}}

        primaryVbPtr = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}

        vbid = {vbid = 799}

        lh = {_M_device = 0x7f56743c6f78, _M_owns = true}

        dummyLock = {<std::__mutex_base> = {_M_mutex = {__data = {__lock = 1, __count = 0, __owner = 39580, __nusers = 1, __kind = 0, __spins = 0, __elision = 0,

                __list = {__prev = 0x0, __next = 0x0}}, __size = "\001\000\000\000\000\000\000\000\234\232\000\000\001", '\000' <repeats 26 times>,

              __align = 1}}, <No data fields>}

        dummyLh = {_M_device = 0x7f56027e1f80, _M_owns = true}

        secondaryCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f55f17bb200}}, <No data fields>}

        secondaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f55b0e09dc0}}, <No data fields>}

        primaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f56341cc880}}, <No data fields>}

        primaryDrops = {

          _M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<Disk---Type <return> to continue, or q <return> to quit---

DocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f55c3d4b000, _M_bucket_count = 1109, _M_before_begin = {

              _M_nxt = 0x7f55b77c67c0}, _M_element_count = 614, _M_rehash_policy = {static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 1109},

            _M_single_bucket = 0x0}}

        secondaryDrops = {

          _M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f55d09ac800, _M_bucket_count = 1109, _M_before_begin = {

              _M_nxt = 0x7f55997ba480}, _M_element_count = 614, _M_rehash_policy = {static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 1109},

            _M_single_bucket = 0x0}}

        originalDroppedKeyCb = {<std::_Maybe_unary_or_binary_function<void, DiskDocKey const&, long, bool, long>> = {<No data fields>}, <std::_Function_base> = {

            static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x7f56842aa000, _M_const_object = 0x7f56842aa000,

                _M_function_pointer = 0x7f56842aa000,

                _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f56842aa000, this adjustment 140008220507024},

              _M_pod_data = "\000\240*\204V\177\000\000\220\273?4V\177\000"}

, _M_manager = 0x817e50

     <std::_Function_handler<void(const DiskDocKey&, long int, bool, long int), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, uint64_t)::<lambda(const DiskDocKey&, int64_t, bool, int64_t)> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},

---Type <return> to continue, or q <return> to quit---

          _M_invoker = 0x81dda0 <std::_Function_handler<void (DiskDocKey const&, long, bool, long), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, unsigned long)::{lambda(DiskDocKey const&, long, bool, long)#1}>::_M_invoke(std::_Any_data const&, DiskDocKey const&, long&&, bool&&, std::_Any_data const&)>}

        nexusCompactionContext = {kvStoreToCompactFirst = 0x7f56842b8480, kvStoreToCompactSecond = 0x7f5674215600,

          firstCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f55f17bb210, _M_refcount = {_M_pi = 0x7f55f17bb200}}, <No data fields>},

          secondCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f56343fbb90, _M_refcount = {_M_pi = 0x7f56343fbb80}}, <No data fields>}, attemptToPruneStaleCallbacks = true}

        firstResult = true

        secondResult = true

        secondaryItr = <optimized out>

#4  0x000000000081c0f9 in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2662

        ctx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f56343fbb90, _M_refcount = {_M_pi = 0x7f56343fbb80}}, <No data fields>}

        shard = <optimized out>

        store = <optimized out>

        result = false

#5  0x000000000081e041 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector<CookieIface const*, std::allocator<CookieIface const*> >&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1364

        err = cb::success

        vb = {

          vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f560c7dca00, _M_refcount = {_M_pi = 0x7f5601b1dce0}}, <No data fields>}, lock = {_M_device = 0x7f563455dcd8, _M_owns = true}}

        success = false

#6  0x000000000071bd76 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:73

        phosphor_internal_category_enabled_54 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_54 = <optimized out>

        phosphor_internal_tpi_54 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {

              phosphor::is_bool, phosphor::is_bool}}}

        phosphor_internal_guard_54 = {tpi = 0x1056da0 <CompactTask::run()::phosphor_internal_tpi_54>, enabled = true, arg1 = 799, arg2 = {<No data fields>}, start = {

            __d = {__r = 3444745905060369}}}

        compactionData = {<std::__pair_base<CompactionConfig, std::vector<CookieIface const*, std::allocator<CookieIface const*> > >> = {<No data fields>}, first = {

            purge_before_ts = 1637817303, purge_before_seq = 0, drop_deletes = false, retain_erroneous_tombstones = true},

          second = {<std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >> = {

              _M_impl = {<std::allocator<CookieIface const*>> = {<__gnu_cxx::new_allocator<CookieIface const*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >::_Vector_impl_data> = {_M_start = 0x0, _M_finish = 0x0,

                  _M_end_of_storage = 0x0}, <No data fields>}}, <No data fields>}}

        reschedule = <optimized out>

#7  0x0000000000a9d092 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68

        guard = {previous = 0x0}

#8  0x0000000000a9a2a5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f56027e2840)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189

        scheduleOverhead = <optimized out>

---Type <return> to continue, or q <return> to quit---

        start = {__d = {__r = 3444745905058911}}

        runAgain = false

        proxy = @0x7f5684404600: <error reading variable>

#9  0x0000000000beee50 in operator() (this=0x7f56027e2840)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

        fn = @0x7f56027e2840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f5684404600, tiny = {

              __data = "\000F@\204V\177\000\000\000\000\000\000\000\000\000\000\220\201\333\217V\177\000\000\234\232\000\000\000\000\000\000\220\001\000\000\000\000\000\000\020)~\002V\177\000", __align = {<No data fields>}}},

          call_ = 0xa9a960 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xa990b0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}

#10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f568f92b400,

    thread=..., task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1786.x86_64.debug, CU 0xa0834c9, DIE 0xa1073fe>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}

        startTime = {__d = {__r = 3444745905055104}}

        stats = {expired = false, waitTime = {__r = 9838815302}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 3444736066239802}}, requestId = 0}

#11 0x0000000000bd6c0a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f568f92b400, thread=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265

        task = {storage_ = {{emptyState = 0 '\000', value = {<folly::ThreadPoolExecutor::Task> = {

                  func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f5684404600, tiny = {

                        __data = "\000F@\204V\177\000\000\000\000\000\000\000\000\000\000\220\201\333\217V\177\000\000\234\232\000\000\000\000\000\000\220\001\000\000\000\000\000\000\020)~\002V\177\000", __align = {<No data fields>}}},

                    call_ = 0xa9a960 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

                    exec_ = 0xa990b0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 3444736066239802}}, expiration_ = {__r = 0},

                  expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x9a9c, tiny = {

                        __data = "\234\232\000\000\000\000\000\000+\232\242", '\000' <repeats 13 times>, "_\376\250\223V\177\000\000pj\274yV\177\000\000@\226\311\223V\177\000", __align = {<No data fields>}}}, call_ = 0x46512d <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>,

                    exec_ = 0x0},

                  context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000',

                queueObserverPayload_ = 140009773111552}}, hasValue = true}}

        guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcbdf2b "CPUThreadPoolExecutor", e_ = 0xcbdf40 ""}}}}

#12 0x0000000000bf1e09 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73

No locals.

#13 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95

---Type <return> to continue, or q <return> to quit---

No locals.

#14 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467

No locals.

#15 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551

No locals.

#16 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401

        fn = <optimized out>

#17 0x0000000000a99f34 in operator() (this=0x7f568fddb380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47

        fn = @0x7f568fddb380: <error reading variable>

#18 operator() (__closure=0x7f568fddb380) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47

        threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615,

                _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

                  _M_p = 0x7f56027e29a0 "WriterPool7"}, _M_string_length = 11, {_M_local_buf = "WriterPool7\000\000\000\000",

                  _M_allocated_capacity = 8021036716417184343}}}, hasValue = true}}

        func = <error reading variable func (Cannot access memory at address 0x7f568fddb380)>

#19 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401

        fn = @0x7f568fddb380: <error reading variable>

#20 0x00007f5691856d40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6

No symbol table info available.

#21 0x00007f569365ee65 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#22 0x00007f5690f9f88d in clone () from /lib64/libc.so.6

No symbol table info available.

git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/qe_r.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_rebalance_in,num_items=5000000,doc_size=256,nodes_init=2,nodes_in=2,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=12,replicas=0,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=50,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=11,num_scopes=1,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=50000,doc_ops=create,create_perc=100,delete_perc=0,update_perc=0,num_collections_to_drop=0 -m rest'