[NexusKVStore] Rebalance exited with mover_crashed "wait_dcp_data_move infinity"

Rebalance exited with reason {mover_crashed,

{unexpected_exit,

{'EXIT',<0.28500.3>,

{socket_closed,

{gen_server,call,

[{'janitor_agent-default',

'ns_1@172.23.121.48'},

{if_rebalance,<0.15650.3>,

{wait_dcp_data_move,

['ns_1@172.23.121.140',

'ns_1@172.23.121.139'],

1004}},

infinity]}}}}}.

Rebalance Operation Id = 3fed6930c3e2aa22f99fa796cf405e1a

(gdb) bt full

#0  0x00007feac4467387 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007feac4468a78 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00000000006d2f22 in cb::handleError(spdlog::logger&, spdlog::level::level_enum, std::basic_string_view<char, std::char_traits<char> >, cb::ErrorHandlingMethod) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:43

No locals.

#3  0x00000000008f981c in NexusKVStore::compactDB(std::unique_lock<std::mutex>&, std::shared_ptr<CompactionContext>) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:1111

        ss = <incomplete type>

        msg = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7fea1006a240 <Address 0x7fea1006a240 out of bounds>}, _M_string_length = 173, {_M_local_buf = "\255", '\000' <repeats 14 times>,

            _M_allocated_capacity = 173}}

        primaryVbPtr = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}

        vbid = {vbid = 539}

        lh = {_M_device = 0x7feac017da78, _M_owns = true}

        dummyLock = {<std::__mutex_base> = {_M_mutex = {__data = {__lock = 1, __count = 0, __owner = 121360, __nusers = 1, __kind = 0, __spins = 0,

                __elision = 0, __list = {__prev = 0x0, __next = 0x0}},

              __size = "\001\000\000\000\000\000\000\000\020\332\001\000\001", '\000' <repeats 26 times>, __align = 1}}, <No data fields>}

        dummyLh = {_M_device = 0x7fea36fe2f80, _M_owns = true}

        secondaryCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7fea74402700}}, <No data fields>}

        secondaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7fea146b2300}}, <No data fields>}

        primaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7fea10e7e9c0}}, <No data fields>}

        primaryDrops = {

          _M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detai---Type <return> to continue, or q <return> to quit---

l::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7fea1295ec00,

            _M_bucket_count = 541, _M_before_begin = {_M_nxt = 0x7fea18089580}, _M_element_count = 266, _M_rehash_policy = {

              static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 541}, _M_single_bucket = 0x0}}

        secondaryDrops = {

          _M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<---Type <return> to continue, or q <return> to quit---

true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7fea74730800,

            _M_bucket_count = 541, _M_before_begin = {_M_nxt = 0x7fea1ef28d40}, _M_element_count = 266, _M_rehash_policy = {

              static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 541}, _M_single_bucket = 0x0}}

        originalDroppedKeyCb = {<std::_Maybe_unary_or_binary_function<void, DiskDocKey const&, long, bool, long>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x7feac01ab000, _M_const_object = 0x7feac01ab000,

                _M_function_pointer = 0x7feac01ab000,

                _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7feac01ab000, this adjustment 140642669571216}, _M_pod_data = "\000\260\032\300\352\177\000\000\220,\\\354\351\177\000"}, _M_manager = 0x819410

     <std::_Function_handler<void(const DiskDocKey&, long int, bool, long int), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, uint64_t)::<lambda(const DiskDocKey&, int64_t, bool, int64_t)> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},

          _M_invoker = 0x81f360 <std::_Function_handler<void (DiskDocKey const&, long, bool, long), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, unsigned long)::{lambda(DiskDocKey const&, long, bool, long)#1}>::_M_invoke(std::_Any_data const&, DiskDocKey const&, long&&, bool&&, std::_Any_data const&)>}

        nexusCompactionContext = {kvStoreToCompactFirst = 0x7feac01b7880, kvStoreToCompactSecond = 0x7feac0108e00,

          firstCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea74402710, _M_refcount = {_M_pi = 0x7fea74402700}}, <No data fields>},

          secondCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe9ec5c2c90, _M_refcount = {_M_pi = 0x7fe9ec5c2c80}}, <No data fields>},

          attemptToPruneStaleCallbacks = true}

        firstResult = true

        secondResult = true

        secondaryItr = <optimized out>

#4  0x000000000081d6b9 in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2660

        ctx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fe9ec5c2c90, _M_refcount = {_M_pi = 0x7fe9ec5c2c80}}, <No data fields>}

        shard = <optimized out>

        store = <optimized out>

        result = false

#5  0x000000000081f601 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector<CookieIface const*, std::allocator<CookieIface const*> >&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1376

        err = cb::success

        vb = {

          vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7fea742fd100, _M_refcount = {_M_pi = 0x7fea27ec32c0}}, <No data fields>}, lock = {_M_device = 0x7fea94163438,

            _M_owns = true}}

---Type <return> to continue, or q <return> to quit---

        success = false

#6  0x000000000071cfc6 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:73

        phosphor_internal_category_enabled_54 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_54 = <optimized out>

        phosphor_internal_tpi_54 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},

          argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}

        phosphor_internal_guard_54 = {tpi = 0x105adc0 <CompactTask::run()::phosphor_internal_tpi_54>, enabled = true, arg1 = 539,

          arg2 = {<No data fields>}, start = {__d = {__r = 6653998839455354}}}

        compactionData = {<std::__pair_base<CompactionConfig, std::vector<CookieIface const*, std::allocator<CookieIface const*> > >> = {<No data fields>}, first = {purge_before_ts = 1638598756, purge_before_seq = 0, drop_deletes = false, retain_erroneous_tombstones = true},

          second = {<std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >> = {

              _M_impl = {<std::allocator<CookieIface const*>> = {<__gnu_cxx::new_allocator<CookieIface const*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >::_Vector_impl_data> = {_M_start = 0x0, _M_finish = 0x0,

                  _M_end_of_storage = 0x0}, <No data fields>}}, <No data fields>}}

        reschedule = <optimized out>

#7  0x0000000000aa0e62 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68

        guard = {previous = 0x0}

#8  0x0000000000a9e075 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fea36fe3840)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189

        scheduleOverhead = <optimized out>

        start = {__d = {__r = 6653998839453742}}

        runAgain = false

        proxy = @0x7feac3161e40: <error reading variable>

#9  0x0000000000bf2c10 in operator() (this=0x7fea36fe3840)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

        fn = @0x7fea36fe3840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7feac3161e40, tiny = {

              __data = "@\036\026\303\352\177\000\000\000\000\000\000\000\000\000\000P\202;\303\352\177\000\000\020\332\001\000\000\000\000\000\300\001\000\000\000\000\000\000\020\071\376\066\352\177\000", __align = {<No data fields>}}},

          call_ = 0xa9e730 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xa9ce80 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}

#10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (

    this=this@entry=0x7feac2f2b400, thread=...,

    task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1851.x86_64.debug, CU 0xa12f680, DIE 0xa1b35b5>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97

        rctx = {

          prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cx---Type <return> to continue, or q <return> to quit---

git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/qe_r.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_rebalance_in,num_items=5000000,doc_size=256,nodes_init=2,nodes_in=1,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=before,sdk_timeout=60,vbuckets=100,key_size=12,replicas=0,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=30,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=3,num_scopes=5,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=12000,doc_ops=create,create_perc=100,delete_perc=0,update_perc=0,num_collections_to_drop=0,get-cbcollect-info=True -m rest'