[Nexus] Memcached aborted in NexusKVStore::compactDB(std::unique_lock<std::mutex>&, std::shared_ptr<CompactionContext>)

(gdb) bt full

#0  0x00007f2fca6c0337 in raise () from /lib64/libc.so.6

No symbol table info available.

#1  0x00007f2fca6c1a28 in abort () from /lib64/libc.so.6

No symbol table info available.

#2  0x00000000006d43d2 in cb::handleError(spdlog::logger&, spdlog::level::level_enum, std::basic_string_view<char, std::char_traits<char> >, cb::ErrorHandlingMethod) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:43

No locals.

#3  0x00000000008f79be in NexusKVStore::compactDB(std::unique_lock<std::mutex>&, std::shared_ptr<CompactionContext>) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:1227

        msg = {static npos = 18446744073709551615,

          _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},

            _M_p = 0x7f2f282310a0 <Address 0x7f2f282310a0 out of bounds>}, _M_string_length = 83, {

            _M_local_buf = "S\000\000\000\000\000\000\000\070\067\064\066\065\000\000", _M_allocated_capacity = 83}}

        primaryVbPtr = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}

        vbid = {vbid = 618}

        nexusLock = {_M_device = 0x7f2fbc29c008, _M_owns = true}

        compactionLock = <optimized out>

        dummyLock = {<std::__mutex_base> = {_M_mutex = {__data = {__lock = 1, __count = 0, __owner = 113210, __nusers = 1, __kind = 0, __spins = 0,

                __elision = 0, __list = {__prev = 0x0, __next = 0x0}},

              __size = "\001\000\000\000\000\000\000\000:\272\001\000\001", '\000' <repeats 26 times>, __align = 1}}, <No data fields>}

        dummyLh = {_M_device = 0x7f2f3cfdec30, _M_owns = true}

        guard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__this = 0x7f2fbc38f2c0, __vbid = {vbid = 618}}}

        secondaryCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2f887be440}}, <No data fields>}

        secondaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2f1f68cf00}}, <No data fields>}

        primaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2ecdafec60}}, <No data fields>}

        primaryDrops = {

          _M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__det---Type <return> to continue, or q <return> to quit---

ail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f2f3cfdecf0,

            _M_bucket_count = 1, _M_before_begin = {_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2,

              _M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}

        secondaryDrops = {

          _M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<Disk---Type <return> to continue, or q <return> to quit---

DocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f2e943d3400,

            _M_bucket_count = 541, _M_before_begin = {_M_nxt = 0x7f2f199ecd40}, _M_element_count = 499, _M_rehash_policy = {

              static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 541}, _M_single_bucket = 0x0}}

        originalDroppedKeyCb = {<std::_Maybe_unary_or_binary_function<void, DiskDocKey const&, long, bool, long>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x7f2fbc2b1000, _M_const_object = 0x7f2fbc2b1000,

                _M_function_pointer = 0x7f2fbc2b1000,

                _M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f2fbc2b1000, this adjustment 139842129448016}, _M_pod_data = "\000\020+\274/\177\000\000PLs\210/\177\000"}, _M_manager = 0x81aff0

     <std::_Function_handler<void(const DiskDocKey&, long int, bool, long int), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, uint64_t)::<lambda(const DiskDocKey&, int64_t, bool, int64_t)> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},

          _M_invoker = 0x821290 <std::_Function_handler<void (DiskDocKey const&, long, bool, long), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, unsigned long)::{lambda(DiskDocKey const&, long, bool, long)#1}>::_M_invoke(std::_Any_data const&, DiskDocKey const&, long&&, bool&&, std::_Any_data const&)>}

        nexusCompactionContext = {kvStoreToCompactFirst = 0x7f2fbc2bd180, kvStoreToCompactSecond = 0x7f2fbc20e800,

          firstCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f887be450, _M_refcount = {_M_pi = 0x7f2f887be440}}, <No data fields>},

          secondCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f88734c50, _M_refcount = {_M_pi = 0x7f2f88734c40}}, <No data fields>},

          attemptToPruneStaleCallbacks = true}

        beforeFirstCompactionHighSeqno = 0

        firstResult = true

        beforeSecondCompactionHighSeqno = 0

        secondResult = false

        secondaryItr = <optimized out>

#4  0x000000000081f29f in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2664

        ctx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f88734c50, _M_refcount = {_M_pi = 0x7f2f88734c40}}, <No data fields>}

        shard = <optimized out>

        store = <optimized out>

        result = false

#5  0x0000000000821535 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector<CookieIface const*, std::allocator<CookieIface const*> >&) ()

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1402

        err = cb::success

        vb = {

---Type <return> to continue, or q <return> to quit---

          vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f29aa9400, _M_refcount = {_M_pi = 0x7f2ed4aa1700}}, <No data fields>}, lock = {_M_device = 0x7f2f88567090,

            _M_owns = true}}

        reschedule = false

#6  0x000000000071ead6 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:71

        phosphor_internal_category_enabled_52 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}

        phosphor_internal_category_enabled_temp_52 = <optimized out>

        phosphor_internal_tpi_52 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},

          argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}

        phosphor_internal_guard_52 = {tpi = 0x105df40 <CompactTask::run()::phosphor_internal_tpi_52>, enabled = true, arg1 = 618,

          arg2 = {<No data fields>}, start = {__d = {__r = 6389160129815625}}}

        compactionData = {<std::__pair_base<CompactionConfig, std::vector<CookieIface const*, std::allocator<CookieIface const*> > >> = {<No data fields>}, first = {purge_before_ts = 1639891526, purge_before_seq = 0, drop_deletes = false, retain_erroneous_tombstones = true,

            internally_requested = false}, second = {<std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >> = {

              _M_impl = {<std::allocator<CookieIface const*>> = {<__gnu_cxx::new_allocator<CookieIface const*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >::_Vector_impl_data> = {_M_start = 0x0, _M_finish = 0x0,

                  _M_end_of_storage = 0x0}, <No data fields>}}, <No data fields>}}

        reschedule = <optimized out>

#7  0x0000000000aa22f2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68

        guard = {previous = 0x0}

#8  0x0000000000a9f505 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f2f3cfdf840)

    at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189

        scheduleOverhead = <optimized out>

        start = {__d = {__r = 6389160129814228}}

        runAgain = false

        proxy = @0x7f2fbc40cc50: <error reading variable>

#9  0x0000000000bf4690 in operator() (this=0x7f2f3cfdf840)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416

        fn = @0x7f2f3cfdf840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f2fbc40cc50, tiny = {

              __data = "P\314@\274/\177\000\000\000\000\000\000\000\000\000\000\220{[\311/\177\000\000:\272\001\000\000\000\000\000\200\001\000\000\000\000\000\000\020\371\375</\177\000", __align = {<No data fields>}}},

          call_ = 0xa9fbc0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,

          exec_ = 0xa9e310 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}

#10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (

    this=this@entry=0x7f2fc912b400, thread=...,

    task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1965.x86_64.debug, CU 0xa1d7cd1, DIE 0xa25bc06>)

    at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.

git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD

guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankush_temp_job.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_swap_rebalance,num_items=500000,doc_size=256,nodes_init=2,nodes_swap=2,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=12,replicas=1,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=50,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=101,num_scopes=1,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=15000,doc_ops=create,create_perc=50,delete_perc=50,update_perc=0,num_collections_to_drop=0 -m rest'