Details
-
Bug
-
Resolution: Not a Bug
-
Major
-
None
-
7.1.0
-
7.1.0-1965
-
Untriaged
-
Centos 64-bit
-
-
1
-
Unknown
Description
Steps to repro:
- Create a 2 node cluster(172.23.122.245, 172.23.122.246(bucket_ram_quota = 2056 MB/node), replicas=1)
- Create 100 non default collections in default scope(collection count is 101(including default collection)
- Start Loading 500k docs in each of 100 non default collections
- while doc loading rebalance out one node 172.23.122.246
- Trigger full compaction
- Drop Few collections/ Recreate few collections
- Observed Memcached aborted in NexusKVStore::compactDB(std::unique_lock<std::mutex>&, std::shared_ptr<CompactionContext>)
Below Core found on node : 172.23.122.245
BackTrace:
(gdb) bt full
|
#0 0x00007f2fca6c0337 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007f2fca6c1a28 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00000000006d43d2 in cb::handleError(spdlog::logger&, spdlog::level::level_enum, std::basic_string_view<char, std::char_traits<char> >, cb::ErrorHandlingMethod) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:43
|
No locals.
|
#3 0x00000000008f79be in NexusKVStore::compactDB(std::unique_lock<std::mutex>&, std::shared_ptr<CompactionContext>) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:1227
|
msg = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f2f282310a0 <Address 0x7f2f282310a0 out of bounds>}, _M_string_length = 83, {
|
_M_local_buf = "S\000\000\000\000\000\000\000\070\067\064\066\065\000\000", _M_allocated_capacity = 83}}
|
primaryVbPtr = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}
|
vbid = {vbid = 618}
|
nexusLock = {_M_device = 0x7f2fbc29c008, _M_owns = true}
|
compactionLock = <optimized out>
|
dummyLock = {<std::__mutex_base> = {_M_mutex = {__data = {__lock = 1, __count = 0, __owner = 113210, __nusers = 1, __kind = 0, __spins = 0,
|
__elision = 0, __list = {__prev = 0x0, __next = 0x0}},
|
__size = "\001\000\000\000\000\000\000\000:\272\001\000\001", '\000' <repeats 26 times>, __align = 1}}, <No data fields>}
|
dummyLh = {_M_device = 0x7f2f3cfdec30, _M_owns = true}
|
guard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__this = 0x7f2fbc38f2c0, __vbid = {vbid = 618}}}
|
secondaryCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2f887be440}}, <No data fields>}
|
secondaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2f1f68cf00}}, <No data fields>}
|
primaryExpiryCb = {<std::__shared_ptr<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<NexusExpiryCB, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7f2ecdafec60}}, <No data fields>}
|
primaryDrops = {
|
_M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__det---Type <return> to continue, or q <return> to quit---
|
ail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f2f3cfdecf0,
|
_M_bucket_count = 1, _M_before_begin = {_M_nxt = 0x0}, _M_element_count = 0, _M_rehash_policy = {static _S_growth_factor = 2,
|
_M_max_load_factor = 1, _M_next_resize = 0}, _M_single_bucket = 0x0}}
|
secondaryDrops = {
|
_M_h = {<std::__detail::_Hashtable_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<true, false, true> >> = {<std::__detail::_Hash_code_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::__detail::_Select1st, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, true>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<DiskDocKey>, true>> = {<std::hash<DiskDocKey>> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<DiskDocKey>, true>> = {<std::equal_to<DiskDocKey>> = {<std::binary_function<DiskDocKey, DiskDocKey, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, false>> = {<std::__detail::_Insert_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<DiskDocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<DiskDocKey, std::pair<DiskDocKey const, long>, std::allocator<std::pair<DiskDocKey const, long> >, std::__detail::_Select1st, std::equal_to<DiskDocKey>, std::hash<Disk---Type <return> to continue, or q <return> to quit---
|
DocKey>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<DiskDocKey const, long>, true> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, _M_buckets = 0x7f2e943d3400,
|
_M_bucket_count = 541, _M_before_begin = {_M_nxt = 0x7f2f199ecd40}, _M_element_count = 499, _M_rehash_policy = {
|
static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 541}, _M_single_bucket = 0x0}}
|
originalDroppedKeyCb = {<std::_Maybe_unary_or_binary_function<void, DiskDocKey const&, long, bool, long>> = {<No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x7f2fbc2b1000, _M_const_object = 0x7f2fbc2b1000,
|
_M_function_pointer = 0x7f2fbc2b1000,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f2fbc2b1000, this adjustment 139842129448016}, _M_pod_data = "\000\020+\274/\177\000\000PLs\210/\177\000"}, _M_manager = 0x81aff0
|
<std::_Function_handler<void(const DiskDocKey&, long int, bool, long int), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, uint64_t)::<lambda(const DiskDocKey&, int64_t, bool, int64_t)> >::_M_manager(std::_Any_data &, const std::_Any_data &, std::_Manager_operation)>},
|
_M_invoker = 0x821290 <std::_Function_handler<void (DiskDocKey const&, long, bool, long), EPBucket::makeCompactionContext(Vbid, CompactionConfig&, unsigned long)::{lambda(DiskDocKey const&, long, bool, long)#1}>::_M_invoke(std::_Any_data const&, DiskDocKey const&, long&&, bool&&, std::_Any_data const&)>}
|
nexusCompactionContext = {kvStoreToCompactFirst = 0x7f2fbc2bd180, kvStoreToCompactSecond = 0x7f2fbc20e800,
|
firstCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f887be450, _M_refcount = {_M_pi = 0x7f2f887be440}}, <No data fields>},
|
secondCtx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f88734c50, _M_refcount = {_M_pi = 0x7f2f88734c40}}, <No data fields>},
|
attemptToPruneStaleCallbacks = true}
|
beforeFirstCompactionHighSeqno = 0
|
firstResult = true
|
beforeSecondCompactionHighSeqno = 0
|
secondResult = false
|
secondaryItr = <optimized out>
|
#4 0x000000000081f29f in EPBucket::compactInternal(LockedVBucketPtr&, CompactionConfig&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.h:2664
|
ctx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f88734c50, _M_refcount = {_M_pi = 0x7f2f88734c40}}, <No data fields>}
|
shard = <optimized out>
|
store = <optimized out>
|
result = false
|
#5 0x0000000000821535 in EPBucket::doCompact(Vbid, CompactionConfig&, std::vector<CookieIface const*, std::allocator<CookieIface const*> >&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:1402
|
err = cb::success
|
vb = {
|
---Type <return> to continue, or q <return> to quit---
|
vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f2f29aa9400, _M_refcount = {_M_pi = 0x7f2ed4aa1700}}, <No data fields>}, lock = {_M_device = 0x7f2f88567090,
|
_M_owns = true}}
|
reschedule = false
|
#6 0x000000000071ead6 in CompactTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/tasks.cc:71
|
phosphor_internal_category_enabled_52 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_52 = <optimized out>
|
phosphor_internal_tpi_52 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},
|
argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}
|
phosphor_internal_guard_52 = {tpi = 0x105df40 <CompactTask::run()::phosphor_internal_tpi_52>, enabled = true, arg1 = 618,
|
arg2 = {<No data fields>}, start = {__d = {__r = 6389160129815625}}}
|
compactionData = {<std::__pair_base<CompactionConfig, std::vector<CookieIface const*, std::allocator<CookieIface const*> > >> = {<No data fields>}, first = {purge_before_ts = 1639891526, purge_before_seq = 0, drop_deletes = false, retain_erroneous_tombstones = true,
|
internally_requested = false}, second = {<std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >> = {
|
_M_impl = {<std::allocator<CookieIface const*>> = {<__gnu_cxx::new_allocator<CookieIface const*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<CookieIface const*, std::allocator<CookieIface const*> >::_Vector_impl_data> = {_M_start = 0x0, _M_finish = 0x0,
|
_M_end_of_storage = 0x0}, <No data fields>}}, <No data fields>}}
|
reschedule = <optimized out>
|
#7 0x0000000000aa22f2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#8 0x0000000000a9f505 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f2f3cfdf840)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 6389160129814228}}
|
runAgain = false
|
proxy = @0x7f2fbc40cc50: <error reading variable>
|
#9 0x0000000000bf4690 in operator() (this=0x7f2f3cfdf840)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f2f3cfdf840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f2fbc40cc50, tiny = {
|
__data = "P\314@\274/\177\000\000\000\000\000\000\000\000\000\000\220{[\311/\177\000\000:\272\001\000\000\000\000\000\200\001\000\000\000\000\000\000\020\371\375</\177\000", __align = {<No data fields>}}},
|
call_ = 0xa9fbc0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xa9e310 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (
|
this=this@entry=0x7f2fc912b400, thread=...,
|
task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1965.x86_64.debug, CU 0xa1d7cd1, DIE 0xa25bc06>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.
|
QE-TEST:
git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD
|
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankush_temp_job.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_swap_rebalance,num_items=500000,doc_size=256,nodes_init=2,nodes_swap=2,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=12,replicas=1,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=50,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=101,num_scopes=1,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=15000,doc_ops=create,create_perc=50,delete_perc=50,update_perc=0,num_collections_to_drop=0 -m rest'
|