Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.1.0
-
7.1.0-1672
-
Untriaged
-
Centos 64-bit
-
1
-
Unknown
-
KV 2021-Nov
Description
Steps to repro:
- Create a 5 node cluster( 172.23.100.38, 172.23.123.5 , 172.23.123.6, 172.23.123.7, 172.23.123.8) .
- Create a magma bucket with 10 scopes, 20 collections in each scope.
- Create 15k items in each collection (total items = 3 million items)
- Rebalance out two nodes (172.23.123.7, 172.23.123.8)
- During rebalance start doc ops (create:update:delete:read)
- Observed Memcached crashed with MagmaKVStore::makeImplicitCompactionContext(Vbid) ()
BackTrace:
Core found on node 172.23.123.7
(gdb) bt full
|
#0 0x000000000087f692 in reset (__p=0x7f9b65ca8390, this=0xd8)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:3029
|
__old_p = <optimized out>
|
#1 operator= (__u=<optimized out>, this=0xd8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:167
|
No locals.
|
#2 operator= (this=0xd8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:212
|
No locals.
|
#3 operator= (this=0xd8) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/unique_ptr.h:371
|
No locals.
|
#4 MagmaKVStore::makeImplicitCompactionContext(Vbid) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:3030
|
config = {purge_before_ts = 1636254632, purge_before_seq = 0, drop_deletes = false, retain_erroneous_tombstones = false}
|
ctx = {<std::__shared_ptr<CompactionContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<CompactionContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}
|
status = <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1672.x86_64.debug, CU 0x59cbb84, DIE 0x5b6ccfd>
|
dropped = <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1672.x86_64.debug, CU 0x59cbb84, DIE 0x5b6cd0e>
|
readState = {status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {
|
_M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {
|
_M_head_impl = 0x7f9b7ffe0275}, <No data fields>}, <No data fields>}}, <No data fields>}}}, vbstate = {static CurrentVersion = 4,
|
maxDeletedSeqno = {counter = {_M_elems = "\037\201[\000\000"}}, highSeqno = 0, purgeSeqno = 140307658755456, lastSnapStart = 0,
|
lastSnapEnd = 11943529, maxCas = 140307658755456, hlcCasEpochSeqno = 10117583, mightContainXattrs = false, supportsNamespaces = 201,
|
version = 32667, persistedCompletedSeqno = 0, persistedPreparedSeqno = 140307658755584, highPreparedSeqno = 140304836133568, maxVisibleSeqno = 1,
|
onDiskPrepares = 11986788, onDiskPrepareBytes = 0, checkpointType = (Memory | unknown: 54), transition = {failovers = {
|
static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 140308047372112, {_M_local_buf = "\200\315(\354\233\177\000\000\002\000\000\000\001\000\000",
|
_M_allocated_capacity = 140307658755456}}, replicationTopology = {m_type = nlohmann::detail::null, m_value = {
|
object = 0x7f9c03528937 <je_sdallocx_default+39>, array = 0x7f9c03528937 <je_sdallocx_default+39>,
|
string = 0x7f9c03528937 <je_sdallocx_default+39>, binary = 0x7f9c03528937 <je_sdallocx_default+39>, boolean = 55,
|
number_integer = 140308047366455, number_unsigned = 140308047366455, number_float = 6.9321386038831801e-310}}, state = 1381558944}},
|
kvstoreRev = 140308047372112}
|
#5 0x000000000087fa62 in MagmaKVStore::compactionCallBack(MagmaKVStore::MagmaCompactionCB&, magma::Slice const&, magma::Slice const&, magma::Slice const&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:310
|
vbid = {vbid = 629}
|
userSanitizedItemStr = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f9b7ffecb30 ""},
|
_M_string_length = 0, {_M_local_buf = "\000p\031n\233\177\000\000\300\b@\364\233\177\000", _M_allocated_capacity = 140305543819264}}
|
#6 0x000000000087ffae in MagmaKVStore::MagmaCompactionCB::operator()(magma::Slice const&, magma::Slice const&, magma::Slice const&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:236
|
No locals.
|
---Type <return> to continue, or q <return> to quit---
|
#7 0x000000000096040d in magma::DocSeqGCContext::Transform (this=0x7f9b6d214300, key=..., value=..., buffer=..., outputValue=0x7f9b7ffecd70)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/magma/kvstore/format.h:44
|
drop = false
|
#8 0x0000000000a112f5 in magma::ItemGCFilter::Process(std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >&, std::vector<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >*) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/mvcc.cc:106
|
newValue = {data = 0x0, len = 0}
|
deleted = <optimized out>
|
oldValue = 0x0
|
kv = {<std::__pair_base<magma::Slice, magma::Slice>> = {<No data fields>}, first = {data = 0x7f9b30988a60 <Address 0x7f9b30988a60 out of bounds>,
|
len = 32}, second = {data = 0x7f9b30988a80 <Address 0x7f9b30988a80 out of bounds>, len = 30}}
|
item = {key = {data = 0x7f9b30988a60 <Address 0x7f9b30988a60 out of bounds>, len = 23}, sn = -8934860185726353408, op = magma::Update}
|
gcOffset = 1
|
lastSn = <optimized out>
|
#9 0x00000000009e84d2 in magma::LSMTree::mergeSSTables(std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >&, magma::ItemGCFilter&, unsigned long, double, std::vector<std::shared_ptr<magma::Table>, std::allocator<std::shared_ptr<magma::Table> > >*) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/magma/lsm/lsm_tree.cc:745
|
kvList = <optimized out>
|
status = {s = {_M_t = {<std::__uniq_ptr_impl<magma::Status::state, std::default_delete<magma::Status::state> >> = {
|
_M_t = {<std::_Tuple_impl<0, magma::Status::state*, std::default_delete<magma::Status::state> >> = {<std::_Tuple_impl<1, std::default_delete<magma::Status::state> >> = {<std::_Head_base<1, std::default_delete<magma::Status::state>, true>> = {<std::default_delete<magma::Status::state>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, magma::Status::state*, false>> = {
|
_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}}
|
filterOutput = {<std::_Vector_base<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >> = {
|
_M_impl = {<std::allocator<std::pair<magma::Slice, magma::Slice> >> = {<__gnu_cxx::new_allocator<std::pair<magma::Slice, magma::Slice> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >::_Vector_impl_data> = {_M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}, <No data fields>}}, <No data fields>}
|
totalPhySize = <optimized out>
|
totalLogicalSize = <optimized out>
|
estimateTables = <optimized out>
|
estimateOutputSize = 1961145
|
maxSSTableSizeWithOverhead = 2306867
|
merger = {<magma::Iterator> = {_vptr.Iterator = 0xc579c0 <vtable for magma::MergeIterator+16>, rstats = {NBytesOutgoing = 0, NReadBytes = 0,
|
NReadIOs = 0, NDataBlockReads = 0, NIndexBlockReads = 0}},
|
iterators = {<std::_Vector_base<magma::Iterator*, std::allocator<magma::Iterator*> >> = {
|
_M_impl = {<std::allocator<magma::Iterator*>> = {<__gnu_cxx::new_allocator<magma::Iterator*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<magma::Iterator*, std::allocator<magma::Iterator*> >::_Vector_impl_data> = {_M_start = 0x7f9b7c353600, _M_finish = 0x7f9b7c353688,
|
_M_end_of_storage = 0x7f9b7c353700}, <No data fields>}}, <No data fields>},
|
ownership = {<std::_Vector_base<std::unique_ptr<magma::Iterator, std::default_delete<magma::Iterator> >, std::allocator<std::unique_ptr<magma::Iterator, std::default_delete<magma::Iterator> > > >> = {
|
_M_impl = {<std::allocator<std::unique_ptr<magma::Iterator, std::default_delete<magma::Iterator> > >> = {<__gnu_cxx::new_allocator<std::unique_ptr<magma::Iterator, std::default_delete<magma::Iterator> > >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::unique_ptr<magma::Iterator, std::defaul---Type <return> to continue, or q <return> to quit---
|
t_delete<magma::Iterator> >, std::allocator<std::unique_ptr<magma::Iterator, std::default_delete<magma::Iterator> > > >::_Vector_impl_data> = {
|
_M_start = 0x7f9b7236d600, _M_finish = 0x7f9b7236d688, _M_end_of_storage = 0x7f9b7236d700}, <No data fields>}}, <No data fields>},
|
queue = {<boost::heap::detail::priority_queue_mutable_wrapper<boost::heap::detail::d_ary_heap<magma::MergeIterator::queueElement, boost::parameter::aux::flat_like_arg_list<boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::mutable_, boost::heap::mutable_<true>, std::integral_constant<bool, true> >, boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::compare, boost::heap::compare<magma::MergeIterator::greaterThan>, std::integral_constant<bool, true> >, boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::arity, boost::heap::arity<2>, std::integral_constant<bool, true> > >, boost::heap::detail::nop_index_updater> >> = {static is_stable = <optimized out>,
|
q_ = {<boost::heap::detail::heap_base<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> >, boost::heap::detail::priority_queue_mutable_wrapper<boost::heap::detail::d_ary_heap<magma::MergeIterator::queueElement, boost::parameter::aux::flat_like_arg_list<boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::mutable_, boost::heap::mutable_<true>, std::integral_constant<bool, true> >, boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::compare, boost::heap::compare<magma::MergeIterator::greaterThan>, std::integral_constant<bool, true> >, boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::arity, boost::heap::arity<2>, std::integral_constant<bool, true> > >, boost::heap::detail::nop_index_updater> >::indirect_cmp, false, unsigned long, false>> = {<boost::heap::detail::priority_queue_mutable_wrapper<boost::heap::detail::d_ary_heap<magma::MergeIterator::queueElement, boost::parameter::aux::flat_like_arg_list<boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::mutable_, boost::heap::mutable_<true>, std::integral_constant<bool, true> >, boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::compare, boost::heap::compare<magma::MergeIterator::greaterThan>, std::integral_constant<bool, true> >, boost::parameter::aux::flat_like_arg_tuple<boost::heap::tag::arity, boost::heap::arity<2>, std::integral_constant<bool, true> > >, boost::heap::detail::nop_index_updater> >::indirect_cmp> = {<magma::MergeIterator::greaterThan> = {iters = 0x7f9b7ffed040,
|
cmp = {<std::_Maybe_unary_or_binary_function<int, magma::Slice const&, magma::Slice const&>> = {<std::binary_function<magma::Slice const&, magma::Slice const&, int>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {
|
_M_unused = {_M_object = 0x7f9b7257b100, _M_const_object = 0x7f9b7257b100, _M_function_pointer = 0x7f9b7257b100,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f9b7257b100, this adjustment 140305327929296}, _M_pod_data = "\000\261Wr\233\177\000\000\320\067;a\233\177\000"},
|
_M_manager = 0x9b5860 <std::_Function_handler<int (magma::Slice const&, magma::Slice const&), magma::KeyVersionCompare>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},
|
_M_invoker = 0x9b57d0 <std::_Function_handler<int (magma::Slice const&, magma::Slice const&), magma::KeyVersionCompare>::_M_invoke(std::_Any_data const&, magma::Slice const&, magma::Slice const&)>}}, <No data fields>}, <boost::heap::detail::size_holder<false, unsigned long>> = {
|
static constant_time_size = <optimized out>}, static is_stable = <optimized out>},
|
q_ = {<std::_Vector_base<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> >, std::allocator<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> > > >> = {
|
_M_impl = {<std::allocator<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> > >> = {<__gnu_cxx::new_allocator<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> > >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> >, std::allocator<std::_List_iterator<std::pair<magma::MergeIterator::queueElement, unsigned long> > > >::_Vector_impl_data> = {_M_start = 0x7f9b72cceb00, _M_finish = 0x7f9b72cceb88,
|
_M_end_of_storage = 0x7f9b72ccec00}, <No data fields>}}, <No data fields>}, static D = 2, static is_stable = <optimized out>},
|
objects = {<std::__cxx11::_List_base<std::pair<magma::MergeIterator::queueElement, unsigned long>, std::allocator<std::pair<magma::MergeIterator::queueElement, unsigned long> > >> = {
|
_M_impl = {<std::allocator<std::_List_node<std::pair<magma::MergeIterator::queueElement, unsigned long> > >> = {<__gnu_cxx::new_allocator<std::_List_node<std::pair<magma::MergeIterator::queueElement, unsigned long> > >> = {<No data fields>}, <No data fields>},
|
_M_node = {<std::__detail::_List_node_base> = {_M_next = 0x7f9b6e75b6b0, _M_prev = 0x7f9b6d6823c0}, _M_size = 17}}}, <No data fields>}},
|
static is_mutable = true, static constant_time_size = <optimized out>, static has_ordered_iterators = <optimized out>,
|
static is_mergable = <optimized out>, static has_reserve = <optimized out>, static is_stable = <optimized out>}, doCaching = false,
|
enableReadAhead = true}
|
---Type <return> to continue, or q <return> to quit---
|
kvReader = {output = {<std::_Vector_base<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >> = {
|
_M_impl = {<std::allocator<std::pair<magma::Slice, magma::Slice> >> = {<__gnu_cxx::new_allocator<std::pair<magma::Slice, magma::Slice> >> = {<No data fields>}, <No data fields>}, <std::_Vector_base<std::pair<magma::Slice, magma::Slice>, std::allocator<std::pair<magma::Slice, magma::Slice> > >::_Vector_impl_data> = {_M_start = 0x7f9b5258eaa0, _M_finish = 0x7f9b5258eac0, _M_end_of_storage = 0x7f9b5258eac0}, <No data fields>}}, <No data fields>},
|
kvSizes = {<std::_Vector_base<unsigned long, std::allocator<unsigned long> >> = {
|
_M_impl = {<std::allocator<unsigned long>> = {<__gnu_cxx::new_allocator<unsigned long>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<unsigned long, std::allocator<unsigned long> >::_Vector_impl_data> = {_M_start = 0x7f9b7c72b400, _M_finish = 0x7f9b7c72b410,
|
_M_end_of_storage = 0x7f9b7c72b410}, <No data fields>}}, <No data fields>},
|
cmp = {<std::_Maybe_unary_or_binary_function<int, magma::Slice const&, magma::Slice const&>> = {<std::binary_function<magma::Slice const&, magma::Slice const&, int>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {
|
_M_object = 0x7f9b6208afa0, _M_const_object = 0x7f9b6208afa0, _M_function_pointer = 0x7f9b6208afa0,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f9b6208afa0, this adjustment 140305844064304},
|
_M_pod_data = "\240\257\bb\233\177\000\000\060\320\376\177\233\177\000"},
|
_M_manager = 0x9b5db0 <std::_Function_handler<int (magma::Slice const&, magma::Slice const&), magma::KeyOnlyCompare>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},
|
_M_invoker = 0x9b5750 <std::_Function_handler<int (magma::Slice const&, magma::Slice const&), magma::KeyOnlyCompare>::_M_invoke(std::_Any_data const&, magma::Slice const&, magma::Slice const&)>}, iter = 0x7f9b7ffed010, buf = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f9b30988a60 <Address 0x7f9b30988a60 out of bounds>}, _M_string_length = 62, {
|
_M_local_buf = "@\000\000\000\000\000\000\000\030C!m\233\177\000", _M_allocated_capacity = 64}}, lastKey = {
|
data = 0x7f9b30988a60 <Address 0x7f9b30988a60 out of bounds>, len = 32}, clearBufOnRead = true, stopIteratorOnFind = false,
|
readUntilFn = {<std::_Maybe_unary_or_binary_function<bool, magma::Slice const&>> = {<std::unary_function<magma::Slice const&, bool>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x7f9b7ffed278,
|
_M_const_object = 0x7f9b7ffed278, _M_function_pointer = 0x7f9b7ffed278,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x7f9b7ffed278, this adjustment 9932005},
|
_M_pod_data = "x\322\376\177\233\177\000\000\345\214\227\000\000\000\000"}, _M_manager = 0x0}, _M_invoker = 0x7f9b6d214300}}
|
sstableCfg = {Path = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f9b7ffed0e0 ""},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}, ReadOnly = false, BCache = 0x0,
|
CacheDataBlocks = false, UseCompression = false, BlockSize = 4096, IndexBlockSize = 4096, EnableFilePreallocation = false,
|
CompressionAlgo = magma::Compression::LZ4, ReadAheadSize = 0, MaxIndexDepth = 2,
|
CompareIndexKey = {<std::_Maybe_unary_or_binary_function<int, magma::Slice const&, magma::Slice const&>> = {<std::binary_function<magma::Slice const&, magma::Slice const&, int>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {
|
_M_unused = {_M_object = 0x9b3120 <magma::compareSlice(magma::Slice const&, magma::Slice const&)>,
|
_M_const_object = 0x9b3120 <magma::compareSlice(magma::Slice const&, magma::Slice const&)>,
|
_M_function_pointer = 0x9b3120 <magma::compareSlice(magma::Slice const&, magma::Slice const&)>,
|
_M_member_pointer = (void (std::_Undefined_class::*)(std::_Undefined_class * const)) 0x9b3120 <magma::compareSlice(magma::Slice const&, magma::Slice const&)>, this adjustment 140308047339234}, _M_pod_data = " 1\233\000\000\000\000\000\342\036R\003\234\177\000"}, _M_manager = 0x92d610
|
<std::_Function_handler<int (magma::Slice const&, magma::Slice const&), int (*)(magma::Slice const&, magma::Slice const&)>::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation)>},
|
_M_invoker = 0x92d490 <std::_Function_handler<int (magma::Slice const&, magma::Slice const&), int (*)(magma::Slice const&, magma::Slice const&)>::_M_invoke(std::_Any_data const&, magma::Slice const&, magma::Slice const&)>},
|
Rebalance Failure:
Rebalance exited with reason {mover_crashed,\n {unexpected_exit,\n {\'EXIT\',<0.27529.0>,\n {{{{nocatch,{error,closed}},\n [{mc_binary,recv_with_data,4,\n [{file,"src/mc_binary.erl"},{line,41}]},\n {mc_binary,quick_active_recv,3,\n [{file,"src/mc_binary.erl"},{line,48}]},\n {mc_binary,quick_stats_loop_enter,5,\n [{file,"src/mc_binary.erl"},{line,100}]},\n {mc_binary,quick_stats,5,\n [{file,"src/mc_binary.erl"},{line,85}]},\n {mc_client_binary,get_dcp_docs_estimate,\n 3,\n [{file,"src/mc_client_binary.erl"},\n {line,754}]},\n {ns_memcached,do_handle_call,3,\n [{file,"src/ns_memcached.erl"},\n {line,581}]},\n {ns_memcached,worker_loop,3,\n [{file,"src/ns_memcached.erl"},\n {line,224}]},\n {proc_lib,init_p_do_apply,3,\n [{file,"proc_lib.erl"},{line,226}]}]},\n {gen_server,call,\n [\'ns_memcached-L5VT2_xKXtFBcLfd_QevK4bT0zv3QiDW6IaZPG42CevEkRkGoe%jkb-22-48000\',\n {get_dcp_docs_estimate,703,\n "replication:ns_1@172.23.123.7->ns_1@172.23.100.38:L5VT2_xKXtFBcLfd_QevK4bT0zv3QiDW6IaZPG42CevEkRkGoe%jkb-22-48000"},\n 180000]}},\n {gen_server,call,\n [{\'janitor_agent-L5VT2_xKXtFBcLfd_QevK4bT0zv3QiDW6IaZPG42CevEkRkGoe%jkb-22-48000\',\n \'ns_1@172.23.123.7\'},\n {if_rebalance,<0.20554.0>,\n {wait_dcp_data_move,\n [\'ns_1@172.23.123.5\',\'ns_1@172.23.123.6\',\n \'ns_1@172.23.100.38\'],\n 703}},\n infinity]}}}}}.\nRebalance Operation Id = 391c27fc16e23388b67b959bf5d7d495'}
|
QE-Test:
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.113832.ini bucket_storage=magma,get-cbcollect-info=True,bucket_eviction_policy=fullEviction,randomize_value=True,rerun=false,infra_log_level=debug,log_level=debug,enable_dp=false,get-cbcollect-info=True,upgrade_version=7.1.0-1672 -t bucket_collections.collections_rebalance.CollectionsRebalance.test_data_load_collections_with_rebalance_out,nodes_init=5,nodes_out=2,bucket_spec=magma_templates.buckets_all_membase_for_rebalance_tests,data_load_stage=during,GROUP=P0_rebalance'
|