Details
-
Task
-
Resolution: Fixed
-
Major
-
7.1.0
-
7.1.0-2150
-
1
-
KV 2022-Jan, KV 2022-Feb
Description
Steps to repro:
- Create a 2 node cluster(172.23.122.245, 172.23.122.246(bucket_ram_quota = 2056 MB/node), replicas=0, durability=majority)
- Create 10 scopes
- Create 10 collections in each of the scopes
- Start Loading 700k docs in each of non default collections
- While doc loading is going on rebalance out node 172.23.122.246
- Remove few collections, recreate deleted collections
- Change replicas to 2 and add node 172.23.122.246 and 172.23.122.247
- Trigger full compaction
- Rebalance was successful
- Rebalance out node 172.23.122.246
- Change replicas to 1 and rebalance
- rebalance was successful
- Change durabilty to majorityandpersistactive
- Again change replica to 2 and rebalance
- Drop few scopes, recreate few scopes
- Trigger full compaction
- Observed Memcached aborted in NexusSecondaryCacheLookup::callback (this=0x7fd0179dcf20, val=...)
Below Core found on node : 172.23.122.247
BackTrace:
(gdb) bt full
|
#0 0x00007fd0f5d43a35 in __memcpy_ssse3_back () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x0000000000a3e44b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) ()
|
No symbol table info available.
|
#2 0x000000000090f58a in assign (__str=..., this=0x7fd09b7ebe60)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:2057
|
No locals.
|
#3 operator= (__str=..., this=0x7fd09b7ebe60) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/basic_string.h:667
|
No locals.
|
#4 operator= (this=0x7fd09b7ebe60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/diskdockey.h:32
|
No locals.
|
#5 operator= (this=0x7fd09b7ebe60) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/callbacks.h:24
|
No locals.
|
#6 operator=<CacheLookup, cb::engine_errc> (__in=..., this=<optimized out>) at /opt/gcc-10.2.0/include/c++/10.2.0/tuple:1219
|
No locals.
|
#7 NexusSecondaryCacheLookup::callback (this=0x7fd0179dcf20, val=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:2057
|
No locals.
|
#8 0x000000000087d865 in MagmaKVStore::scan(BySeqnoScanContext&) const ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/magma-kvstore/magma-kvstore.cc:1749
|
lookup = {key = {keydata = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7fd0983e6860 <Address 0x7fd0983e6860 out of bounds>}, _M_string_length = 23, {
|
_M_local_buf = "\027\000\000\000\000\000\000\000\300\202\006G\320\177\000", _M_allocated_capacity = 23}}}, bySeqno = 21997, vbid = {
|
vbid = 1017}}
|
metaSlice = {data = 0x7fd0e02949ac <Address 0x7fd0e02949ac out of bounds>, len = 28}
|
rv = {item = {_M_t = {<std::__uniq_ptr_impl<Item, std::default_delete<Item> >> = {
|
_M_t = {<std::_Tuple_impl<0, Item*, std::default_delete<Item> >> = {<std::_Tuple_impl<1, std::default_delete<Item> >> = {<std::_Head_base<1, std::default_delete<Item>, true>> = {<std::default_delete<Item>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, Item*, false>> = {_M_head_impl = 0x7fd002e46280}, <No data fields>}, <No data fields>}}, <No data fields>}}, id = 18446744073709551615,
|
status = cb::success, partial = false}
|
keySlice = {data = 0x7fd0e0294994 <Address 0x7fd0e0294994 out of bounds>, len = 23}
|
valSlice = {data = 0x7fcfcafd1622 <Address 0x7fcfcafd1622 out of bounds>, len = 0}
|
seqno = 21997
|
diskKey = {keydata = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7fd070402920 <Address 0x7fd070402920 out of bounds>}, _M_string_length = 23, {
|
_M_local_buf = "\027\000\000\000\000\000\000\000ection\000", _M_allocated_capacity = 23}}}
|
---Type <return> to continue, or q <return> to quit---
|
docKey = {<DocKeyInterface<DocKey>> = {<No data fields>}, buffer = {static npos = <optimized out>,
|
buf = 0x7fd070402920 <Address 0x7fd070402920 out of bounds>, len = 23}, encoding = Yes}
|
itm = {_M_t = {<std::__uniq_ptr_impl<Item, std::default_delete<Item> >> = {
|
_M_t = {<std::_Tuple_impl<0, Item*, std::default_delete<Item> >> = {<std::_Tuple_impl<1, std::default_delete<Item> >> = {<std::_Head_base<1, std::default_delete<Item>, true>> = {<std::default_delete<Item>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, Item*, false>> = {_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
callbackStatus = <optimized out>
|
startSeqno = <optimized out>
|
onlyKeys = <optimized out>
|
mctx = @0x7fd025217f40: <error reading variable>
|
#9 0x00000000008f35de in NexusKVStore::scan(BySeqnoScanContext&) const ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:2358
|
nexusCtx = @0x7fcfeb61ab80: <error reading variable>
|
primaryCtx = @0x7fcffeae14e0: <error reading variable>
|
secondaryCtx = @0x7fd025217f40: <error reading variable>
|
primaryScanResult = scan_again
|
secondaryScanResult = <optimized out>
|
primaryScanCallback = <optimized out>
|
primaryCacheLookup = <optimized out>
|
#10 0x00000000008d628d in DCPBackfillBySeqnoDisk::scan() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_by_seqno_disk.cc:173
|
stream = {<std::__shared_ptr<ActiveStream, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<ActiveStream, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7fcff4da6000}}, <No data fields>}
|
kvstore = 0x7fd0e025c860
|
bySeqnoCtx = @0x7fcfeb61ab80: <error reading variable>
|
|
#11 0x00000000008d9c42 in DCPBackfillDisk::run() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill_disk.cc:131
|
lh = {_M_device = @0x7fd09878c588}
|
runtimeGuard = {<folly::detail::ScopeGuardImplBase> = {dismissed_ = false}, function_ = {__start = {__d = {__r = 8530938467628964}},
|
__this = 0x7fd09878c580}}
|
#12 0x00000000008db661 in BackfillManager::backfill() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:344
|
lh = {_M_device = <optimized out>, _M_owns = false}
|
backfill = {_M_t = {<std::__uniq_ptr_impl<DCPBackfillIface, std::default_delete<DCPBackfillIface> >> = {
|
_M_t = {<std::_Tuple_impl<0, DCPBackfillIface*, std::default_delete<DCPBackfillIface> >> = {<std::_Tuple_impl<1, std::default_delete<DCPBackfillIface> >> = {<std::_Head_base<1, std::default_delete<DCPBackfillIface>, true>> = {<std::default_delete<DCPBackfillIface>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, DCPBackfillIface*, false>> = {
|
_M_head_impl = 0x7fd09878c5d8}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
source = BackfillManager::Active
|
---Type <return> to continue, or q <return> to quit---
|
status = <optimized out>
|
#13 0x00000000008dbe04 in BackfillManagerTask::run() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/backfill-manager.cc:78
|
phosphor_internal_category_enabled_67 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_67 = <optimized out>
|
phosphor_internal_tpi_67 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},
|
argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}
|
phosphor_internal_guard_67 = {tpi = 0x1073320 <BackfillManagerTask::run()::phosphor_internal_tpi_67>, enabled = true,
|
arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 8530938467626820}}}
|
manager = {<std::__shared_ptr<BackfillManager, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<BackfillManager, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = <optimized out>, _M_refcount = {_M_pi = 0x7fcffbf59200}}, <No data fields>}
|
status = <optimized out>
|
#14 0x0000000000ab1962 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
|
|
|
guard = {previous = 0x0}
|
|
#15 0x0000000000aaea65 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fd09b7ec840)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 8530938467625523}}
|
runAgain = false
|
proxy = @0x7fd0e803aed0: <error reading variable>
|
#16 0x0000000000c02f10 in operator() (this=0x7fd09b7ec840)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7fd09b7ec840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fd0e803aed0, tiny = {
|
__data = "\320\256\003\350\320\177\000\000\000\000\000\000\000\000\000\000\260+-\350\320\177\000\000T\326", '\000' <repeats 14 times>, "\020\311~\233\320\177\000", __align = {<No data fields>}}},
|
call_ = 0xaaf120 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xaad870 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
|
#17 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (
|
this=this@entry=0x7fd0f4551000, thread=...,
|
task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-2150.x86_64.debug, CU 0xa344264, DIE 0xa3c8199>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 8530938467622417}}
|
stats = {expired = false, waitTime = {__r = 14844}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 8530938467607573}}, requestId = 0}
|
---Type <return> to continue, or q <return> to quit---
|
#18 0x0000000000bed84a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fd0f4551000, thread=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = -48 '\320', value = {<folly::ThreadPoolExecutor::Task> = {
|
func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fd0e803aed0, tiny = {
|
__data = "\320\256\003\350\320\177\000\000\000\000\000\000\000\000\000\000\260+-\350\320\177\000\000T\326", '\000' <repeats 14 times>, "\020\311~\233\320\177\000", __align = {<No data fields>}}},
|
call_ = 0xaaf120 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xaad870 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 8530938467607573}}, expiration_ = {
|
__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0xd654,
|
tiny = {
|
__data = "T\326\000\000\000\000\000\000K\344\243", '\000' <repeats 13 times>, "_\336}\370\320\177\000\000p\332\376\276\320\177\000\000@v\236\370\320\177\000", __align = {<No data fields>}}}, call_ = 0x466655
|
<folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},
|
context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,
|
priority_ = 0 '\000', queueObserverPayload_ = 140535448749520}}, hasValue = true}}
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xcd4eab "CPUThreadPoolExecutor",
|
e_ = 0xcd4ec0 ""}}}}
|
#19 0x0000000000c05ec9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>)
|
at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#20 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#21 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#22 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#23 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#24 0x0000000000aae6f4 in operator() (this=0x7fd0f49d8cc0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
fn = @0x7fd0f49d8cc0: <error reading variable>
|
---Type <return> to continue, or q <return> to quit---
|
#25 operator() (__closure=0x7fd0f49d8cc0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7fd09b7ec9a0 "AuxIoPool5"}, _M_string_length = 10, {_M_local_buf = "AuxIoPool5\000\000\000\000\000",
|
_M_allocated_capacity = 8029725099529106753}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7fd0f49d8cc0)>
|
#26 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = @0x7fd0f49d8cc0: <error reading variable>
|
#27 0x00007fd0f65a4d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
|
No locals.
|
#28 0x00007fd0f83ace65 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#29 0x00007fd0f5ced88d in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
QE-TEST:
git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD
|
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/qe_r.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_rebalance_out,num_items=700000,doc_size=512,nodes_init=2,nodes_out=1,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=22,replicas=0,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=50,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=10,num_scopes=10,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=30000,doc_ops=create,create_perc=0,delete_perc=50,update_perc=50,num_collections_to_drop=5,get-cbcollect-info=True -m rest'
|
Attachments
Issue Links
- relates to
-
MB-50492 [NexusKVStore] Memcached aborted in NexusKVStore::scan(BySeqnoScanContext&) const ()
- Closed