Details
-
Bug
-
Resolution: User Error
-
Major
-
None
-
7.1.0
-
7.1.0-1881
-
Untriaged
-
Centos 64-bit
-
-
1
-
Unknown
Description
Steps to repro:
- Create a 3 node cluster(172.23.122.245, 172.23.122.246, 172.23.122.247 (bucket_ram_quota = 2056 MB/node), replicas=2)
- Create non default scope(scope1)
- Create 100 non default collections in default scope(collection count is 11(including default collection) and 100 collections in scope 1
- Load 200K docs in each of 200 non default collections
- After data loading, gracefully failover a node
- while failover is going on sigkill memcached(kill -9 <process id>)
- Observed Memcached aborted in NexusKVStore::getCollectionsManifest(Vbid) const ()
Below Core found on node : 172.23.122.245(c367bb78-6ab6-486a-e64e9d90-e73ea843.dmp)
BackTrace:
(gdb) bt full
|
#0 0x00007fb66b2b1337 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007fb66b2b2a28 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00000000006d36e2 in cb::handleError(spdlog::logger&, spdlog::level::level_enum, std::basic_string_view<char, std::char_traits<char> >, cb::ErrorHandlingMethod)
|
() at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:43
|
No locals.
|
#3 0x00000000008ec90a in handleError (msg=..., this=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore-config.h:39
|
No locals.
|
#4 NexusKVStore::getCollectionsManifest(Vbid) const ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:2363
|
msg = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7fb5dc347000 <Address 0x7fb5dc347000 out of bounds>}, _M_string_length = 20676, {_M_local_buf = "\304P", '\000' <repeats 13 times>,
|
_M_allocated_capacity = 20676}}
|
primaryResult = <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1881.x86_64.debug, CU 0x67c3dc4, DIE 0x692fd5b>
|
primaryManifest = <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1881.x86_64.debug, CU 0x67c3dc4, DIE 0x692fd6e>
|
secondaryResult = <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1881.x86_64.debug, CU 0x67c3dc4, DIE 0x692fd8b>
|
secondaryManifest = <unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1881.x86_64.debug, CU 0x67c3dc4, DIE 0x692fd9c>
|
#5 0x0000000000746636 in Warmup::createVBuckets(unsigned short) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:76
|
getManifestStatus = <optimized out>
|
persistedManifest = <optimized out>
|
bucket = <optimized out>
|
table = {_M_t = {<std::__uniq_ptr_impl<FailoverTable, std::default_delete<FailoverTable> >> = {
|
_M_t = {<std::_Tuple_impl<0, FailoverTable*, std::default_delete<FailoverTable> >> = {<std::_Tuple_impl<1, std::default_delete<FailoverTable> >> = {<std::_Head_base<1, std::default_delete<FailoverTable>, true>> = {<std::default_delete<FailoverTable>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, FailoverTable*, false>> = {_M_head_impl = 0x7fb5de3fb680}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
shard = 0x7fb5e2402b30
|
manifest = {_M_t = {<std::__uniq_ptr_impl<Collections::VB::Manifest, std::default_delete<Collections::VB::Manifest> >> = {
|
_M_t = {<std::_Tuple_impl<0, Collections::VB::Manifest*, std::default_delete<Collections::VB::Manifest> >> = {<std::_Tuple_impl<1, std::default_delete<Collections::VB::Manifest> >> = {<std::_Head_base<1, std::default_delete<Collections::VB::Manifest>, true>> = {<std::default_delete<Collections::VB::Manifest>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, Collections::VB::Manifest*, false>> = {
|
_M_head_impl = 0x0}, <No data fields>}, <No data fields>}}, <No data fields>}}
|
topology = <optimized out>
|
vbid = {vbid = 212}
|
vbs = @0x7fb5de9d1488: <error reading variable>
|
vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}
|
itr = @0x7fb5de9d1480: <error reading variable>
|
__for_range = <optimized out>
|
maxEntries = <optimized out>
|
#6 0x0000000000751961 in WarmupCreateVBuckets::run (this=0x7fb5ddc2cfd0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:194
|
phosphor_internal_category_enabled_193 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_193 = <optimized out>
|
---Type <return> to continue, or q <return> to quit---
|
phosphor_internal_tpi_193 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}}, argument_types = {_M_elems = {
|
phosphor::is_bool, phosphor::is_bool}}}
|
phosphor_internal_guard_193 = {tpi = 0x105f8a0 <WarmupCreateVBuckets::run()::phosphor_internal_tpi_193>, enabled = true, arg1 = {<No data fields>},
|
arg2 = {<No data fields>}, start = {__d = {__r = 5311065419398342}}}
|
#7 0x0000000000aa3ea2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#8 0x0000000000aa10b5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fb656feb840)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 5311065419389614}}
|
runAgain = false
|
proxy = @0x7fb669f777e0: <error reading variable>
|
#9 0x0000000000bf5c50 in operator() (this=0x7fb656feb840)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7fb656feb840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fb669f777e0, tiny = {
|
__data = "\340w\367i\266\177\000\000\320W\an\266\177\000\000\060\000\000\000\000\000\000\000\337\326\000\000\000\000\000\000 \000\000\000\000\000\000\000\020\271\376V\266\177\000", __align = {<No data fields>}}},
|
call_ = 0xaa1770 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xa9fec0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#10 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7fb669d0a800,
|
thread=..., task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1881.x86_64.debug, CU 0xa195f2a, DIE 0xa219e5f>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 5311065419373607}}
|
stats = {expired = false, waitTime = {__r = 442590}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 5311065418931017}}, requestId = 0}
|
#11 0x0000000000bdda0a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fb669d0a800, thread=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = -32 '\340', value = {<folly::ThreadPoolExecutor::Task> = {
|
func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fb669f777e0, tiny = {
|
__data = "\340w\367i\266\177\000\000\320W\an\266\177\000\000\060\000\000\000\000\000\000\000\337\326\000\000\000\000\000\000 \000\000\000\000\000\000\000\020\271\376V\266\177\000", __align = {<No data fields>}}},
|
call_ = 0xaa1770 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xa9fec0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 5311065418931017}}, expiration_ = {__r = 0},
|
expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0xd6df, tiny = {
|
__data = "\337\326\000\000\000\000\000\000;\b\243", '\000' <repeats 13 times>, "_\236\346m\266\177\000\000p\312\216h\266\177\000\000@6\an\266\177\000", __align = {<No data fields>}}}, call_ = 0x46590d <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>,
|
exec_ = 0x0},
|
context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000',
|
---Type <return> to continue, or q <return> to quit--
|
QE-TEST:
git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD
|
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankush_temp_job.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_rebalance_in,num_items=200000,doc_size=256,nodes_init=2,nodes_in=1,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=20,replicas=0,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=30,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=101,num_scopes=2,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=20000,doc_ops=create,create_perc=100,delete_perc=0,update_perc=0,num_collections_to_drop=0 -m rest'
|