Details
-
Bug
-
Resolution: Fixed
-
Major
-
7.1.0
-
7.1.0-1965
-
Untriaged
-
Centos 64-bit
-
1
-
Unknown
-
KV 2021-Dec, KV 2022-Jan
Description
Steps to repro:
- Create a 1 node cluster(172.23.122.245 (bucket_ram_quota = 2056 MB/node), replicas=0)
- Create 2 non default scopes (scope1 & scope2)
- Create 5 non default collections in all the scopes including default scope
- Start Loading 5M docs in each of non default collections(all scopes)
- While doc load is going on tried adding a node which was already part of a cluster(Prepare Join failed) and Memcached aborted in NexusKVStore::listPersistedVbuckets() ()
*BackTrace:
#0 0x00007fd9bb5da337 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007fd9bb5dba28 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00000000006d43d2 in cb::handleError(spdlog::logger&, spdlog::level::level_enum, std::basic_string_view<char, std::char_traits<char> >, cb::ErrorHandlingMethod) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/error_handler.cc:43
|
No locals.
|
#3 0x00000000008fb401 in handleError (msg=..., this=0x7fd9a46dc5c0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore-config.h:39
|
No locals.
|
#4 NexusKVStore::listPersistedVbuckets() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvstore/nexus-kvstore/nexus-kvstore.cc:892
|
msg = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7fd916749c00 <Address 0x7fd916749c00 out of bounds>}, _M_string_length = 1104, {
|
_M_local_buf = "P\004\000\000\000\000\000\000&v\000\000\000\000\000", _M_allocated_capacity = 1104}}
|
vbid = {vbid = 0}
|
i = 0
|
primaryVbStates = {<std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >> = {
|
_M_impl = {<std::allocator<vbucket_state*>> = {<__gnu_cxx::new_allocator<vbucket_state*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >::_Vector_impl_data> = {_M_start = 0x7fd8c3811400, _M_finish = 0x7fd8c3811800,
|
_M_end_of_storage = 0x7fd8c3811800}, <No data fields>}}, <No data fields>}
|
secondaryVbStates = {<std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >> = {
|
_M_impl = {<std::allocator<vbucket_state*>> = {<__gnu_cxx::new_allocator<vbucket_state*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >::_Vector_impl_data> = {_M_start = 0x7fd91f2f6400, _M_finish = 0x7fd91f2f6800,
|
_M_end_of_storage = 0x7fd91f2f6800}, <No data fields>}}, <No data fields>}
|
#5 0x000000000074521d in Warmup::populateShardVbStates() ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/kvshard.h:72
|
kvStoreVbStates = {<std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >> = {
|
_M_impl = {<std::allocator<vbucket_state*>> = {<__gnu_cxx::new_allocator<vbucket_state*>> = {<No data fields>}, <No data fields>}, <std::_Vector_base<vbucket_state*, std::allocator<vbucket_state*> >::_Vector_impl_data> = {_M_start = 0x7fd8c3811400, _M_finish = 0x7fd8c3811800,
|
_M_end_of_storage = 0x7fd8c3811800}, <No data fields>}}, <No data fields>}
|
i = 0
|
numKvs = 8
|
#6 0x000000000074680c in Warmup::initialize() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:1292
|
session_stats = {m_type = nlohmann::detail::object, m_value = {object = 0x7fd9b4167d20, array = 0x7fd9b4167d20, string = 0x7fd9b4167d20,
|
binary = 0x7fd9b4167d20, boolean = 32, number_integer = 140573006003488, number_unsigned = 140573006003488,
|
number_float = 6.9452292998958726e-310}}
|
---Type <return> to continue, or q <return> to quit---
|
it = {m_object = <optimized out>, m_it = {object_iterator = {_M_node = <optimized out>}, array_iterator = {_M_current = 0x0},
|
primitive_iterator = {static begin_value = 0, static end_value = 1, m_it = <optimized out>}}}
|
#7 0x000000000075111d in WarmupInitialize::run (this=0x7fd8ccb15040)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/warmup.cc:163
|
phosphor_internal_category_enabled_162 = {_M_b = {_M_p = 0x0}, static is_always_lock_free = <optimized out>}
|
phosphor_internal_category_enabled_temp_162 = <optimized out>
|
phosphor_internal_tpi_162 = {category = 0x0, name = 0x0, type = phosphor::AsyncStart, argument_names = {_M_elems = {0x0, 0x0}},
|
argument_types = {_M_elems = {phosphor::is_bool, phosphor::is_bool}}}
|
phosphor_internal_guard_162 = {tpi = 0x105ea80 <WarmupInitialize::run()::phosphor_internal_tpi_162>, enabled = true,
|
arg1 = {<No data fields>}, arg2 = {<No data fields>}, start = {__d = {__r = 6404794305504528}}}
|
#8 0x0000000000aa22f2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#9 0x0000000000a9f505 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7fd9967ea840)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 6404794305502636}}
|
runAgain = false
|
proxy = @0x7fd9ac40d580: <error reading variable>
|
#10 0x0000000000bf4690 in operator() (this=0x7fd9967ea840)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7fd9967ea840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fd9ac40d580, tiny = {
|
__data = "\200\325@\254\331\177\000\000\000\000\000\000\000\000\000\000\220{;\272\331\177\000\000 ,\001\000\000\000\000\000X\000\000\000\000\000\000\000\020\251~\226\331\177\000", __align = {<No data fields>}}},
|
call_ = 0xa9fbc0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0xa9e310 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#11 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (
|
this=this@entry=0x7fd9b9f2a800, thread=...,
|
task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1965.x86_64.debug, CU 0xa1d7cd1, DIE 0xa25bc06>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 6404794305497956}}
|
stats = {expired = false, waitTime = {__r = 22556}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 6404794305475400}}, requestId = 0}
|
#12 0x0000000000bdc44a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fd9b9f2a800, thread=...)
|
QE-TEST:
git fetch "https://review.couchbase.org/TAF" refs/changes/88/166488/1 && git checkout FETCH_HEAD
|
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankush_temp_job.ini -p bucket_storage=couchstore,rerun=false,bucket_eviction_policy=fullEviction,init_loading=False -t storage.magma.magma_rebalance.MagmaRebalance.test_data_load_collections_with_swap_rebalance,num_items=500000,doc_size=256,nodes_init=2,nodes_swap=2,standard_buckets=1,magma_buckets=0,bucket_storage=couchstore,data_load_stage=during,sdk_timeout=60,vbuckets=1024,key_size=12,replicas=1,infra_log_level=debug,log_level=debug,skip_cleaup=True,randomize_value=True,bucket_eviction_policy=fullEviction,infra_log_level=debug,log_level=debug,init_loading=False,fragmentation=50,skip_cleanup=True,autoCompactionDefined=true,iterations=1,enable_dp=True,num_collections=101,num_scopes=1,bucket_ram_quota=2056,skip_cleanup=True,sdk_client_pool=False,ops_rate=15000,doc_ops=create,create_perc=50,delete_perc=50,update_perc=0,num_collections_to_drop=0 -m rest'
|
Attachments
For Gerrit Dashboard: MB-50194 | ||||||
---|---|---|---|---|---|---|
# | Subject | Branch | Project | Status | CR | V |
168239,3 | MB-50194: Pass Vbid to compareVBucketStates | master | kv_engine | Status: MERGED | +2 | +1 |
168240,5 | MB-50194: Shard NexusKVStore::purgeSeqno by vbid | master | kv_engine | Status: MERGED | +2 | +1 |
168298,4 | MB-50194: Populate NexusKVStore purgeSeqno cache on warmup | master | kv_engine | Status: MERGED | +2 | +1 |