Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.1.0
-
7.1.0-1040
-
Untriaged
-
Centos 64-bit
-
1
-
No
Description
Steps to Recreate:
- Create a 4 node cluster with 10 magma bucket with replica=1
- Keep deleting , recreating 8 out of 10 magma buckets
- During Step 2, Create load is going on for two buckets(which are not getting deleted)
- While load and "deletion/recreation of buckets" is running in step 2 and Step 3, on all 4 nodes keep klilling memcached
- Oberved memcahced crashed in KVBucket::KVBucket(EventuallyPersistentEngine&)
BackTrace:
Core found on Node 172.23.121.134
(gdb) bt full
|
#0 FollyExecutorPool::registerTaskable(Taskable&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:824
|
guard = {engine = 0x0}
|
#1 0x00000000006d6154 in KVBucket::KVBucket(EventuallyPersistentEngine&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.h:800
|
NoopSyncWriteCompleteCb = {<std::_Maybe_unary_or_binary_function<void, CookieIface const*, cb::engine_errc>> = {<std::binary_function<CookieIface const*, cb::engine_errc, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16,
|
static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0,
|
_M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}
|
std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}
|
aliveVBStates = {bits = {<std::_Base_bitset<1>> = {_M_w = 0}, <No data fields>}}
|
globalBucketLoggerName = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
NoopSeqnoAckCb = {<std::_Maybe_unary_or_binary_function<void, Vbid, long>> = {<std::binary_function<Vbid, long, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0,
|
_M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>},
|
_M_manager = 0x0}, _M_invoker = 0x0}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_alloc_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_exception_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
flatbuffers::flatbuffer_version_string = 0xbab20e "FlatBuffers 1.10.0"
|
#2 0x00000000007e5d66 in EPBucket::EPBucket(EventuallyPersistentEngine&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_bucket.cc:240
|
cb::userdataStartTag = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
NoopSeqnoAckCb = {<std::_Maybe_unary_or_binary_function<void, Vbid, long>> = {<std::binary_function<Vbid, long, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0,
|
_M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>},
|
_M_manager = 0x0}, _M_invoker = 0x0}
|
globalBucketLoggerName = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
NoopSyncWriteCompleteCb = {<std::_Maybe_unary_or_binary_function<void, CookieIface const*, cb::engine_errc>> = {<std::binary_function<CookieIface const*, cb::engine_errc, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16,
|
static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0,
|
_M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}
|
---Type <return> to continue, or q <return> to quit---
|
aliveVBStates = {bits = {<std::_Base_bitset<1>> = {_M_w = 0}, <No data fields>}}
|
std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}
|
cb::userdataEndTag = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
flatbuffers::flatbuffer_version_string = 0xbab20e "FlatBuffers 1.10.0"
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_alloc_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_exception_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
#3 0x000000000069a5d8 in EventuallyPersistentEngine::makeBucket(Configuration&) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:6531
|
aliveVBStates = {bits = {<std::_Base_bitset<1>> = {_M_w = 0}, <No data fields>}}
|
std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}
|
globalBucketLoggerName = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
cb::userdataEndTag = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
NoopSeqnoAckCb = {<std::_Maybe_unary_or_binary_function<void, Vbid, long>> = {<std::binary_function<Vbid, long, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0,
|
_M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>},
|
_M_manager = 0x0}, _M_invoker = 0x0}
|
NoopSyncWriteCompleteCb = {<std::_Maybe_unary_or_binary_function<void, CookieIface const*, cb::engine_errc>> = {<std::binary_function<CookieIface const*, cb::engine_errc, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16,
|
static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0,
|
_M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}
|
cb::userdataStartTag = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_exception_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
makeExitBorderGuard = {<No data fields>}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_alloc_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
flatbuffers::flatbuffer_version_string = 0xbab20e "FlatBuffers 1.10.0"
|
#4 0x000000000069acc0 in EventuallyPersistentEngine::initialize(char const*) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/ep_engine.cc:2024
|
---Type <return> to continue, or q <return> to quit---
|
aliveVBStates = {bits = {<std::_Base_bitset<1>> = {_M_w = 0}, <No data fields>}}
|
std::__ioinit = {static _S_refcount = 0, static _S_synced_with_stdio = true}
|
globalBucketLoggerName = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
cb::userdataEndTag = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
NoopSeqnoAckCb = {<std::_Maybe_unary_or_binary_function<void, Vbid, long>> = {<std::binary_function<Vbid, long, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16, static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0,
|
_M_const_object = 0x0, _M_function_pointer = 0x0, _M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>},
|
_M_manager = 0x0}, _M_invoker = 0x0}
|
NoopSyncWriteCompleteCb = {<std::_Maybe_unary_or_binary_function<void, CookieIface const*, cb::engine_errc>> = {<std::binary_function<CookieIface const*, cb::engine_errc, void>> = {<No data fields>}, <No data fields>}, <std::_Function_base> = {static _M_max_size = 16,
|
static _M_max_align = 8, _M_functor = {_M_unused = {_M_object = 0x0, _M_const_object = 0x0, _M_function_pointer = 0x0,
|
_M_member_pointer = NULL}, _M_pod_data = '\000' <repeats 15 times>}, _M_manager = 0x0}, _M_invoker = 0x0}
|
cb::userdataStartTag = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x0},
|
_M_string_length = 0, {_M_local_buf = '\000' <repeats 15 times>, _M_allocated_capacity = 0}}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_exception_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
makeExitBorderGuard = {<No data fields>}
|
boost::exception_detail::exception_ptr_static_exception_object<boost::exception_detail::bad_alloc_>::e = {ptr_ = {px = 0x0, pn = {
|
pi_ = 0x0}}}
|
flatbuffers::flatbuffer_version_string = 0xbab20e "FlatBuffers 1.10.0"
|
#5 0x00000000005ac788 in BucketManager::create(Cookie&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, BucketType) ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/buckets.cc:221
|
#6 0x0000000000611ede in operator() (__closure=0x7f817513fd60) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/char_traits.h:322
|
connection = @0x7f81751ccf00: <error reading variable>
|
status = <optimized out>
|
t = <error reading variable t (Cannot access memory at address 0x7f817513fda8)>
|
cfg = <error reading variable cfg (Cannot access memory at address 0x7f817513fd88)>
|
---Type <return> to continue, or q <return> to quit---
|
nm = <error reading variable nm (Cannot access memory at address 0x7f817513fd68)>
|
client = <error reading variable client (Cannot access memory at address 0x7f817513fd60)>
|
#7 __invoke_impl<void, CreateRemoveBucketCommandContext::create()::<lambda()>&> (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60
|
No locals.
|
#8 __invoke_r<void, CreateRemoveBucketCommandContext::create()::<lambda()>&> (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:110
|
No locals.
|
#9 std::_Function_handler<void (), CreateRemoveBucketCommandContext::create()::{lambda()#1}>::_M_invoke(std::_Any_data const&) ()
|
at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:291
|
No locals.
|
#10 0x0000000000610494 in operator() (this=0x7f816844c448) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/std_function.h:248
|
No locals.
|
#11 OneShotTask::run (this=0x7f816844c3d0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/daemon/one_shot_task.h:50
|
No locals.
|
#12 0x00000000009fb012 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#13 0x00000000009f8135 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f8139fe9540)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 12905361421149905}}
|
runAgain = false
|
proxy = @0x7f817534c540: <error reading variable>
|
#14 0x0000000000b53030 in operator() (this=0x7f8139fe9540)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f8139fe9540: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f817534c540, tiny = {
|
__data = "@\305\064u\201\177\000\000\000\000\000\000\000\000\000\000@\305\064u\201\177\000\000\207+\000\000\000\000\000\000\070\000\000\000\000\000\000\000\020\226\376\071\201\177\000", __align = {<No data fields>}}},
|
call_ = 0x9f87f0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0x9f6e20 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#15 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (
|
this=0x7f8175146c00, thread=..., task=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached.debug, CU 0x6458a2c, DIE 0x649e2ef>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 12905361421147623}}
|
stats = {expired = false, waitTime = {__r = 15480}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 12905361421132143}}, requestId = 0}
|
---Type <return> to continue, or q <return> to quit---
|
#16 0x0000000000b3adea in folly::CPUThreadPoolExecutor::threadRun (this=0x7f8175146c00, thread=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xc02ecb "CPUThreadPoolExecutor",
|
e_ = 0xc02ee0 ""}}}}
|
#17 0x0000000000b55fe9 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>)
|
at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#18 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#19 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#20 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#21 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#22 0x00000000009f7cb4 in operator() (this=0x7f81751590c0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
fn = @0x7f81751590c0: <error reading variable>
|
#23 operator() (__closure=0x7f81751590c0) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
|
_M_p = 0x7f8139fe96a0 "NonIoPool1"}, _M_string_length = 10, {_M_local_buf = "NonIoPool1\000\000\000\000\000",
|
_M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7f81751590c0)>
|
#24 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = @0x7f81751590c0: <error reading variable>
|
#25 0x00007f8177158d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
|
No locals.
|
#26 0x00007f8178f74ea5 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#27 0x00007f81768a18dd in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
QE-Test:
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/ankushr.ini bucket_storage=magma,rerun=false,bucket_eviction_policy=fullEviction,randomize_value=True,enable_dp=True,log_level=debug,infra_log_level=debug -t storage.magma.magma_kvstore.KVStoreTests.test_create_delete_bucket_n_times,num_items=10000,doc_size=256,nodes_init=4,sdk_timeout=60,vbuckets=1024,key_size=12,standard_buckets=10,magma_buckets=10,num_delete_buckets=8,test_itr=3,doc_ops="create",crash_memcached=True,GROUP=crash;P0'
|
Attachments
Issue Links
- relates to
-
-
- Closed
-