Details
-
Bug
-
Resolution: Fixed
-
Critical
-
Cheshire-Cat
-
7.0.0-4554
-
Untriaged
-
1
-
Unknown
Description
Steps to repro:*
- Created a 5 node cluster
- Create 10 scopes and 20 collections in each scope
- Created 15K items in each collection
- Gracefully failed over 2 nodes(recovery type == Full)
- Observed Memcached crashes in GlobalTask::execute (this=<optimized out>)
BackTrace:
(gdb) bt full
|
#0 0x00007fe435ae2387 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007fe435ae3a78 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00007fe436640195 in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x000000000055ca82 in backtrace_terminate_handler () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:93
|
No locals.
|
#4 0x00007fe43663df86 in __cxxabiv1::__terminate (handler=<optimized out>) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:47
|
No locals.
|
#5 0x00007fe43663dfd1 in std::terminate () at /tmp/deploy/gcc-7.3.0/libstdc++-v3/libsupc++/eh_terminate.cc:57
|
No locals.
|
#6 0x00007fe439e51713 in GlobalTask::execute (this=<optimized out>) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/globaltask.cc:78
|
No locals.
|
#7 0x00007fe439e4b602 in operator() (__closure=0x7fe3bf7e5040) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:195
|
executedAt = <optimized out>
|
end = <optimized out>
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 817300638303029}}
|
runAgain = false
|
proxy = @0x7fe36a3b85d0: <error reading variable>
|
#8 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:387
|
fn = @0x7fe3bf7e5040: {__proxy = @0x7fe36a3b85d0}
|
#9 0x00007fe439fc2f90 in operator() (this=0x7fe3bf7e5040)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7fe3bf7e5040: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7fe36a3b85d0, tiny = {
|
__data = "\320\205;j\343\177\000\000\000\000\000\000\000\000\000\000\320-Hj\343\177\000\000\200&\000\330\343\177\000\000\001\000\000\000\000\000\000\000\220P~\277\343\177\000", __align = {<No data fields>}}},
|
call_ = 0x7fe439e4b540 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Data&)>,
|
exec_ = 0x7fe439e49090 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
task=<unknown type in /usr/lib/debug/opt/couchbase/lib/libep.so.debug, CU 0x35631ce, DIE 0x35a9612>)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 817300638300175}}
|
stats = {expired = false, waitTime = {__r = 29167371}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 817300609132804}}, requestId = 0}
|
#11 0x00007fe439fab46a in folly::CPUThreadPoolExecutor::threadRun (this=0x7fe430924400, thread=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0x7fe43a03325b "CPUThreadPoolExecutor", e_ = 0x7fe43a033270 ""}}}}
|
#12 0x00007fe439fc5f49 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_p---Type <return> to continue, or q <return> to quit---
|
tr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#13 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#14 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#15 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#16 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#17 0x00007fe439e49ec8 in operator() (this=0x7fe430048100)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416
|
fn = @0x7fe430048100: <error reading variable>
|
#18 operator() (__closure=0x7fe430048100) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/folly_executorpool.cc:54
|
threadNameOpt = {storage_ = {{emptyState = -112 '\220', value = {static npos = 18446744073709551615,
|
_M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7fe3bf7e5190 "WriterPool0"},
|
_M_string_length = 11, {_M_local_buf = "WriterPool0\000\000\000\000", _M_allocated_capacity = 8021036716417184343}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7fe430048100)>
|
#19 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (
|
p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = @0x7fe430048100: <error reading variable>
|
#20 0x00007fe439e49da3 in operator() (this=0x7fe43080e090)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:416
|
fn = @0x7fe43080e090: <error reading variable>
|
#21 operator() (__closure=0x7fe43080e080)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54
|
func = <error reading variable func (Cannot access memory at address 0x7fe43080e090)>
|
priority = <error reading variable priority (Cannot access memory at address 0x7fe43080e080)>
|
#22 folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&)
|
(p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = @0x7fe43080e080: <error reading variable>
|
#23 0x00007fe436668dcf in std::execute_native_thread_routine (__p=0x7fe430a60b30) at /tmp/deploy/gcc-7.3.0/libstdc++-v3/src/c++11/thread.cc:83
|
__t = {_M_t = {
|
_M_t = {<std::_Tuple_impl<0, std::thread::_State*, std::default_delete<std::thread::_State> >> = {<std::_Tuple_impl<1, std::default_delete<std::thread::_State> >> = {<std::_Head_base<1, std::default_delete<std::thread::_State>, true>> = {<std::default_delete<std::thread::_State>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::_Head_base<0, std::thread::_State*, false>> = {_M_head_impl = 0x7fe430a60b30}, <No data fields>}, <No data fields>}}}
|
#24 0x00007fe435e81ea5 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#25 0x00007fe435baa8dd in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
QE test:
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.1431.ini bucket_storage=magma,get-cbcollect-info=True,bucket_eviction_policy=fullEviction,randomize_value=True,rerun=false,infra_log_level=debug,log_level=debug,get-cbcollect-info=True,infra_log_level=critical,log_level=error,upgrade_version=7.0.0-4554 -t bucket_collections.collections_rebalance.CollectionsRebalance.test_data_load_collections_with_graceful_failover_recovery,nodes_init=5,nodes_failover=2,recovery_type=full,bucket_spec=magma_templates.buckets_all_membase_for_rebalance_tests_more_collections,data_load_spec=volume_test_load_with_CRUD_on_collections,data_load_stage=during,GROUP=failover_with_collection_crud'
|
Jenkins Job:
http://qa.sc.couchbase.com/job/test_suite_executor-TAF/95921/consoleText