Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.2.0
-
7.2.0-5225
-
Untriaged
-
0
-
Unknown
Description
Steps To Recreate:
- Create a 4 node cluster(172.23.106.216(kv), 172.23.106.219(kv), 172.23.97.83(kv) 172.23.97.82 (n1q1 index))
- Create a magma buckets with (bucket_history_retention_seconds=0,bucket_history_retention_bytes=200000000000,)
replicas = 2, just default collections - Create 5000000 docs(doc_size = 256 bytes)
- Stop persistence on node 172.23.106.216
- Start doc ops on node 172.23.106.216
- Kill memcached on 172.23.106.216 to ensure rollback of replicas vbuckets on other kv nodes
- Repeat steps from 4 to 6 few times
- Observed Memcached crashed in Collections::VB::Filter::checkAndUpdateSystemEvent (this=<optimized out>, item=...)
Core Dump was found on node all three KV nodes 172.23.106.216, 172.23.106.219, 172.23.97.83 , Below is the back trace on node 172.23.106.216
8:19:52 PM
BackTrace:
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
|
set = {__val = {0, 3475201638897249123, 0 <repeats 14 times>}}
|
pid = <optimized out>
|
tid = <optimized out>
|
ret = <optimized out>
|
#1 0x00007f23a3584535 in __GI_abort () at abort.c:79
|
save_stage = 1
|
act = {__sigaction_handler = {sa_handler = 0x7f2389334320, sa_sigaction = 0x7f2389334320}, sa_mask = {__val = {139791042729600, 3432,
|
139791042712224, 139791042729731, 139791041415023, 51, 185727536, 139791062457992, 139790602435272, 185726496, 17473825302914,
|
106446469651664, 139791097801630, 139791042729600, 106034161095208, 105759275781040}}, sa_flags = -1536377840,
|
sa_restorer = 0xb11fa30 <stderr>}
|
sigs = {__val = {32, 0 <repeats 15 times>}}
|
#2 0x00007f23a46c163c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95
|
terminating = false
|
t = <optimized out>
|
#3 0x00000000034461ca in backtrace_terminate_handler ()
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88
|
No locals.
|
#4 0x00007f23a46cc8f6 in __cxxabiv1::__terminate (handler=<optimized out>)
|
at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48
|
No locals.
|
#5 0x00007f23a46cc961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58
|
No locals.
|
#6 0x00007f23a46ccbf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0xb11f408 <typeinfo for std::invalid_argument>,
|
dest=0x5db940 <std::invalid_argument::~invalid_argument()@plt>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95
|
globals = <optimized out>
|
header = 0x60d00002dc50
|
#7 0x00000000008ef740 in Collections::VB::Filter::checkAndUpdateSystemEvent (this=<optimized out>, item=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_filter.cc:278
|
No locals.
|
#8 0x000000000243f3f0 in Collections::VB::Filter::checkAndUpdateSlow (this=this@entry=0x618000190348, item=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_filter.cc:222
|
allowed = false
|
#9 0x00000000024cf115 in Collections::VB::Filter::checkAndUpdate (item=..., this=0x618000190348)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_filter.h:121
|
No locals.
|
#10 Collections::VB::Filter::checkAndUpdate (item=..., this=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/collections/vbucket_filter.h:110
|
No locals.
|
--Type <RET> for more, q to quit, c to continue without paging--
|
#11 ActiveStream::processItems (this=0x618000190090, outstandingItemsResult=..., streamMutex=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:1329
|
qi = <error reading variable>
|
__for_range = <optimized out>
|
__for_begin = <optimized out>
|
__for_end = <optimized out>
|
mutations = <optimized out>
|
visibleSeqno = <optimized out>
|
highNonVisibleSeqno = <optimized out>
|
#12 0x00000000024d1981 in ActiveStream::nextCheckpointItemTask (this=0x618000190090, streamMutex=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:975
|
res = <optimized out>
|
producer = <optimized out>
|
vbucket = <optimized out>
|
#13 0x00000000024d2c7e in ActiveStream::nextCheckpointItemTask (this=this@entry=0x618000190090)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream.cc:959
|
lh = <optimized out>
|
#14 0x0000000002534137 in ActiveStreamCheckpointProcessorTask::run (this=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/dcp/active_stream_checkpoint_processor_task.cc:52
|
as = 0x618000190090
|
rh = <optimized out>
|
streams = <optimized out>
|
iterations = <optimized out>
|
expected = <optimized out>
|
#15 0x00000000031356ea in GlobalTask::execute (this=0x61a0004efd00, threadName=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:98
|
guard = <optimized out>
|
executedAt = <optimized out>
|
scheduleOverhead = <optimized out>
|
start = <optimized out>
|
runAgain = <optimized out>
|
end = <optimized out>
|
runtime = <optimized out>
|
#16 0x000000000310585b in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f2389336c90)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:309
|
runAgain = <optimized out>
|
proxy = <error reading variable>
|
proxy = <optimized out>
|
runAgain = <optimized out>
|
--Type <RET> for more, q to quit, c to continue without paging--
|
_logger_ = <optimized out>
|
#17 folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:387
|
fn = @0x7f2389336c90: {__proxy = @0x60b00019f390}
|
#18 0x000000000313cdef in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f2389336c90)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42
|
fn = @0x7f2389336c90: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x60b00019f390, tiny = {
|
__data = "\220\363\031\000\260`\000\000\330\215\bqAGI\027\250\363S\223#\177\000\000\347{b\003\000\000\000\000\b\377\063\211#\177\000\000\306\323d\003\000\000\000", __align = {<No data fields>}}},
|
call_ = 0x3105410 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>,
|
exec_ = 0x30eb800 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
fn = <optimized out>
|
#19 operator() (__closure=<optimized out>)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/cancellable_cpu_executor.cc:42
|
task = <optimized out>
|
this = <optimized out>
|
#20 0x0000000003637700 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x7f2389336ec0)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f2389336ec0: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x61f000002a80, tiny = {
|
__data = "\200*\000\000\360a", '\000' <repeats 26 times>, "pH&\246#\177\000\000\240o3\211#\177\000", __align = {<No data fields>}}},
|
call_ = 0x313e7c0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,
|
exec_ = 0x31398e0 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}
|
fn = <optimized out>
|
#21 folly::ThreadPoolExecutor::runTask (this=this@entry=0x61f000002b80, thread=..., task=...)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {
|
prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 3174412600422520}}
|
stats = {expired = false, waitTime = {__r = 26720}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 3174412600395800}}, requestId = 0}
|
#22 0x0000000003621bda in folly::CPUThreadPoolExecutor::threadRun (this=0x61f000002b80, thread=...)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = -128 '\200', value = {<folly::ThreadPoolExecutor::Task> = {
|
func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x61f000002a80, tiny = {
|
__data = "\200*\000\000\360a", '\000' <repeats 26 times>, "pH&\246#\177\000\000\240o3\211#\177\000",
|
--Type <RET> for more, q to quit, c to continue without paging--
|
__align = {<No data fields>}}},
|
call_ = 0x313e7c0 <folly::detail::function::FunctionTraits<void()>::callSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Data &)>,
|
exec_ = 0x31398e0 <folly::detail::function::execSmall<CancellableCPUExecutor::add(GlobalTask*, folly::Func)::<lambda()> >(folly::detail::function::Op, folly::detail::function::Data *, folly::detail::function::Data *)>}, enqueueTime_ = {__d = {__r = 3174412600395800}},
|
expiration_ = {__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {
|
big = 0x7f2389336fd0, tiny = {
|
__data = "\320o3\211#\177\000\000yt\t\003\000\000\000\000X'&\246#\177", '\000' <repeats 18 times>, "\350m&q\344\017\000",
|
__align = {<No data fields>}}}
|
, call_ = 0x646512
|
<folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0},
|
context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false,
|
priority_ = 0 '\000', queueObserverPayload_ = 139790593938192}}, hasValue = true}}
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0x3d4c8c7 "CPUThreadPoolExecutor",
|
e_ = 0x3d4c8dc ""}}}}
|
#23 0x000000000363a6b9 in std::__invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>)
|
at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#24 std::__invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#25 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (__args=..., this=<optimized out>)
|
at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#26 std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)>::operator()<, void>() (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#27 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/cbdeps-ws/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#28 0x00000000030f95e7 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x6060000210e0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49
|
fn = <error reading variable>
|
fn = <optimized out>
|
#29 CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x6060000210e0)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:49
|
--Type <RET> for more, q to quit, c to continue without paging--
|
threadNameOpt = <optimized out>
|
func = <error reading variable func (Cannot access memory at address 0x6060000210e0)>
|
func = <optimized out>
|
threadNameOpt = <optimized out>
|
#30 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = <error reading variable>
|
#31 0x00000000030ecc87 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x60700029cc90)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54
|
fn = <error reading variable>
|
fn = <optimized out>
|
#32 folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x60700029cc80)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/PriorityThreadFactory.h:54
|
func = <error reading variable func (Cannot access memory at address 0x60700029cc90)>
|
priority = <error reading variable priority (Cannot access memory at address 0x60700029cc80)>
|
func = <optimized out>
|
priority = <optimized out>
|
#33 folly::detail::function::FunctionTraits<void ()>::callBig<folly::PriorityThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = <error reading variable>
|
#34 0x00000000030eb677 in folly::detail::function::FunctionTraits<void ()>::operator()() (this=0x60b000015550)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40
|
fn = <error reading variable>
|
fn = <optimized out>
|
#35 folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}::operator()() (__closure=0x60b000015550)
|
at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/executors/thread_factory/NamedThreadFactory.h:40
|
name = <error reading variable name (Cannot access memory at address 0x60b000015590)>
|
func = <error reading variable func (Cannot access memory at address 0x60b000015550)>
|
name = <optimized out>
|
func = <optimized out>
|
#36 std::__invoke_impl<void, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_other, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&) (__f=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:60
|
No locals.
|
--Type <RET> for more, q to quit, c to continue without paging--
|
#37 std::__invoke<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(std::__invoke_result&&, (folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}&&)...) (__fn=...) at /opt/gcc-10.2.0/include/c++/10.2.0/bits/invoke.h:95
|
No locals.
|
#38 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x60b000015550) at /opt/gcc-10.2.0/include/c++/10.2.0/thread:264
|
No locals.
|
#39 std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> >::operator()() (
|
this=0x60b000015550) at /opt/gcc-10.2.0/include/c++/10.2.0/thread:271
|
No locals.
|
#40 std::thread::_State_impl<std::thread::_Invoker<std::tuple<folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}> > >::_M_run() (this=0x60b000015540) at /opt/gcc-10.2.0/include/c++/10.2.0/thread:215
|
No locals.
|
#41 0x00007f23a46f5d40 in std::execute_native_thread_routine (__p=0x60b000015540)
|
at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80
|
__t = <optimized out>
|
#42 0x00007f23a5e2cfa3 in start_thread (arg=<optimized out>) at pthread_create.c:486
|
ret = <optimized out>
|
pd = <optimized out>
|
now = <optimized out>
|
unwind_buf = {cancel_jmp_buf = {{jmp_buf = {139790602499840, 800351138724411860, 140725591647006, 140725591647007, 139790602499840,
|
140725591647168, -766604279673816620, -766701710085359148}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {
|
prev = 0x0, cleanup = 0x0, canceltype = 0}}}
|
not_first_call = <optimized out>
|
#43 0x00007f23a365aeff in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
|
No locals.
|
QE-TEST:
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/testexec.69645.ini bucket_storage=magma,dcp_services=n1ql:index,rerun=false,GROUP=P1;new_snapshot;default,divisor=10,randomize_value=True,bucket_eviction_policy=fullEviction,divisor=20,infra_log_level=debug,log_level=debug,enable_dp=false,get-cbcollect-info=True,autoCompactionDefined=true,bucket_history_retention_bytes=200000000000,upgrade_version=7.2.0-5225 -t storage.magma.magma_rollback.MagmaRollbackTests.test_magma_rollback_to_new_snapshot,num_items=5000000,doc_size=256,nodes_init=3,sdk_timeout=60,num_rollbacks=5,vbuckets=1024,rollback_items=100000,duration=2,key_size=12,doc_ops=create:expiry,replicas=2,GROUP=P1;new_snapshot;default'
|
|
|
Test Input params:
|
{'doc_size': '256', 'conf_file': 'conf/magma/rollback.conf', 'upgrade_version': '7.2.0-5225', 'spec': 'rollback', 'rerun': 'false', 'num_nodes': 4, 'rollback_items': '100000', 'duration': '2', 'GROUP': 'P1;new_snapshot;default', 'enable_dp': 'false', 'doc_ops': 'create:expiry', 'divisor': '20', 'sdk_timeout': '60', 'case_number': 1, 'dcp_services': 'n1ql:index', 'cluster_name': 'testexec.69645', 'bucket_history_retention_bytes': '200000000000', 'ini': '/tmp/testexec.69645.ini', 'get-cbcollect-info': 'True', 'replicas': '2', 'bucket_storage': 'magma', 'log_level': 'debug', 'vbuckets': '1024', 'bucket_eviction_policy': 'fullEviction', 'logs_folder': '/data/workspace/debian-p0-magma-vset00-00-rollback_to_new_snapshot_7.0_P1/logs/testrunner-23-Mar-04_04-50-40/test_1', 'num_rollbacks': '5', 'nodes_init': '3', 'num_items': '5000000', 'autoCompactionDefined': 'true', 'randomize_value': 'True', 'key_size': '12', 'infra_log_level': 'debug'}
|