Details
-
Bug
-
Resolution: Fixed
-
Critical
-
7.1.0
-
7.1.0-1745
-
Triaged
-
-
1
-
Unknown
-
KV 2021-Nov, KV 2021-Dec, KV 2022-Feb
Description
Steps:
- Step 1: Create a 9 node cluster
- Step 2: Create required buckets and collections.
- Step 3: Create 1000000 items sequentially
- Step 4: Update 1000000 RandonKey keys to create 50 percent fragmentation
- Step 5: Create 1000000 items sequentially
- Step 6: Update 1000000 RandonKey keys to create 50 percent fragmentation
- Step 7: Start a asny load with durability=Majority asynchronously.
- Step 7: Rebalance in with async Loading of docs. Abort and resume Rebl at 20%, 40%, 60%, 80%
- Step 8: Crash Magma/memc with async Loading of docs
- Step 9: Rebalance Out with async Loading of docs. Abort and resume Rebl at 20%, 40%. On Resumption rebalance failed. Memcached crashed.
|
172.23.121.126 |
running: //opt/couchbase/bin/minidump-2-core /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.dmp > /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core
|
|
|
running: gdb --batch /opt/couchbase/bin/memcached -c /opt/couchbase/var/lib/couchbase/crash/e299ceb3-8985-4ce0-0c4601b6-3fda1379.core -ex "bt full" -ex quit
|
|
|
Core was generated by `/opt/couchbase/bin/memcached -C /opt/couchbase/var/lib/couchbase/config/memcach'.
|
#0 0x00007f56e2643387 in raise () from /lib64/libc.so.6
|
#0 0x00007f56e2643387 in raise () from /lib64/libc.so.6
|
No symbol table info available.
|
#1 0x00007f56e2644a78 in abort () from /lib64/libc.so.6
|
No symbol table info available.
|
#2 0x00007f56e2f8e63c in __gnu_cxx::__verbose_terminate_handler() [clone .cold] () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#3 0x0000000000afd3db in backtrace_terminate_handler() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/utilities/terminate_handler.cc:88
|
No locals.
|
#4 0x00007f56e2f998f6 in __cxxabiv1::__terminate(void (*)()) () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#5 0x00007f56e2f99961 in std::terminate() () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#6 0x00007f56e2f99bf4 in __cxa_throw () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#7 0x00000000004c272e in ActiveDurabilityMonitor::State::updateHighCompletedSeqno (this=0x7f55ef0d6e00) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:1620
|
e = @0x7f5678000940: <incomplete type>
|
#8 0x00000000007f784c in ActiveDurabilityMonitor::commit(DurabilityMonitor::ActiveSyncWrite const&) () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:919
|
s = {holder_ = {<folly::LockedPtrBase<folly::Synchronized<std::unique_ptr<ActiveDurabilityMonitor::State, std::default_delete<ActiveDurabilityMonitor::State> >, folly::SharedMutexImpl<false, void, std::atomic, false, false> >, folly::SharedMutexImpl<false, void, std::atomic, false, false>, folly::LockPolicyExclusive>> = {parent_ = 0x7f54ec53f1e8}, static AllowsConcurrentAccess = false}}
|
key = <optimized out>
|
cHandle = {<Collections::VB::ReadHandle> = {readLock = {lock_ = 0x7f56d4395158, token_ = {type_ = folly::SharedMutexToken::DEFERRED_SHARED, slot_ = 8}}, manifest = 0x7f56d43950c0}, itr = {<folly::f14::detail::BaseIter<std::pair<CollectionID const, Collections::VB::ManifestEntry> const*, std::pair<CollectionID const, Collections::VB::ManifestEntry>*>> = {<No data fields>}, underlying_ = {itemPtr_ = 0x7f5650d1ad50, index_ = 8}}, key = {<DocKeyInterface<DocKey>> = {<No data fields>}, buffer = {static npos = <optimized out>, buf = 0x7f55f3c60d20 <Address 0x7f55f3c60d20 out of bounds>, len = 19}, encoding = Yes}}
|
prepareEnd = {__d = {__r = 8069434606129862}}
|
cookie = <optimized out>
|
result = <optimized out>
|
__func__ = "commit"
|
index = <optimized out>
|
commitDuration = <optimized out>
|
#9 0x00000000007fe273 in ActiveDurabilityMonitor::processCompletedSyncWriteQueue() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/durability/active_durability_monitor.cc:488
|
sw = {storage_ = {{emptyState = 0 '\000', value = {<DurabilityMonitor::SyncWrite> = {item = {value = 0x7f55e5e75f00}, startTime = {__d = {__r = 8069346703426060}}, status = ToCommit}, cookie = 0x0, firstChain = {ackCount = {<ThrowExceptionPolicy<unsigned char>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5683fed2e0 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000&V\177\000", _M_allocated_capacity = 7812730926796664437}}}, val = 0 '\000'}, chainPtr = 0x0}, secondChain = {ackCount = {<ThrowExceptionPolicy<unsigned char>> = {label = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5683fed310 "unlabelled"}, _M_string_length = 10, {_M_local_buf = "unlabelled\000&V\177\000", _M_allocated_capacity = 7812730926796664437}}}, val = 0 '\000'}, chainPtr = 0x0}, expiryTime = {<std::_Optional_base<std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1, 1000000000> > >, true, true>> = {<std::_Optional_base_impl<std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1, 1000000000> > >, std::_Optional_base<std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1, 1000000000> > >, true, true> >> = {<No data fields>}, _M_payload = {<std::_Optional_payload_base<std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1, 1000000000> > > >> = {_M_payload = {_M_empty = {<No data fields>}, _M_value = {__d = {__r = 0}}}, _M_engaged = false}, <No data fields>}}, <std::_Enable_copy_move<true, true, true, true, std::optional<std::chrono::time_point<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1, 1000000000> > > > >> = {<No data fields>}, <No data fields>}}}, hasValue = true}}
|
lock = <optimized out>
|
#10 0x000000000071f484 in VBucket::processResolvedSyncWrites() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vbucket.cc:710
|
rlh = {lock_ = 0x7f55e6baf7f8, token_ = {type_ = folly::SharedMutexToken::INLINE_SHARED, slot_ = 115}}
|
#11 0x0000000000840158 in VBNotifiableTask::run() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/engines/ep/src/vb_notifiable_task.cc:53
|
vb = {<std::__shared_ptr<VBucket, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<VBucket, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x7f55e6baf600, _M_refcount = {_M_pi = 0x7f56439d2f60}}, <No data fields>}
|
runtime = <optimized out>
|
startTime = {__d = {__r = 8069434606121978}}
|
pendingVb = {vbid = 494}
|
#12 0x0000000000a6d9d2 in GlobalTask::execute() () at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/globaltask.cc:68
|
guard = {previous = 0x0}
|
#13 0x0000000000a6abe5 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}::operator()() const (__closure=0x7f5683fed840) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:189
|
scheduleOverhead = <optimized out>
|
start = {__d = {__r = 8069434606120059}}
|
runAgain = false
|
proxy = @0x7f56e15c3c10: <error reading variable>
|
#14 0x0000000000bbf750 in operator() (this=0x7f5683fed840) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:416
|
fn = @0x7f5683fed840: {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f56e15c3c10, tiny = {__data = "\020<\\\341V\177\000\000\000\000\000\000\000\000\000\000\260\"<EV\177\000\000\063\313\001\000\000\000\000\000\220\000\000\000\000\000\000\000\020\331\376\203V\177\000", __align = {<No data fields>}}}, call_ = 0xa6b2a0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>, exec_ = 0xa699f0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}
|
#15 folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) (this=this@entry=0x7f56e1157c00, thread=..., task=task@entry=<unknown type in /usr/lib/debug/opt/couchbase/bin/memcached-7.1.0-1745.x86_64.debug, CU 0xa026ee0, DIE 0xa0aae15>) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/ThreadPoolExecutor.cpp:97
|
rctx = {prev_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}
|
startTime = {__d = {__r = 8069434606115758}}
|
stats = {expired = false, waitTime = {__r = 14806}, runTime = {__r = 0}, enqueueTime = {__d = {__r = 8069434606100952}}, requestId = 0}
|
#16 0x0000000000ba750a in folly::CPUThreadPoolExecutor::threadRun (this=0x7f56e1157c00, thread=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/executors/CPUThreadPoolExecutor.cpp:265
|
task = {storage_ = {{emptyState = 16 '\020', value = {<folly::ThreadPoolExecutor::Task> = {func_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x7f56e15c3c10, tiny = {__data = "\020<\\\341V\177\000\000\000\000\000\000\000\000\000\000\260\"<EV\177\000\000\063\313\001\000\000\000\000\000\220\000\000\000\000\000\000\000\020\331\376\203V\177\000", __align = {<No data fields>}}}, call_ = 0xa6b2a0 <folly::detail::function::FunctionTraits<void ()>::callSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Data&)>, exec_ = 0xa699f0 <folly::detail::function::execSmall<FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#2}>(folly::detail::function::Op, folly::detail::function::Data*, folly::detail::function::Data)>}, enqueueTime_ = {__d = {__r = 8069434606100952}}, expiration_ = {__r = 0}, expireCallback_ = {<folly::detail::function::FunctionTraits<void()>> = {<No data fields>}, data_ = {big = 0x1cb33, tiny = {__data = "3\313\001\000\000\000\000\000\333)\242", '\000' <repeats 13 times>, "_\276\037\345V\177\000\000p\352>\316V\177\000\000@V@\345V\177\000", __align = {<No data fields>}}}, call_ = 0x463cfd <folly::detail::function::FunctionTraits<void ()>::uninitCall(folly::detail::function::Data&)>, exec_ = 0x0}, context_ = {<std::__shared_ptr<folly::RequestContext, (__gnu_cxx::_Lock_policy)2>> = {<std::__shared_ptr_access<folly::RequestContext, (__gnu_cxx::_Lock_policy)2, false, false>> = {<No data fields>}, _M_ptr = 0x0, _M_refcount = {_M_pi = 0x0}}, <No data fields>}}, poison = false, priority_ = 0 '\000', queueObserverPayload_ = 140011140538112}}, hasValue = true}}
|
guard = {list_ = {forbid = true, prev = 0x0, curr = {name = {static npos = <optimized out>, b_ = 0xc7b62b "CPUThreadPoolExecutor", e_ = 0xc7b640 ""}}}}
|
#17 0x0000000000bc2709 in __invoke_impl<void, void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__t=<optimized out>, __f=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:73
|
No locals.
|
#18 __invoke<void (folly::ThreadPoolExecutor::*&)(std::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::shared_ptr<folly::ThreadPoolExecutor::Thread>&> (__fn=<optimized out>) at /usr/local/include/c++/7.3.0/bits/invoke.h:95
|
No locals.
|
#19 __call<void, 0, 1> (__args=<optimized out>, this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:467
|
No locals.
|
#20 operator()<> (this=<optimized out>) at /usr/local/include/c++/7.3.0/functional:551
|
No locals.
|
#21 folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/cbdeps-platform-build-old/deps/packages/build/folly/folly-prefix/src/folly/folly/Function.h:401
|
fn = <optimized out>
|
#22 0x0000000000a6a874 in operator() (this=0x7f56e15e4080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
fn = @0x7f56e15e4080: <error reading variable>
|
#23 operator() (__closure=0x7f56e15e4080) at /home/couchbase/jenkins/workspace/couchbase-server-unix/kv_engine/executor/folly_executorpool.cc:47
|
threadNameOpt = {storage_ = {{emptyState = -96 '\240', value = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5683fed9a0 "NonIoPool0"}, _M_string_length = 10, {_M_local_buf = "NonIoPool0\000\000\000\000\000", _M_allocated_capacity = 8029725099528449870}}}, hasValue = true}}
|
func = <error reading variable func (Cannot access memory at address 0x7f56e15e4080)>
|
#24 folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) (p=...) at /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/tlm/deps/folly.exploded/include/folly/Function.h:401
|
fn = @0x7f56e15e4080: <error reading variable>
|
#25 0x00007f56e2fc2d40 in execute_native_thread_routine () from /opt/couchbase/bin/../lib/libstdc++.so.6
|
No symbol table info available.
|
#26 0x00007f56e4dcaea5 in start_thread () from /lib64/libpthread.so.0
|
No symbol table info available.
|
#27 0x00007f56e270b8dd in clone () from /lib64/libc.so.6
|
No symbol table info available.
|
|
QE Test |
guides/gradlew --refresh-dependencies testrunner -P jython=/opt/jython/bin/jython -P 'args=-i /tmp/magma_temp_job4.ini -p bucket_storage=magma,bucket_eviction_policy=fullEviction,rerun=False,iterations=2,sdk_timeout=60,log_level=debug,infra_log_level=debug,skip_cleanup=True -t aGoodDoctor.Hospital.Murphy.SystemTestMagma,nodes_init=9,graceful=True,skip_cleanup=True,num_items=1000000,num_buckets=1,bucket_names=GleamBook,doc_size=1024,key_size=18,assert_crashes_on_load=True,num_collections=20,maxttl=10,num_indexes=5,pc=10,index_nodes=5,query_nodes=0,cbas_nodes=0,fts_nodes=0,ops_rate=50000,doc_ops=create:update:delete:read,durability=Majority,crashes=10 -m rest'
|
Attachments
Issue Links
- is duplicated by
-
-
- Closed
-
- relates to
-
-
- Closed
-
-
-
- Closed
-
-
-
- Closed
-