Details
-
Bug
-
Resolution: Duplicate
-
Critical
-
None
-
7.0.2
-
Untriaged
-
Centos 64-bit
-
-
1
-
Unknown
Description
Build - 7.0.2-6643
Test :
-test tests/integration/cheshirecat/test_cheshirecat_kv_gsi_coll_xdcr_backup_sgw_fts_itemct_txns_eventing_cbas_scale3.yml -scope tests/integration/cheshirecat/scope_cheshirecat_with_backup.yml
|
Day - 5
Scale - 3
On 172.23.104.155
diag.log
2021-09-07T16:27:37.729-07:00, ns_vbucket_mover:0:critical:message(ns_1@172.23.104.155) - Worker <0.12308.1499> (for action {move, |
{843, |
['ns_1@172.23.104.69', |
'ns_1@172.23.99.25'], |
['ns_1@172.23.106.100', |
'ns_1@172.23.104.69'], |
[]}}) exited with reason {unexpected_exit,
|
{'EXIT', |
<0.17158.1499>, |
{{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,
|
{error,
|
closed}},
|
[{mc_client_binary,
|
cmd_vocal_recv,
|
5, |
[{file,
|
"src/mc_client_binary.erl"}, |
{line,
|
152}]}, |
{mc_client_binary,
|
auth,
|
2, |
[{file,
|
"src/mc_client_binary.erl"}, |
{line,
|
193}]}, |
{dcp_proxy,
|
connect,
|
5, |
[{file,
|
"src/dcp_proxy.erl"}, |
{line,
|
254}]}, |
{dcp_proxy,
|
maybe_connect,
|
2, |
[{file,
|
"src/dcp_proxy.erl"}, |
{line,
|
236}]}, |
{dcp_producer_conn,
|
init,
|
2, |
[{file,
|
"src/dcp_producer_conn.erl"}, |
{line,
|
25}]}, |
{dcp_proxy,
|
init,
|
1, |
[{file,
|
"src/dcp_proxy.erl"}, |
{line,
|
52}]}, |
{gen_server,
|
init_it,
|
2, |
[{file,
|
"gen_server.erl"}, |
{line,
|
374}]}, |
{gen_server,
|
init_it,
|
6, |
[{file,
|
"gen_server.erl"}, |
{line,
|
342}]}]}}}, |
[{dcp_replicator,
|
init,
|
1, |
[{file,
|
"src/dcp_replicator.erl"}, |
{line,
|
50}]}, |
{gen_server,
|
init_it,
|
2, |
[{file,
|
"gen_server.erl"}, |
{line,
|
374}]}, |
{gen_server,
|
init_it,
|
6, |
[{file,
|
"gen_server.erl"}, |
{line,
|
342}]}, |
{proc_lib,
|
init_p_do_apply,
|
3, |
[{file,
|
"proc_lib.erl"}, |
{line,
|
249}]}]}, |
{child,
|
undefined,
|
{'ns_1@172.23.104.157', |
[collections,
|
del_times,
|
del_user_xattr,
|
json,
|
set_consumer_name,
|
snappy,
|
xattr]},
|
{dcp_replicator,
|
start_link,
|
['ns_1@172.23.104.157', |
"bucket8", |
[collections,
|
del_times,
|
del_user_xattr,
|
json,
|
set_consumer_name,
|
snappy,
|
xattr]]},
|
temporary,
|
60000, |
worker,
|
[dcp_replicator]}}}},
|
[{dcp_sup,
|
start_replicator,
|
2, |
[{file,
|
"src/dcp_sup.erl"}, |
{line,
|
51}]}, |
{dcp_sup,
|
'-manage_replicators/2-lc$^3/1-3-', |
2, |
[{file,
|
"src/dcp_sup.erl"}, |
{line,
|
101}]}, |
{dcp_replication_manager,
|
handle_call,
|
3, |
[{file,
|
"src/dcp_replication_manager.erl"}, |
{line,
|
83}]}, |
{gen_server,
|
try_handle_call,
|
4, |
[{file,
|
"gen_server.erl"}, |
{line,
|
661}]}, |
{gen_server,
|
handle_msg,
|
6, |
[{file,
|
"gen_server.erl"}, |
{line,
|
690}]}, |
{proc_lib,
|
init_p_do_apply,
|
3, |
[{file,
|
"proc_lib.erl"}, |
{line,
|
249}]}]}, |
{gen_server,
|
call,
|
['dcp_replication_manager-bucket8', |
{manage_replicators,
|
['ns_1@172.23.104.157']}, |
infinity]}},
|
{gen_server,
|
call,
|
['replication_manager-bucket8', |
{change_vbucket_replication,
|
673, |
'ns_1@172.23.104.157'}, |
infinity]}},
|
{gen_server,
|
call,
|
[{'janitor_agent-bucket8', |
'ns_1@172.23.106.100'}, |
{if_rebalance,
|
<0.10102.1499>, |
{update_vbucket_state,
|
843, |
active,
|
undefined,
|
undefined,
|
undefined}},
|
infinity]}}}}
|
2021-09-07T16:27:37.766-07:00, ns_log:0:info:message(ns_1@172.23.104.157) - Service 'memcached' exited with status 134. Restarting. Messages: |
2021-09-07T16:27:16.227375-07:00 CRITICAL #18 /opt/couchbase/bin/memcached() [0x400000+0x2a77f7] |
2021-09-07T16:27:16.227389-07:00 CRITICAL #19 /opt/couchbase/bin/memcached() [0x400000+0x2a2940] |
2021-09-07T16:27:16.227411-07:00 CRITICAL #20 /opt/couchbase/bin/memcached() [0x400000+0x716650] |
2021-09-07T16:27:16.227436-07:00 CRITICAL #21 /opt/couchbase/bin/memcached() [0x400000+0x6fe40a] |
2021-09-07T16:27:16.227496-07:00 CRITICAL #22 /opt/couchbase/bin/memcached() [0x400000+0x719609] |
2021-09-07T16:27:16.227516-07:00 CRITICAL #23 /opt/couchbase/bin/memcached() [0x400000+0x2a101d] |
2021-09-07T16:27:16.227553-07:00 CRITICAL #24 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fbdac993000+0xcdd40] |
2021-09-07T16:27:16.227562-07:00 CRITICAL #25 /lib64/libpthread.so.0() [0x7fbdae642000+0x7e25] |
2021-09-07T16:27:16.227597-07:00 CRITICAL #26 /lib64/libc.so.6(clone+0x6d) [0x7fbdac0ac000+0xfebad] |
[*** LOG ERROR ***] [2021-09-07 16:27:16] [spdlog_file_logger] async log: thread pool doesn't exist anymore |
2021-09-07T16:27:37.778-07:00, ns_orchestrator:0:critical:message(ns_1@172.23.104.155) - Rebalance exited with reason {mover_crashed, |
{unexpected_exit,
|
{'EXIT',<0.17158.1499>, |
{{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,{error,closed}},
|
[{mc_client_binary,cmd_vocal_recv,
|
5, |
[{file,
|
"src/mc_client_binary.erl"}, |
{line,152}]}, |
{mc_client_binary,auth,2, |
[{file,
|
"src/mc_client_binary.erl"}, |
{line,193}]}, |
{dcp_proxy,connect,5, |
[{file,"src/dcp_proxy.erl"}, |
{line,254}]}, |
{dcp_proxy,maybe_connect,2, |
[{file,"src/dcp_proxy.erl"}, |
{line,236}]}, |
{dcp_producer_conn,init,2, |
[{file,
|
"src/dcp_producer_conn.erl"}, |
{line,25}]}, |
{dcp_proxy,init,1, |
[{file,"src/dcp_proxy.erl"}, |
{line,52}]}, |
{gen_server,init_it,2, |
[{file,"gen_server.erl"}, |
{line,374}]}, |
{gen_server,init_it,6, |
[{file,"gen_server.erl"}, |
{line,342}]}]}}}, |
[{dcp_replicator,init,1, |
[{file,"src/dcp_replicator.erl"}, |
{line,50}]}, |
{gen_server,init_it,2, |
[{file,"gen_server.erl"}, |
{line,374}]}, |
{gen_server,init_it,6, |
[{file,"gen_server.erl"}, |
{line,342}]}, |
{proc_lib,init_p_do_apply,3, |
[{file,"proc_lib.erl"}, |
{line,249}]}]}, |
{child,undefined,
|
{'ns_1@172.23.104.157', |
[collections,del_times,
|
del_user_xattr,json,
|
set_consumer_name,snappy,xattr]},
|
{dcp_replicator,start_link,
|
['ns_1@172.23.104.157',"bucket8", |
[collections,del_times,
|
del_user_xattr,json,
|
set_consumer_name,snappy,xattr]]},
|
temporary,60000,worker, |
[dcp_replicator]}}}},
|
[{dcp_sup,start_replicator,2, |
[{file,"src/dcp_sup.erl"},{line,51}]}, |
{dcp_sup,
|
'-manage_replicators/2-lc$^3/1-3-',2, |
[{file,"src/dcp_sup.erl"},{line,101}]}, |
{dcp_replication_manager,handle_call,3, |
[{file,
|
"src/dcp_replication_manager.erl"}, |
{line,83}]}, |
{gen_server,try_handle_call,4, |
[{file,"gen_server.erl"},{line,661}]}, |
{gen_server,handle_msg,6, |
[{file,"gen_server.erl"},{line,690}]}, |
{proc_lib,init_p_do_apply,3, |
[{file,"proc_lib.erl"},{line,249}]}]}, |
{gen_server,call,
|
['dcp_replication_manager-bucket8', |
{manage_replicators,
|
['ns_1@172.23.104.157']}, |
infinity]}},
|
{gen_server,call,
|
['replication_manager-bucket8', |
{change_vbucket_replication,673, |
'ns_1@172.23.104.157'}, |
infinity]}},
|
{gen_server,call,
|
[{'janitor_agent-bucket8', |
'ns_1@172.23.106.100'}, |
{if_rebalance,<0.10102.1499>, |
{update_vbucket_state,843,active, |
undefined,undefined,undefined}},
|
infinity]}}}}}.
|
Rebalance Operation Id = c5a91f24a46828f69bd5d900456f28a7
|
On 172.23.104.157
memcached.log
2021-09-07T16:27:16.226531-07:00 CRITICAL Breakpad caught a crash (Couchbase version 7.0.2-6643). Writing crash dump to /opt/couchbase/var/lib/couchbase/crash/f2a10f35-2839-4719-d5fcee9f-de539915.dmp before terminating. |
2021-09-07T16:27:16.226569-07:00 CRITICAL Stack backtrace of crashed thread: |
2021-09-07T16:27:16.226902-07:00 CRITICAL #0 /opt/couchbase/bin/memcached() [0x400000+0x67ae73] |
2021-09-07T16:27:16.226932-07:00 CRITICAL #1 /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler12GenerateDumpEPNS0_12CrashContextE+0x3ea) [0x400000+0x69be8a] |
2021-09-07T16:27:16.226963-07:00 CRITICAL #2 /opt/couchbase/bin/memcached(_ZN15google_breakpad16ExceptionHandler13SignalHandlerEiP9siginfo_tPv+0xb8) [0x400000+0x69c1c8] |
2021-09-07T16:27:16.226973-07:00 CRITICAL #3 /lib64/libpthread.so.0() [0x7fbdae642000+0xf6d0] |
2021-09-07T16:27:16.226998-07:00 CRITICAL #4 /lib64/libc.so.6(gsignal+0x37) [0x7fbdac0ac000+0x36277] |
2021-09-07T16:27:16.227018-07:00 CRITICAL #5 /lib64/libc.so.6(abort+0x148) [0x7fbdac0ac000+0x37968] |
2021-09-07T16:27:16.227054-07:00 CRITICAL #6 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fbdac993000+0x9963c] |
2021-09-07T16:27:16.227080-07:00 CRITICAL #7 /opt/couchbase/bin/memcached() [0x400000+0x688aaf] |
2021-09-07T16:27:16.227104-07:00 CRITICAL #8 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fbdac993000+0xa48f6] |
2021-09-07T16:27:16.227217-07:00 CRITICAL #9 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fbdac993000+0xa4961] |
2021-09-07T16:27:16.227241-07:00 CRITICAL #10 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fbdac993000+0xa4bf4] |
2021-09-07T16:27:16.227261-07:00 CRITICAL #11 /opt/couchbase/bin/memcached() [0x400000+0xb3197] |
2021-09-07T16:27:16.227276-07:00 CRITICAL #12 /opt/couchbase/bin/memcached() [0x400000+0xce670] |
2021-09-07T16:27:16.227294-07:00 CRITICAL #13 /opt/couchbase/bin/memcached() [0x400000+0x482e29] |
2021-09-07T16:27:16.227310-07:00 CRITICAL #14 /opt/couchbase/bin/memcached() [0x400000+0x485d7d] |
2021-09-07T16:27:16.227326-07:00 CRITICAL #15 /opt/couchbase/bin/memcached() [0x400000+0x4865cc] |
2021-09-07T16:27:16.227340-07:00 CRITICAL #16 /opt/couchbase/bin/memcached() [0x400000+0x4866e4] |
2021-09-07T16:27:16.227352-07:00 CRITICAL #17 /opt/couchbase/bin/memcached() [0x400000+0x492bb9] |
2021-09-07T16:27:16.227375-07:00 CRITICAL #18 /opt/couchbase/bin/memcached() [0x400000+0x2a77f7] |
2021-09-07T16:27:16.227389-07:00 CRITICAL #19 /opt/couchbase/bin/memcached() [0x400000+0x2a2940] |
2021-09-07T16:27:16.227411-07:00 CRITICAL #20 /opt/couchbase/bin/memcached() [0x400000+0x716650] |
2021-09-07T16:27:16.227436-07:00 CRITICAL #21 /opt/couchbase/bin/memcached() [0x400000+0x6fe40a] |
2021-09-07T16:27:16.227496-07:00 CRITICAL #22 /opt/couchbase/bin/memcached() [0x400000+0x719609] |
2021-09-07T16:27:16.227516-07:00 CRITICAL #23 /opt/couchbase/bin/memcached() [0x400000+0x2a101d] |
2021-09-07T16:27:16.227553-07:00 CRITICAL #24 /opt/couchbase/bin/../lib/libstdc++.so.6() [0x7fbdac993000+0xcdd40] |
2021-09-07T16:27:16.227562-07:00 CRITICAL #25 /lib64/libpthread.so.0() [0x7fbdae642000+0x7e25] |
2021-09-07T16:27:16.227597-07:00 CRITICAL #26 /lib64/libc.so.6(clone+0x6d) [0x7fbdac0ac000+0xfebad] |
Stack trace
#0 0x00007fbdac0e2277 in raise () from /lib64/libc.so.6 |
#1 0x00007fbdac0e3968 in abort () from /lib64/libc.so.6 |
#2 0x00007fbdaca2c63c in __gnu_cxx::__verbose_terminate_handler () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/vterminate.cc:95 |
#3 0x0000000000a88aaf in backtrace_terminate_handler() () |
#4 0x00007fbdaca378f6 in __cxxabiv1::__terminate(void (*)()) () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:48 |
#5 0x00007fbdaca37961 in std::terminate () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_terminate.cc:58 |
#6 0x00007fbdaca37bf4 in __cxxabiv1::__cxa_throw (obj=<optimized out>, tinfo=0xf6edd0 <typeinfo for boost::exception_detail::error_info_injector<std::logic_error>>, dest=0x7e5030 <boost::exception_detail::error_info_injector<std::logic_error>::~error_info_injector()>) at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/libsupc++/eh_throw.cc:95 |
#7 0x00000000004b3197 in void cb::throwWithTrace<std::logic_error>(std::logic_error const&) () |
#8 0x00000000004ce670 in ThrowExceptionPolicy<unsigned long>::nonMonotonic(unsigned long const&, unsigned long const&) [clone .isra.0] [clone .cold] () |
#9 0x0000000000882e29 in ActiveStream::snapshot(CheckpointType, std::deque<std::unique_ptr<DcpResponse, std::default_delete<DcpResponse> >, std::allocator<std::unique_ptr<DcpResponse, std::default_delete<DcpResponse> > > >&, std::optional<unsigned long>, unsigned long, std::optional<unsigned long>) () |
#10 0x0000000000885d7d in ActiveStream::processItems(ActiveStream::OutstandingItemsResult&, std::lock_guard<std::mutex> const&) () |
#11 0x00000000008865cc in ActiveStream::nextCheckpointItemTask(std::lock_guard<std::mutex> const&) () |
#12 0x00000000008866e4 in ActiveStream::nextCheckpointItemTask() () |
#13 0x0000000000892bb9 in ActiveStreamCheckpointProcessorTask::run() () |
#14 0x00000000006a77f7 in GlobalTask::execute() () |
#15 0x00000000006a2940 in FollyExecutorPool::TaskProxy::scheduleViaCPUPool()::{lambda()#1}::operator()() const () |
#16 0x0000000000b16650 in folly::ThreadPoolExecutor::runTask(std::shared_ptr<folly::ThreadPoolExecutor::Thread> const&, folly::ThreadPoolExecutor::Task&&) () |
#17 0x0000000000afe40a in folly::CPUThreadPoolExecutor::threadRun(std::shared_ptr<folly::ThreadPoolExecutor::Thread>) () |
#18 0x0000000000b19609 in void folly::detail::function::FunctionTraits<void ()>::callBig<std::_Bind<void (folly::ThreadPoolExecutor::*(folly::ThreadPoolExecutor*, std::shared_ptr<folly::ThreadPoolExecutor::Thread>))(std::shared_ptr<folly::ThreadPoolExecutor::Thread>)> >(folly::detail::function::Data&) () |
#19 0x00000000006a101d in void folly::detail::function::FunctionTraits<void ()>::callBig<CBRegisteredThreadFactory::newThread(folly::Function<void ()>&&)::{lambda()#1}>(folly::detail::function::Data&) () |
#20 0x00007fbdaca60d40 in execute_native_thread_routine () at /tmp/deploy/objdir/../gcc-10.2.0/libstdc++-v3/src/c++11/thread.cc:80 |
#21 0x00007fbdae649e25 in start_thread () from /lib64/libpthread.so.0 |
#22 0x00007fbdac1aabad in clone () from /lib64/libc.so.6 |
Attachments
Issue Links
- duplicates
-
-
- Closed
-