Details
-
Bug
-
Resolution: Incomplete
-
Critical
-
None
-
7.0.2
-
Untriaged
-
1
-
Unknown
Description
7.0.2-6522
Test:
-test tests/integration/cheshirecat/test_cheshirecat_kv_gsi_coll_xdcr_backup_sgw_fts_itemct_txns_eventing_cbas_scale3.yml -scope tests/integration/cheshirecat/scope_cheshirecat_with_backup.yml
Scale 3
1st iteration
Following rebalance exit observed:
2021-08-17T20:48:16.644-07:00, ns_vbucket_mover:0:critical:message(ns_1@172.23.97.74) - Worker <0.19173.325> (for action {move,
|
{811,
|
['ns_1@172.23.123.25',undefined],
|
['ns_1@172.23.97.112',
|
'ns_1@172.23.123.25'],
|
[]}}) exited with reason {unexpected_exit,
|
{'EXIT',
|
<0.19827.325>,
|
{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,
|
{error,
|
timeout}},
|
[{mc_client_binary,
|
cmd_vocal_recv,
|
5,
|
[{file,
|
"src/mc_client_binary.erl"},
|
{line,
|
152}]},
|
{mc_client_binary,
|
auth,
|
2,
|
[{file,
|
"src/mc_client_binary.erl"},
|
{line,
|
193}]},
|
{dcp_proxy,
|
connect,
|
5,
|
[{file,
|
"src/dcp_proxy.erl"},
|
{line,
|
254}]},
|
{dcp_proxy,
|
maybe_connect,
|
2,
|
[{file,
|
"src/dcp_proxy.erl"},
|
{line,
|
236}]},
|
{dcp_producer_conn,
|
init,
|
2,
|
[{file,
|
"src/dcp_producer_conn.erl"},
|
{line,
|
25}]},
|
{dcp_proxy,
|
init,
|
1,
|
[{file,
|
"src/dcp_proxy.erl"},
|
{line,
|
52}]},
|
{gen_server,
|
init_it,
|
2,
|
[{file,
|
"gen_server.erl"},
|
{line,
|
374}]},
|
{gen_server,
|
init_it,
|
6,
|
[{file,
|
"gen_server.erl"},
|
{line,
|
342}]}]}}},
|
[{dcp_replicator,
|
init,
|
1,
|
[{file,
|
"src/dcp_replicator.erl"},
|
{line,
|
54}]},
|
{gen_server,
|
init_it,
|
2,
|
[{file,
|
"gen_server.erl"},
|
{line,
|
374}]},
|
{gen_server,
|
init_it,
|
6,
|
[{file,
|
"gen_server.erl"},
|
{line,
|
342}]},
|
{proc_lib,
|
init_p_do_apply,
|
3,
|
[{file,
|
"proc_lib.erl"},
|
{line,
|
249}]}]},
|
{child,
|
undefined,
|
{'ns_1@172.23.120.73',
|
[collections,
|
del_times,
|
del_user_xattr,
|
json,
|
set_consumer_name,
|
snappy,
|
xattr]},
|
{dcp_replicator,
|
start_link,
|
['ns_1@172.23.120.73',
|
"bucket6",
|
[collections,
|
del_times,
|
del_user_xattr,
|
json,
|
set_consumer_name,
|
snappy,
|
xattr]]},
|
temporary,
|
60000,
|
worker,
|
[dcp_replicator]}}}},
|
[{dcp_sup,
|
start_replicator,
|
2,
|
[{file,
|
"src/dcp_sup.erl"},
|
{line,
|
51}]},
|
{dcp_sup,
|
'-manage_replicators/2-lc$^3/1-3-',
|
2,
|
[{file,
|
"src/dcp_sup.erl"},
|
{line,
|
101}]},
|
{dcp_replication_manager,
|
handle_call,
|
3,
|
[{file,
|
"src/dcp_replication_manager.erl"},
|
{line,
|
83}]},
|
{gen_server,
|
try_handle_call,
|
4,
|
[{file,
|
"gen_server.erl"},
|
{line,
|
661}]},
|
{gen_server,
|
handle_msg,
|
6,
|
[{file,
|
"gen_server.erl"},
|
{line,
|
690}]},
|
{proc_lib,
|
init_p_do_apply,
|
3,
|
[{file,
|
"proc_lib.erl"},
|
{line,
|
249}]}]},
|
{gen_server,
|
call,
|
['dcp_replication_manager-bucket6',
|
{get_replicator_pid,
|
921},
|
infinity]}},
|
{gen_server,
|
call,
|
[{'janitor_agent-bucket6',
|
'ns_1@172.23.97.112'},
|
{if_rebalance,
|
<0.13926.325>,
|
{update_vbucket_state,
|
811,
|
active,
|
undefined,
|
undefined,
|
undefined}},
|
infinity]}}}}
|
2021-08-17T20:48:16.683-07:00, ns_orchestrator:0:critical:message(ns_1@172.23.97.74) - Rebalance exited with reason {mover_crashed,
|
{unexpected_exit,
|
{'EXIT',<0.19827.325>,
|
{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,{error,timeout}},
|
[{mc_client_binary,cmd_vocal_recv,
|
5,
|
[{file,"src/mc_client_binary.erl"},
|
{line,152}]},
|
{mc_client_binary,auth,2,
|
[{file,"src/mc_client_binary.erl"},
|
{line,193}]},
|
{dcp_proxy,connect,5,
|
[{file,"src/dcp_proxy.erl"},
|
{line,254}]},
|
{dcp_proxy,maybe_connect,2,
|
[{file,"src/dcp_proxy.erl"},
|
{line,236}]},
|
{dcp_producer_conn,init,2,
|
[{file,
|
"src/dcp_producer_conn.erl"},
|
{line,25}]},
|
{dcp_proxy,init,1,
|
[{file,"src/dcp_proxy.erl"},
|
{line,52}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},
|
{line,374}]},
|
{gen_server,init_it,6,
|
[{file,"gen_server.erl"},
|
{line,342}]}]}}},
|
[{dcp_replicator,init,1,
|
[{file,"src/dcp_replicator.erl"},
|
{line,54}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},{line,374}]},
|
{gen_server,init_it,6,
|
[{file,"gen_server.erl"},{line,342}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,249}]}]},
|
{child,undefined,
|
{'ns_1@172.23.120.73',
|
[collections,del_times,del_user_xattr,
|
json,set_consumer_name,snappy,xattr]},
|
{dcp_replicator,start_link,
|
['ns_1@172.23.120.73',"bucket6",
|
[collections,del_times,
|
del_user_xattr,json,
|
set_consumer_name,snappy,xattr]]},
|
temporary,60000,worker,
|
[dcp_replicator]}}}},
|
[{dcp_sup,start_replicator,2,
|
[{file,"src/dcp_sup.erl"},{line,51}]},
|
{dcp_sup,
|
'-manage_replicators/2-lc$^3/1-3-',2,
|
[{file,"src/dcp_sup.erl"},{line,101}]},
|
{dcp_replication_manager,handle_call,3,
|
[{file,"src/dcp_replication_manager.erl"},
|
{line,83}]},
|
{gen_server,try_handle_call,4,
|
[{file,"gen_server.erl"},{line,661}]},
|
{gen_server,handle_msg,6,
|
[{file,"gen_server.erl"},{line,690}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,249}]}]},
|
{gen_server,call,
|
['dcp_replication_manager-bucket6',
|
{get_replicator_pid,921},
|
infinity]}},
|
{gen_server,call,
|
[{'janitor_agent-bucket6',
|
'ns_1@172.23.97.112'},
|
{if_rebalance,<0.13926.325>,
|
{update_vbucket_state,811,active,
|
undefined,undefined,undefined}},
|
infinity]}}}}}.
|
Logs:
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.106.134.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.106.136.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.58.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.73.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.74.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.75.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.77.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.81.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.120.86.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.121.77.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.123.24.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.123.25.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.123.26.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.123.31.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.123.32.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.123.33.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.96.122.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.96.14.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.96.243.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.96.254.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.96.48.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.105.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.110.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.112.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.148.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.149.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.150.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.241.zip
url : https://cb-jira.s3.us-east-2.amazonaws.com/logs/systestmon-1629299013/collectinfo-2021-08-18T150334-ns_1%40172.23.97.74.zip