Details
-
Bug
-
Resolution: User Error
-
Critical
-
6.5.0
-
Untriaged
-
Unknown
Description
Observed rebalance failure while running eventing test
Build: 6.5.0-3458
./testrunner -i /tmp/testexec.2053.ini -p get-cbcollect-info=True,GROUP=source_bucket_mutation -t eventing.eventing_rebalance.EventingRebalance.test_kv_swap_rebalance_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=20,handler_code=source_bucket_mutation,source_bucket_mutation=True,GROUP=source_bucket_mutation
|
Rebalance failure
2019-06-10T17:33:22.394-07:00, ns_rebalancer:0:info:message(ns_1@172.23.104.229) - Started rebalancing bucket metadata
|
2019-06-10T17:33:22.535-07:00, ns_memcached:0:info:message(ns_1@172.23.109.8) - Bucket "metadata" loaded on node 'ns_1@172.23.109.8' in 0 seconds.
|
2019-06-10T17:33:23.975-07:00, ns_vbucket_mover:0:info:message(ns_1@172.23.104.229) - Bucket "metadata" rebalance appears to be swap rebalance
|
2019-06-10T17:35:44.068-07:00, ns_vbucket_mover:0:critical:message(ns_1@172.23.104.229) - <0.28175.1> exited with {unexpected_exit,
|
{'EXIT',<0.28203.1>,
|
{{bulk_set_vbucket_state_failed,
|
[{'ns_1@172.23.109.8',
|
{'EXIT',
|
{{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,{error,timeout}},
|
[{mc_client_binary,cmd_vocal_recv,
|
5,
|
[{file,"src/mc_client_binary.erl"},
|
{line,152}]},
|
{mc_client_binary,cmd_vocal,3,
|
[{file,"src/mc_client_binary.erl"},
|
{line,137}]},
|
{dcp_commands,open_connection,5,
|
[{file,"src/dcp_commands.erl"},
|
{line,79}]},
|
{dcp_proxy,connect,5,
|
[{file,"src/dcp_proxy.erl"},
|
{line,247}]},
|
{dcp_proxy,maybe_connect,2,
|
[{file,"src/dcp_proxy.erl"},
|
{line,221}]},
|
{dcp_consumer_conn,init,2,
|
[{file,
|
"src/dcp_consumer_conn.erl"},
|
{line,57}]},
|
{dcp_proxy,init,1,
|
[{file,"src/dcp_proxy.erl"},
|
{line,59}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},
|
{line,365}]}]}}},
|
[{dcp_replicator,init,1,
|
[{file,"src/dcp_replicator.erl"},
|
{line,48}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},{line,365}]},
|
{gen_server,init_it,6,
|
[{file,"gen_server.erl"},{line,333}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,247}]}]},
|
{child,undefined,
|
{'ns_1@172.23.104.231',
|
[del_times,json,set_consumer_name,
|
snappy,xattr]},
|
{dcp_replicator,start_link,
|
['ns_1@172.23.104.231',"metadata",
|
[del_times,json,set_consumer_name,
|
snappy,xattr]]},
|
temporary,60000,worker,
|
[dcp_replicator]}}}},
|
[{dcp_sup,start_replicator,2,
|
[{file,"src/dcp_sup.erl"},{line,57}]},
|
{dcp_sup,
|
'-manage_replicators/2-lc$^3/1-3-',2,
|
[{file,"src/dcp_sup.erl"},{line,103}]},
|
{dcp_replication_manager,handle_call,3,
|
[{file,"src/dcp_replication_manager.erl"},
|
{line,89}]},
|
{gen_server,try_handle_call,4,
|
[{file,"gen_server.erl"},{line,636}]},
|
{gen_server,handle_msg,6,
|
[{file,"gen_server.erl"},{line,665}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},{line,247}]}]},
|
{gen_server,call,
|
['dcp_replication_manager-metadata',
|
{manage_replicators,
|
['ns_1@172.23.104.231']},
|
infinity]}},
|
{gen_server,call,
|
['replication_manager-metadata',
|
{change_vbucket_replication,551,
|
'ns_1@172.23.104.231'},
|
infinity]}},
|
{gen_server,call,
|
[{'janitor_agent-metadata',
|
'ns_1@172.23.109.8'},
|
{if_rebalance,<0.16241.0>,
|
{update_vbucket_state,550,pending,passive,
|
'ns_1@172.23.104.231'}},
|
infinity]}}}}]},
|
[{janitor_agent,bulk_set_vbucket_state,4,
|
[{file,"src/janitor_agent.erl"},{line,426}]},
|
{proc_lib,init_p,3,
|
[{file,"proc_lib.erl"},{line,232}]}]}}}
|
2019-06-10T17:35:44.073-07:00, ns_orchestrator:0:critical:message(ns_1@172.23.104.229) - Rebalance exited with reason {mover_crashed,
|
{unexpected_exit,
|
{'EXIT',<0.28203.1>,
|
{{bulk_set_vbucket_state_failed,
|
[{'ns_1@172.23.109.8',
|
{'EXIT',
|
{{{{{case_clause,
|
{error,
|
{{{badmatch,
|
{error,
|
{{badmatch,{error,timeout}},
|
[{mc_client_binary,
|
cmd_vocal_recv,5,
|
[{file,
|
"src/mc_client_binary.erl"},
|
{line,152}]},
|
{mc_client_binary,cmd_vocal,
|
3,
|
[{file,
|
"src/mc_client_binary.erl"},
|
{line,137}]},
|
{dcp_commands,
|
open_connection,5,
|
[{file,
|
"src/dcp_commands.erl"},
|
{line,79}]},
|
{dcp_proxy,connect,5,
|
[{file,"src/dcp_proxy.erl"},
|
{line,247}]},
|
{dcp_proxy,maybe_connect,2,
|
[{file,"src/dcp_proxy.erl"},
|
{line,221}]},
|
{dcp_consumer_conn,init,2,
|
[{file,
|
"src/dcp_consumer_conn.erl"},
|
{line,57}]},
|
{dcp_proxy,init,1,
|
[{file,"src/dcp_proxy.erl"},
|
{line,59}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},
|
{line,365}]}]}}},
|
[{dcp_replicator,init,1,
|
[{file,
|
"src/dcp_replicator.erl"},
|
{line,48}]},
|
{gen_server,init_it,2,
|
[{file,"gen_server.erl"},
|
{line,365}]},
|
{gen_server,init_it,6,
|
[{file,"gen_server.erl"},
|
{line,333}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},
|
{line,247}]}]},
|
{child,undefined,
|
{'ns_1@172.23.104.231',
|
[del_times,json,
|
set_consumer_name,snappy,
|
xattr]},
|
{dcp_replicator,start_link,
|
['ns_1@172.23.104.231',
|
"metadata",
|
[del_times,json,
|
set_consumer_name,snappy,
|
xattr]]},
|
temporary,60000,worker,
|
[dcp_replicator]}}}},
|
[{dcp_sup,start_replicator,2,
|
[{file,"src/dcp_sup.erl"},
|
{line,57}]},
|
{dcp_sup,
|
'-manage_replicators/2-lc$^3/1-3-',
|
2,
|
[{file,"src/dcp_sup.erl"},
|
{line,103}]},
|
{dcp_replication_manager,
|
handle_call,3,
|
[{file,
|
"src/dcp_replication_manager.erl"},
|
{line,89}]},
|
{gen_server,try_handle_call,4,
|
[{file,"gen_server.erl"},
|
{line,636}]},
|
{gen_server,handle_msg,6,
|
[{file,"gen_server.erl"},
|
{line,665}]},
|
{proc_lib,init_p_do_apply,3,
|
[{file,"proc_lib.erl"},
|
{line,247}]}]},
|
{gen_server,call,
|
['dcp_replication_manager-metadata',
|
{manage_replicators,
|
['ns_1@172.23.104.231']},
|
infinity]}},
|
{gen_server,call,
|
['replication_manager-metadata',
|
{change_vbucket_replication,551,
|
'ns_1@172.23.104.231'},
|
infinity]}},
|
{gen_server,call,
|
[{'janitor_agent-metadata',
|
'ns_1@172.23.109.8'},
|
{if_rebalance,<0.16241.0>,
|
{update_vbucket_state,550,pending,
|
passive,'ns_1@172.23.104.231'}},
|
infinity]}}}}]},
|
[{janitor_agent,bulk_set_vbucket_state,4,
|
[{file,"src/janitor_agent.erl"},
|
{line,426}]},
|
{proc_lib,init_p,3,
|
[{file,"proc_lib.erl"},{line,232}]}]}}}}. Operation Id = 8d10774786e654ce866b0dc83c237272
|
logs:
