Rebalance exited with reason mover_crashed

./testrunner -i /tmp/testexec.21506.ini -p get-cbcollect-info=True,GROUP=bucket_op -t eventing.eventing_rebalance.EventingRebalance.test_kv_rebalance_in_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=10,GROUP=bucket_op

./testrunner -i /tmp/testexec.21506.ini -p get-cbcollect-info=True,GROUP=bucket_op -t eventing.eventing_rebalance.EventingRebalance.test_kv_swap_rebalance_when_existing_eventing_node_is_processing_mutations,nodes_init=6,services_init=kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=10,GROUP=bucket_op

{u'node': u'ns_1@172.23.104.73', u'code': 0, u'text': u'Rebalance exited with reason \{mover_crashed,\n\{unexpected_exit,\n \{\'EXIT\',<0.29869.4>,\n  \{wait_seqno_persisted_failed,"src_bucket",\n   335,18,\n   [\{\'ns_1@172.23.106.131\',\n \{\'EXIT\',\n\{\{\{badmatch,\n[\{<23261.6908.1>,\n \{done,exit,\n  \{\{\{badmatch,\{error,closed}},\n[\{dcp_proxy,handle_packet,2,\n [\{file,"src/dcp_proxy.erl"},\n  \{line,172}]},\n\{dcp_proxy,process_data_loop,3,\n [\{file,"src/dcp_proxy.erl"},\n  \{line,334}]},\n\{dcp_proxy,handle_info,2,\n [\{file,"src/dcp_proxy.erl"},\n  \{line,100}]},\n\{gen_server,handle_msg,5,\n [\{file,"gen_server.erl"},\n  \{line,604}]},\n\{proc_lib,init_p_do_apply,3,\n [\{file,"proc_lib.erl"},\n  \{line,239}]}]},\n   \{gen_server,call,\n[<23261.6208.1>,\n\{setup_streams,\n [335,500,845,846,847,848,849,\n  850,851,852,853]},\ninfinity]}},\n  [\{gen_server,call,3,\n[\{file,"gen_server.erl"},\n\{line,188}]},\n   \{dcp_replicator,\n\'-spawn_and_wait/1-fun-0-\',1,\n[\{file,"src/dcp_replicator.erl"},\n\{line,226}]}]}}]},\n  [\{misc,\nsync_shutdown_many_i_am_trapping_exits,\n1,\n[\{file,"src/misc.erl"},\{line,1303}]},\n\{dcp_replicator,spawn_and_wait,1,\n[\{file,"src/dcp_replicator.erl"},\n \{line,248}]},\n\{dcp_replicator,handle_call,3,\n[\{file,"src/dcp_replicator.erl"},\n \{line,121}]},\n\{gen_server,handle_msg,5,\n[\{file,"gen_server.erl"},\{line,585}]},\n\{proc_lib,init_p_do_apply,3,\n[\{file,"proc_lib.erl"},\{line,239}]}]},\n \{gen_server,call,\n  [\{\'janitor_agent-src_bucket\',\n\'ns_1@172.23.106.131\'},\n\{if_rebalance,<0.26561.4>,\n\{wait_seqno_persisted,335,18}},\ninfinity]}}}}]}}}}', u'shortText': u'message', u'serverTime': u'2018-05-12T10:08:55.334Z', u'module': u'ns_orchestrator', u'tstamp': 1526144935334, u'type': u'critical'}

[2018-05-12 10:47:57,515] - [rest_client:3132] ERROR - {u'node': u'ns_1@172.23.104.73', u'code': 0, u'text': u'Rebalance exited with reason {mover_crashed,\n{unexpected_exit,\n {\'EXIT\',<0.27490.0>,\n  {bulk_set_vbucket_state_failed,\n   [{\'ns_1@172.23.104.80\',\ng{\'EXIT\',\n{{badmatch,wrong_rebalancer_pid},\n [{janitor_agent,set_vbucket_state,7,\n   [{file,"src/janitor_agent.erl"},\n    {line,405}]},\n  {janitor_agent,\n   \'-bulk_set_vbucket_state/4-fun-0-\',4,\n   [{file,"src/janitor_agent.erl"},\n    {line,392}]},\n  {async,\'-async_init/4-fun-2-\',3,\n   [{file,"src/async.erl"},\n    {line,208}]}]}}}]}}}}', u'shortText': u'message', u'serverTime': u'2018-05-12T10:47:55.861Z', u'module': u'ns_orchestrator', u'tstamp': 1526147275861, u'type': u'critical'}