Rebalance failed with mover crashed

./testrunner -i /tmp/testexec.30142.ini -p get-cbcollect-info=True,GROUP=rebalance_multi,num_functions=5 -t eventing.eventing_rebalance.EventingRebalance.test_kv_eventing_rebalance_with_multiple_functions_deployed,nodes_init=7,services_init=kv-kv-kv-eventing-eventing-eventing-index:n1ql,dataset=default,groups=simple,reset_services=True,doc-per-day=1,services_in=index,n1ql,server_out=2,skip_validation=False,GROUP=rebalance_multi

ERROR - {u'node': u'ns_1@172.23.105.46', u'code': 0, u'text': u'Rebalance exited with reason {mover_crashed,\n                              {unexpected_exit,\n                               {\'EXIT\',<0.11469.3>,\n                                {{bulk_set_vbucket_state_failed,\n                                  [{\'ns_1@172.23.105.47\',\n                                    {\'EXIT\',\n                                     {{{{{case_clause,\n                                          {error,\n                                           {{{badmatch,\n                                              {error,\n                                               {{badmatch,{error,timeout}},\n                                                [{mc_client_binary,\n                                                  cmd_vocal_recv,5,\n                                                  [{file,\n                                                    "src/mc_client_binary.erl"},\n                                                   {line,155}]},\n                                                 {mc_client_binary,cmd_vocal,\n                                                  3,\n                                                  [{file,\n                                                    "src/mc_client_binary.erl"},\n                                                   {line,140}]},\n                                                 {dcp_commands,\n                                                  open_connection,5,\n                                                  [{file,\n                                                    "src/dcp_commands.erl"},\n                                                   {line,79}]},\n                                                 {dcp_proxy,connect,5,\n                                                  [{file,"src/dcp_proxy.erl"},\n                                                   {line,247}]},\n                                                 {dcp_proxy,maybe_connect,2,\n                                                  [{file,"src/dcp_proxy.erl"},\n                                                   {line,221}]},\n                                                 {dcp_producer_conn,init,2,\n                                                  [{file,\n                                                    "src/dcp_producer_conn.erl"},\n                                                   {line,31}]},\n                                                 {dcp_proxy,init,1,\n                                                  [{file,"src/dcp_proxy.erl"},\n                                                   {line,59}]},\n                                                 {gen_server,init_it,2,\n                                                  [{file,"gen_server.erl"},\n                                                   {line,365}]}]}}},\n                                             [{dcp_replicator,init,1,\n                                               [{file,\n                                                 "src/dcp_replicator.erl"},\n                                                {line,50}]},\n                                              {gen_server,init_it,2,\n                                               [{file,"gen_server.erl"},\n                                                {line,365}]},\n                                              {gen_server,init_it,6,\n                                               [{file,"gen_server.erl"},\n                                                {line,333}]},\n                                              {proc_lib,init_p_do_apply,3,\n                                               [{file,"proc_lib.erl"},\n                                                {line,247}]}]},\n                                            {child,undefined,\n                                             {\'ns_1@172.23.105.49\',\n                                              [del_times,json,\n                                               set_consumer_name,snappy,\n                                               xattr]},\n                                             {dcp_replicator,start_link,\n                                              [\'ns_1@172.23.105.49\',\n                                               "src_bucket",\n                                               [del_times,json,\n                                                set_consumer_name,snappy,\n                                                xattr]]},\n                                             temporary,60000,worker,\n                                             [dcp_replicator]}}}},\n                                         [{dcp_sup,start_replicator,2,\n                                           [{file,"src/dcp_sup.erl"},\n                                            {line,57}]},\n                                          {dcp_sup,\n                                           \'-manage_replicators/2-lc$^3/1-3-\',\n                                           2,\n                                           [{file,"src/dcp_sup.erl"},\n                                            {line,101}]},\n                                          {dcp_replication_manager,\n                                           handle_call,3,\n                                           [{file,\n                                             "src/dcp_replication_manager.erl"},\n                                            {line,89}]},\n                                          {gen_server,try_handle_call,4,\n                                           [{file,"gen_server.erl"},\n                                            {line,636}]},\n                                          {gen_server,handle_msg,6,\n                                           [{file,"gen_server.erl"},\n                                            {line,665}]},\n                                          {proc_lib,init_p_do_apply,3,\n                                           [{file,"proc_lib.erl"},\n                                            {line,247}]}]},\n                                        {gen_server,call,\n                                         [\'dcp_replication_manager-src_bucket\',\n                                          {manage_replicators,\n                                           [\'ns_1@172.23.105.49\']},\n                                          infinity]}},\n                                       {gen_server,call,\n                                        [\'replication_manager-src_bucket\',\n                                         {change_vbucket_replication,986,\n                                          \'ns_1@172.23.105.49\'},\n                                         infinity]}},\n                                      {gen_server,call,\n                                       [{\'janitor_agent-src_bucket\',\n                                         \'ns_1@172.23.105.47\'},\n                                        {if_rebalance,<0.4428.3>,\n                                         {update_vbucket_state,987,pending,\n                                          passive,\'ns_1@172.23.105.49\'}},\n                                        infinity]}}}}]},\n                                 [{janitor_agent,bulk_set_vbucket_state,4,\n                                   [{file,"src/janitor_agent.erl"},\n                                    {line,409}]},\n                                  {proc_lib,init_p,3,\n                                   [{file,"proc_lib.erl"},{line,232}]}]}}}}. Operation Id = 864953299c293516072a9372ada891ca', u'shortText': u'message', u'serverTime': u'2019-05-04T05:54:42.439Z', u'module': u'ns_orchestrator', u'tstamp': 1556974482439, u'type': u'critical'}