rebalance failed with bulk_set_vbucket_state_failed

./testrunner -i centos_x64--rebalance_out.ini stop-on-failure=False,force_kill_memached=False,verify_unacked_bytes=True,std_vbucker_dist=8,chk_max_items=1000,chk_period=30 -t rebalance.rebalanceinout.RebalanceInOutTests.test_incremental_rebalance_out_in_with_mutation,init_num_nodes=3,items=400000,standard_buckets=2,sasl_buckets=2,standard_bucket_priority=low:high,sasl_bucket_priority=low:high,GROUP=IN_OUT;P0

[2016-04-07 17:59:50,650] - [rest_client:2444] ERROR - {u'node': u'ns_1@10.6.2.144', u'code': 2, u'text': u'Rebalance exited with reason {unexpected_exit,\n                              {\'EXIT\',<0.31308.97>,\n                               {bulk_set_vbucket_state_failed,\n                                [{\'ns_1@10.6.2.149\',\n                                  {\'EXIT\',\n                                   {{{{{case_clause,\n                                        {error,\n                                         {{{badmatch,\n                                            {error,\n                                             {{badmatch,{error,closed}},\n                                              [{mc_client_binary,\n                                                cmd_vocal_recv,5,\n                                                [{file,\n                                                  "src/mc_client_binary.erl"},\n                                                 {line,156}]},\n                                               {mc_client_binary,cmd_vocal,3,\n                                                [{file,\n                                                  "src/mc_client_binary.erl"},\n                                                 {line,141}]},\n                                               {dcp_commands,open_connection,\n                                                4,\n                                                [{file,"src/dcp_commands.erl"},\n                                                 {line,60}]},\n                                               {dcp_proxy,connect,4,\n                                                [{file,"src/dcp_proxy.erl"},\n                                                 {line,184}]},\n                                               {dcp_proxy,maybe_connect,1,\n                                                [{file,"src/dcp_proxy.erl"},\n                                                 {line,170}]},\n                                               {dcp_producer_conn,init,2,\n                                                [{file,\n                                                  "src/dcp_producer_conn.erl"},\n                                                 {line,30}]},\n                                               {dcp_proxy,init,1,\n                                                [{file,"src/dcp_proxy.erl"},\n                                                 {line,47}]},\n                                               {gen_server,init_it,6,\n                                                [{file,"gen_server.erl"},\n                                                 {line,304}]}]}}},\n                                           [{dcp_replicator,init,1,\n                                             [{file,"src/dcp_replicator.erl"},\n                                              {line,50}]},\n                                            {gen_server,init_it,6,\n                                             [{file,"gen_server.erl"},\n                                              {line,304}]},\n                                            {proc_lib,init_p_do_apply,3,\n                                             [{file,"proc_lib.erl"},\n                                              {line,239}]}]},\n                                          {child,undefined,\'ns_1@10.6.2.144\',\n                                           {dcp_replicator,start_link,\n                                            [\'ns_1@10.6.2.144\',"bucket0"]},\n                                           temporary,60000,worker,\n                                           [dcp_replicator]}}}},\n                                       [{dcp_sup,start_replicator,2,\n                                         [{file,"src/dcp_sup.erl"},{line,53}]},\n                                        {dcp_sup,\n                                         \'-manage_replicators/2-lc$^2/1-2-\',2,\n                                         [{file,"src/dcp_sup.erl"},{line,69}]},\n                                        {dcp_replication_manager,handle_call,\n                                         3,\n                                         [{file,\n                                           "src/dcp_replication_manager.erl"},\n                                          {line,87}]},\n                                        {gen_server,handle_msg,5,\n                                         [{file,"gen_server.erl"},{line,585}]},\n                                        {proc_lib,init_p_do_apply,3,\n                                         [{file,"proc_lib.erl"},{line,239}]}]},\n                                      {gen_server,call,\n                                       [\'dcp_replication_manager-bucket0\',\n                                        {manage_replicators,\n                                         [\'ns_1@10.6.2.144\',\'ns_1@10.6.2.146\',\n                                          \'ns_1@10.6.2.147\',\'ns_1@10.6.2.148\',\n                                          \'ns_1@10.6.2.150\']},\n                                        infinity]}},\n                                     {gen_server,call,\n                                      [\'replication_manager-bucket0\',\n                                       {change_vbucket_replication,814,\n                                        \'ns_1@10.6.2.144\'},\n                                       infinity]}},\n                                    {gen_server,call,\n                                     [{\'janitor_agent-bucket0\',\n                                       \'ns_1@10.6.2.149\'},\n                                      {if_rebalance,<0.17043.97>,\n                                       {update_vbucket_state,814,pending,\n                                        passive,\'ns_1@10.6.2.144\'}},\n                                      infinity]}}}}]}}}\n', u'shortText': u'message', u'serverTime': u'2016-04-07T17:59:50.055Z', u'module': u'ns_orchestrator', u'tstamp': 1460077190055, u'type': u'info'}

[2016-04-07 17:59:50,653] - [rest_client:2444] ERROR - {u'node': u'ns_1@10.6.2.144', u'code': 0, u'text': u'<0.31336.97> exited with {unexpected_exit,\n                          {\'EXIT\',<0.31308.97>,\n                           {bulk_set_vbucket_state_failed,\n                            [{\'ns_1@10.6.2.149\',\n                              {\'EXIT\',\n                               {{{{{case_clause,\n                                    {error,\n                                     {{{badmatch,\n                                        {error,\n                                         {{badmatch,{error,closed}},\n                                          [{mc_client_binary,cmd_vocal_recv,\n                                            5,\n                                            [{file,"src/mc_client_binary.erl"},\n                                             {line,156}]},\n                                           {mc_client_binary,cmd_vocal,3,\n                                            [{file,"src/mc_client_binary.erl"},\n                                             {line,141}]},\n                                           {dcp_commands,open_connection,4,\n                                            [{file,"src/dcp_commands.erl"},\n                                             {line,60}]},\n                                           {dcp_proxy,connect,4,\n                                            [{file,"src/dcp_proxy.erl"},\n                                             {line,184}]},\n                                           {dcp_proxy,maybe_connect,1,\n                                            [{file,"src/dcp_proxy.erl"},\n                                             {line,170}]},\n                                           {dcp_producer_conn,init,2,\n                                            [{file,\n                                              "src/dcp_producer_conn.erl"},\n                                             {line,30}]},\n                                           {dcp_proxy,init,1,\n                                            [{file,"src/dcp_proxy.erl"},\n                                             {line,47}]},\n                                           {gen_server,init_it,6,\n                                            [{file,"gen_server.erl"},\n                                             {line,304}]}]}}},\n                                       [{dcp_replicator,init,1,\n                                         [{file,"src/dcp_replicator.erl"},\n                                          {line,50}]},\n                                        {gen_server,init_it,6,\n                                         [{file,"gen_server.erl"},{line,304}]},\n                                        {proc_lib,init_p_do_apply,3,\n                                         [{file,"proc_lib.erl"},{line,239}]}]},\n                                      {child,undefined,\'ns_1@10.6.2.144\',\n                                       {dcp_replicator,start_link,\n                                        [\'ns_1@10.6.2.144\',"bucket0"]},\n                                       temporary,60000,worker,\n                                       [dcp_replicator]}}}},\n                                   [{dcp_sup,start_replicator,2,\n                                     [{file,"src/dcp_sup.erl"},{line,53}]},\n                                    {dcp_sup,\n                                     \'-manage_replicators/2-lc$^2/1-2-\',2,\n                                     [{file,"src/dcp_sup.erl"},{line,69}]},\n                                    {dcp_replication_manager,handle_call,3,\n                                     [{file,"src/dcp_replication_manager.erl"},\n                                      {line,87}]},\n                                    {gen_server,handle_msg,5,\n                                     [{file,"gen_server.erl"},{line,585}]},\n                                    {proc_lib,init_p_do_apply,3,\n                                     [{file,"proc_lib.erl"},{line,239}]}]},\n                                  {gen_server,call,\n                                   [\'dcp_replication_manager-bucket0\',\n                                    {manage_replicators,\n                                     [\'ns_1@10.6.2.144\',\'ns_1@10.6.2.146\',\n                                      \'ns_1@10.6.2.147\',\'ns_1@10.6.2.148\',\n                                      \'ns_1@10.6.2.150\']},\n                                    infinity]}},\n                                 {gen_server,call,\n                                  [\'replication_manager-bucket0\',\n                                   {change_vbucket_replication,814,\n                                    \'ns_1@10.6.2.144\'},\n                                   infinity]}},\n                                {gen_server,call,\n                                 [{\'janitor_agent-bucket0\',\'ns_1@10.6.2.149\'},\n                                  {if_rebalance,<0.17043.97>,\n                                   {update_vbucket_state,814,pending,passive,\n                                    \'ns_1@10.6.2.144\'}},\n                                  infinity]}}}}]}}}', u'shortText': u'message', u'serverTime': u'2016-04-07T17:59:50.037Z', u'module': u'ns_vbucket_mover', u'tstamp': 1460077190037, u'type': u'critical'}