rebalance failed bulk_set_vbucket_state_failed

-t swaprebalance.SwapRebalanceBasicTests.do_test,replica=1,num-buckets=3,num-swap=2,keys-count=1000000

[2013-05-29 23:34:54,242] - [rest_client:1016] ERROR - {u'node': u'ns_1@10.3.121.93', u'code': 2, u'text': u"Rebalance exited with reason bulk_set_vbucket_state_failed,\n [{'ns_1@10.3.121.93',\n {'EXIT',\n {{{{unexpected_reason,\n {{badmatch,{error,closed,\n [

{mc_binary,quick_stats_recv,3}

,\n

{mc_binary,quick_stats_loop,5}

,\n

{mc_binary,quick_stats,5}

,\n

{mc_client_binary,\n get_zero_open_checkpoint_vbuckets,3}

,\n

{ebucketmigrator_srv,handle_call,3}

,\n

{gen_server,handle_msg,5},\n {proc_lib,init_p_do_apply,3}]}},\n [{misc,executing_on_new_process,1},\n {tap_replication_manager,\n change_vbucket_filter,4},\n {tap_replication_manager,\n '-do_set_incoming_replication_map/3-lc$^2/1-2-',\n 2},\n {tap_replication_manager,\n do_set_incoming_replication_map,3},\n {tap_replication_manager,handle_call,3},\n {gen_server,handle_msg,5}

,\n

{proc_lib,init_p_do_apply,3}]},\n {gen_server,call,\n ['tap_replication_manager-bucket-2',\n {change_vbucket_replication,783,\n 'ns_1@10.3.121.96'},\n infinity]}},\n {gen_server,call,\n [{'janitor_agent-bucket-2',\n 'ns_1@10.3.121.93'},\n {if_rebalance,<0.2078.113>,\n {update_vbucket_state,783,replica,\n undefined,'ns_1@10.3.121.96'}},\n infinity]}}}}]},\n [{janitor_agent,bulk_set_vbucket_state,4},\n {ns_vbucket_mover,\n update_replication_post_move,3},\n {ns_vbucket_mover,on_move_done,2},\n {gen_server,handle_msg,5},\n {proc_lib,init_p_do_apply,3}

]}\n", u'shortText': u'message', u'module': u'ns_orchestrator', u'tstamp': 1369895707119, u'type': u'info'}
[2013-05-29 23:34:54,243] - [rest_client:1016] ERROR -

{u'node': u'ns_1@10.3.121.96', u'code': 0, u'text': u'Port server memcached on node \'babysitter_of_ns_1@127.0.0.1\' exited with status 137. Restarting. Messages: Wed May 29 23:34:44.126792 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 794, cookie 0x59ca2c0\nWed May 29 23:34:44.148193 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22527 - Reset vbucket 515 was completed succecssfully.\nWed May 29 23:34:44.166105 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 516, cookie 0x5978dc0\nWed May 29 23:34:44.189898 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22528 - disconnected\nWed May 29 23:34:44.286413 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22524"\nWed May 29 23:34:44.286646 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22528"\nWed May 29 23:34:44.320876 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 793, cookie 0x5a1b8c0\nWed May 29 23:34:44.347300 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22525 - disconnected\nWed May 29 23:34:44.502155 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22529 - disconnected\nWed May 29 23:34:44.556501 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22526 - disconnected\nWed May 29 23:34:44.579156 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22522 - disconnected\nWed May 29 23:34:44.710812 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22530 - disconnected\nWed May 29 23:34:44.947708 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22522"\nWed May 29 23:34:44.947924 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22525"\nWed May 29 23:34:44.947879 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:44.949458 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22526"\nWed May 29 23:34:44.949559 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22529"\nWed May 29 23:34:44.949665 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22530"\nWed May 29 23:34:44.990566 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 515, cookie 0x5a1b8c0\nWed May 29 23:34:45.335431 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22527 - disconnected\nWed May 29 23:34:45.480212 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22531 - Reset vbucket 792 was completed succecssfully.\nWed May 29 23:34:45.738067 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22532 - Reset vbucket 514 was completed succecssfully.\nWed May 29 23:34:45.776281 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22527"\nWed May 29 23:34:45.776635 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:46.378671 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:46.959837 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 792, cookie 0x5a1a000\nWed May 29 23:34:47.119746 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22531 - disconnected\nWed May 29 23:34:47.281992 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22533 - disconnected\nWed May 29 23:34:47.297888 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 514, cookie 0x5a1a000\nWed May 29 23:34:47.339256 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22534 - Reset vbucket 791 was completed succecssfully.\nWed May 29 23:34:47.517277 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22531"\nWed May 29 23:34:47.517335 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22533"\nWed May 29 23:34:47.663035 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22532 - disconnected\nWed May 29 23:34:47.663928 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22532"\nWed May 29 23:34:47.664379 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:47.779927 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22535 - Reset vbucket 513 was completed succecssfully.\nWed May 29 23:34:48.212763 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22536 - Reset vbucket 790 was completed succecssfully.\nWed May 29 23:34:48.481576 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22537 - Reset vbucket 789 was completed succecssfully.\nWed May 29 23:34:48.754921 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22538 - Reset vbucket 788 was completed succecssfully.\nWed May 29 23:34:49.049998 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22539 - Reset vbucket 787 was completed succecssfully.\nWed May 29 23:34:49.376595 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22540 - Reset vbucket 786 was completed succecssfully.\nWed May 29 23:34:49.469913 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 790, cookie 0x5a89600\nWed May 29 23:34:49.546094 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 788, cookie 0x5ad38c0\nWed May 29 23:34:49.556600 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22536 - disconnected\nWed May 29 23:34:49.614257 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22538 - disconnected\nWed May 29 23:34:49.631136 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22541 - disconnected\nWed May 29 23:34:49.736618 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22542 - disconnected\nWed May 29 23:34:49.847391 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22536"\nWed May 29 23:34:49.847470 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22538"\nWed May 29 23:34:49.847493 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22541"\nWed May 29 23:34:49.847503 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22542"\nWed May 29 23:34:49.979179 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22543 - Reset vbucket 785 was completed succecssfully.\nWed May 29 23:34:50.358457 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 513, cookie 0x5a89600\nWed May 29 23:34:50.448193 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 786, cookie 0x5a1b8c0\nWed May 29 23:34:50.467751 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 787, cookie 0x59358c0\nWed May 29 23:34:50.511413 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 789, cookie 0x5978dc0\nWed May 29 23:34:50.515376 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 791, cookie 0x5ab0b00\nWed May 29 23:34:50.557676 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22535 - disconnected\nWed May 29 23:34:50.774439 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22535"\nWed May 29 23:34:50.774474 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:50.957393 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22540 - disconnected\nWed May 29 23:34:50.964958 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22539 - disconnected\nWed May 29 23:34:50.977201 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22534 - disconnected\nWed May 29 23:34:50.986823 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22537 - disconnected\nWed May 29 23:34:52.172346 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22534"\nWed May 29 23:34:52.172455 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22537"\nWed May 29 23:34:52.172479 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22539"\nWed May 29 23:34:52.172495 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22540"\nWed May 29 23:34:52.189993 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22545 - disconnected\nWed May 29 23:34:52.191608 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22545"\nWed May 29 23:34:52.191849 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:52.192569 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22544 - disconnected\nWed May 29 23:34:52.207717 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22546 - disconnected\nWed May 29 23:34:52.217385 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22547 - disconnected\nWed May 29 23:34:52.396606 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22544"\nWed May 29 23:34:52.396656 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22546"\nWed May 29 23:34:52.396668 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22547"\nWed May 29 23:34:56.094475 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:56.110812 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22548 - Reset vbucket 784 was completed succecssfully.\nWed May 29 23:34:56.383898 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:56.456891 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 785, cookie 0x59cb600\nWed May 29 23:34:56.583117 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:56.629439 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 784, cookie 0x59f22c0\nWed May 29 23:34:56.673897 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22543 - disconnected\nWed May 29 23:34:56.711990 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22543"\nWed May 29 23:34:56.757115 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22548 - disconnected\nWed May 29 23:34:56.808952 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22549 - disconnected\nWed May 29 23:34:56.858781 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22548"\nWed May 29 23:34:56.858861 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22549"\nWed May 29 23:34:56.859012 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:56.860995 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22550 - disconnected\nWed May 29 23:34:56.995835 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22550"\nWed May 29 23:34:57.807126 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22551 - Reset vbucket 783 was completed succecssfully.\nWed May 29 23:34:57.846500 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:34:58.882421 PDT 3: (bucket-2) Notified the completion of checkpoint persistence for vbucket 783, cookie 0x5a5cdc0\nWed May 29 23:35:00.180538 PDT 3: (bucket-2) TAP (Producer) eq_tapq:replication_ns_1@10.3.121.93 - Sending TAP_OPAQUE with command "complete_vb_filter_change" and vbucket 0\nWed May 29 23:35:00.416154 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22551 - disconnected\nWed May 29 23:35:00.423006 PDT 3: (bucket-2) Schedule cleanup of "eq_tapq:anon_22551"\nWed May 29 23:35:00.554309 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22552 - disconnected\nWed May 29 23:35:00.899342 PDT 3: (bucket-2) TAP (Consumer) eq_tapq:anon_22553 - Reset vbucket 782 was completed succecssfully.', u'shortText': u'message', u'module': u'ns_log', u'tstamp': 1369895703802, u'type': u'info'}

\