[ns_server:warn] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:218] Janitor run exited for bucket "admatter" with reason shutdown [user:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:idle:387] Starting failing over 'ns_1@10.22.64.9' [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 10 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 11 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 76 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 77 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 114 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 115 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 115 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 116 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 117 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 118 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 128 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 128 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 129 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 130 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 143 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 146 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 217 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 217 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 218 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 218 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 219 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 219 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 220 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 220 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 253 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 254 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 255 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 256 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 271 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 271 state to active [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 272 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 273 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 274 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:58] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 302 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 328 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 328 state to active [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 350 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 351 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 352 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 358 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 358 state to active [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 402 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 403 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 415 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 415 state to active [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 441 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 441 state to active [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 464 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 482 in "campaign_imp" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.1555.0>:ns_memcached:do_handle_call:369] Changed vbucket 482 state to active [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 483 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 484 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 485 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 491 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 492 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 493 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 510 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 520 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 521 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 573 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 597 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 598 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 599 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 600 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 601 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 633 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 634 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 635 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 636 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 637 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 638 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 642 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 649 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 670 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 679 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 680 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 681 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 699 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 701 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 705 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 706 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 727 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 728 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 734 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 737 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 738 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 760 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 761 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 785 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 855 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 860 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 864 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 868 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 874 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 875 in "campaign_imp" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 876 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 880 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 881 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 886 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 888 in "campaign_imp" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 889 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 893 in "campaign_imp" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 894 in "campaign_imp" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 895 in "campaign_imp" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 896 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 912 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 913 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 921 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 945 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 946 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 947 in "campaign_imp" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 952 in "campaign_imp" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 963 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:37:59] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 964 in "campaign_imp" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:02] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.25' [error_logger:error] [2013-01-29 14:38:18] [ns_1@10.22.64.12:error_logger:ale_error_logger_handler:log_msg:76] ** Node 'ns_1@10.22.64.9' not responding ** ** Removing (timedout) connection ** [user:warn] [2013-01-29 14:38:18] [ns_1@10.22.64.12:ns_node_disco:ns_node_disco:handle_info:150] Node 'ns_1@10.22.64.12' saw that node 'ns_1@10.22.64.9' went down. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:ns_node_disco_events:ns_node_disco_log:handle_event:46] ns_node_disco_log: nodes changed: ['ns_1@10.22.64.12','ns_1@10.22.64.13', 'ns_1@10.22.64.14','ns_1@10.22.64.22', 'ns_1@10.22.64.23','ns_1@10.22.64.24', 'ns_1@10.22.64.25','ns_1@10.22.64.26', 'ns_1@10.22.64.27'] [rebalance:error] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_rebalancer:failover:104] Janitor cleanup of "campaign_imp" failed after failover of 'ns_1@10.22.64.9': {exit, {{nodedown, 'ns_1@10.22.64.9'}, {gen_server, call, [{'ns_vbm_sup-campaign_imp', 'ns_1@10.22.64.9'}, which_children, infinity]}}} [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 80 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 103 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 103 state to active [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 119 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 120 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 121 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 122 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 123 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 124 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 125 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 137 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 138 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 139 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 140 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 141 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 142 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 143 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 153 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 153 state to active [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 154 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 154 state to active [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 155 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 155 state to active [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 156 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 157 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 158 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 159 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 170 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 262 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 263 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 263 state to active [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 264 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 271 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 342 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 344 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 345 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 346 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 347 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 348 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 349 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 367 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 372 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 405 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 406 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 407 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 408 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 409 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 410 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 414 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 428 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 429 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 430 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 431 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 432 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 435 in "default" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 436 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:18] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 442 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 474 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 475 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 505 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 508 in "default" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 553 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 597 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 598 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 598 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 599 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 600 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 601 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 602 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 603 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 608 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 609 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 612 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 613 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 795 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 796 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 797 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 798 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 799 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 800 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 801 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 807 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 810 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 811 in "default" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 814 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 815 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 816 in "default" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 821 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 822 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 823 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 824 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 825 in "default" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 887 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 897 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 911 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 924 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 938 in "default" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 939 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 940 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 941 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 950 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 950 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 951 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 951 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 952 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 952 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 953 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 953 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 954 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 954 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 955 in "default" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.9370.0>:ns_memcached:do_handle_call:369] Changed vbucket 955 state to active [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 1014 in "default" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 1015 in "default" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:38:19] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Producer) eq_tapq:replication_ns_1@10.22.64.9 - disconnected, keep alive for 300 seconds [ns_server:info] [2013-01-29 14:38:48] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.22' [ns_server:info] [2013-01-29 14:38:49] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[12,13,337,338,467,532,778,787,809,908,909,948], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.12', "default"} [error_logger:error] [2013-01-29 14:38:49] [ns_1@10.22.64.12:error_logger:ale_error_logger_handler:log_report:72] =========================SUPERVISOR REPORT========================= Supervisor: {local,'ns_vbm_sup-default'} Context: shutdown_error Reason: killed Offender: [{pid,<0.16185.249>}, {name, {child_id, [12,13,337,338,467,532,778,787,809,908,909,948], 'ns_1@10.22.64.9'}}, {mfargs, {ebucketmigrator_srv,start_link, [{"10.22.64.12",11209}, {"10.22.64.9",11209}, [{username,"default"}, {password,[]}, {vbuckets, [12,13,337,338,467,532,778,787,809,908,909, 948]}, {takeover,false}, {suffix,"ns_1@10.22.64.9"}, {passed_downstream_retriever, #Fun}]]}}, {restart_type,permanent}, {shutdown,60000}, {child_type,worker}] [ns_server:info] [2013-01-29 14:38:59] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.27' [ns_server:info] [2013-01-29 14:39:19] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[165,234,476,501,718,719,726,775,788,927,979], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.13', "default"} [ns_server:info] [2013-01-29 14:39:21] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.26' [ns_server:info] [2013-01-29 14:39:49] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[22,32,96,97,98,99,100,101,277,377,534], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.14', "default"} [ns_server:info] [2013-01-29 14:40:07] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.14' [ns_server:info] [2013-01-29 14:40:19] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[60,61,252,394,451,452,453,454,515,708,893], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.22', "default"} [ns_server:info] [2013-01-29 14:40:23] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.26' [ns_server:info] [2013-01-29 14:40:49] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[53,66,67,85,86,87,88,89,90,275,789], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.23', "default"} [ns_server:info] [2013-01-29 14:41:18] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.23' [ns_server:info] [2013-01-29 14:41:19] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[160,202,203,524,525,526,527,528,529,530,743], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.24', "default"} [ns_server:info] [2013-01-29 14:41:23] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.23' [ns_server:info] [2013-01-29 14:41:41] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.25' [ns_server:info] [2013-01-29 14:41:49] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[144,145,194,224,276,684,685,768,769,770,771,826], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.25', "default"} [ns_server:info] [2013-01-29 14:41:53] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[43,112,136,371,614,696,802,803,804,805,806,964], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.26', "default"} [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[2,217,218,219,412,413,671,702,945,1003,1017], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.27', "default"} [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 80 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 103 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 103 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 119 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 120 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 121 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 122 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 123 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 124 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 125 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 137 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 138 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 139 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 140 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 141 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 142 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 143 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 153 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 153 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 154 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 154 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 155 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 155 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 156 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 157 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 158 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 159 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 170 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 262 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 263 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 263 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 264 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 271 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 342 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 344 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 345 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 346 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 347 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 348 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 349 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 367 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 372 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 405 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 406 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 407 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 408 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 409 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 410 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 414 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 428 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 429 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 430 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 431 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 432 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 435 in "admatter" on 'ns_1@10.22.64.13' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 436 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 442 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 474 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 475 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 505 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 508 in "admatter" on 'ns_1@10.22.64.23' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 553 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 597 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 598 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 598 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 599 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 600 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 601 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 602 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 603 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 608 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 609 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 612 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 613 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 795 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 796 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 797 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 798 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 799 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 800 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 801 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 807 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 810 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 811 in "admatter" on 'ns_1@10.22.64.27' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 814 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 815 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 816 in "admatter" on 'ns_1@10.22.64.14' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 821 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 822 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 823 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 824 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 825 in "admatter" on 'ns_1@10.22.64.24' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 887 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 897 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 911 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 924 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 938 in "admatter" on 'ns_1@10.22.64.25' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 939 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 940 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 941 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 950 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 950 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 951 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 951 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 952 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 952 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 953 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 953 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 954 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 954 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 955 in "admatter" on 'ns_1@10.22.64.12' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.3730.0>:ns_memcached:do_handle_call:369] Changed vbucket 955 state to active [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 1014 in "admatter" on 'ns_1@10.22.64.22' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:<0.4488.244>:ns_janitor:do_sanify_chain:162] Setting vbucket 1015 in "admatter" on 'ns_1@10.22.64.26' from replica to active. [ns_server:info] [2013-01-29 14:41:54] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5167 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5167" [ns_server:info] [2013-01-29 14:41:55] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Producer) eq_tapq:replication_ns_1@10.22.64.9 - disconnected, keep alive for 300 seconds [ns_server:info] [2013-01-29 14:41:58] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5231 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5231" memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5246 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5246" [ns_server:info] [2013-01-29 14:41:58] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5278 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5278" [rebalance:warn] [2013-01-29 14:41:59] [ns_1@10.22.64.12:<0.9357.249>:ebucketmigrator_srv:do_confirm_sent_messages:337] Got error while trying to read close ack:{error,closed} [ns_server:info] [2013-01-29 14:41:59] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[12,13,337,338,467,532,778,787,809,908,909,948], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.12', "admatter"} [ns_server:info] [2013-01-29 14:41:59] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5446 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5446" memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5157 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5157" [ns_server:info] [2013-01-29 14:42:04] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5135 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5135" [ns_server:info] [2013-01-29 14:42:05] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5515 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5515" [user:info] [2013-01-29 14:42:06] [ns_1@10.22.64.12:ns_node_disco:ns_node_disco:handle_info:144] Node 'ns_1@10.22.64.12' saw that node 'ns_1@10.22.64.9' came up. [ns_server:info] [2013-01-29 14:42:06] [ns_1@10.22.64.12:ns_node_disco_events:ns_node_disco_log:handle_event:46] ns_node_disco_log: nodes changed: ['ns_1@10.22.64.12','ns_1@10.22.64.13', 'ns_1@10.22.64.14','ns_1@10.22.64.22', 'ns_1@10.22.64.23','ns_1@10.22.64.24', 'ns_1@10.22.64.25','ns_1@10.22.64.26', 'ns_1@10.22.64.27','ns_1@10.22.64.9'] [ns_server:info] [2013-01-29 14:42:07] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[22,32,96,97,98,99,100,101,277,377,534], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.14', "admatter"} [ns_server:info] [2013-01-29 14:42:12] [ns_1@10.22.64.12:ns_config_rep:ns_config_rep:do_pull:336] Pulling config from: 'ns_1@10.22.64.26' [ns_server:info] [2013-01-29 14:42:12] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5272 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5272" [ns_server:info] [2013-01-29 14:42:16] [ns_1@10.22.64.12:ns_port_memcached:ns_port_server:log:166] memcached<0.804.0>: TAP (Consumer) eq_tapq:anon_5263 - disconnected memcached<0.804.0>: Schedule cleanup of "eq_tapq:anon_5263" [ns_server:info] [2013-01-29 14:42:26] [ns_1@10.22.64.12:<0.4488.244>:ns_vbm_sup:kill_child:218] Stopped replicator:{child_id,[60,61,252,394,451,452,453,454,515,708,893], 'ns_1@10.22.64.9'} on {'ns_1@10.22.64.22', "admatter"} [user:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:idle:390] Failed over 'ns_1@10.22.64.9': ok [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [ns_server:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4488.244>:ns_orchestrator:handle_info:209] Skipping janitor in state janitor_running: {janitor_state, ["default"], <0.7050.855>} [user:info] [2013-01-29 14:42:27] [ns_1@10.22.64.12:<0.4494.244>:auto_failover:handle_info:241] Node ('ns_1@10.22.64.9') was automatically failovered. [{last_heard,{1359,499347,31623}},